feat: add durable safe retries with uncertainty tracking and SDK examples

internal/storage/mysql/retry_upgrade_integration_test.go

@@ -0,0 +1,116 @@
+package mysql
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	core "git.yuntuo.hk/k2safe/orbit/model"
+	mysqldriver "github.com/go-sql-driver/mysql"
+)
+
+// 在独立库中重建真实 v4，再走正式迁移入口；不改变主测试库的结构或迁移记录。
+func TestIntegrationSafeRetryUpgrade(t *testing.T) {
+	dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
+	if dsn == "" {
+		t.Skip("requires disposable MySQL with CREATE DATABASE permission")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	admin, err := Open(ctx, dsn, 2, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer admin.Close()
+	name := "orbit_retry_upgrade_" + string(executionID(t))
+	if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+name+"`"); err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		if _, err := admin.ExecContext(cleanup, "DROP DATABASE `"+name+"`"); err != nil {
+			t.Error(err)
+		}
+	}()
+	cfg, err := mysqldriver.ParseDSN(dsn)
+	if err != nil {
+		t.Fatal("invalid test DSN")
+	}
+	cfg.DBName = name
+	db, err := Open(ctx, cfg.FormatDSN(), 4, 2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	exec := func(q string, args ...any) {
+		t.Helper()
+		if _, err := db.ExecContext(ctx, q, args...); err != nil {
+			t.Fatal(err)
+		}
+	}
+	exec(`CREATE TABLE orbit_schema_migrations (version BIGINT UNSIGNED PRIMARY KEY,name VARCHAR(191) NOT NULL,checksum CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,dirty BOOLEAN NOT NULL,applied_at DATETIME(6) NULL) ENGINE=InnoDB`)
+	migrations, err := Migrations()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, m := range migrations {
+		if m.Version > 4 {
+			break
+		}
+		statements, err := splitSQL(m.SQL)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, statement := range statements {
+			exec(statement)
+		}
+		exec(`INSERT INTO orbit_schema_migrations(version,name,checksum,dirty,applied_at) VALUES (?,?,?,FALSE,UTC_TIMESTAMP(6))`, m.Version, m.Name, m.Checksum)
+	}
+	app, ns, queue := executionID(t), executionID(t), executionID(t)
+	exec(`INSERT INTO orbit_applications(id,name) VALUES (?,?)`, app, string(app))
+	exec(`INSERT INTO orbit_namespaces(id,application_id,name,environment) VALUES (?,?,?,'test')`, ns, app, string(ns))
+	exec(`INSERT INTO pulse_queues(namespace_id,id,name,max_concurrency,rate_per_second,policy) VALUES (?,?,?,4,10,'{}')`, ns, queue, string(queue))
+	policy := `{"retry":{"max_attempts":3,"initial_delay_ms":1000,"max_delay_ms":30000,"attempt_timeout_ms":5000}}`
+	ids := map[string]core.ID{}
+	for _, state := range []string{"NEEDS_REVIEW", "WAITING", "SUCCEEDED", "FAILED", "CANCELLED"} {
+		id := executionID(t)
+		ids[state] = id
+		var completed *time.Time
+		if state == "SUCCEEDED" || state == "FAILED" || state == "CANCELLED" {
+			now := time.Now().UTC()
+			completed = &now
+		}
+		exec(`INSERT INTO pulse_tasks(namespace_id,id,queue_id,state,handler,handler_version,payload_version,payload,policy_snapshot,available_at,deadline_at,attempt_timeout_ms,max_attempts,completed_at) VALUES (?,?,?,?,'object.prepare','v1','v1','{}',?,UTC_TIMESTAMP(6),DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 1 HOUR),5000,3,?)`, ns, id, queue, state, policy, completed)
+	}
+	// MySQL 原生 JSON 会规范化文本，以迁移前实际存储值核对冻结策略。
+	if err := db.QueryRowContext(ctx, `SELECT policy_snapshot FROM pulse_tasks WHERE namespace_id=? AND id=?`, ns, ids["WAITING"]).Scan(&policy); err != nil {
+		t.Fatal(err)
+	}
+	if err := Migrate(ctx, db); err != nil {
+		t.Fatal(err)
+	}
+	if err := CheckSchema(ctx, db); err != nil {
+		t.Fatal(err)
+	}
+	for state, id := range ids {
+		var gotState, gotPolicy string
+		var uncertain bool
+		var revision uint64
+		if err := db.QueryRowContext(ctx, `SELECT state,effect_uncertain,policy_snapshot,revision FROM pulse_tasks WHERE namespace_id=? AND id=?`, ns, id).Scan(&gotState, &uncertain, &gotPolicy, &revision); err != nil {
+			t.Fatal(err)
+		}
+		if gotState != state || uncertain != (state == "NEEDS_REVIEW") || gotPolicy != policy || revision != 1 {
+			t.Fatalf("upgrade changed frozen data or lost uncertainty for %s", state)
+		}
+	}
+	// 再次运行不重复修改回填或策略。
+	if err := Migrate(ctx, db); err != nil {
+		t.Fatal(err)
+	}
+	var count int
+	if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_tasks WHERE effect_uncertain=TRUE`).Scan(&count); err != nil || count != 1 {
+		t.Fatalf("upgrade is not idempotent: %d %v", count, err)
+	}
+}