feat: serialize keyed tasks across queues with durable fencing tokens

internal/storage/mysql/application_resources_upgrade_integration_test.go

@@ -63,9 +63,12 @@ func TestIntegrationApplicationResourcesUpgrade(t *testing.T) {
 	finished := retryStart(t, g)
 	retryFinish(t, g, finished, "SUCCEEDED", false)
 	assertApplicationCounts(t, f, 4, 2, 1)
-	// v10 只新增此表；其余列、索引与数据均是 v9 结构。
+	// 还原后续新增表和字段，得到真实 v9 结构；现有任务内容保持不变。
 	f.exec(`DROP TABLE orbit_application_resources`)
-	f.exec(`DELETE FROM orbit_schema_migrations WHERE version=10`)
+	f.exec(`ALTER TABLE pulse_tasks DROP CHECK ck_task_concurrency_key, DROP COLUMN concurrency_key, DROP COLUMN concurrency_key_digest`)
+	f.exec(`ALTER TABLE pulse_attempts DROP COLUMN concurrency_token`)
+	f.exec(`ALTER TABLE pulse_worker_sessions DROP COLUMN concurrency_keys`)
+	f.exec(`DELETE FROM orbit_schema_migrations WHERE version>=10`)
 	if err := Migrate(ctx, db); err != nil {
 		t.Fatal(err)
 	}

internal/storage/mysql/keyed_concurrency.go

@@ -0,0 +1,71 @@
+package mysql
+
+import (
+	"context"
+	"crypto/sha256"
+	"database/sql"
+	"errors"
+
+	"git.yuntuo.hk/k2safe/orbit/internal/fault"
+	core "git.yuntuo.hk/k2safe/orbit/model"
+)
+
+func concurrencyDigest(key string) *core.Digest {
+	if key == "" {
+		return nil
+	}
+	digest := core.Digest(sha256.Sum256(append([]byte("orbit-concurrency-key-v1\x00"), []byte(key)...)))
+	return &digest
+}
+
+// concurrencyKeyValue 在 Go 中判断空字节串，避免 MySQL 文本排序规则将某些字符视为空。
+func concurrencyKeyValue(key string) *string {
+	if key == "" {
+		return nil
+	}
+	return &key
+}
+
+// keyEligibility 在 LIMIT 前排除已占用资源，避免前 16 个同键任务挡住其他资源。
+// 调用者已锁应用额度与队列；命名空间中的所有领取都遵守此顺序。
+const keyEligibility = `(concurrency_key_digest IS NULL OR NOT EXISTS (
+ SELECT 1 FROM pulse_concurrency_limits k WHERE k.namespace_id=pulse_tasks.namespace_id
+ AND k.scope_key='key' AND k.key_digest=pulse_tasks.concurrency_key_digest AND k.used_permits>=k.max_permits))`
+
+// reserveConcurrencyKey 为一个显式资源键预占单个许可。零 token 表示未启用或暂不可用。
+// 序号使用资源行单调 revision；释放也递增，序号允许有间隔，禁止删除或重置资源行。
+func reserveConcurrencyKey(ctx context.Context, tx *sql.Tx, ns core.ID, key string) (core.ID, uint64, error) {
+	if key == "" {
+		return "", 0, nil
+	}
+	id, err := core.NewID()
+	if err != nil {
+		return "", 0, err
+	}
+	digest := concurrencyDigest(key)
+	_, err = tx.ExecContext(ctx, `INSERT INTO pulse_concurrency_limits(namespace_id,id,scope_key,key_digest,max_permits) VALUES (?,?,'key',?,1) ON DUPLICATE KEY UPDATE id=pulse_concurrency_limits.id`, ns, id, digest)
+	if err != nil {
+		return "", 0, err
+	}
+	var used, max uint32
+	var revision uint64
+	err = tx.QueryRowContext(ctx, `SELECT id,used_permits,max_permits,revision FROM pulse_concurrency_limits WHERE namespace_id=? AND scope_key='key' AND key_digest=? FOR UPDATE`, ns, digest).Scan(&id, &used, &max, &revision)
+	if err != nil {
+		return "", 0, err
+	}
+	if max != 1 {
+		return "", 0, errors.New("key permit limit must be one")
+	}
+	if used != 0 {
+		return id, 0, nil
+	}
+	// 为当前授权及后续释放各保留一次递增空间，不能溢出后重用旧序号。
+	if revision > ^uint64(0)-2 {
+		return "", 0, fault.Conflict
+	}
+	r, err := tx.ExecContext(ctx, `UPDATE pulse_concurrency_limits SET used_permits=1,revision=revision+1,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=? AND used_permits=0 AND revision=?`, ns, id, revision)
+	if err := exactlyOne(r, err); err != nil {
+		return "", 0, err
+	}
+	return id, revision + 1, nil
+}

internal/storage/mysql/keyed_concurrency_integration_test.go

@@ -0,0 +1,333 @@
+package mysql
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	core "git.yuntuo.hk/k2safe/orbit/model"
+	pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
+	"git.yuntuo.hk/k2safe/orbit/pulse/service"
+	"git.yuntuo.hk/k2safe/orbit/pulse/worker"
+)
+
+func keyedWorker(t *testing.T, f *executionFixture, queue core.ID, node *worker.Service) *executionFixture {
+	t.Helper()
+	g := *f
+	g.resources.QueueID = queue
+	g.a = node
+	g.hello.InstanceID = executionID(t)
+	g.hello.QueueID = queue
+	g.hello.ConcurrencyKeys = true
+	var err error
+	g.binding, _, err = node.Connect(g.ctx, g.token, g.hello)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return &g
+}
+func keyedSubmit(t *testing.T, f *executionFixture, key string, max uint32) service.TaskView {
+	t.Helper()
+	out, err := f.tasks.Submit(f.ctx, f.token, string(executionID(t)), service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`), ConcurrencyKey: key, Retry: pulse.RetryPolicy{MaxAttempts: max, InitialDelayMS: 1000, MaxDelayMS: 30000, AttemptTimeoutMS: 30000}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return out.Task
+}
+func keyedUsed(t *testing.T, f *executionFixture, key string) uint32 {
+	t.Helper()
+	var used uint32
+	if err := f.db.QueryRowContext(f.ctx, `SELECT used_permits FROM pulse_concurrency_limits WHERE namespace_id=? AND scope_key='key' AND key_digest=?`, f.resources.NamespaceID, concurrencyDigest(key)).Scan(&used); err != nil {
+		t.Fatal(err)
+	}
+	return used
+}
+func noKeyClaim(t *testing.T, f *executionFixture) {
+	t.Helper()
+	if a, err := f.a.Claim(f.ctx, f.token, f.binding); err != nil || a != nil {
+		t.Fatal("expected no eligible claim", a, err)
+	}
+}
+func TestIntegrationKeyedConcurrency(t *testing.T) {
+	dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
+	if dsn == "" {
+		t.Skip("requires disposable MySQL")
+	}
+	ctx, stop := context.WithTimeout(context.Background(), 70*time.Second)
+	defer stop()
+	db, err := Open(ctx, dsn, 32, 8)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	if err := Migrate(ctx, db); err != nil {
+		t.Fatal(err)
+	}
+	fixture := func(t *testing.T) *executionFixture {
+		f := newExecutionFixture(t, ctx, db)
+		return keyedWorker(t, f, f.resources.QueueID, f.a)
+	}
+	t.Run("legacy_workers_and_tracking_keys_keep_their_contract", func(t *testing.T) {
+		old := newExecutionFixture(t, ctx, db)
+		keyed := keyedSubmit(t, old, "resource/one", 1)
+		noKeyClaim(t, old)
+		if old.get(keyed.ID).AttemptCount != 0 {
+			t.Fatal("legacy worker consumed keyed attempt")
+		}
+		for i := 0; i < 2; i++ {
+			if _, err := old.tasks.Submit(ctx, old.token, string(executionID(t)), service.SubmitInput{QueueID: old.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`), BusinessKey: "same-tracking-key"}); err != nil {
+				t.Fatal(err)
+			}
+		}
+		a, _ := old.claim()
+		b, _ := old.claim()
+		if a.ConcurrencyToken != 0 || b.ConcurrencyToken != 0 {
+			t.Fatal("tracking key changed execution semantics")
+		}
+		f := keyedWorker(t, old, old.resources.QueueID, old.b)
+		a, _ = f.claim()
+		if a.Task.ID != keyed.ID || a.ConcurrencyToken == 0 {
+			t.Fatal("opted-in worker did not receive key token")
+		}
+		hello := f.hello
+		hello.ConcurrencyKeys = false
+		_, _, err := f.a.Connect(ctx, f.token, hello)
+		expectWorkerCode(t, err, "STATE_CONFLICT")
+		// 旧 flag=false 摘要必须继续允许旧进程重连。
+		if _, _, err := old.a.Connect(ctx, old.token, old.hello); err != nil {
+			t.Fatal("legacy registration digest changed", err)
+		}
+	})
+	t.Run("cross_queue_race_single_permit_and_monotonic_tokens", func(t *testing.T) {
+		f := fixture(t)
+		q, err := f.tasks.CreateQueue(ctx, f.token, "second-queue", service.QueueInput{Name: "second", MaxConcurrency: 4, RatePerSecond: 100})
+		if err != nil {
+			t.Fatal(err)
+		}
+		g := keyedWorker(t, f, q.Queue.ID, f.b)
+		keyedSubmit(t, f, "object/42", 1)
+		keyedSubmit(t, g, "object/42", 1)
+		type claim struct {
+			f   *executionFixture
+			a   *worker.Assignment
+			err error
+		}
+		ch := make(chan claim, 2)
+		var wg sync.WaitGroup
+		for _, target := range []*executionFixture{f, g} {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				a, err := target.a.Claim(ctx, target.token, target.binding)
+				ch <- claim{target, a, err}
+			}()
+		}
+		wg.Wait()
+		close(ch)
+		var won claim
+		for v := range ch {
+			if v.err != nil {
+				t.Fatal(v.err)
+			}
+			if v.a != nil {
+				if won.a != nil {
+					t.Fatal("same key dispatched twice")
+				}
+				won = v
+			}
+		}
+		if won.a == nil || won.a.ConcurrencyToken == 0 || keyedUsed(t, f, "object/42") != 1 {
+			t.Fatal("missing key permit")
+		}
+		assertApplicationCounts(t, f, 2, 1, 1)
+		if f.count("pulse_permits", "attempt_id=? AND released_at IS NULL", won.a.AttemptID) != 2 {
+			t.Fatal("expected queue and key permits")
+		}
+		task := won.a.Task
+		if _, err := won.f.tasks.CancelTask(ctx, won.f.token, task.ID, "cancel", service.CancelInput{ExpectedRevision: task.Revision, Reason: "release unstarted"}); err != nil {
+			t.Fatal(err)
+		}
+		other := f
+		if won.f == f {
+			other = g
+		}
+		next, ref := other.claim()
+		if next.ConcurrencyToken <= won.a.ConcurrencyToken {
+			t.Fatal("key sequence reset across tasks")
+		}
+		message := executionID(t)
+		if _, err := other.a.Start(ctx, other.token, other.binding, message, ref); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := other.a.Start(ctx, other.token, other.binding, message, ref); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := other.a.Renew(ctx, other.token, other.binding, executionID(t), ref); err != nil {
+			t.Fatal(err)
+		}
+		history, err := other.tasks.GetTaskAttempt(ctx, other.token, ref.TaskID, ref.AttemptID)
+		if err != nil || history.ConcurrencyToken != next.ConcurrencyToken {
+			t.Fatal("token not preserved in history", err)
+		}
+		retryFinish(t, other, ref, "SUCCEEDED", false)
+		assertApplicationCounts(t, f, 0, 0, 0)
+		if keyedUsed(t, f, "object/42") != 0 {
+			t.Fatal("key permit leaked")
+		}
+	})
+	t.Run("busy_keys_do_not_hide_later_keys_or_other_namespaces", func(t *testing.T) {
+		f := fixture(t)
+		keyedSubmit(t, f, "busy", 1)
+		f.claim()
+		for i := 0; i < 20; i++ {
+			keyedSubmit(t, f, "busy", 1)
+		}
+		free := keyedSubmit(t, f, "free", 1)
+		a, _ := f.claim()
+		if a.Task.ID != free.ID {
+			t.Fatal("blocked prefix hid free key")
+		}
+		if f.count("pulse_tasks", "concurrency_key='busy' AND state='WAITING' AND attempt_count<>0") != 0 {
+			t.Fatal("blocked keys consumed attempts")
+		}
+		g := newExecutionFixture(t, ctx, db, f.resources.ApplicationID)
+		g = keyedWorker(t, g, g.resources.QueueID, g.a)
+		keyedSubmit(t, g, "busy", 1)
+		a, _ = g.claim()
+		if a.ConcurrencyToken == 0 {
+			t.Fatal("namespace isolation lost")
+		}
+	})
+	t.Run("unknown_late_result_and_retry_release_exactly_once", func(t *testing.T) {
+		f := fixture(t)
+		task := keyedSubmit(t, f, "resource/retry", 2)
+		first, ref := f.claim()
+		if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
+			t.Fatal(err)
+		}
+		retryFinish(t, f, ref, "FAILED", true)
+		if keyedUsed(t, f, "resource/retry") != 0 {
+			t.Fatal("retry holds key while waiting")
+		}
+		makeRetryDue(f, task.ID)
+		second, ref2 := f.claim()
+		if second.ConcurrencyToken <= first.ConcurrencyToken {
+			t.Fatal("retry reused token")
+		}
+		if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref2); err != nil {
+			t.Fatal(err)
+		}
+		retryFinish(t, f, ref2, "UNKNOWN", false)
+		assertApplicationCounts(t, f, 1, 0, 0)
+		if f.get(task.ID).State != pulse.TaskNeedsReview {
+			t.Fatal("unknown result lost")
+		}
+		keyedSubmit(t, f, "resource/retry", 1)
+		third, ref3 := f.claim()
+		if third.ConcurrencyToken <= second.ConcurrencyToken {
+			t.Fatal("key epoch did not advance")
+		}
+		late, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref2, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)})
+		if err != nil || late.Receipt.Disposition != "EVIDENCE_ONLY" {
+			t.Fatal("late result applied", err)
+		}
+		if keyedUsed(t, f, "resource/retry") != 1 {
+			t.Fatal("late result released another attempt's key")
+		}
+		f.expire(ref3)
+		if _, err := f.storeB.Recover(ctx, 100); err != nil {
+			t.Fatal(err)
+		}
+		if keyedUsed(t, f, "resource/retry") != 0 {
+			t.Fatal("expiry leaked key")
+		}
+		assertApplicationCounts(t, f, 1, 0, 0)
+	})
+	t.Run("claim_rollback_and_exact_utf8_key_identity", func(t *testing.T) {
+		f := fixture(t)
+		keyedSubmit(t, f, "asset/A", 1)
+		trigger := "test_key_event_" + string(executionID(t))
+		f.exec("CREATE TRIGGER " + trigger + " BEFORE INSERT ON orbit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected claim failure'")
+		out, err := f.a.Claim(ctx, f.token, f.binding)
+		f.exec("DROP TRIGGER " + trigger)
+		if err == nil || out != nil {
+			t.Fatal("failed claim exposed permit")
+		}
+		assertApplicationCounts(t, f, 1, 0, 0)
+		if f.count("pulse_attempts", "1=1") != 0 || f.count("pulse_permits", "1=1") != 0 || f.count("pulse_concurrency_limits", "scope_key='key'") != 0 {
+			t.Fatal("claim did not fully roll back")
+		}
+		f.claim()
+		for _, key := range []string{"asset/a", "asset/é", "asset/e\u0301"} {
+			keyedSubmit(t, f, key, 1)
+			f.claim()
+		}
+		if f.count("pulse_concurrency_limits", "scope_key='key' AND used_permits=1") != 4 {
+			t.Fatal("resource keys normalized unexpectedly")
+		}
+	})
+	t.Run("sequence_exhaustion_is_not_allowed_to_wrap", func(t *testing.T) {
+		f := fixture(t)
+		keyedSubmit(t, f, "exhausted", 1)
+		ref := retryStart(t, f)
+		retryFinish(t, f, ref, "SUCCEEDED", false)
+		f.exec(`UPDATE pulse_concurrency_limits SET revision=18446744073709551613 WHERE namespace_id=? AND scope_key='key'`, f.resources.NamespaceID)
+		keyedSubmit(t, f, "exhausted", 1)
+		a, ref := f.claim()
+		if a.ConcurrencyToken != ^uint64(0)-1 {
+			t.Fatal("high token truncated")
+		}
+		if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
+			t.Fatal(err)
+		}
+		retryFinish(t, f, ref, "SUCCEEDED", false)
+		waiting := keyedSubmit(t, f, "exhausted", 1)
+		_, err := f.a.Claim(ctx, f.token, f.binding)
+		expectWorkerCode(t, err, "STATE_CONFLICT")
+		if f.get(waiting.ID).AttemptCount != 0 {
+			t.Fatal("exhausted sequence consumed task attempt")
+		}
+	})
+	t.Run("nonempty_keys_do_not_use_text_collation_for_null", func(t *testing.T) {
+		f := fixture(t)
+		for _, key := range []string{"\u0301", "\u200b"} {
+			task := keyedSubmit(t, f, key, 1)
+			if task.ConcurrencyKey != key {
+				t.Fatal("nonempty key collapsed by database collation")
+			}
+			a, _ := f.claim()
+			if a.Task.ID != task.ID || a.Task.ConcurrencyKey != key || a.ConcurrencyToken == 0 {
+				t.Fatal("exact key did not reach assignment")
+			}
+		}
+	})
+	t.Run("schedule_template_and_key_validation", func(t *testing.T) {
+		f := fixture(t)
+		sf := newScheduleFixture(t, f)
+		def := sf.definition()
+		def.Task.ConcurrencyKey = "schedule/resource"
+		v := sf.create(def)
+		sf.due(v, 2*time.Second)
+		sf.tick()
+		a, ref := f.claim()
+		if a.Task.ConcurrencyKey != def.Task.ConcurrencyKey || a.ConcurrencyToken == 0 {
+			t.Fatal("schedule omitted frozen key")
+		}
+		noKeyClaim(t, f)
+		if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
+			t.Fatal(err)
+		}
+		retryFinish(t, f, ref, "SUCCEEDED", false)
+		b, _ := f.claim()
+		if b.ConcurrencyToken <= a.ConcurrencyToken {
+			t.Fatal("schedule reused key token")
+		}
+		for _, key := range []string{" leading", "trailing ", "line\nfeed", "\u0000"} {
+			_, err := f.tasks.Submit(ctx, f.token, string(executionID(t)), service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`), ConcurrencyKey: key})
+			expectWorkerCode(t, err, "INVALID_ARGUMENT")
+		}
+	})
+}

internal/storage/mysql/migrations/000011_keyed_concurrency.up.sql

@@ -0,0 +1,10 @@
+-- 显式资源键互斥：旧业务关联键语义不变；旧任务、会话与尝试默认不启用。
+-- 不使用外键，归属、许可与单调序号在任务事务内维护。
+ALTER TABLE pulse_tasks
+ ADD COLUMN concurrency_key VARCHAR(191) NULL COMMENT '显式互斥资源键，同命名空间跨队列共享；空表示不启用，不代替业务幂等',
+ ADD COLUMN concurrency_key_digest BINARY(32) NULL COMMENT '资源键精确 UTF-8 字节的域分隔 SHA-256；与资源键同时为空或有值',
+ ADD CONSTRAINT ck_task_concurrency_key CHECK ((concurrency_key IS NULL) = (concurrency_key_digest IS NULL));
+ALTER TABLE pulse_attempts
+ ADD COLUMN concurrency_token BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '当前资源键的单调许可序号，零表示未启用；重连不变，新尝试递增，不可单独证明执行授权';
+ALTER TABLE pulse_worker_sessions
+ ADD COLUMN concurrency_keys BOOLEAN NOT NULL DEFAULT FALSE COMMENT '是否明确支持资源键互斥与序号交付；未声明的旧 Worker 不领取带资源键任务';

internal/storage/mysql/no_foreign_keys_integration_test.go

@@ -133,12 +133,18 @@ func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
 		if err := rows.Scan(scan...); err != nil {
 			t.Fatal(err)
 		}
-		result := make([]string, len(cols))
+		result := make([]string, 0, len(cols))
 		for i, value := range values {
+			if cols[i] == "concurrency_key" || cols[i] == "concurrency_key_digest" {
+				if value.Valid {
+					t.Fatal("legacy task unexpectedly enabled keyed concurrency")
+				}
+				continue
+			}
 			if value.Valid {
-				result[i] = cols[i] + "=" + value.String
+				result = append(result, cols[i]+"="+value.String)
 			} else {
-				result[i] = cols[i] + "=<NULL>"
+				result = append(result, cols[i]+"=<NULL>")
 			}
 		}
 		if rows.Next() || rows.Err() != nil {

internal/storage/mysql/schedule_tick.go

@@ -224,7 +224,7 @@ func (s *ScheduleStore) saveOccurrence(ctx context.Context, tx *sql.Tx, v schedu
 		if err != nil {
 			return err
 		}
-		_, err = tx.ExecContext(ctx, `INSERT INTO pulse_tasks(namespace_id,id,queue_id,handler,handler_version,payload_version,payload,policy_snapshot,recovery_safety,business_key,owner_kind,owner_id,available_at,deadline_at,attempt_timeout_ms,max_attempts,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,NULLIF(?,''),'schedule',?,?,?,?,?,?,?)`, v.NamespaceID, value, v.QueueID, in.Handler, in.HandlerVersion, in.PayloadVersion, string(in.Payload), string(policy), in.RecoverySafety, in.BusinessKey, v.ID, now, now.Add(time.Duration(in.TaskTimeoutMS)*time.Millisecond), in.Retry.AttemptTimeoutMS, in.Retry.MaxAttempts, now, now)
+		_, err = tx.ExecContext(ctx, `INSERT INTO pulse_tasks(namespace_id,id,queue_id,handler,handler_version,payload_version,payload,policy_snapshot,recovery_safety,business_key,concurrency_key,concurrency_key_digest,owner_kind,owner_id,available_at,deadline_at,attempt_timeout_ms,max_attempts,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,NULLIF(?,''),?,?,'schedule',?,?,?,?,?,?,?)`, v.NamespaceID, value, v.QueueID, in.Handler, in.HandlerVersion, in.PayloadVersion, string(in.Payload), string(policy), in.RecoverySafety, in.BusinessKey, concurrencyKeyValue(in.ConcurrencyKey), concurrencyDigest(in.ConcurrencyKey), v.ID, now, now.Add(time.Duration(in.TaskTimeoutMS)*time.Millisecond), in.Retry.AttemptTimeoutMS, in.Retry.MaxAttempts, now, now)
 		if err != nil {
 			return err
 		}

internal/storage/mysql/task_history.go

@@ -91,11 +91,11 @@ func historyAttemptExists(ctx context.Context, tx *sql.Tx, ns, task, attempt cor
 	return err
 }
 
-const attemptHistoryColumns = `a.id,a.task_id,a.epoch,a.state,a.revision,a.lease_until,a.authorized_at,a.ended_at,COALESCE(a.error_code,''),a.retryable,a.created_at,a.updated_at`
+const attemptHistoryColumns = `a.id,a.task_id,a.epoch,a.state,a.revision,a.lease_until,a.authorized_at,a.ended_at,COALESCE(a.error_code,''),a.retryable,a.created_at,a.updated_at,a.concurrency_token`
 
 func scanAttemptSummary(row interface{ Scan(...any) error }) (service.AttemptSummary, error) {
 	var a service.AttemptSummary
-	err := row.Scan(&a.ID, &a.TaskID, &a.Epoch, &a.State, &a.Revision, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Retryable, &a.CreatedAt, &a.UpdatedAt)
+	err := row.Scan(&a.ID, &a.TaskID, &a.Epoch, &a.State, &a.Revision, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Retryable, &a.CreatedAt, &a.UpdatedAt, &a.ConcurrencyToken)
 	return a, err
 }
 func (s *TaskStore) ListTaskAttempts(ctx context.Context, t auth.Token, task core.ID, in service.HistoryPageInput) (service.AttemptPage, error) {
@@ -148,7 +148,7 @@ func (s *TaskStore) GetTaskAttempt(ctx context.Context, t auth.Token, task, atte
 			return err
 		}
 		var result []byte
-		err := tx.QueryRowContext(ctx, `SELECT `+attemptHistoryColumns+`,a.session_id,w.worker_instance_id,a.result FROM pulse_attempts a JOIN pulse_worker_sessions w ON w.namespace_id=a.namespace_id AND w.id=a.session_id WHERE a.namespace_id=? AND a.task_id=? AND a.id=?`, t.NamespaceID, task, attempt).Scan(&out.ID, &out.TaskID, &out.Epoch, &out.State, &out.Revision, &out.LeaseUntil, &out.AuthorizedAt, &out.EndedAt, &out.ErrorCode, &out.Retryable, &out.CreatedAt, &out.UpdatedAt, &out.SessionID, &out.WorkerInstanceID, &result)
+		err := tx.QueryRowContext(ctx, `SELECT `+attemptHistoryColumns+`,a.session_id,w.worker_instance_id,a.result FROM pulse_attempts a JOIN pulse_worker_sessions w ON w.namespace_id=a.namespace_id AND w.id=a.session_id WHERE a.namespace_id=? AND a.task_id=? AND a.id=?`, t.NamespaceID, task, attempt).Scan(&out.ID, &out.TaskID, &out.Epoch, &out.State, &out.Revision, &out.LeaseUntil, &out.AuthorizedAt, &out.EndedAt, &out.ErrorCode, &out.Retryable, &out.CreatedAt, &out.UpdatedAt, &out.ConcurrencyToken, &out.SessionID, &out.WorkerInstanceID, &result)
 		if errors.Is(err, sql.ErrNoRows) {
 			return fault.NotFound
 		}

internal/storage/mysql/task_history_integration_test.go

@@ -43,8 +43,17 @@ func TestIntegrationTaskHistory(t *testing.T) {
 		t.Fatal(err)
 	}
 	f.expire(first)
-	if _, err := f.storeA.Recover(ctx, 10); err != nil {
-		t.Fatal(err)
+	// 扫描按全库到期顺序分批；完整套件可能留下比本任务更早到期的其他夹具。
+	for round := 0; ; round++ {
+		if _, err := f.storeA.Recover(ctx, 10); err != nil {
+			t.Fatal(err)
+		}
+		if f.get(task.ID).State == "WAITING" {
+			break
+		}
+		if round == 19 {
+			t.Fatal("history fixture was not recovered within bounded batches")
+		}
 	}
 	f.exec(`UPDATE pulse_tasks SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, task.ID)
 	_, second := f.claim()

internal/storage/mysql/task_store.go

@@ -195,8 +195,8 @@ func (s *TaskStore) Submit(ctx context.Context, token auth.Token, key string, in
 		if err := reserveQueueTask(ctx, tx, token.NamespaceID, in.QueueID); err != nil {
 			return err
 		}
-		_, err = tx.ExecContext(ctx, `INSERT INTO pulse_tasks(namespace_id,id,queue_id,handler,handler_version,payload_version,payload,policy_snapshot,recovery_safety,business_key,available_at,start_deadline_at,deadline_at,attempt_timeout_ms,max_attempts,created_at,updated_at)
-            VALUES (?,?,?,?,?,?,?,?,?,NULLIF(?,''),?,?,?,?,?,?,?)`, token.NamespaceID, taskID, in.QueueID, in.Handler, in.HandlerVersion, in.PayloadVersion, string(in.Payload), string(policy), in.RecoverySafety, in.BusinessKey, available, in.StartDeadlineAt, deadline, in.Retry.AttemptTimeoutMS, in.Retry.MaxAttempts, now, now)
+		_, err = tx.ExecContext(ctx, `INSERT INTO pulse_tasks(namespace_id,id,queue_id,handler,handler_version,payload_version,payload,policy_snapshot,recovery_safety,business_key,concurrency_key,concurrency_key_digest,available_at,start_deadline_at,deadline_at,attempt_timeout_ms,max_attempts,created_at,updated_at)
+            VALUES (?,?,?,?,?,?,?,?,?,NULLIF(?,''),?,?,?,?,?,?,?,?,?)`, token.NamespaceID, taskID, in.QueueID, in.Handler, in.HandlerVersion, in.PayloadVersion, string(in.Payload), string(policy), in.RecoverySafety, in.BusinessKey, concurrencyKeyValue(in.ConcurrencyKey), concurrencyDigest(in.ConcurrencyKey), available, in.StartDeadlineAt, deadline, in.Retry.AttemptTimeoutMS, in.Retry.MaxAttempts, now, now)
 		if err != nil {
 			return err
 		}
@@ -215,14 +215,14 @@ func (s *TaskStore) Submit(ctx context.Context, token auth.Token, key string, in
 	return out, err
 }
 
-const taskColumns = `id,namespace_id,queue_id,state,revision,handler,handler_version,payload_version,payload,result,COALESCE(business_key,''),available_at,start_deadline_at,deadline_at,recovery_safety,policy_snapshot,attempt_count,cancel_requested_at,completed_at,created_at,updated_at,effect_uncertain`
+const taskColumns = `id,namespace_id,queue_id,state,revision,handler,handler_version,payload_version,payload,result,COALESCE(business_key,''),COALESCE(concurrency_key,''),available_at,start_deadline_at,deadline_at,recovery_safety,policy_snapshot,attempt_count,cancel_requested_at,completed_at,created_at,updated_at,effect_uncertain`
 
 type scanner interface{ Scan(...any) error }
 
 func scanTask(row scanner) (service.TaskView, error) {
 	var v service.TaskView
 	var policy, payload, result []byte
-	err := row.Scan(&v.ID, &v.NamespaceID, &v.QueueID, &v.State, &v.Revision, &v.Handler, &v.HandlerVersion, &v.PayloadVersion, &payload, &result, &v.BusinessKey, &v.AvailableAt, &v.StartDeadlineAt, &v.DeadlineAt, &v.RecoverySafety, &policy, &v.AttemptCount, &v.CancelRequestedAt, &v.CompletedAt, &v.CreatedAt, &v.UpdatedAt, &v.EffectUncertain)
+	err := row.Scan(&v.ID, &v.NamespaceID, &v.QueueID, &v.State, &v.Revision, &v.Handler, &v.HandlerVersion, &v.PayloadVersion, &payload, &result, &v.BusinessKey, &v.ConcurrencyKey, &v.AvailableAt, &v.StartDeadlineAt, &v.DeadlineAt, &v.RecoverySafety, &policy, &v.AttemptCount, &v.CancelRequestedAt, &v.CompletedAt, &v.CreatedAt, &v.UpdatedAt, &v.EffectUncertain)
 	if err != nil {
 		return v, err
 	}
@@ -272,7 +272,7 @@ func (s *TaskStore) ListTasks(ctx context.Context, token auth.Token, in service.
 		if !p.AllowsQueue(in.QueueID) {
 			return fault.Forbidden
 		}
-		q := `SELECT id,queue_id,state,revision,handler,handler_version,COALESCE(business_key,''),available_at,deadline_at,attempt_count,created_at,updated_at,effect_uncertain FROM pulse_tasks WHERE namespace_id=? AND queue_id=?`
+		q := `SELECT id,queue_id,state,revision,handler,handler_version,COALESCE(business_key,''),COALESCE(concurrency_key,''),available_at,deadline_at,attempt_count,created_at,updated_at,effect_uncertain FROM pulse_tasks WHERE namespace_id=? AND queue_id=?`
 		args := []any{token.NamespaceID, in.QueueID}
 		if in.State != "" {
 			q += " AND state=?"
@@ -296,7 +296,7 @@ func (s *TaskStore) ListTasks(ctx context.Context, token auth.Token, in service.
 		defer rows.Close()
 		for rows.Next() {
 			var v service.TaskSummary
-			if err := rows.Scan(&v.ID, &v.QueueID, &v.State, &v.Revision, &v.Handler, &v.HandlerVersion, &v.BusinessKey, &v.AvailableAt, &v.DeadlineAt, &v.AttemptCount, &v.CreatedAt, &v.UpdatedAt, &v.EffectUncertain); err != nil {
+			if err := rows.Scan(&v.ID, &v.QueueID, &v.State, &v.Revision, &v.Handler, &v.HandlerVersion, &v.BusinessKey, &v.ConcurrencyKey, &v.AvailableAt, &v.DeadlineAt, &v.AttemptCount, &v.CreatedAt, &v.UpdatedAt, &v.EffectUncertain); err != nil {
 				return err
 			}
 			out.Items = append(out.Items, v)

internal/storage/mysql/worker_execution.go

@@ -148,7 +148,11 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
 		if len(matches) == 0 {
 			return nil
 		}
-		rows, err := tx.QueryContext(ctx, `SELECT id,gate_id FROM pulse_tasks WHERE namespace_id=? AND queue_id=? AND state='WAITING' AND available_at<=UTC_TIMESTAMP(6) AND deadline_at>UTC_TIMESTAMP(6) AND (start_deadline_at IS NULL OR start_deadline_at>UTC_TIMESTAMP(6)) AND attempt_count<max_attempts AND (gate_id IS NULL OR EXISTS (SELECT 1 FROM pulse_execution_gates g WHERE g.namespace_id=pulse_tasks.namespace_id AND g.id=pulse_tasks.gate_id AND g.paused=FALSE)) AND (`+strings.Join(matches, " OR ")+`) ORDER BY available_at,priority DESC,id LIMIT 16`, args...)
+		keyFilter := keyEligibility
+		if !state.concurrencyKeys {
+			keyFilter = "concurrency_key_digest IS NULL"
+		}
+		rows, err := tx.QueryContext(ctx, `SELECT id,gate_id FROM pulse_tasks WHERE namespace_id=? AND queue_id=? AND state='WAITING' AND available_at<=UTC_TIMESTAMP(6) AND deadline_at>UTC_TIMESTAMP(6) AND (start_deadline_at IS NULL OR start_deadline_at>UTC_TIMESTAMP(6)) AND attempt_count<max_attempts AND (gate_id IS NULL OR EXISTS (SELECT 1 FROM pulse_execution_gates g WHERE g.namespace_id=pulse_tasks.namespace_id AND g.id=pulse_tasks.gate_id AND g.paused=FALSE)) AND (`+strings.Join(matches, " OR ")+`) AND `+keyFilter+` ORDER BY available_at,priority DESC,id LIMIT 16`, args...)
 		if err != nil {
 			return err
 		}
@@ -203,6 +207,13 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
 			if task.epoch == ^uint64(0) {
 				return fault.Conflict
 			}
+			keyLimitID, concurrencyToken, err := reserveConcurrencyKey(ctx, tx, token.NamespaceID, task.view.ConcurrencyKey)
+			if err != nil {
+				return err
+			}
+			if task.view.ConcurrencyKey != "" && concurrencyToken == 0 {
+				continue
+			}
 			attempt, err := core.NewID()
 			if err != nil {
 				return err
@@ -215,7 +226,7 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
 			if task.view.StartDeadlineAt != nil {
 				until = minimumTime(until, *task.view.StartDeadlineAt)
 			}
-			_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempts(namespace_id,id,task_id,epoch,session_id,token_digest,lease_until) VALUES (?,?,?,?,?,?,?)`, token.NamespaceID, attempt, locked, task.epoch+1, b.SessionID, digest, until)
+			_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempts(namespace_id,id,task_id,epoch,session_id,token_digest,lease_until,concurrency_token) VALUES (?,?,?,?,?,?,?,?)`, token.NamespaceID, attempt, locked, task.epoch+1, b.SessionID, digest, until, concurrencyToken)
 			if err != nil {
 				return err
 			}
@@ -227,6 +238,15 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
 			if err != nil {
 				return err
 			}
+			if keyLimitID != "" {
+				id, err := core.NewID()
+				if err != nil {
+					return err
+				}
+				if _, err := tx.ExecContext(ctx, `INSERT INTO pulse_permits(namespace_id,id,limit_id,attempt_id,lease_until) VALUES (?,?,?,?,?)`, token.NamespaceID, id, keyLimitID, attempt, until); err != nil {
+					return err
+				}
+			}
 			changed, err := tx.ExecContext(ctx, `UPDATE pulse_concurrency_limits SET used_permits=used_permits+1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND used_permits<max_permits`, now, token.NamespaceID, limit)
 			if err := exactlyOne(changed, err); err != nil {
 				return err
@@ -245,7 +265,7 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
 			if err := emitEvent(ctx, tx, token.NamespaceID, locked, view.Revision, "task", "task.leased", map[string]any{"attempt_id": attempt, "epoch": fmt.Sprint(task.epoch + 1), "session_id": b.SessionID, "lease_until": until}); err != nil {
 				return err
 			}
-			assignment = &worker.Assignment{Task: view, AttemptID: attempt, Epoch: task.epoch + 1, LeaseSecret: secret, LeaseUntil: until, ServerTime: now}
+			assignment = &worker.Assignment{ConcurrencyToken: concurrencyToken, Task: view, AttemptID: attempt, Epoch: task.epoch + 1, LeaseSecret: secret, LeaseUntil: until, ServerTime: now}
 			return nil
 		}
 		return nil
@@ -328,7 +348,7 @@ func (s *ExecutionStore) lockExecution(ctx context.Context, tx *sql.Tx, token au
 func readAttempt(ctx context.Context, tx *sql.Tx, ns, id core.ID) (pulse.Attempt, error) {
 	var a pulse.Attempt
 	a.NamespaceID = ns
-	err := tx.QueryRowContext(ctx, `SELECT id,revision,task_id,epoch,session_id,token_digest,state,lease_until,authorized_at,ended_at,error_code,retryable,result,result_digest FROM pulse_attempts WHERE namespace_id=? AND id=? FOR UPDATE`, ns, id).Scan(&a.ID, &a.Revision, &a.TaskID, &a.Epoch, &a.SessionID, &a.TokenDigest, &a.State, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Retryable, &a.Result, &a.ResultDigest)
+	err := tx.QueryRowContext(ctx, `SELECT id,revision,task_id,epoch,session_id,token_digest,state,lease_until,authorized_at,ended_at,error_code,retryable,result,result_digest,concurrency_token FROM pulse_attempts WHERE namespace_id=? AND id=? FOR UPDATE`, ns, id).Scan(&a.ID, &a.Revision, &a.TaskID, &a.Epoch, &a.SessionID, &a.TokenDigest, &a.State, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Retryable, &a.Result, &a.ResultDigest, &a.ConcurrencyToken)
 	if errors.Is(err, sql.ErrNoRows) {
 		err = fault.NotFound
 	}

internal/storage/mysql/worker_sessions.go

@@ -50,12 +50,13 @@ func (s *ExecutionStore) transaction(ctx context.Context, token auth.Token, acti
 }
 
 type sessionState struct {
-	queue        core.ID
-	capacity     uint32
-	sequence     uint64
-	draining     bool
-	expires      time.Time
-	capabilities []worker.Capability
+	queue           core.ID
+	capacity        uint32
+	sequence        uint64
+	draining        bool
+	expires         time.Time
+	capabilities    []worker.Capability
+	concurrencyKeys bool
 }
 
 func (s *ExecutionStore) welcome(b worker.Binding, state sessionState, now time.Time) worker.Welcome {
@@ -69,9 +70,10 @@ func (s *ExecutionStore) Connect(ctx context.Context, token auth.Token, in worke
 		return binding, out, err
 	}
 	registration, err := requestDigest(struct {
-		Queue        core.ID
-		Capabilities []worker.Capability
-	}{in.QueueID, in.Capabilities})
+		Queue           core.ID
+		Capabilities    []worker.Capability
+		ConcurrencyKeys bool `json:"ConcurrencyKeys,omitempty"`
+	}{in.QueueID, in.Capabilities, in.ConcurrencyKeys})
 	if err != nil {
 		return binding, out, err
 	}
@@ -116,12 +118,12 @@ func (s *ExecutionStore) Connect(ctx context.Context, token auth.Token, in worke
 			return err
 		}
 		binding = worker.Binding{InstanceID: in.InstanceID, SessionID: session, Epoch: epoch + 1, Proof: proof}
-		state := sessionState{queue: in.QueueID, capacity: in.Capacity, draining: draining, expires: now.Add(s.options.SessionTTL), capabilities: in.Capabilities}
+		state := sessionState{queue: in.QueueID, capacity: in.Capacity, draining: draining, expires: now.Add(s.options.SessionTTL), capabilities: in.Capabilities, concurrencyKeys: in.ConcurrencyKeys}
 		caps, err := json.Marshal(in.Capabilities)
 		if err != nil {
 			return err
 		}
-		_, err = tx.ExecContext(ctx, `INSERT INTO pulse_worker_sessions(namespace_id,id,worker_instance_id,credential_id,node_id,session_epoch,capabilities,capacity,draining,expires_at) VALUES (?,?,?,?,?,?,?,?,?,?)`, token.NamespaceID, session, in.InstanceID, token.CredentialID, s.node, binding.Epoch, string(caps), in.Capacity, draining, state.expires)
+		_, err = tx.ExecContext(ctx, `INSERT INTO pulse_worker_sessions(namespace_id,id,worker_instance_id,credential_id,node_id,session_epoch,capabilities,capacity,draining,expires_at,concurrency_keys) VALUES (?,?,?,?,?,?,?,?,?,?,?)`, token.NamespaceID, session, in.InstanceID, token.CredentialID, s.node, binding.Epoch, string(caps), in.Capacity, draining, state.expires, in.ConcurrencyKeys)
 		if err != nil {
 			return err
 		}
@@ -172,7 +174,7 @@ func (s *ExecutionStore) lockSession(ctx context.Context, tx *sql.Tx, token auth
 	var node, instance, sessionCredential core.ID
 	var sessionEpoch uint64
 	var raw []byte
-	err = tx.QueryRowContext(ctx, `SELECT node_id,worker_instance_id,credential_id,session_epoch,capacity,capacity_sequence,capabilities,expires_at FROM pulse_worker_sessions WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, b.SessionID).Scan(&node, &instance, &sessionCredential, &sessionEpoch, &state.capacity, &state.sequence, &raw, &state.expires)
+	err = tx.QueryRowContext(ctx, `SELECT node_id,worker_instance_id,credential_id,session_epoch,capacity,capacity_sequence,capabilities,expires_at,concurrency_keys FROM pulse_worker_sessions WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, b.SessionID).Scan(&node, &instance, &sessionCredential, &sessionEpoch, &state.capacity, &state.sequence, &raw, &state.expires, &state.concurrencyKeys)
 	if errors.Is(err, sql.ErrNoRows) {
 		return state, worker.StaleSession
 	}