feat: enforce durable queue resource limits and backpressure

internal/storage/mysql/migrations/000009_queue_resources.up.sql

@@ -0,0 +1,15 @@
+-- 停止旧服务后迁移；未终结任务包含等待、预占、运行和待核查，不能因重试重复计数。
+-- 无数据库外键。额度与任务状态由服务端在同一队列事务内维护。
+ALTER TABLE pulse_queues
+ ADD COLUMN max_unfinished_tasks INT UNSIGNED NOT NULL DEFAULT 100000 COMMENT '队列允许积压的未终结任务上限；降额不删除已有任务',
+ ADD COLUMN unfinished_tasks BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '未终结任务数，包含待核查；受理增加、首次进入终态减少，重试不重复计数',
+ ADD COLUMN start_window_until DATETIME(6) NULL COMMENT '数据库时间的一秒开始限流窗口结束时刻；空表示尚未开始计数',
+ ADD COLUMN starts_in_window INT UNSIGNED NOT NULL DEFAULT 0 COMMENT '当前窗口已提交的开始授权次数，回执重报不重复计数，降额不清零',
+ ADD CONSTRAINT ck_queue_unfinished_limit CHECK (max_unfinished_tasks > 0),
+ ADD CONSTRAINT ck_queue_start_window CHECK (start_window_until IS NOT NULL OR starts_in_window=0);
+
+UPDATE pulse_queues q LEFT JOIN (
+ SELECT namespace_id,queue_id,COUNT(*) AS unfinished FROM pulse_tasks
+ WHERE state IN ('WAITING','LEASED','RUNNING','NEEDS_REVIEW') GROUP BY namespace_id,queue_id
+) t ON t.namespace_id=q.namespace_id AND t.queue_id=q.id
+SET q.unfinished_tasks=COALESCE(t.unfinished,0);

internal/storage/mysql/queue_resources.go

@@ -0,0 +1,131 @@
+package mysql
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"time"
+
+	"git.yuntuo.hk/k2safe/orbit/internal/fault"
+	core "git.yuntuo.hk/k2safe/orbit/model"
+	"git.yuntuo.hk/k2safe/orbit/platform/auth"
+	"git.yuntuo.hk/k2safe/orbit/pulse/service"
+	"git.yuntuo.hk/k2safe/orbit/pulse/worker"
+)
+
+// 以下额度操作要求调用者已取得队列排他锁，且在同一事务中写入任务/尝试。
+func reserveQueueTask(ctx context.Context, tx *sql.Tx, ns, queue core.ID) error {
+	r, err := tx.ExecContext(ctx, `UPDATE pulse_queues SET unfinished_tasks=unfinished_tasks+1 WHERE namespace_id=? AND id=? AND unfinished_tasks<max_unfinished_tasks`, ns, queue)
+	if err != nil {
+		return err
+	}
+	n, err := r.RowsAffected()
+	if err != nil {
+		return err
+	}
+	if n != 1 {
+		return fault.QueueFull
+	}
+	return nil
+}
+
+func releaseQueueTask(ctx context.Context, tx *sql.Tx, ns, queue core.ID) error {
+	r, err := tx.ExecContext(ctx, `UPDATE pulse_queues SET unfinished_tasks=unfinished_tasks-1 WHERE namespace_id=? AND id=? AND unfinished_tasks>0`, ns, queue)
+	return exactlyOne(r, err)
+}
+
+// queueStartBudget 只读取当前窗口，不把空轮询写成新的窗口。
+// 窗口由第一次成功 Start 建立；以数据库时间计算，节点重启不清零。
+func queueStartBudget(ctx context.Context, tx *sql.Tx, ns, queue core.ID, now time.Time) (uint32, uint32, *time.Time, error) {
+	var rate, used uint32
+	var until *time.Time
+	err := tx.QueryRowContext(ctx, `SELECT rate_per_second,starts_in_window,start_window_until FROM pulse_queues WHERE namespace_id=? AND id=?`, ns, queue).Scan(&rate, &used, &until)
+	if err != nil {
+		return 0, 0, nil, err
+	}
+	if until == nil || !until.After(now) {
+		used, until = 0, nil
+	}
+	return rate, used, until, nil
+}
+
+// queueCanClaim 把尚未 Start 的预占也算入当前窗口可用空间，避免限流时空耗尝试预算。
+func queueCanClaim(ctx context.Context, tx *sql.Tx, ns, queue core.ID, now time.Time) (bool, error) {
+	rate, used, _, err := queueStartBudget(ctx, tx, ns, queue, now)
+	if err != nil || used >= rate {
+		return false, err
+	}
+	var reserved uint64
+	err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_tasks WHERE namespace_id=? AND queue_id=? AND state='LEASED'`, ns, queue).Scan(&reserved)
+	return err == nil && reserved < uint64(rate-used), err
+}
+
+func consumeQueueStart(ctx context.Context, tx *sql.Tx, ns, queue core.ID, now time.Time) error {
+	rate, used, until, err := queueStartBudget(ctx, tx, ns, queue, now)
+	if err != nil {
+		return err
+	}
+	if used >= rate {
+		return worker.RateLimited
+	}
+	if until == nil {
+		end := now.Add(time.Second)
+		until = &end
+	}
+	r, err := tx.ExecContext(ctx, `UPDATE pulse_queues SET start_window_until=?,starts_in_window=? WHERE namespace_id=? AND id=?`, until, used+1, ns, queue)
+	return exactlyOne(r, err)
+}
+
+// ConfigureQueue 完整替换控制参数，运行计数不参与配置版本，也不因配置变更重置。
+func (s *TaskStore) ConfigureQueue(ctx context.Context, token auth.Token, id core.ID, key string, in service.QueueConfigInput) (service.QueueResult, error) {
+	var out service.QueueResult
+	err := s.transaction(ctx, token, "queues:manage", func(tx *sql.Tx, p auth.Permissions) error {
+		if !p.AllowsQueue(id) {
+			return fault.NotFound
+		}
+		identity, cached, err := reserveIdentity(ctx, tx, token.NamespaceID, "queue.configure", string(id), key, "queue", id, in)
+		if err != nil {
+			return err
+		}
+		if cached != nil {
+			if err := json.Unmarshal(cached, &out); err != nil {
+				return err
+			}
+			out.Deduplicated = true
+			return nil
+		}
+		if _, _, err := lockQueue(ctx, tx, token.NamespaceID, id); err != nil {
+			return err
+		}
+		before, err := readQueue(ctx, tx, token.NamespaceID, id)
+		if err != nil {
+			return err
+		}
+		if before.Revision != in.ExpectedRevision {
+			return fault.Conflict
+		}
+		now, err := dbNow(ctx, tx)
+		if err != nil {
+			return err
+		}
+		r, err := tx.ExecContext(ctx, `UPDATE pulse_queues SET paused=?,max_concurrency=?,rate_per_second=?,max_unfinished_tasks=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, in.Paused, in.MaxConcurrency, in.RatePerSecond, in.MaxUnfinishedTasks, now, token.NamespaceID, id, in.ExpectedRevision)
+		if err := exactlyOne(r, err); err != nil {
+			return err
+		}
+		if _, _, err := queueLimit(ctx, tx, token.NamespaceID, id, in.MaxConcurrency); err != nil {
+			return err
+		}
+		out.Queue, err = readQueue(ctx, tx, token.NamespaceID, id)
+		if err != nil {
+			return err
+		}
+		if err := appendAudit(ctx, tx, token, identity, id, "queue", "queue.configure", in.Reason, before, out.Queue); err != nil {
+			return err
+		}
+		if err := emitEvent(ctx, tx, token.NamespaceID, id, out.Queue.Revision, "queue", "queue.configured", out.Queue); err != nil {
+			return err
+		}
+		return saveIdentity(ctx, tx, token.NamespaceID, identity, out)
+	})
+	return out, err
+}

internal/storage/mysql/queue_resources_integration_test.go

@@ -0,0 +1,376 @@
+package mysql
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"git.yuntuo.hk/k2safe/orbit/internal/fault"
+	"git.yuntuo.hk/k2safe/orbit/platform/auth"
+	pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
+	"git.yuntuo.hk/k2safe/orbit/pulse/service"
+	"git.yuntuo.hk/k2safe/orbit/pulse/worker"
+)
+
+func configureResources(t *testing.T, f *executionFixture, rate, concurrency, unfinished uint32, paused bool) service.QueueView {
+	t.Helper()
+	q, err := f.tasks.GetQueue(f.ctx, f.token, f.resources.QueueID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := f.tasks.ConfigureQueue(f.ctx, f.token, q.ID, string(executionID(t)), service.QueueConfigInput{ExpectedRevision: q.Revision, Reason: "controlled resource regression", Paused: paused, MaxConcurrency: concurrency, RatePerSecond: rate, MaxUnfinishedTasks: unfinished})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return out.Queue
+}
+func assertQueueCount(t *testing.T, f *executionFixture, expected uint64) {
+	t.Helper()
+	q, err := f.tasks.GetQueue(f.ctx, f.token, f.resources.QueueID)
+	if err != nil || q.UnfinishedTasks != expected {
+		t.Fatalf("queue count=%d want=%d err=%v", q.UnfinishedTasks, expected, err)
+	}
+	actual := f.count("pulse_tasks", "queue_id=? AND state IN ('WAITING','LEASED','RUNNING','NEEDS_REVIEW')", q.ID)
+	if uint64(actual) != expected {
+		t.Fatalf("counter differs from actual states: %d", actual)
+	}
+}
+func waitStartWindow(t *testing.T, f *executionFixture) {
+	t.Helper()
+	var until, now time.Time
+	if err := f.db.QueryRowContext(f.ctx, `SELECT start_window_until,UTC_TIMESTAMP(6) FROM pulse_queues WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, f.resources.QueueID).Scan(&until, &now); err != nil {
+		t.Fatal(err)
+	}
+	timer := time.NewTimer(max(time.Millisecond, until.Sub(now)+10*time.Millisecond))
+	defer timer.Stop()
+	select {
+	case <-timer.C:
+	case <-f.ctx.Done():
+		t.Fatal(f.ctx.Err())
+	}
+}
+func TestIntegrationQueueResources(t *testing.T) {
+	dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
+	if dsn == "" {
+		t.Skip("requires disposable MySQL")
+	}
+	ctx, stop := context.WithTimeout(context.Background(), 60*time.Second)
+	defer stop()
+	db, err := Open(ctx, dsn, 32, 8)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	if err := Migrate(ctx, db); err != nil {
+		t.Fatal(err)
+	}
+	fixture := func(t *testing.T) *executionFixture { return newExecutionFixture(t, ctx, db) }
+	t.Run("concurrent_nodes_share_start_budget", func(t *testing.T) {
+		f := fixture(t)
+		configureResources(t, f, 4, 4, 10, false)
+		hello := f.hello
+		hello.InstanceID = executionID(t)
+		b, _, err := f.b.Connect(ctx, f.token, hello)
+		if err != nil {
+			t.Fatal(err)
+		}
+		type item struct {
+			api     *worker.Service
+			binding worker.Binding
+			ref     worker.AttemptRef
+		}
+		var items []item
+		for i := 0; i < 4; i++ {
+			f.submit(1)
+			api, binding := f.a, f.binding
+			if i%2 == 1 {
+				api, binding = f.b, b
+			}
+			a, err := api.Claim(ctx, f.token, binding)
+			if err != nil || a == nil {
+				t.Fatal("claim reservation failed", err)
+			}
+			items = append(items, item{api, binding, worker.AttemptRef{TaskID: a.Task.ID, AttemptID: a.AttemptID, Epoch: a.Epoch, LeaseSecret: a.LeaseSecret}})
+		}
+		configureResources(t, f, 2, 4, 10, false)
+		start := make(chan struct{})
+		results := make(chan error, 4)
+		var wg sync.WaitGroup
+		for _, item := range items {
+			message := executionID(t)
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				<-start
+				_, err := item.api.Start(ctx, f.token, item.binding, message, item.ref)
+				results <- err
+			}()
+		}
+		close(start)
+		wg.Wait()
+		close(results)
+		allowed := 0
+		for err := range results {
+			if err == nil {
+				allowed++
+			} else if fault.Code(err) != "RATE_LIMITED" {
+				t.Fatal(err)
+			}
+		}
+		if allowed != 2 {
+			t.Fatalf("concurrent starts=%d, want 2", allowed)
+		}
+		assertQueueCount(t, f, 4)
+	})
+	t.Run("start_budget_rollback_and_configuration_scope", func(t *testing.T) {
+		f := fixture(t)
+		q := configureResources(t, f, 1, 4, 10, false)
+		f.submit(1)
+		_, ref := f.claim()
+		message := executionID(t)
+		f.exec(`CREATE TRIGGER orbit_start_budget_fail BEFORE INSERT ON orbit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='controlled rollback'`)
+		_, err := f.a.Start(ctx, f.token, f.binding, message, ref)
+		f.exec(`DROP TRIGGER orbit_start_budget_fail`)
+		if err == nil || f.get(ref.TaskID).State != pulse.TaskLeased {
+			t.Fatal("failed start committed")
+		}
+		var used uint32
+		if err := db.QueryRowContext(ctx, `SELECT starts_in_window FROM pulse_queues WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, q.ID).Scan(&used); err != nil || used != 0 {
+			t.Fatal("rolled back start spent budget")
+		}
+		if _, err := f.a.Start(ctx, f.token, f.binding, message, ref); err != nil {
+			t.Fatal(err)
+		}
+		other := fixture(t)
+		in := service.QueueConfigInput{ExpectedRevision: q.Revision, Reason: "scope", MaxConcurrency: 4, RatePerSecond: 1, MaxUnfinishedTasks: 10}
+		_, err = other.tasks.ConfigureQueue(ctx, other.token, q.ID, "cross-namespace", in)
+		expectWorkerCode(t, err, "NOT_FOUND")
+		raw, _ := json.Marshal(auth.Permissions{Actions: []string{"queues:manage"}, Queues: []string{string(other.resources.QueueID)}})
+		f.exec(`UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?`, string(raw), f.resources.NamespaceID, f.resources.CredentialID)
+		_, err = f.tasks.ConfigureQueue(ctx, f.token, q.ID, "denied-queue", in)
+		expectWorkerCode(t, err, "NOT_FOUND")
+		if f.count("orbit_idempotency_keys", "operation='queue.configure' AND scope_key=?", string(q.ID)) != 1 {
+			t.Fatal("denied config left identity")
+		}
+	})
+	t.Run("concurrent_admission_duplicate_and_release", func(t *testing.T) {
+		f := fixture(t)
+		configureResources(t, f, 10, 4, 3, false)
+		input := service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)}
+		type result struct {
+			key string
+			out service.SubmitResult
+			err error
+		}
+		results := make(chan result, 16)
+		var wg sync.WaitGroup
+		for i := 0; i < 16; i++ {
+			key := string(executionID(t))
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				out, err := f.tasks.Submit(ctx, f.token, key, input)
+				results <- result{key, out, err}
+			}()
+		}
+		wg.Wait()
+		close(results)
+		var accepted []result
+		for r := range results {
+			if r.err == nil {
+				accepted = append(accepted, r)
+			} else {
+				expectWorkerCode(t, r.err, "QUEUE_FULL")
+			}
+		}
+		if len(accepted) != 3 || f.count("orbit_idempotency_keys", "operation='task.submit'") != 3 {
+			t.Fatal("admission exceeded limit or left rejected identities")
+		}
+		assertQueueCount(t, f, 3)
+		again, err := f.tasks.Submit(ctx, f.token, accepted[0].key, input)
+		if err != nil || !again.Deduplicated || again.Task.ID != accepted[0].out.Task.ID {
+			t.Fatal("full queue rejected original receipt")
+		}
+		_, err = f.tasks.CancelTask(ctx, f.token, again.Task.ID, "release", service.CancelInput{ExpectedRevision: again.Task.Revision, Reason: "release slot"})
+		if err != nil {
+			t.Fatal(err)
+		}
+		assertQueueCount(t, f, 2)
+		if _, err := f.tasks.Submit(ctx, f.token, "new-after-release", input); err != nil {
+			t.Fatal(err)
+		}
+		assertQueueCount(t, f, 3)
+	})
+	t.Run("configuration_cas_audit_and_pause", func(t *testing.T) {
+		f := fixture(t)
+		q := configureResources(t, f, 10, 4, 10, true)
+		task := f.submit(1)
+		if a, err := f.a.Claim(ctx, f.token, f.binding); err != nil || a != nil {
+			t.Fatal("paused queue dispatched")
+		}
+		in := service.QueueConfigInput{ExpectedRevision: q.Revision, Reason: "resume", MaxConcurrency: 4, RatePerSecond: 10, MaxUnfinishedTasks: 10}
+		first, err := f.tasks.ConfigureQueue(ctx, f.token, q.ID, "resume", in)
+		if err != nil {
+			t.Fatal(err)
+		}
+		again, err := f.tasks.ConfigureQueue(ctx, f.token, q.ID, "resume", in)
+		if err != nil || !again.Deduplicated || again.Queue.Revision != first.Queue.Revision {
+			t.Fatal("config receipt changed")
+		}
+		_, err = f.tasks.ConfigureQueue(ctx, f.token, q.ID, "stale", in)
+		expectWorkerCode(t, err, "STATE_CONFLICT")
+		_, ref := f.claim()
+		configureResources(t, f, 10, 4, 10, true)
+		_, err = f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
+		expectWorkerCode(t, err, "EXECUTION_PAUSED")
+		_, err = f.tasks.CancelTask(ctx, f.token, task.ID, "cancel-paused", service.CancelInput{ExpectedRevision: f.get(task.ID).Revision, Reason: "cancel while paused"})
+		if err != nil {
+			t.Fatal(err)
+		}
+		assertQueueCount(t, f, 0)
+		if f.count("orbit_audit_events", "resource_id=? AND action='queue.configure'", q.ID) != 3 {
+			t.Fatal("configuration audit count")
+		}
+	})
+	t.Run("rate_window_survives_reconnect_and_receipt_replay", func(t *testing.T) {
+		f := fixture(t)
+		configureResources(t, f, 1, 4, 10, false)
+		f.submit(1)
+		f.submit(1)
+		_, first := f.claim()
+		if a, err := f.a.Claim(ctx, f.token, f.binding); err != nil || a != nil {
+			t.Fatal("unstarted reservation did not throttle claims")
+		}
+		message := executionID(t)
+		if _, err := f.a.Start(ctx, f.token, f.binding, message, first); err != nil {
+			t.Fatal(err)
+		}
+		if out, err := f.a.Start(ctx, f.token, f.binding, message, first); err != nil || !out.Deduplicated {
+			t.Fatal("start replay rejected by quota")
+		}
+		binding, _, err := f.b.Connect(ctx, f.token, f.hello)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err := f.b.Resume(ctx, f.token, binding, first); err != nil {
+			t.Fatal(err)
+		}
+		f.a, f.binding = f.b, binding
+		if a, err := f.a.Claim(ctx, f.token, f.binding); err != nil || a != nil {
+			t.Fatal("node handoff reset rate window")
+		}
+		waitStartWindow(t, f)
+		_, second := f.claim()
+		if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), second); err != nil {
+			t.Fatal(err)
+		}
+		var gap int64
+		if err := db.QueryRowContext(ctx, `SELECT TIMESTAMPDIFF(MICROSECOND,a.authorized_at,b.authorized_at) FROM pulse_attempts a JOIN pulse_attempts b ON b.namespace_id=a.namespace_id WHERE a.namespace_id=? AND a.id=? AND b.id=?`, f.resources.NamespaceID, first.AttemptID, second.AttemptID).Scan(&gap); err != nil || gap < 1000000 {
+			t.Fatalf("rate gap=%d err=%v", gap, err)
+		}
+		retryFinish(t, f, first, "SUCCEEDED", false)
+		retryFinish(t, f, second, "SUCCEEDED", false)
+		assertQueueCount(t, f, 0)
+	})
+	t.Run("lower_limits_preserve_inflight_and_throttled_start", func(t *testing.T) {
+		f := fixture(t)
+		configureResources(t, f, 2, 4, 10, false)
+		f.submit(1)
+		f.submit(1)
+		f.submit(1)
+		_, a := f.claim()
+		_, b := f.claim()
+		configureResources(t, f, 1, 1, 1, false)
+		if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), a); err != nil {
+			t.Fatal(err)
+		}
+		message := executionID(t)
+		_, err := f.a.Start(ctx, f.token, f.binding, message, b)
+		expectWorkerCode(t, err, "RATE_LIMITED")
+		if f.get(b.TaskID).State != pulse.TaskLeased || f.get(b.TaskID).AttemptCount != 1 || f.used() != 2 {
+			t.Fatal("rate rejection consumed or revoked work")
+		}
+		retryFinish(t, f, a, "SUCCEEDED", false)
+		if got, err := f.a.Claim(ctx, f.token, f.binding); err != nil || got != nil {
+			t.Fatal("lowered concurrency admitted excess")
+		}
+		waitStartWindow(t, f)
+		if _, err := f.a.Start(ctx, f.token, f.binding, message, b); err != nil {
+			t.Fatal(err)
+		}
+		retryFinish(t, f, b, "SUCCEEDED", false)
+		assertQueueCount(t, f, 1)
+		waitStartWindow(t, f)
+		f.claim()
+	})
+	t.Run("rollback_retry_unknown_and_expiry_accounting", func(t *testing.T) {
+		f := fixture(t)
+		configureResources(t, f, 10, 4, 2, false)
+		f.exec(`CREATE TRIGGER orbit_resource_test_fail BEFORE INSERT ON orbit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='controlled rollback'`)
+		_, err := f.tasks.Submit(ctx, f.token, "rollback", service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)})
+		f.exec(`DROP TRIGGER orbit_resource_test_fail`)
+		if err == nil {
+			t.Fatal("injected failure ignored")
+		}
+		assertQueueCount(t, f, 0)
+		task := retryTask(t, f, 2, pulse.RecoveryIdempotent)
+		ref := retryStart(t, f)
+		retryFinish(t, f, ref, "FAILED", true)
+		assertQueueCount(t, f, 1)
+		makeRetryDue(f, task.ID)
+		ref = retryStart(t, f)
+		retryFinish(t, f, ref, "UNKNOWN", false)
+		assertQueueCount(t, f, 1)
+		waiting := f.submit(1)
+		f.exec(`UPDATE pulse_tasks SET available_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 2 SECOND),deadline_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, waiting.ID)
+		if _, err := f.storeA.Recover(ctx, 100); err != nil {
+			t.Fatal(err)
+		}
+		assertQueueCount(t, f, 1)
+		if _, err := f.storeA.Recover(ctx, 100); err != nil {
+			t.Fatal(err)
+		}
+		assertQueueCount(t, f, 1)
+	})
+	t.Run("schedule_backpressure_preserves_unaccepted_cursor", func(t *testing.T) {
+		f := fixture(t)
+		configureResources(t, f, 10, 4, 1, false)
+		sf := newScheduleFixture(t, f)
+		v := sf.create(sf.definition())
+		sf.due(v, 2*time.Second)
+		sf.tick()
+		assertQueueCount(t, f, 1)
+		items := sf.occurrences(v.ID)
+		if len(items) != 1 || items[0].TaskID == nil {
+			t.Fatal("schedule exceeded backlog limit")
+		}
+		next := sf.current(v.ID).NextFireAt
+		if next == nil || !next.Equal(items[0].ScheduledAt.Add(time.Second)) {
+			t.Fatal("backpressure skipped unaccepted point")
+		}
+		id := *items[0].TaskID
+		_, err := f.tasks.CancelTask(ctx, f.token, id, "free-schedule", service.CancelInput{ExpectedRevision: 1, Reason: "free slot"})
+		if err != nil {
+			t.Fatal(err)
+		}
+		sf.due(v, 0)
+		sf.tick()
+		items = sf.occurrences(v.ID)
+		if len(items) != 2 {
+			t.Fatal("schedule did not resume after release")
+		}
+		found := false
+		for _, item := range items {
+			if item.ScheduledAt.Equal(*next) {
+				found = true
+			}
+		}
+		if !found {
+			t.Fatal("resumed wrong original time")
+		}
+		assertQueueCount(t, f, 1)
+	})
+}

internal/storage/mysql/retry_upgrade_integration_test.go

@@ -94,6 +94,12 @@ func TestIntegrationSafeRetryUpgrade(t *testing.T) {
 	if err := CheckSchema(ctx, db); err != nil {
 		t.Fatal(err)
 	}
+	// 000009 必须回填 WAITING 和 NEEDS_REVIEW，并保留既有队列配置版本。
+	var unfinished uint64
+	var maxUnfinished, queueRevision uint64
+	if err := db.QueryRowContext(ctx, `SELECT unfinished_tasks,max_unfinished_tasks,revision FROM pulse_queues WHERE namespace_id=? AND id=?`, ns, queue).Scan(&unfinished, &maxUnfinished, &queueRevision); err != nil || unfinished != 2 || maxUnfinished != 100000 || queueRevision != 1 {
+		t.Fatalf("resource upgrade changed queue or missed states: count=%d max=%d revision=%d err=%v", unfinished, maxUnfinished, queueRevision, err)
+	}
 	for state, id := range ids {
 		var gotState, gotPolicy string
 		var uncertain bool

internal/storage/mysql/schedule_tick.go

@@ -153,8 +153,15 @@ func (s *ScheduleStore) tickOne(ctx context.Context, ns, id, queue core.ID) (int
 			if err != nil {
 				return err
 			}
+			backpressured := false
 			for _, decision := range batch.Decisions {
 				if err := s.saveOccurrence(ctx, tx, v, version, decision, now); err != nil {
+					if errors.Is(err, fault.QueueFull) {
+						// 已提交本批较早的决定仍保留；满额的原触发点留待下次按冻结策略处理。
+						t := decision.ScheduledAt
+						next, backpressured = &t, true
+						break
+					}
 					return err
 				}
 				if decision.Kind == "SKIPPED" {
@@ -163,7 +170,9 @@ func (s *ScheduleStore) tickOne(ctx context.Context, ns, id, queue core.ID) (int
 					fired++
 				}
 			}
-			next = batch.Next
+			if !backpressured {
+				next = batch.Next
+			}
 			if next == nil && cursor < v.CurrentVersion {
 				cursor++
 				needsInitialization = true
@@ -205,6 +214,9 @@ func (s *ScheduleStore) saveOccurrence(ctx context.Context, tx *sql.Tx, v schedu
 		if in.QueueID != v.QueueID {
 			return errors.New("schedule template queue mismatch")
 		}
+		if err := reserveQueueTask(ctx, tx, v.NamespaceID, v.QueueID); err != nil {
+			return err
+		}
 		policy, err := json.Marshal(in.Retry)
 		if err != nil {
 			return err

internal/storage/mysql/task_cancel.go

@@ -101,6 +101,11 @@ func (s *TaskStore) CancelTask(ctx context.Context, token auth.Token, id core.ID
 		if err := exactlyOne(result, err); err != nil {
 			return err
 		}
+		if next.Terminal() {
+			if err := releaseQueueTask(ctx, tx, token.NamespaceID, view.QueueID); err != nil {
+				return err
+			}
+		}
 		out.Task, err = readTask(ctx, tx, token.NamespaceID, id, false)
 		if err != nil {
 			return err

internal/storage/mysql/task_store.go

@@ -171,7 +171,7 @@ func (s *TaskStore) Submit(ctx context.Context, token auth.Token, key string, in
 			return nil
 		}
 		var queue core.ID
-		if err := tx.QueryRowContext(ctx, "SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR SHARE", token.NamespaceID, in.QueueID).Scan(&queue); errors.Is(err, sql.ErrNoRows) {
+		if err := tx.QueryRowContext(ctx, "SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE", token.NamespaceID, in.QueueID).Scan(&queue); errors.Is(err, sql.ErrNoRows) {
 			return fault.NotFound
 		} else if err != nil {
 			return err
@@ -195,6 +195,9 @@ func (s *TaskStore) Submit(ctx context.Context, token auth.Token, key string, in
 		if err != nil {
 			return err
 		}
+		if err := reserveQueueTask(ctx, tx, token.NamespaceID, in.QueueID); err != nil {
+			return err
+		}
 		_, err = tx.ExecContext(ctx, `INSERT INTO pulse_tasks(namespace_id,id,queue_id,handler,handler_version,payload_version,payload,policy_snapshot,recovery_safety,business_key,available_at,start_deadline_at,deadline_at,attempt_timeout_ms,max_attempts,created_at,updated_at)
             VALUES (?,?,?,?,?,?,?,?,?,NULLIF(?,''),?,?,?,?,?,?,?)`, token.NamespaceID, taskID, in.QueueID, in.Handler, in.HandlerVersion, in.PayloadVersion, string(in.Payload), string(policy), in.RecoverySafety, in.BusinessKey, available, in.StartDeadlineAt, deadline, in.Retry.AttemptTimeoutMS, in.Retry.MaxAttempts, now, now)
 		if err != nil {
@@ -338,7 +341,11 @@ func (s *TaskStore) CreateQueue(ctx context.Context, token auth.Token, key strin
 			out.Deduplicated = true
 			return nil
 		}
-		_, err = tx.ExecContext(ctx, "INSERT INTO pulse_queues(namespace_id,id,name,max_concurrency,rate_per_second,policy) VALUES (?,?,?,?,?,'{}')", token.NamespaceID, id, in.Name, in.MaxConcurrency, in.RatePerSecond)
+		maxUnfinished := in.MaxUnfinishedTasks
+		if maxUnfinished == 0 {
+			maxUnfinished = 100000
+		}
+		_, err = tx.ExecContext(ctx, "INSERT INTO pulse_queues(namespace_id,id,name,max_concurrency,rate_per_second,max_unfinished_tasks,policy) VALUES (?,?,?,?,?,?,'{}')", token.NamespaceID, id, in.Name, in.MaxConcurrency, in.RatePerSecond, maxUnfinished)
 		var dbErr *driver.MySQLError
 		if errors.As(err, &dbErr) && dbErr.Number == 1062 {
 			return fault.Conflict
@@ -359,7 +366,7 @@ func (s *TaskStore) CreateQueue(ctx context.Context, token auth.Token, key strin
 }
 func readQueue(ctx context.Context, tx *sql.Tx, ns, id core.ID) (service.QueueView, error) {
 	var q service.QueueView
-	err := tx.QueryRowContext(ctx, "SELECT id,namespace_id,name,paused,max_concurrency,rate_per_second,revision FROM pulse_queues WHERE namespace_id=? AND id=?", ns, id).Scan(&q.ID, &q.NamespaceID, &q.Name, &q.Paused, &q.MaxConcurrency, &q.RatePerSecond, &q.Revision)
+	err := tx.QueryRowContext(ctx, "SELECT id,namespace_id,name,paused,max_concurrency,rate_per_second,max_unfinished_tasks,unfinished_tasks,revision FROM pulse_queues WHERE namespace_id=? AND id=?", ns, id).Scan(&q.ID, &q.NamespaceID, &q.Name, &q.Paused, &q.MaxConcurrency, &q.RatePerSecond, &q.MaxUnfinishedTasks, &q.UnfinishedTasks, &q.Revision)
 	if errors.Is(err, sql.ErrNoRows) {
 		err = fault.NotFound
 	}

internal/storage/mysql/worker_execution.go

@@ -81,8 +81,13 @@ func queueLimit(ctx context.Context, tx *sql.Tx, ns, queue core.ID, max uint32)
 	var used uint32
 	var configured uint32
 	err = tx.QueryRowContext(ctx, `SELECT id,used_permits,max_permits FROM pulse_concurrency_limits WHERE namespace_id=? AND scope_key='queue' AND key_digest=? FOR UPDATE`, ns, digest).Scan(&id, &used, &configured)
-	if err == nil && configured != max {
-		err = errors.New("queue limit configuration requires coordinated update")
+	// 降额时保留已有许可；队列上限控制新的领取，存储上限至少容纳在途数。
+	target := max
+	if used > target {
+		target = used
+	}
+	if err == nil && configured != target {
+		_, err = tx.ExecContext(ctx, `UPDATE pulse_concurrency_limits SET max_permits=?,revision=revision+1,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, target, ns, id)
 	}
 	return id, used, err
 }
@@ -123,6 +128,14 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
 		if used >= max {
 			return nil
 		}
+		now, err := dbNow(ctx, tx)
+		if err != nil {
+			return err
+		}
+		canClaim, err := queueCanClaim(ctx, tx, token.NamespaceID, state.queue, now)
+		if err != nil || !canClaim {
+			return err
+		}
 		args := []any{token.NamespaceID, state.queue}
 		matches := make([]string, 0, len(state.capabilities))
 		for _, c := range state.capabilities {
@@ -465,6 +478,9 @@ func (s *ExecutionStore) executionCommand(ctx context.Context, token auth.Token,
 			if task.view.StartDeadlineAt != nil && !task.view.StartDeadlineAt.After(now) {
 				return worker.LeaseLost
 			}
+			if err := consumeQueueStart(ctx, tx, token.NamespaceID, task.view.QueueID, now); err != nil {
+				return err
+			}
 			authorized = &now
 		} else if a.State != pulse.AttemptRunning || authorized == nil {
 			return fault.Conflict

internal/storage/mysql/worker_recovery.go

@@ -98,6 +98,11 @@ func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.Recover
 				if err := exactlyOne(result, err); err != nil {
 					return err
 				}
+				if changed.Terminal() {
+					if err := releaseQueueTask(ctx, tx, c.ns, c.queue); err != nil {
+						return err
+					}
+				}
 				return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", resultEvent(changed), map[string]any{"reason": "START_DEADLINE_EXCEEDED", "node_id": s.node, "effect_uncertain": task.view.EffectUncertain})
 			}
 			if task.view.State != pulse.TaskLeased && task.view.State != pulse.TaskRunning {
@@ -155,6 +160,11 @@ func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.Recover
 			if err := releasePermits(ctx, tx, c.ns, attempt.ID, now); err != nil {
 				return err
 			}
+			if next.Terminal() {
+				if err := releaseQueueTask(ctx, tx, c.ns, c.queue); err != nil {
+					return err
+				}
+			}
 			changed = next
 			return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", resultEvent(next), map[string]any{"attempt_id": attempt.ID, "epoch": fmt.Sprint(attempt.Epoch), "reason": reason, "node_id": s.node, "effect_uncertain": decision.Uncertain, "decision": decision.Reason, "next_available_at": decision.NextAt, "delay_ms": decision.DelayMS, "attempt_count": task.view.AttemptCount, "max_attempts": task.maxAttempts})
 		})

internal/storage/mysql/worker_results.go

@@ -112,6 +112,11 @@ func (s *ExecutionStore) Finish(ctx context.Context, token auth.Token, b worker.
 		if err := releasePermits(ctx, tx, token.NamespaceID, a.ID, now); err != nil {
 			return err
 		}
+		if taskState.Terminal() {
+			if err := releaseQueueTask(ctx, tx, token.NamespaceID, task.view.QueueID); err != nil {
+				return err
+			}
+		}
 		event := map[string]any{"attempt_id": a.ID, "epoch": fmt.Sprint(a.Epoch), "state": taskState, "outcome": in.Outcome, "error_code": in.ErrorCode, "retryable": in.Retryable, "effect_uncertain": decision.Uncertain, "decision": decision.Reason, "next_available_at": decision.NextAt, "delay_ms": decision.DelayMS, "attempt_count": task.view.AttemptCount, "max_attempts": task.maxAttempts}
 		if err := emitEvent(ctx, tx, token.NamespaceID, in.Ref.TaskID, task.view.Revision+1, "task", resultEvent(taskState), event); err != nil {
 			return err