feat: enforce durable queue resource limits and backpressure

sdk/go/examples/resources/main.go

@@ -0,0 +1,150 @@
+// resources 演示队列暂停、积压背压、幂等核对、释放空间与限速执行。
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+func run() error {
+	token := os.Getenv("ORBIT_TOKEN")
+	if path := os.Getenv("ORBIT_CREDENTIALS_FILE"); path != "" && token == "" {
+		raw, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		var c struct {
+			Token string `json:"token"`
+		}
+		if err := json.Unmarshal(raw, &c); err != nil {
+			return err
+		}
+		token = c.Token
+	}
+	api, err := orbit.NewClient(os.Getenv("ORBIT_URL"), orbit.Options{Token: token})
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
+	defer cancel()
+	op, err := orbit.NewWorkerMessageID()
+	if err != nil {
+		return err
+	}
+	name := "resources." + op
+	created, err := api.CreateQueue(ctx, "queue-"+op, orbit.CreateQueueRequest{Name: name, MaxConcurrency: 2, RatePerSecond: 1, MaxUnfinishedTasks: 2})
+	if err != nil {
+		return err
+	}
+	q := created.Queue
+	configure := func(paused bool) error {
+		out, err := api.ConfigureQueue(ctx, q.ID, fmt.Sprintf("configure-%s-%t", op, paused), orbit.ConfigureQueueRequest{ExpectedRevision: q.Revision, Reason: "resource control example", Paused: paused, MaxConcurrency: 2, RatePerSecond: 1, MaxUnfinishedTasks: 2})
+		if err == nil {
+			q = out.Queue
+		}
+		return err
+	}
+	if err := configure(true); err != nil {
+		return err
+	}
+	in := orbit.SubmitTaskRequest{QueueID: q.ID, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"object_ref":"objects/example","n":9007199254740993}`), TaskTimeoutMS: 20000, Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: 10000}}
+	first, err := api.SubmitTask(ctx, "first-"+op, in)
+	if err != nil {
+		return err
+	}
+	second, err := api.SubmitTask(ctx, "second-"+op, in)
+	if err != nil {
+		return err
+	}
+	_, err = api.SubmitTask(ctx, "third-"+op, in)
+	var rejected *orbit.APIError
+	if !errors.As(err, &rejected) || rejected.Code != "QUEUE_FULL" {
+		return fmt.Errorf("expected QUEUE_FULL, got %v", err)
+	}
+	fmt.Printf("backpressure=%s http=%d\n", rejected.Code, rejected.StatusCode)
+	duplicate, err := api.SubmitTask(ctx, "first-"+op, in)
+	if err != nil {
+		return err
+	}
+	if !duplicate.Deduplicated || duplicate.Task.ID != first.Task.ID {
+		return errors.New("original receipt was not preserved")
+	}
+	if _, err := api.CancelTask(ctx, first.Task.ID, "cancel-"+op, orbit.CancelTaskRequest{ExpectedRevision: first.Task.Revision, Reason: "free one unfinished slot"}); err != nil {
+		return err
+	}
+	// 明确 429 表示未受理；释放空间后仍使用第三个任务的原幂等键与原内容。
+	third, err := api.SubmitTask(ctx, "third-"+op, in)
+	if err != nil {
+		return err
+	}
+	fmt.Println("duplicate=true rejected_request_retried=true")
+	w, err := orbit.NewWorker(api, orbit.WorkerOptions{QueueID: q.ID, Capacity: 2, ShutdownTimeout: 5 * time.Second, Handlers: []orbit.WorkerHandler{{Capability: orbit.WorkerCapability{Handler: name, HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(ctx context.Context, e orbit.WorkerExecution) (json.RawMessage, error) {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+		return e.Task.Payload, nil
+	}}}})
+	if err != nil {
+		return err
+	}
+	run, stop := context.WithCancel(ctx)
+	done := make(chan error, 1)
+	go func() { done <- w.Run(run) }()
+	waited := false
+	defer func() {
+		stop()
+		if !waited {
+			<-done
+		}
+	}()
+	if err := configure(false); err != nil {
+		return err
+	}
+	for _, id := range []string{second.Task.ID, third.Task.ID} {
+		for {
+			task, err := api.GetTask(ctx, id)
+			if err != nil {
+				return err
+			}
+			if task.State == orbit.TaskSucceeded {
+				fmt.Printf("task=%s state=%s attempts=%d\n", id, task.State, task.AttemptCount)
+				break
+			}
+			if task.State == orbit.TaskFailed || task.State == orbit.TaskNeedsReview || task.State == orbit.TaskCancelled {
+				return fmt.Errorf("unexpected task state: %s", task.State)
+			}
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-time.After(80 * time.Millisecond):
+			}
+		}
+	}
+	q, err = api.GetQueue(ctx, q.ID)
+	if err != nil {
+		return err
+	}
+	if q.UnfinishedTasks != 0 {
+		return errors.New("queue still has unfinished work")
+	}
+	stop()
+	err = <-done
+	waited = true
+	if err != nil {
+		return err
+	}
+	fmt.Printf("unfinished=%d rate_per_second=%d worker_drained=true\n", q.UnfinishedTasks, q.RatePerSecond)
+	return nil
+}

sdk/go/queue_resources_integration_test.go

@@ -0,0 +1,115 @@
+package orbit
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"testing"
+	"time"
+)
+
+func TestIntegrationQueueResourcesSDK(t *testing.T) {
+	api, _, _ := integrationWorkerClient(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
+	defer cancel()
+	name := "resources." + workerTestID(t)
+	created, err := api.CreateQueue(ctx, "create-"+name, CreateQueueRequest{Name: name, MaxConcurrency: 2, RatePerSecond: 1, MaxUnfinishedTasks: 2})
+	if err != nil {
+		t.Fatal(err)
+	}
+	q := created.Queue
+	configure := func(paused bool) {
+		t.Helper()
+		out, err := api.ConfigureQueue(ctx, q.ID, workerTestID(t), ConfigureQueueRequest{ExpectedRevision: q.Revision, Reason: "SDK resource regression", Paused: paused, MaxConcurrency: 2, RatePerSecond: 1, MaxUnfinishedTasks: 2})
+		if err != nil {
+			t.Fatal(err)
+		}
+		q = out.Queue
+	}
+	configure(true)
+	// 丢失 paused 不能被当成解除暂停；这是原始 HTTP 对象，SDK 本身总会发送 bool。
+	var ignored CreateQueueResult
+	err = api.call(ctx, "POST", "/v1/queues/"+q.ID+":configure", workerTestID(t), map[string]any{"expected_revision": "2", "reason": "missing pause", "max_concurrency": 2, "rate_per_second": 1, "max_unfinished_tasks": 2}, &ignored)
+	var apiErr *APIError
+	if !errors.As(err, &apiErr) || apiErr.StatusCode != 400 {
+		t.Fatalf("missing paused accepted: %v", err)
+	}
+	input := SubmitTaskRequest{QueueID: q.ID, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`)}
+	first, err := api.SubmitTask(ctx, name+"-one", input)
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := api.SubmitTask(ctx, name+"-two", input)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = api.SubmitTask(ctx, name+"-three", input)
+	if !errors.As(err, &apiErr) || apiErr.StatusCode != 429 || apiErr.Code != "QUEUE_FULL" || apiErr.RetryAfter != time.Second {
+		t.Fatalf("queue did not backpressure: %v", err)
+	}
+	if duplicate, err := api.SubmitTask(ctx, name+"-one", input); err != nil || !duplicate.Deduplicated || duplicate.Task.ID != first.Task.ID {
+		t.Fatal("full queue lost original identity")
+	}
+	if _, err := api.CancelTask(ctx, first.Task.ID, name+"-cancel", CancelTaskRequest{ExpectedRevision: first.Task.Revision, Reason: "free backlog slot"}); err != nil {
+		t.Fatal(err)
+	}
+	third, err := api.SubmitTask(ctx, name+"-three", input)
+	if err != nil {
+		t.Fatal(err)
+	}
+	w, err := NewWorker(api, WorkerOptions{QueueID: q.ID, Capacity: 2, Handlers: []WorkerHandler{runtimeHandler(name, func(ctx context.Context, e WorkerExecution) (json.RawMessage, error) { return e.Task.Payload, nil })}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	run, stop := context.WithCancel(ctx)
+	done := make(chan error, 1)
+	go func() { done <- w.Run(run) }()
+	waited := false
+	defer func() {
+		stop()
+		if !waited {
+			<-done
+		}
+	}()
+	select {
+	case <-time.After(350 * time.Millisecond):
+	case <-ctx.Done():
+		t.Fatal(ctx.Err())
+	}
+	current, err := api.GetTask(ctx, second.Task.ID)
+	if err != nil || current.AttemptCount != 0 {
+		t.Fatal("paused queue consumed attempt")
+	}
+	configure(false)
+	for _, id := range []string{second.Task.ID, third.Task.ID} {
+		finished := waitRuntimeState(t, ctx, api, id, TaskSucceeded)
+		if finished.AttemptCount != 1 || string(finished.Result) != `{"n":9007199254740993}` {
+			t.Fatal("resource controls altered execution")
+		}
+	}
+	var authorized []time.Time
+	for _, id := range []string{second.Task.ID, third.Task.ID} {
+		attempts, err := api.ListTaskAttempts(ctx, id, HistoryPageRequest{})
+		if err != nil || len(attempts.Items) != 1 || attempts.Items[0].AuthorizedAt == nil {
+			t.Fatal("missing start history")
+		}
+		authorized = append(authorized, *attempts.Items[0].AuthorizedAt)
+	}
+	gap := authorized[1].Sub(authorized[0])
+	if gap < 0 {
+		gap = -gap
+	}
+	if gap < time.Second {
+		t.Fatalf("start rate exceeded: %s", gap)
+	}
+	q, err = api.GetQueue(ctx, q.ID)
+	if err != nil || q.UnfinishedTasks != 0 || q.Revision != 3 {
+		t.Fatal("quota count or configuration revision drifted")
+	}
+	stop()
+	err = <-done
+	waited = true
+	if err != nil {
+		t.Fatal(err)
+	}
+}

sdk/go/tasks.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"net/url"
 	"strconv"
+	"time"
 )
 
 // APIError 提供可分支处理的错误码和请求追踪标识，不包含内部数据库错误。
@@ -18,6 +19,8 @@ type APIError struct {
 	Code       string
 	Message    string
 	RequestID  string
+	// RetryAfter 服务端建议的最小退避时间；缺失、无效或超过一小时返回零，SDK 不自动重试。
+	RetryAfter time.Duration
 }
 
 func (e *APIError) Error() string {
@@ -56,7 +59,7 @@ func (c *Client) ListTasks(ctx context.Context, in ListTasksRequest) (TaskPage,
 	return out, err
 }
 
-// CancelTask 当前只取消 WAITING 状态且不归属流程的任务；重复请求使用原键与原版本。
+// CancelTask 保存取消意图；运行中等待结果确认，重复请求保留原键与原版本。
 func (c *Client) CancelTask(ctx context.Context, id, key string, in CancelTaskRequest) (CancelTaskResult, error) {
 	var out CancelTaskResult
 	if !validID(id) {
@@ -150,7 +153,11 @@ func (c *Client) call(ctx context.Context, method, path, key string, in, out any
 	decoded := json.Unmarshal(raw, &envelope)
 	if response.StatusCode < 200 || response.StatusCode >= 300 {
 		if decoded == nil && envelope.Error != nil && len(envelope.Error.Code) <= 64 && len(envelope.Error.Message) <= 1024 && validID(envelope.RequestID) {
-			return &APIError{response.StatusCode, envelope.Error.Code, envelope.Error.Message, envelope.RequestID}
+			var retryAfter time.Duration
+			if seconds, err := strconv.ParseUint(response.Header.Get("Retry-After"), 10, 32); err == nil && seconds > 0 && seconds <= 3600 {
+				retryAfter = time.Duration(seconds) * time.Second
+			}
+			return &APIError{StatusCode: response.StatusCode, Code: envelope.Error.Code, Message: envelope.Error.Message, RequestID: envelope.RequestID, RetryAfter: retryAfter}
 		}
 		return &HTTPError{StatusCode: response.StatusCode}
 	}
@@ -159,3 +166,13 @@ func (c *Client) call(ctx context.Context, method, path, key string, in, out any
 	}
 	return json.Unmarshal(envelope.Data, out)
 }
+
+// ConfigureQueue 完整替换队列控制参数；重复请求保留原版本及幂等键。
+func (c *Client) ConfigureQueue(ctx context.Context, id, key string, in ConfigureQueueRequest) (CreateQueueResult, error) {
+	var out CreateQueueResult
+	if !validID(id) {
+		return out, errors.New("invalid queue id")
+	}
+	err := c.call(ctx, "POST", "/v1/queues/"+id+":configure", key, in, &out)
+	return out, err
+}

sdk/go/types.go

@@ -144,14 +144,20 @@ type TaskPage struct {
 	NextCursor string `json:"next_cursor,omitempty"`
 }
 type CreateQueueRequest struct {
+	// MaxUnfinishedTasks 未终结任务上限，包含待核查；省略或零使用 100000。
+	MaxUnfinishedTasks uint32 `json:"max_unfinished_tasks,omitempty"`
 	// Name 队列名称，同一命名空间内唯一。
 	Name string `json:"name"`
-	// MaxConcurrency 最大同时执行数量，供执行阶段使用。
+	// MaxConcurrency 允许占用的有效执行许可上限。
 	MaxConcurrency uint32 `json:"max_concurrency"`
-	// RatePerSecond 每秒开始数量上限，供执行阶段使用。
+	// RatePerSecond 数据库一秒窗口最多提交的开始授权数。
 	RatePerSecond uint32 `json:"rate_per_second"`
 }
 type Queue struct {
+	// MaxUnfinishedTasks 未终结任务上限，只限制新的受理。
+	MaxUnfinishedTasks uint32 `json:"max_unfinished_tasks"`
+	// UnfinishedTasks 未终结任务数，线上为十进制字符串；变化不增加配置版本。
+	UnfinishedTasks uint64 `json:"unfinished_tasks,string"`
 	// ID 资源标识，32 位小写十六进制字符串。
 	ID string `json:"id"`
 	// NamespaceID 命名空间标识，由凭据确定隔离范围。
@@ -160,9 +166,9 @@ type Queue struct {
 	Name string `json:"name"`
 	// Paused 是否停止新执行；不禁止受理新任务。
 	Paused bool `json:"paused"`
-	// MaxConcurrency 最大同时执行数量，供执行阶段使用。
+	// MaxConcurrency 允许占用的有效执行许可上限。
 	MaxConcurrency uint32 `json:"max_concurrency"`
-	// RatePerSecond 每秒开始数量上限，供执行阶段使用。
+	// RatePerSecond 数据库一秒窗口最多提交的开始授权数。
 	RatePerSecond uint32 `json:"rate_per_second"`
 	// Revision 并发版本，JSON 使用十进制字符串避免跨语言精度丢失。
 	Revision uint64 `json:"revision,string"`
@@ -202,3 +208,19 @@ type RetryPolicy struct {
 	MaxDelayMS       uint64 `json:"max_delay_ms"`       // MaxDelayMS 间隔上限。
 	AttemptTimeoutMS uint64 `json:"attempt_timeout_ms"` // AttemptTimeoutMS 单次尝试时限。
 }
+
+// ConfigureQueueRequest 完整替换控制配置，必须保留同一操作的原版本及原因。
+type ConfigureQueueRequest struct {
+	// ExpectedRevision 非零的队列配置版本。
+	ExpectedRevision uint64 `json:"expected_revision,string"`
+	// Reason 管理原因，写入不可变审计。
+	Reason string `json:"reason"`
+	// Paused 停止新的领取和开始；在途续租、结果与受理继续。
+	Paused bool `json:"paused"`
+	// MaxConcurrency 有效并发上限，降低时已有许可自然收尾。
+	MaxConcurrency uint32 `json:"max_concurrency"`
+	// RatePerSecond 数据库一秒窗口最多开始授权数，降低时保留原计数。
+	RatePerSecond uint32 `json:"rate_per_second"`
+	// MaxUnfinishedTasks 未终结任务上限，降低时不删除历史或在途任务。
+	MaxUnfinishedTasks uint32 `json:"max_unfinished_tasks"`
+}

sdk/go/worker_process_integration_test.go

@@ -147,6 +147,49 @@ func awaitProcessDrop(t *testing.T, ctx context.Context, g *workerFaultGateway,
 	}
 }
 
+func TestIntegrationWorkerProcessQueueBackpressure(t *testing.T) {
+	a := startOrbitProcess(t)
+	_, _, token := integrationWorkerClient(t)
+	api := processClient(t, a, token)
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	name := "quota-crash-" + workerTestID(t)
+	q, err := api.CreateQueue(ctx, name, CreateQueueRequest{Name: name, MaxConcurrency: 1, RatePerSecond: 1, MaxUnfinishedTasks: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	in := SubmitTaskRequest{QueueID: q.Queue.ID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)}
+	first, err := api.SubmitTask(ctx, name+"-first", in)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.kill(t)
+	b := startOrbitProcess(t)
+	api = processClient(t, b, token)
+	if a.nodeID == b.nodeID {
+		t.Fatal("node identity reused")
+	}
+	_, err = api.SubmitTask(ctx, name+"-second", in)
+	var rejected *APIError
+	if !errors.As(err, &rejected) || rejected.Code != "QUEUE_FULL" {
+		t.Fatalf("restart lost quota: %v", err)
+	}
+	dupe, err := api.SubmitTask(ctx, name+"-first", in)
+	if err != nil || !dupe.Deduplicated || dupe.Task.ID != first.Task.ID {
+		t.Fatal("restart lost original receipt")
+	}
+	if _, err := api.CancelTask(ctx, first.Task.ID, name+"-release", CancelTaskRequest{ExpectedRevision: first.Task.Revision, Reason: "free recovered queue"}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := api.SubmitTask(ctx, name+"-second", in); err != nil {
+		t.Fatal(err)
+	}
+	current, err := api.GetQueue(ctx, q.Queue.ID)
+	if err != nil || current.UnfinishedTasks != 1 {
+		t.Fatal("recovered quota counter mismatch")
+	}
+}
+
 // 两次在真实数据库提交后丢 ACK 并强杀节点，由另外的真实进程核对原操作。
 func TestIntegrationWorkerProcessACKRecovery(t *testing.T) {
 	a := startOrbitProcess(t)

sdk/go/worker_runtime.go

@@ -497,7 +497,7 @@ func (r *workerRuntime) receive(reply runtimeReply) error {
 					r.remove(j)
 				}
 				return nil
-			case "EXECUTION_PAUSED", "UNAVAILABLE":
+			case "EXECUTION_PAUSED", "RATE_LIMITED", "UNAVAILABLE":
 				if j.retryDelay == 0 {
 					j.retryDelay = 200 * time.Millisecond
 				} else {