feat: add durable safe retries with uncertainty tracking and SDK examples

2026-09-17 15:43:44 +08:00
parent 9e3d4be5cc
commit d65cf96ea7
44 changed files with 1300 additions and 77 deletions
--- a/sdk/go/examples/retry/main.go
+++ b/sdk/go/examples/retry/main.go
@@ -0,0 +1,149 @@
+// retry 演示明确失败的安全重试：同一任务两次尝试，退避由平台持久化管理。
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+func run() error {
+	token, queue := os.Getenv("ORBIT_TOKEN"), os.Getenv("ORBIT_QUEUE_ID")
+	if path := os.Getenv("ORBIT_CREDENTIALS_FILE"); path != "" {
+		raw, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("read private credentials: %w", err)
+		}
+		var credentials struct {
+			Token   string `json:"token"`
+			QueueID string `json:"queue_id"`
+		}
+		if json.Unmarshal(raw, &credentials) != nil {
+			return errors.New("invalid credentials file")
+		}
+		if token == "" {
+			token = credentials.Token
+		}
+		if queue == "" {
+			queue = credentials.QueueID
+		}
+	}
+	client, err := orbit.NewClient(os.Getenv("ORBIT_URL"), orbit.Options{Token: token})
+	if err != nil {
+		return err
+	}
+	operation, err := orbit.NewWorkerMessageID()
+	if err != nil {
+		return err
+	}
+	handler := "text.normalize.retry." + operation
+	var calls atomic.Uint32
+	process := func(ctx context.Context, e orbit.WorkerExecution) (json.RawMessage, error) {
+		// 第一次在产生业务效果前明确拒绝，可安全重试；计数仅用于制造示例故障。
+		if calls.Add(1) == 1 {
+			return nil, &orbit.WorkerFailure{Code: "TEMPORARY_REJECTION", Retryable: true}
+		}
+		return normalize(ctx, e)
+	}
+	w, err := orbit.NewWorker(client, orbit.WorkerOptions{QueueID: queue, Capacity: 1, ShutdownTimeout: 5 * time.Second, Handlers: []orbit.WorkerHandler{{Capability: orbit.WorkerCapability{Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: process}}})
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	// 示例先受理，再启动消费者；生产通常是独立的长驻 Worker 进程。
+	var tasks []orbit.Task
+	for i, text := range []string{"  safe retry  "} {
+		payload, _ := json.Marshal(struct {
+			Text string `json:"text"`
+		}{text})
+		accepted, err := client.SubmitTask(ctx, fmt.Sprintf("retry-%s-%d", operation, i), orbit.SubmitTaskRequest{QueueID: queue, Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1", Payload: payload, TaskTimeoutMS: 60000, Retry: orbit.RetryPolicy{MaxAttempts: 3, AttemptTimeoutMS: 10000, InitialDelayMS: 500, MaxDelayMS: 2000}, RecoverySafety: orbit.RecoveryReconcileOnly})
+		if err != nil {
+			return err
+		}
+		tasks = append(tasks, accepted.Task)
+	}
+	workerCtx, stop := context.WithCancel(ctx)
+	done := make(chan error, 1)
+	go func() { done <- w.Run(workerCtx) }()
+	// 任一调用失败时也等待有界排空，不在 main 返回时静默丢掉执行中的 Worker。
+	waited := false
+	defer func() {
+		stop()
+		if !waited {
+			<-done
+		}
+	}()
+	for _, task := range tasks {
+		for {
+			current, err := client.GetTask(ctx, task.ID)
+			if err != nil {
+				return err
+			}
+			if current.State == orbit.TaskSucceeded {
+				history, err := client.ListTaskAttempts(ctx, current.ID, orbit.HistoryPageRequest{})
+				if err != nil {
+					return err
+				}
+				if current.AttemptCount != 2 || len(history.Items) != 2 || history.Items[1].State != orbit.AttemptFailed || !history.Items[1].Retryable || current.EffectUncertain {
+					return errors.New("retry history mismatch")
+				}
+				fmt.Printf("task=%s state=%s attempts=%d first_retryable=%t result=%s\n", current.ID, current.State, current.AttemptCount, history.Items[1].Retryable, current.Result)
+				break
+			}
+			if current.State == orbit.TaskFailed || current.State == orbit.TaskNeedsReview {
+				return fmt.Errorf("task %s requires attention: %s", current.ID, current.State)
+			}
+			select {
+			case err := <-done:
+				waited = true
+				return fmt.Errorf("Worker stopped before completion: %v", err)
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-time.After(100 * time.Millisecond):
+			}
+		}
+	}
+	stop()
+	err = <-done
+	waited = true
+	if err != nil {
+		return err
+	}
+	stats := w.Stats()
+	if stats.Started != 2 || stats.ReportsConfirmed != 2 || stats.Running != 0 {
+		return errors.New("unexpected Worker completion counts")
+	}
+	fmt.Printf("worker_drained=true started=%d confirmed=%d running=%d\n", stats.Started, stats.ReportsConfirmed, stats.Running)
+	return nil
+}
+func normalize(ctx context.Context, execution orbit.WorkerExecution) (json.RawMessage, error) {
+	var in struct {
+		Text string `json:"text"`
+	}
+	if err := json.Unmarshal(execution.Task.Payload, &in); err != nil {
+		return nil, &orbit.WorkerFailure{Code: "INVALID_TEXT"}
+	}
+	// 模拟可取消的短工作；真实 I/O 也必须使用传入的 ctx。
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case <-time.After(200 * time.Millisecond):
+	}
+	return json.Marshal(struct {
+		Text string `json:"text"`
+	}{Text: strings.ToUpper(strings.TrimSpace(in.Text))})
+}
--- a/sdk/go/history_types.go
+++ b/sdk/go/history_types.go
@@ -21,6 +21,7 @@ type AttemptSummary struct {
 	LeaseUntil   time.Time    `json:"lease_until"`             // LeaseUntil 数据库记录的租约期限；查询不能授权业务执行。
 	AuthorizedAt *time.Time   `json:"authorized_at,omitempty"` // AuthorizedAt 开始批准时间；空表示未批准，不证明副作用已经发生。
 	EndedAt      *time.Time   `json:"ended_at,omitempty"`      // EndedAt 尝试结束时间；仍在执行时为空。
+	Retryable    bool         `json:"retryable,omitempty"`     // Retryable 该明确失败报告是否声明可安全重试；不代表平台必定安排重试。
 	ErrorCode    string       `json:"error_code,omitempty"`    // ErrorCode 稳定错误分类；无错误时为空。
 	CreatedAt    time.Time    `json:"created_at"`              // CreatedAt 创建时间，UTC 微秒精度。
 	UpdatedAt    time.Time    `json:"updated_at"`              // UpdatedAt 最新变更时间，不能代替版本校验。
@@ -47,6 +48,7 @@ type AttemptEvidenceSummary struct {
 	AttemptID   string    `json:"attempt_id"`           // AttemptID 所属执行尝试。
 	Outcome     string    `json:"outcome"`              // Outcome Worker 声明的 SUCCEEDED、FAILED 或 UNKNOWN，不代表平台认可。
 	ErrorCode   string    `json:"error_code,omitempty"` // ErrorCode 原报告的稳定分类，可为空。
+	Retryable   bool      `json:"retryable,omitempty"`  // Retryable 原报告的重试声明；证据不触发重试。
 	Reason      string    `json:"reason"`               // Reason LEASE_LOST、NOT_STARTED 或 RESULT_CONFLICT。
 	Disposition string    `json:"disposition"`          // Disposition 固定为 EVIDENCE_ONLY，表示采集时仅保存证据。
 	CreatedAt   time.Time `json:"created_at"`           // CreatedAt 首次收到时间，重复报告不改变它。
--- a/sdk/go/types.go
+++ b/sdk/go/types.go
@@ -54,6 +54,8 @@ type Task struct {
 	Payload json.RawMessage `json:"payload"`
 	// Result 业务 JSON 结果；未取得结果时为空并省略。
 	Result json.RawMessage `json:"result,omitempty"`
+	// EffectUncertain 曾有执行效果尚未核实；重试与预算耗尽不会把未知伪装成明确失败。
+	EffectUncertain bool `json:"effect_uncertain,omitempty"`
 	// BusinessKey 可选业务关联键，不自动提供业务效果幂等。
 	BusinessKey string `json:"business_key,omitempty"`
 	// AvailableAt 最早可执行时间，UTC。
@@ -120,6 +122,8 @@ type TaskSummary struct {
 	Handler string `json:"handler"`
 	// HandlerVersion 受理时固定的处理器版本。
 	HandlerVersion string `json:"handler_version"`
+	// EffectUncertain 曾有执行效果尚未核实；重试与预算耗尽不会把未知伪装成明确失败。
+	EffectUncertain bool `json:"effect_uncertain,omitempty"`
 	// BusinessKey 可选业务关联键，不自动提供业务效果幂等。
 	BusinessKey string `json:"business_key,omitempty"`
 	// AvailableAt 最早可执行时间，UTC。
--- a/sdk/go/worker_connection.go
+++ b/sdk/go/worker_connection.go
@@ -332,6 +332,9 @@ func (w *WorkerConnection) Renew(ctx context.Context, id string, ref WorkerAttem
 // Finish 保存业务结果/证据；迟到结果可返回 EVIDENCE_ONLY，不能视为任务成功终结。
 func (w *WorkerConnection) Finish(ctx context.Context, id string, in WorkerFinish) (WorkerCommandResult, error) {
 	var out WorkerCommandResult
+	if in.Retryable && in.Outcome != WorkerFailed {
+		return out, errors.New("retryable is only valid for known failure")
+	}
 	if (in.Outcome != WorkerSucceeded && in.Outcome != WorkerFailed && in.Outcome != WorkerUnknown) || len(in.Result) > 256<<10 || bytes.Equal(bytes.TrimSpace(in.Result), []byte("null")) || validateWorkerJSON(in.Result) != nil {
 		return out, errors.New("invalid Worker result outcome or JSON")
 	}
--- a/sdk/go/worker_handler.go
+++ b/sdk/go/worker_handler.go
@@ -52,7 +52,7 @@ func (w *Worker) invoke(ctx context.Context, handler WorkerHandler, execution Wo
 				details = json.RawMessage(`{}`)
 			}
 			if validHandlerResult(details) {
-				out = WorkerFinish{Ref: ref, Outcome: WorkerFailed, ErrorCode: known.Code, Result: details}
+				out = WorkerFinish{Ref: ref, Outcome: WorkerFailed, ErrorCode: known.Code, Result: details, Retryable: known.Retryable}
 			} else {
 				out = unknownHandlerResult(ref, "INVALID_HANDLER_RESULT")
 			}
--- a/sdk/go/worker_retry_integration_test.go
+++ b/sdk/go/worker_retry_integration_test.go
@@ -0,0 +1,104 @@
+package orbit
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestIntegrationWorkerRuntimeSafeRetry(t *testing.T) {
+	api, queue, token := integrationWorkerClient(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second)
+	defer cancel()
+	for _, tc := range []struct {
+		name    string
+		safety  RecoverySafety
+		final   TaskState
+		calls   int32
+		unknown bool
+	}{
+		{"known", RecoveryReconcileOnly, TaskSucceeded, 2, false},
+		{"unknown_idempotent", RecoveryIdempotent, TaskSucceeded, 2, true},
+		{"unknown_reconcile", RecoveryReconcileOnly, TaskNeedsReview, 1, true},
+		{"exhausted", RecoveryReconcileOnly, TaskFailed, 2, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			g := newWorkerFaultGateway(t, api.baseURL)
+			// 首次失败已落库并安排重试后丢确认，运行器必须核对同一回执而不是重置尝试。
+			if tc.name == "known" {
+				g.drop.Store("task.finish")
+			}
+			client, err := NewClient(g.server.URL, Options{Token: token, HTTPClient: g.server.Client()})
+			if err != nil {
+				t.Fatal(err)
+			}
+			name := "retry." + tc.name + "." + workerTestID(t)
+			var calls atomic.Int32
+			var epochs [2]uint64
+			w, err := NewWorker(client, WorkerOptions{QueueID: queue, Capacity: 1, Handlers: []WorkerHandler{runtimeHandler(name, func(_ context.Context, e WorkerExecution) (json.RawMessage, error) {
+				n := calls.Add(1)
+				if n > 2 {
+					return nil, errors.New("retry exceeded test budget")
+				}
+				epochs[n-1] = e.Epoch
+				if n == 1 || tc.name == "exhausted" {
+					if tc.unknown {
+						return nil, errors.New("simulated unknown outcome")
+					}
+					return nil, &WorkerFailure{Code: "TEMPORARY_REJECTION", Retryable: true, Details: json.RawMessage(`{"reason_ref":"diagnostics/temporary"}`)}
+				}
+				return json.RawMessage(`{"n":9007199254740993}`), nil
+			})}})
+			if err != nil {
+				t.Fatal(err)
+			}
+			accepted, err := api.SubmitTask(ctx, "retry-"+workerTestID(t), SubmitTaskRequest{QueueID: queue, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`), RecoverySafety: tc.safety, TaskTimeoutMS: 30000, Retry: RetryPolicy{MaxAttempts: 2, AttemptTimeoutMS: 5000, InitialDelayMS: 200, MaxDelayMS: 1000}})
+			if err != nil {
+				t.Fatal(err)
+			}
+			run, stop := context.WithCancel(ctx)
+			defer stop()
+			done := make(chan error, 1)
+			go func() { done <- w.Run(run) }()
+			current := waitRuntimeState(t, ctx, api, accepted.Task.ID, tc.final)
+			stop()
+			select {
+			case err := <-done:
+				if err != nil {
+					t.Fatal(err)
+				}
+			case <-ctx.Done():
+				t.Fatal("retry Worker did not drain")
+			}
+			if calls.Load() != tc.calls || current.AttemptCount != uint32(tc.calls) || w.Stats().Started != uint64(tc.calls) || epochs[0] != 1 || (tc.calls == 2 && epochs[1] != 2) {
+				t.Fatal("retry bypassed attempt identity or budget")
+			}
+			if current.EffectUncertain != (tc.final == TaskNeedsReview) {
+				t.Fatal("uncertainty does not match authoritative result")
+			}
+			attempts, err := api.ListTaskAttempts(ctx, current.ID, HistoryPageRequest{})
+			if err != nil || len(attempts.Items) != int(tc.calls) {
+				t.Fatal("SDK lost retry history")
+			}
+			oldest := attempts.Items[len(attempts.Items)-1]
+			if oldest.Retryable == tc.unknown {
+				t.Fatal("SDK lost retry classification")
+			}
+			if tc.calls == 2 {
+				second := attempts.Items[0]
+				if oldest.EndedAt == nil || second.CreatedAt.Before(oldest.EndedAt.Add(100*time.Millisecond)) {
+					t.Fatal("dispatch ignored retry delay")
+				}
+			}
+			if tc.name == "known" {
+				g.expectDrop(t, "task.finish")
+				if w.Stats().Reconnects == 0 {
+					t.Fatal("retry receipt did not recover lost ACK")
+				}
+			}
+		})
+	}
+}
--- a/sdk/go/worker_runtime_types.go
+++ b/sdk/go/worker_runtime_types.go
@@ -34,8 +34,9 @@ type WorkerHandler struct {

 // WorkerFailure 仅用于接入方能够确认的业务失败，不用于网络超时或未知外部效果。
 type WorkerFailure struct {
-	Code    string          // Code 1–64 位稳定 ASCII 分类，禁止放原始异常正文。
-	Details json.RawMessage // Details 可选非 null JSON，最多 256 KiB；空值使用空对象。
+	Code      string          // Code 1–64 位稳定 ASCII 分类，禁止放原始异常正文。
+	Details   json.RawMessage // Details 可选非 null JSON，最多 256 KiB；空值使用空对象。
+	Retryable bool            // Retryable 声明该明确失败可安全重试；默认假，不能用它掩盖网络超时或部分副作用未知。
 }

 func (e *WorkerFailure) Error() string { return "Worker handler reported a known failure" }
--- a/sdk/go/worker_types.go
+++ b/sdk/go/worker_types.go
@@ -97,9 +97,10 @@ type WorkerFinish struct {
 	Outcome   WorkerOutcome   `json:"outcome"`              // Outcome 明确成功、明确失败或效果未知。
 	Result    json.RawMessage `json:"result"`               // Result 非 null JSON，最多 256 KiB；保留精确数字文本。
 	ErrorCode string          `json:"error_code,omitempty"` // ErrorCode 失败/未知时必填稳定分类，成功时为空；禁止原始异常正文。
+	Retryable bool            `json:"retryable,omitempty"`  // Retryable 仅明确 FAILED 可声明安全重试；默认假，不适用于 UNKNOWN。
 }

-// WorkerOutcome 描述业务效果；UNKNOWN 必须进入核查而不是盲目重跑。
+// WorkerOutcome 描述业务效果；UNKNOWN 仅在任务声明幂等恢复且预算足够时可重试，否则转核查。
 type WorkerOutcome string

 const (