package orbit import ( "context" "encoding/json" "errors" "sync/atomic" "testing" "time" ) func TestIntegrationWorkerRuntimeSafeRetry(t *testing.T) { api, queue, token := integrationWorkerClient(t) ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second) defer cancel() for _, tc := range []struct { name string safety RecoverySafety final TaskState calls int32 unknown bool }{ {"known", RecoveryReconcileOnly, TaskSucceeded, 2, false}, {"unknown_idempotent", RecoveryIdempotent, TaskSucceeded, 2, true}, {"unknown_reconcile", RecoveryReconcileOnly, TaskNeedsReview, 1, true}, {"exhausted", RecoveryReconcileOnly, TaskFailed, 2, false}, } { t.Run(tc.name, func(t *testing.T) { g := newWorkerFaultGateway(t, api.baseURL) // 首次失败已落库并安排重试后丢确认,运行器必须核对同一回执而不是重置尝试。 if tc.name == "known" { g.drop.Store("task.finish") } client, err := NewClient(g.server.URL, Options{Token: token, HTTPClient: g.server.Client()}) if err != nil { t.Fatal(err) } name := "retry." + tc.name + "." + workerTestID(t) var calls atomic.Int32 var epochs [2]uint64 w, err := NewWorker(client, WorkerOptions{QueueID: queue, Capacity: 1, Handlers: []WorkerHandler{runtimeHandler(name, func(_ context.Context, e WorkerExecution) (json.RawMessage, error) { n := calls.Add(1) if n > 2 { return nil, errors.New("retry exceeded test budget") } epochs[n-1] = e.Epoch if n == 1 || tc.name == "exhausted" { if tc.unknown { return nil, errors.New("simulated unknown outcome") } return nil, &WorkerFailure{Code: "TEMPORARY_REJECTION", Retryable: true, Details: json.RawMessage(`{"reason_ref":"diagnostics/temporary"}`)} } return json.RawMessage(`{"n":9007199254740993}`), nil })}}) if err != nil { t.Fatal(err) } accepted, err := api.SubmitTask(ctx, "retry-"+workerTestID(t), SubmitTaskRequest{QueueID: queue, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`), RecoverySafety: tc.safety, TaskTimeoutMS: 30000, Retry: RetryPolicy{MaxAttempts: 2, AttemptTimeoutMS: 5000, InitialDelayMS: 200, MaxDelayMS: 1000}}) if err != nil { t.Fatal(err) } run, stop := context.WithCancel(ctx) defer stop() done := make(chan error, 1) go func() { done <- w.Run(run) }() current := waitRuntimeState(t, ctx, api, accepted.Task.ID, tc.final) stop() select { case err := <-done: if err != nil { t.Fatal(err) } case <-ctx.Done(): t.Fatal("retry Worker did not drain") } if calls.Load() != tc.calls || current.AttemptCount != uint32(tc.calls) || w.Stats().Started != uint64(tc.calls) || epochs[0] != 1 || (tc.calls == 2 && epochs[1] != 2) { t.Fatal("retry bypassed attempt identity or budget") } if current.EffectUncertain != (tc.final == TaskNeedsReview) { t.Fatal("uncertainty does not match authoritative result") } attempts, err := api.ListTaskAttempts(ctx, current.ID, HistoryPageRequest{}) if err != nil || len(attempts.Items) != int(tc.calls) { t.Fatal("SDK lost retry history") } oldest := attempts.Items[len(attempts.Items)-1] if oldest.Retryable == tc.unknown { t.Fatal("SDK lost retry classification") } if tc.calls == 2 { second := attempts.Items[0] if oldest.EndedAt == nil || second.CreatedAt.Before(oldest.EndedAt.Add(100*time.Millisecond)) { t.Fatal("dispatch ignored retry delay") } } if tc.name == "known" { g.expectDrop(t, "task.finish") if w.Stats().Reconnects == 0 { t.Fatal("retry receipt did not recover lost ACK") } } }) } }