105 lines
3.6 KiB
Go
105 lines
3.6 KiB
Go
package orbit
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestIntegrationWorkerRuntimeSafeRetry(t *testing.T) {
|
|
api, queue, token := integrationWorkerClient(t)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second)
|
|
defer cancel()
|
|
for _, tc := range []struct {
|
|
name string
|
|
safety RecoverySafety
|
|
final TaskState
|
|
calls int32
|
|
unknown bool
|
|
}{
|
|
{"known", RecoveryReconcileOnly, TaskSucceeded, 2, false},
|
|
{"unknown_idempotent", RecoveryIdempotent, TaskSucceeded, 2, true},
|
|
{"unknown_reconcile", RecoveryReconcileOnly, TaskNeedsReview, 1, true},
|
|
{"exhausted", RecoveryReconcileOnly, TaskFailed, 2, false},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
g := newWorkerFaultGateway(t, api.baseURL)
|
|
// 首次失败已落库并安排重试后丢确认，运行器必须核对同一回执而不是重置尝试。
|
|
if tc.name == "known" {
|
|
g.drop.Store("task.finish")
|
|
}
|
|
client, err := NewClient(g.server.URL, Options{Token: token, HTTPClient: g.server.Client()})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
name := "retry." + tc.name + "." + workerTestID(t)
|
|
var calls atomic.Int32
|
|
var epochs [2]uint64
|
|
w, err := NewWorker(client, WorkerOptions{QueueID: queue, Capacity: 1, Handlers: []WorkerHandler{runtimeHandler(name, func(_ context.Context, e WorkerExecution) (json.RawMessage, error) {
|
|
n := calls.Add(1)
|
|
if n > 2 {
|
|
return nil, errors.New("retry exceeded test budget")
|
|
}
|
|
epochs[n-1] = e.Epoch
|
|
if n == 1 || tc.name == "exhausted" {
|
|
if tc.unknown {
|
|
return nil, errors.New("simulated unknown outcome")
|
|
}
|
|
return nil, &WorkerFailure{Code: "TEMPORARY_REJECTION", Retryable: true, Details: json.RawMessage(`{"reason_ref":"diagnostics/temporary"}`)}
|
|
}
|
|
return json.RawMessage(`{"n":9007199254740993}`), nil
|
|
})}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
accepted, err := api.SubmitTask(ctx, "retry-"+workerTestID(t), SubmitTaskRequest{QueueID: queue, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`), RecoverySafety: tc.safety, TaskTimeoutMS: 30000, Retry: RetryPolicy{MaxAttempts: 2, AttemptTimeoutMS: 5000, InitialDelayMS: 200, MaxDelayMS: 1000}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
run, stop := context.WithCancel(ctx)
|
|
defer stop()
|
|
done := make(chan error, 1)
|
|
go func() { done <- w.Run(run) }()
|
|
current := waitRuntimeState(t, ctx, api, accepted.Task.ID, tc.final)
|
|
stop()
|
|
select {
|
|
case err := <-done:
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
case <-ctx.Done():
|
|
t.Fatal("retry Worker did not drain")
|
|
}
|
|
if calls.Load() != tc.calls || current.AttemptCount != uint32(tc.calls) || w.Stats().Started != uint64(tc.calls) || epochs[0] != 1 || (tc.calls == 2 && epochs[1] != 2) {
|
|
t.Fatal("retry bypassed attempt identity or budget")
|
|
}
|
|
if current.EffectUncertain != (tc.final == TaskNeedsReview) {
|
|
t.Fatal("uncertainty does not match authoritative result")
|
|
}
|
|
attempts, err := api.ListTaskAttempts(ctx, current.ID, HistoryPageRequest{})
|
|
if err != nil || len(attempts.Items) != int(tc.calls) {
|
|
t.Fatal("SDK lost retry history")
|
|
}
|
|
oldest := attempts.Items[len(attempts.Items)-1]
|
|
if oldest.Retryable == tc.unknown {
|
|
t.Fatal("SDK lost retry classification")
|
|
}
|
|
if tc.calls == 2 {
|
|
second := attempts.Items[0]
|
|
if oldest.EndedAt == nil || second.CreatedAt.Before(oldest.EndedAt.Add(100*time.Millisecond)) {
|
|
t.Fatal("dispatch ignored retry delay")
|
|
}
|
|
}
|
|
if tc.name == "known" {
|
|
g.expectDrop(t, "task.finish")
|
|
if w.Stats().Reconnects == 0 {
|
|
t.Fatal("retry receipt did not recover lost ACK")
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|