feat: add durable relative task delays and outage recovery regression
This commit is contained in:
192
sdk/go/retry_process_integration_test.go
Normal file
192
sdk/go/retry_process_integration_test.go
Normal file
@@ -0,0 +1,192 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 首次失败已提交并进入退避后，强杀唯一服务进程；两个新进程仅靠持久化状态接管。
|
||||
// 所有时间自然流逝，不修改数据库的退避、截止或租约字段。
|
||||
func TestIntegrationWorkerProcessRetryBackoffRecovery(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
unknown bool
|
||||
expire bool
|
||||
final TaskState
|
||||
}{
|
||||
{"known_then_success", false, false, TaskSucceeded},
|
||||
{"unknown_then_exhausted", true, false, TaskNeedsReview},
|
||||
{"known_deadline_during_outage", false, true, TaskFailed},
|
||||
{"unknown_deadline_during_outage", true, true, TaskNeedsReview},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
a := startOrbitProcess(t)
|
||||
_, _, token := integrationWorkerClient(t)
|
||||
api := processClient(t, a, token)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
name := "retry-crash." + workerTestID(t)
|
||||
q, err := api.CreateQueue(ctx, name, CreateQueueRequest{Name: name, MaxConcurrency: 2, RatePerSecond: 100})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
connect := func(client *Client) *WorkerConnection {
|
||||
t.Helper()
|
||||
r, err := NewWorkerRegistration(q.Queue.ID, 1, []WorkerCapability{{Handler: name, HandlerVersion: "v1", PayloadVersion: "v1"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w, err := client.DialWorker(ctx, r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = w.Close() })
|
||||
if _, err := w.Capacity(ctx, workerTestID(t), WorkerCapacity{Sequence: 1, Capacity: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return w
|
||||
}
|
||||
in := SubmitTaskRequest{QueueID: q.Queue.ID, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`), RecoverySafety: RecoveryReconcileOnly, TaskTimeoutMS: 40000, Retry: RetryPolicy{MaxAttempts: 2, AttemptTimeoutMS: 3000, InitialDelayMS: 8000, MaxDelayMS: 8000}}
|
||||
if tc.unknown {
|
||||
in.RecoverySafety = RecoveryIdempotent
|
||||
}
|
||||
if tc.expire {
|
||||
in.TaskTimeoutMS = 5000
|
||||
in.Retry.InitialDelayMS, in.Retry.MaxDelayMS = 1000, 1000
|
||||
}
|
||||
accepted, err := api.SubmitTask(ctx, workerTestID(t), in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := connect(api)
|
||||
first, err := w.NextAssignment(ctx)
|
||||
if err != nil || first.Task.ID != accepted.Task.ID || first.Epoch != 1 {
|
||||
t.Fatal("first attempt identity", err)
|
||||
}
|
||||
if _, err := w.Start(ctx, workerTestID(t), first.Attempt()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
finish := WorkerFinish{Ref: first.Attempt(), Outcome: WorkerFailed, Retryable: true, ErrorCode: "TEMPORARY_REJECTION", Result: json.RawMessage(`{"evidence_ref":"diagnostics/first"}`)}
|
||||
if tc.unknown {
|
||||
finish.Outcome, finish.Retryable, finish.ErrorCode = WorkerUnknown, false, "RESULT_UNKNOWN"
|
||||
}
|
||||
receipt, err := w.Finish(ctx, workerTestID(t), finish)
|
||||
if err != nil || receipt.Receipt.TaskState != TaskWaiting || receipt.Lease.Active {
|
||||
t.Fatal("first result did not durably enter backoff", err)
|
||||
}
|
||||
pending, err := api.GetTask(ctx, accepted.Task.ID)
|
||||
if err != nil || pending.State != TaskWaiting || pending.AttemptCount != 1 || pending.EffectUncertain != tc.unknown || !pending.AvailableAt.After(receipt.Lease.ServerTime) {
|
||||
t.Fatal("retry facts were not persisted", err)
|
||||
}
|
||||
old, err := api.GetTaskAttempt(ctx, pending.ID, first.AttemptID)
|
||||
if err != nil || old.EndedAt == nil || old.Retryable == tc.unknown || string(old.Result) != string(finish.Result) {
|
||||
t.Fatal("first attempt evidence missing", err)
|
||||
}
|
||||
if delay := pending.AvailableAt.Sub(*old.EndedAt); delay < time.Duration(in.Retry.InitialDelayMS/2)*time.Millisecond || delay > time.Duration(in.Retry.MaxDelayMS)*time.Millisecond {
|
||||
t.Fatal("persisted backoff outside policy")
|
||||
}
|
||||
a.kill(t)
|
||||
_ = w.Close()
|
||||
if tc.expire {
|
||||
// 用已确认的数据库时间差等待，恢复后的状态/历史仍以数据库事实作断言。
|
||||
timer := time.NewTimer(pending.DeadlineAt.Sub(receipt.Lease.ServerTime) + 100*time.Millisecond)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("outage exceeded test budget")
|
||||
}
|
||||
}
|
||||
b, c := startOrbitProcess(t), startOrbitProcess(t)
|
||||
if a.nodeID == b.nodeID || a.nodeID == c.nodeID || b.nodeID == c.nodeID {
|
||||
t.Fatal("replacement reused node identity")
|
||||
}
|
||||
apiB, apiC := processClient(t, b, token), processClient(t, c, token)
|
||||
if !tc.expire {
|
||||
recovered, err := apiB.GetTask(ctx, pending.ID)
|
||||
if err != nil || !reflect.DeepEqual(recovered, pending) {
|
||||
t.Fatal("restart changed frozen backoff, budget, deadline or uncertainty", err)
|
||||
}
|
||||
}
|
||||
workers := []*WorkerConnection{connect(apiB), connect(apiC)}
|
||||
var current Task
|
||||
wantAttempts := uint32(1)
|
||||
if !tc.expire {
|
||||
type assignmentResult struct {
|
||||
index int
|
||||
assignment WorkerAssignment
|
||||
err error
|
||||
}
|
||||
wait, stop := context.WithCancel(ctx)
|
||||
defer stop()
|
||||
results := make(chan assignmentResult, 2)
|
||||
for i, worker := range workers {
|
||||
go func() {
|
||||
assignment, err := worker.NextAssignment(wait)
|
||||
results <- assignmentResult{i, assignment, err}
|
||||
}()
|
||||
}
|
||||
winner := <-results
|
||||
second := winner.assignment
|
||||
if winner.err != nil || second.Task.ID != pending.ID || second.AttemptID == first.AttemptID || second.Epoch != 2 {
|
||||
t.Fatal("retry did not preserve task identity and consumed budget", winner.err)
|
||||
}
|
||||
history, err := apiB.GetTaskAttempt(ctx, pending.ID, second.AttemptID)
|
||||
if err != nil || history.CreatedAt.Before(pending.AvailableAt) || !history.CreatedAt.Before(pending.DeadlineAt) {
|
||||
t.Fatal("retry ignored persisted timing", err)
|
||||
}
|
||||
owner := workers[winner.index]
|
||||
if _, err := owner.Start(ctx, workerTestID(t), second.Attempt()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
last := WorkerFinish{Ref: second.Attempt(), Outcome: WorkerSucceeded, Result: json.RawMessage(`{"n":9007199254740993}`)}
|
||||
if tc.unknown {
|
||||
last.Outcome, last.Retryable, last.ErrorCode = WorkerFailed, true, "TEMPORARY_REJECTION"
|
||||
}
|
||||
if _, err := owner.Finish(ctx, workerTestID(t), last); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stop()
|
||||
loser := <-results
|
||||
if !errors.Is(loser.err, context.Canceled) {
|
||||
t.Fatal("two replacement nodes received a retry assignment", loser.err)
|
||||
}
|
||||
wantAttempts = 2
|
||||
}
|
||||
current = waitRuntimeState(t, ctx, apiB, pending.ID, tc.final)
|
||||
if current.AttemptCount != wantAttempts || current.EffectUncertain != tc.unknown || current.Retry != pending.Retry || !current.DeadlineAt.Equal(pending.DeadlineAt) || current.RecoverySafety != pending.RecoverySafety || string(current.Payload) != string(pending.Payload) {
|
||||
t.Fatal("recovery reset policy, input, deadline, attempts or uncertainty")
|
||||
}
|
||||
// 两个节点仍在线且都有可用容量，预算/截止应阻止任何额外领取。
|
||||
for _, worker := range workers {
|
||||
probe, stop := context.WithTimeout(ctx, 250*time.Millisecond)
|
||||
_, err := worker.NextAssignment(probe)
|
||||
stop()
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatal("unexpected assignment after final decision", err)
|
||||
}
|
||||
}
|
||||
attempts, err := apiC.ListTaskAttempts(ctx, pending.ID, HistoryPageRequest{})
|
||||
if err != nil || len(attempts.Items) != int(wantAttempts) || attempts.NextCursor != "" {
|
||||
t.Fatal("unexpected execution history after failover", err)
|
||||
}
|
||||
preserved, err := apiC.GetTaskAttempt(ctx, pending.ID, first.AttemptID)
|
||||
if err != nil || !reflect.DeepEqual(preserved, old) {
|
||||
t.Fatal("restart or subsequent execution rewrote first result", err)
|
||||
}
|
||||
queue, err := apiC.GetQueue(ctx, q.Queue.ID)
|
||||
wantUnfinished := uint64(0)
|
||||
if tc.final == TaskNeedsReview {
|
||||
wantUnfinished = 1
|
||||
}
|
||||
if err != nil || queue.UnfinishedTasks != wantUnfinished {
|
||||
t.Fatal("recovery lost unfinished-task accounting", err)
|
||||
}
|
||||
t.Logf("sole node killed; two fresh nodes recovered state=%s attempts=%d uncertainty=%t; original evidence and frozen policy preserved", current.State, wantAttempts, current.EffectUncertain)
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user