193 lines
8.0 KiB
Go
193 lines
8.0 KiB
Go
package orbit
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// 首次失败已提交并进入退避后，强杀唯一服务进程；两个新进程仅靠持久化状态接管。
|
|
// 所有时间自然流逝，不修改数据库的退避、截止或租约字段。
|
|
func TestIntegrationWorkerProcessRetryBackoffRecovery(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
unknown bool
|
|
expire bool
|
|
final TaskState
|
|
}{
|
|
{"known_then_success", false, false, TaskSucceeded},
|
|
{"unknown_then_exhausted", true, false, TaskNeedsReview},
|
|
{"known_deadline_during_outage", false, true, TaskFailed},
|
|
{"unknown_deadline_during_outage", true, true, TaskNeedsReview},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
a := startOrbitProcess(t)
|
|
_, _, token := integrationWorkerClient(t)
|
|
api := processClient(t, a, token)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
|
defer cancel()
|
|
name := "retry-crash." + workerTestID(t)
|
|
q, err := api.CreateQueue(ctx, name, CreateQueueRequest{Name: name, MaxConcurrency: 2, RatePerSecond: 100})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
connect := func(client *Client) *WorkerConnection {
|
|
t.Helper()
|
|
r, err := NewWorkerRegistration(q.Queue.ID, 1, []WorkerCapability{{Handler: name, HandlerVersion: "v1", PayloadVersion: "v1"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w, err := client.DialWorker(ctx, r)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = w.Close() })
|
|
if _, err := w.Capacity(ctx, workerTestID(t), WorkerCapacity{Sequence: 1, Capacity: 1}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return w
|
|
}
|
|
in := SubmitTaskRequest{QueueID: q.Queue.ID, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`), RecoverySafety: RecoveryReconcileOnly, TaskTimeoutMS: 40000, Retry: RetryPolicy{MaxAttempts: 2, AttemptTimeoutMS: 3000, InitialDelayMS: 8000, MaxDelayMS: 8000}}
|
|
if tc.unknown {
|
|
in.RecoverySafety = RecoveryIdempotent
|
|
}
|
|
if tc.expire {
|
|
in.TaskTimeoutMS = 5000
|
|
in.Retry.InitialDelayMS, in.Retry.MaxDelayMS = 1000, 1000
|
|
}
|
|
accepted, err := api.SubmitTask(ctx, workerTestID(t), in)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w := connect(api)
|
|
first, err := w.NextAssignment(ctx)
|
|
if err != nil || first.Task.ID != accepted.Task.ID || first.Epoch != 1 {
|
|
t.Fatal("first attempt identity", err)
|
|
}
|
|
if _, err := w.Start(ctx, workerTestID(t), first.Attempt()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
finish := WorkerFinish{Ref: first.Attempt(), Outcome: WorkerFailed, Retryable: true, ErrorCode: "TEMPORARY_REJECTION", Result: json.RawMessage(`{"evidence_ref":"diagnostics/first"}`)}
|
|
if tc.unknown {
|
|
finish.Outcome, finish.Retryable, finish.ErrorCode = WorkerUnknown, false, "RESULT_UNKNOWN"
|
|
}
|
|
receipt, err := w.Finish(ctx, workerTestID(t), finish)
|
|
if err != nil || receipt.Receipt.TaskState != TaskWaiting || receipt.Lease.Active {
|
|
t.Fatal("first result did not durably enter backoff", err)
|
|
}
|
|
pending, err := api.GetTask(ctx, accepted.Task.ID)
|
|
if err != nil || pending.State != TaskWaiting || pending.AttemptCount != 1 || pending.EffectUncertain != tc.unknown || !pending.AvailableAt.After(receipt.Lease.ServerTime) {
|
|
t.Fatal("retry facts were not persisted", err)
|
|
}
|
|
old, err := api.GetTaskAttempt(ctx, pending.ID, first.AttemptID)
|
|
if err != nil || old.EndedAt == nil || old.Retryable == tc.unknown || string(old.Result) != string(finish.Result) {
|
|
t.Fatal("first attempt evidence missing", err)
|
|
}
|
|
if delay := pending.AvailableAt.Sub(*old.EndedAt); delay < time.Duration(in.Retry.InitialDelayMS/2)*time.Millisecond || delay > time.Duration(in.Retry.MaxDelayMS)*time.Millisecond {
|
|
t.Fatal("persisted backoff outside policy")
|
|
}
|
|
a.kill(t)
|
|
_ = w.Close()
|
|
if tc.expire {
|
|
// 用已确认的数据库时间差等待，恢复后的状态/历史仍以数据库事实作断言。
|
|
timer := time.NewTimer(pending.DeadlineAt.Sub(receipt.Lease.ServerTime) + 100*time.Millisecond)
|
|
defer timer.Stop()
|
|
select {
|
|
case <-timer.C:
|
|
case <-ctx.Done():
|
|
t.Fatal("outage exceeded test budget")
|
|
}
|
|
}
|
|
b, c := startOrbitProcess(t), startOrbitProcess(t)
|
|
if a.nodeID == b.nodeID || a.nodeID == c.nodeID || b.nodeID == c.nodeID {
|
|
t.Fatal("replacement reused node identity")
|
|
}
|
|
apiB, apiC := processClient(t, b, token), processClient(t, c, token)
|
|
if !tc.expire {
|
|
recovered, err := apiB.GetTask(ctx, pending.ID)
|
|
if err != nil || !reflect.DeepEqual(recovered, pending) {
|
|
t.Fatal("restart changed frozen backoff, budget, deadline or uncertainty", err)
|
|
}
|
|
}
|
|
workers := []*WorkerConnection{connect(apiB), connect(apiC)}
|
|
var current Task
|
|
wantAttempts := uint32(1)
|
|
if !tc.expire {
|
|
type assignmentResult struct {
|
|
index int
|
|
assignment WorkerAssignment
|
|
err error
|
|
}
|
|
wait, stop := context.WithCancel(ctx)
|
|
defer stop()
|
|
results := make(chan assignmentResult, 2)
|
|
for i, worker := range workers {
|
|
go func() {
|
|
assignment, err := worker.NextAssignment(wait)
|
|
results <- assignmentResult{i, assignment, err}
|
|
}()
|
|
}
|
|
winner := <-results
|
|
second := winner.assignment
|
|
if winner.err != nil || second.Task.ID != pending.ID || second.AttemptID == first.AttemptID || second.Epoch != 2 {
|
|
t.Fatal("retry did not preserve task identity and consumed budget", winner.err)
|
|
}
|
|
history, err := apiB.GetTaskAttempt(ctx, pending.ID, second.AttemptID)
|
|
if err != nil || history.CreatedAt.Before(pending.AvailableAt) || !history.CreatedAt.Before(pending.DeadlineAt) {
|
|
t.Fatal("retry ignored persisted timing", err)
|
|
}
|
|
owner := workers[winner.index]
|
|
if _, err := owner.Start(ctx, workerTestID(t), second.Attempt()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
last := WorkerFinish{Ref: second.Attempt(), Outcome: WorkerSucceeded, Result: json.RawMessage(`{"n":9007199254740993}`)}
|
|
if tc.unknown {
|
|
last.Outcome, last.Retryable, last.ErrorCode = WorkerFailed, true, "TEMPORARY_REJECTION"
|
|
}
|
|
if _, err := owner.Finish(ctx, workerTestID(t), last); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stop()
|
|
loser := <-results
|
|
if !errors.Is(loser.err, context.Canceled) {
|
|
t.Fatal("two replacement nodes received a retry assignment", loser.err)
|
|
}
|
|
wantAttempts = 2
|
|
}
|
|
current = waitRuntimeState(t, ctx, apiB, pending.ID, tc.final)
|
|
if current.AttemptCount != wantAttempts || current.EffectUncertain != tc.unknown || current.Retry != pending.Retry || !current.DeadlineAt.Equal(pending.DeadlineAt) || current.RecoverySafety != pending.RecoverySafety || string(current.Payload) != string(pending.Payload) {
|
|
t.Fatal("recovery reset policy, input, deadline, attempts or uncertainty")
|
|
}
|
|
// 两个节点仍在线且都有可用容量，预算/截止应阻止任何额外领取。
|
|
for _, worker := range workers {
|
|
probe, stop := context.WithTimeout(ctx, 250*time.Millisecond)
|
|
_, err := worker.NextAssignment(probe)
|
|
stop()
|
|
if !errors.Is(err, context.DeadlineExceeded) {
|
|
t.Fatal("unexpected assignment after final decision", err)
|
|
}
|
|
}
|
|
attempts, err := apiC.ListTaskAttempts(ctx, pending.ID, HistoryPageRequest{})
|
|
if err != nil || len(attempts.Items) != int(wantAttempts) || attempts.NextCursor != "" {
|
|
t.Fatal("unexpected execution history after failover", err)
|
|
}
|
|
preserved, err := apiC.GetTaskAttempt(ctx, pending.ID, first.AttemptID)
|
|
if err != nil || !reflect.DeepEqual(preserved, old) {
|
|
t.Fatal("restart or subsequent execution rewrote first result", err)
|
|
}
|
|
queue, err := apiC.GetQueue(ctx, q.Queue.ID)
|
|
wantUnfinished := uint64(0)
|
|
if tc.final == TaskNeedsReview {
|
|
wantUnfinished = 1
|
|
}
|
|
if err != nil || queue.UnfinishedTasks != wantUnfinished {
|
|
t.Fatal("recovery lost unfinished-task accounting", err)
|
|
}
|
|
t.Logf("sole node killed; two fresh nodes recovered state=%s attempts=%d uncertainty=%t; original evidence and frozen policy preserved", current.State, wantAttempts, current.EffectUncertain)
|
|
})
|
|
}
|
|
}
|