test: cover dispatch loss and workflow commit fault boundaries
This commit is contained in:
116
sdk/go/dispatch_process_integration_test.go
Normal file
116
sdk/go/dispatch_process_integration_test.go
Normal file
@@ -0,0 +1,116 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A04：数据库已提交派发，但真实WS帧被网关截留，业务Worker从未收到该尝试。
|
||||
func TestIntegrationWorkerProcessUndeliveredAssignment(t *testing.T) {
|
||||
a := startOrbitProcess(t)
|
||||
_, _, token := integrationWorkerClient(t)
|
||||
api := processClient(t, a, token)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Second)
|
||||
defer cancel()
|
||||
name := "dispatch.crash." + workerTestID(t)
|
||||
queue, err := api.CreateQueue(ctx, name, CreateQueueRequest{Name: name, MaxConcurrency: 1, RatePerSecond: 20, MaxUnfinishedTasks: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in := SubmitTaskRequest{QueueID: queue.Queue.ID, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`), TaskTimeoutMS: 45000, Retry: RetryPolicy{MaxAttempts: 2, AttemptTimeoutMS: 5000, InitialDelayMS: 100, MaxDelayMS: 100}, RecoverySafety: RecoveryReconcileOnly}
|
||||
key := workerTestID(t)
|
||||
accepted, err := api.SubmitTask(ctx, key, in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := newWorkerFaultGateway(t, a.url)
|
||||
g.holdAssignment.Store(true)
|
||||
client, err := NewClient(g.server.URL, Options{Token: token, HTTPClient: g.server.Client()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var calls atomic.Int32
|
||||
w, err := NewWorker(client, WorkerOptions{QueueID: queue.Queue.ID, Capacity: 1, ReconnectMaxDelay: time.Second, Handlers: []WorkerHandler{runtimeHandler(name, func(_ context.Context, e WorkerExecution) (json.RawMessage, error) {
|
||||
calls.Add(1)
|
||||
return e.Task.Payload, nil
|
||||
})}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run, stop := context.WithCancel(ctx)
|
||||
defer stop()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- w.Run(run) }()
|
||||
var held WorkerAssignment
|
||||
select {
|
||||
case held = <-g.assignmentsHeld:
|
||||
case err := <-done:
|
||||
t.Fatal("Worker stopped before assignment fault", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal("assignment fault was not reached")
|
||||
}
|
||||
if held.Task.ID != accepted.Task.ID || calls.Load() != 0 || w.Stats().Started != 0 {
|
||||
t.Fatal("fault was not before Worker execution")
|
||||
}
|
||||
leased, err := api.GetTask(ctx, held.Task.ID)
|
||||
if err != nil || leased.State != TaskLeased || leased.AttemptCount != 1 {
|
||||
t.Fatal("dispatch was not durably committed", err)
|
||||
}
|
||||
first, err := api.GetTaskAttempt(ctx, held.Task.ID, held.AttemptID)
|
||||
if err != nil || first.State != AttemptLeased || first.AuthorizedAt != nil || first.EndedAt != nil {
|
||||
t.Fatal("intercepted assignment already authorized", err)
|
||||
}
|
||||
a.kill(t)
|
||||
close(g.releaseAssignment)
|
||||
b := startOrbitProcess(t)
|
||||
if a.nodeID == b.nodeID {
|
||||
t.Fatal("node identity reused")
|
||||
}
|
||||
api = processClient(t, b, token)
|
||||
receipt, err := api.SubmitTask(ctx, key, in)
|
||||
if err != nil || !receipt.Deduplicated || receipt.Task.ID != accepted.Task.ID || !receipt.Task.DeadlineAt.Equal(accepted.Task.DeadlineAt) {
|
||||
t.Fatal("dispatch crash lost admission receipt", err)
|
||||
}
|
||||
// 保持Worker离线，等真实原租约自然过期；不使用SQL改时钟或伪造恢复结果。
|
||||
recovered := waitRuntimeState(t, ctx, api, held.Task.ID, TaskWaiting)
|
||||
old, err := api.GetTaskAttempt(ctx, held.Task.ID, held.AttemptID)
|
||||
if err != nil || old.State != AttemptExpired || old.AuthorizedAt != nil || old.EndedAt == nil || old.EndedAt.Before(first.LeaseUntil) || recovered.EffectUncertain || recovered.AttemptCount != 1 || calls.Load() != 0 {
|
||||
t.Fatal("unreceived assignment recovery fabricated an execution", err)
|
||||
}
|
||||
g.target.Store(b.url)
|
||||
g.offline.Store(false)
|
||||
finished := waitRuntimeState(t, ctx, api, held.Task.ID, TaskSucceeded)
|
||||
for w.Stats().ReportsConfirmed != 1 {
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatal(err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal("result receipt not recovered")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
stop()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Worker did not drain")
|
||||
}
|
||||
history, err := api.ListTaskAttempts(ctx, held.Task.ID, HistoryPageRequest{})
|
||||
if err != nil || len(history.Items) != 2 || history.NextCursor != "" || history.Items[1].ID != held.AttemptID || history.Items[1].AuthorizedAt != nil || history.Items[1].State != AttemptExpired || history.Items[0].State != AttemptSucceeded || history.Items[0].Epoch != held.Epoch+1 {
|
||||
t.Fatal("dispatch fault duplicated or replaced history", err)
|
||||
}
|
||||
if calls.Load() != 1 || w.Stats().Started != 1 || finished.AttemptCount != 2 || finished.EffectUncertain || !finished.DeadlineAt.Equal(accepted.Task.DeadlineAt) || finished.Retry != accepted.Task.Retry || string(finished.Result) != string(in.Payload) {
|
||||
t.Fatal("dispatch recovery reset budget or executed twice")
|
||||
}
|
||||
resources, err := api.GetQueue(ctx, queue.Queue.ID)
|
||||
if err != nil || resources.UnfinishedTasks != 0 {
|
||||
t.Fatal("dispatch recovery leaked unfinished task quota", err)
|
||||
}
|
||||
t.Log("A04: committed LEASED frame never forwarded to Worker; original node SIGKILL confirmed; natural lease expiry preserved unauthorized history; new node completed original task with exactly one handler call and two budgeted attempts")
|
||||
}
|
||||
@@ -53,6 +53,9 @@ type workerFaultGateway struct {
|
||||
target atomic.Value
|
||||
dropOffline atomic.Bool
|
||||
dropCancel atomic.Bool
|
||||
holdAssignment atomic.Bool
|
||||
assignmentsHeld chan WorkerAssignment
|
||||
releaseAssignment chan struct{}
|
||||
offline atomic.Bool
|
||||
mu sync.Mutex
|
||||
active map[*websocket.Conn]struct{}
|
||||
@@ -66,7 +69,7 @@ func newWorkerFaultGateway(t *testing.T, upstream string) *workerFaultGateway {
|
||||
|
||||
func newProtocolFaultGateway(t *testing.T, upstream, path, protocol string) *workerFaultGateway {
|
||||
t.Helper()
|
||||
g := &workerFaultGateway{dropped: make(chan string, 4), active: make(map[*websocket.Conn]struct{})}
|
||||
g := &workerFaultGateway{dropped: make(chan string, 4), active: make(map[*websocket.Conn]struct{}), assignmentsHeld: make(chan WorkerAssignment, 1), releaseAssignment: make(chan struct{})}
|
||||
g.drop.Store("")
|
||||
g.target.Store(upstream)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
@@ -150,6 +153,20 @@ func newProtocolFaultGateway(t *testing.T, upstream, path, protocol string) *wor
|
||||
g.dropped <- "task.cancel"
|
||||
return
|
||||
}
|
||||
if f.Type == "task.assign" && g.holdAssignment.CompareAndSwap(true, false) {
|
||||
var assignment WorkerAssignment
|
||||
if json.Unmarshal(f.Body, &assignment) != nil {
|
||||
return
|
||||
}
|
||||
// 真实派发帧只到隔离网关，不向Worker发送；测试确认节点退出后丢弃。
|
||||
g.offline.Store(true)
|
||||
g.assignmentsHeld <- assignment
|
||||
select {
|
||||
case <-g.releaseAssignment:
|
||||
case <-bridge.Done():
|
||||
}
|
||||
return
|
||||
}
|
||||
if f.Type == "ack" && operation != "" && g.drop.CompareAndSwap(operation, "") {
|
||||
if g.dropOffline.Load() {
|
||||
g.offline.Store(true)
|
||||
|
||||
Reference in New Issue
Block a user