test: cover dispatch loss and workflow commit fault boundaries

sdk/go/dispatch_process_integration_test.go

@@ -0,0 +1,116 @@
+package orbit
+
+import (
+	"context"
+	"encoding/json"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// A04：数据库已提交派发，但真实WS帧被网关截留，业务Worker从未收到该尝试。
+func TestIntegrationWorkerProcessUndeliveredAssignment(t *testing.T) {
+	a := startOrbitProcess(t)
+	_, _, token := integrationWorkerClient(t)
+	api := processClient(t, a, token)
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Second)
+	defer cancel()
+	name := "dispatch.crash." + workerTestID(t)
+	queue, err := api.CreateQueue(ctx, name, CreateQueueRequest{Name: name, MaxConcurrency: 1, RatePerSecond: 20, MaxUnfinishedTasks: 2})
+	if err != nil {
+		t.Fatal(err)
+	}
+	in := SubmitTaskRequest{QueueID: queue.Queue.ID, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`), TaskTimeoutMS: 45000, Retry: RetryPolicy{MaxAttempts: 2, AttemptTimeoutMS: 5000, InitialDelayMS: 100, MaxDelayMS: 100}, RecoverySafety: RecoveryReconcileOnly}
+	key := workerTestID(t)
+	accepted, err := api.SubmitTask(ctx, key, in)
+	if err != nil {
+		t.Fatal(err)
+	}
+	g := newWorkerFaultGateway(t, a.url)
+	g.holdAssignment.Store(true)
+	client, err := NewClient(g.server.URL, Options{Token: token, HTTPClient: g.server.Client()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var calls atomic.Int32
+	w, err := NewWorker(client, WorkerOptions{QueueID: queue.Queue.ID, Capacity: 1, ReconnectMaxDelay: time.Second, Handlers: []WorkerHandler{runtimeHandler(name, func(_ context.Context, e WorkerExecution) (json.RawMessage, error) {
+		calls.Add(1)
+		return e.Task.Payload, nil
+	})}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	run, stop := context.WithCancel(ctx)
+	defer stop()
+	done := make(chan error, 1)
+	go func() { done <- w.Run(run) }()
+	var held WorkerAssignment
+	select {
+	case held = <-g.assignmentsHeld:
+	case err := <-done:
+		t.Fatal("Worker stopped before assignment fault", err)
+	case <-ctx.Done():
+		t.Fatal("assignment fault was not reached")
+	}
+	if held.Task.ID != accepted.Task.ID || calls.Load() != 0 || w.Stats().Started != 0 {
+		t.Fatal("fault was not before Worker execution")
+	}
+	leased, err := api.GetTask(ctx, held.Task.ID)
+	if err != nil || leased.State != TaskLeased || leased.AttemptCount != 1 {
+		t.Fatal("dispatch was not durably committed", err)
+	}
+	first, err := api.GetTaskAttempt(ctx, held.Task.ID, held.AttemptID)
+	if err != nil || first.State != AttemptLeased || first.AuthorizedAt != nil || first.EndedAt != nil {
+		t.Fatal("intercepted assignment already authorized", err)
+	}
+	a.kill(t)
+	close(g.releaseAssignment)
+	b := startOrbitProcess(t)
+	if a.nodeID == b.nodeID {
+		t.Fatal("node identity reused")
+	}
+	api = processClient(t, b, token)
+	receipt, err := api.SubmitTask(ctx, key, in)
+	if err != nil || !receipt.Deduplicated || receipt.Task.ID != accepted.Task.ID || !receipt.Task.DeadlineAt.Equal(accepted.Task.DeadlineAt) {
+		t.Fatal("dispatch crash lost admission receipt", err)
+	}
+	// 保持Worker离线，等真实原租约自然过期；不使用SQL改时钟或伪造恢复结果。
+	recovered := waitRuntimeState(t, ctx, api, held.Task.ID, TaskWaiting)
+	old, err := api.GetTaskAttempt(ctx, held.Task.ID, held.AttemptID)
+	if err != nil || old.State != AttemptExpired || old.AuthorizedAt != nil || old.EndedAt == nil || old.EndedAt.Before(first.LeaseUntil) || recovered.EffectUncertain || recovered.AttemptCount != 1 || calls.Load() != 0 {
+		t.Fatal("unreceived assignment recovery fabricated an execution", err)
+	}
+	g.target.Store(b.url)
+	g.offline.Store(false)
+	finished := waitRuntimeState(t, ctx, api, held.Task.ID, TaskSucceeded)
+	for w.Stats().ReportsConfirmed != 1 {
+		select {
+		case err := <-done:
+			t.Fatal(err)
+		case <-ctx.Done():
+			t.Fatal("result receipt not recovered")
+		case <-time.After(20 * time.Millisecond):
+		}
+	}
+	stop()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Fatal(err)
+		}
+	case <-ctx.Done():
+		t.Fatal("Worker did not drain")
+	}
+	history, err := api.ListTaskAttempts(ctx, held.Task.ID, HistoryPageRequest{})
+	if err != nil || len(history.Items) != 2 || history.NextCursor != "" || history.Items[1].ID != held.AttemptID || history.Items[1].AuthorizedAt != nil || history.Items[1].State != AttemptExpired || history.Items[0].State != AttemptSucceeded || history.Items[0].Epoch != held.Epoch+1 {
+		t.Fatal("dispatch fault duplicated or replaced history", err)
+	}
+	if calls.Load() != 1 || w.Stats().Started != 1 || finished.AttemptCount != 2 || finished.EffectUncertain || !finished.DeadlineAt.Equal(accepted.Task.DeadlineAt) || finished.Retry != accepted.Task.Retry || string(finished.Result) != string(in.Payload) {
+		t.Fatal("dispatch recovery reset budget or executed twice")
+	}
+	resources, err := api.GetQueue(ctx, queue.Queue.ID)
+	if err != nil || resources.UnfinishedTasks != 0 {
+		t.Fatal("dispatch recovery leaked unfinished task quota", err)
+	}
+	t.Log("A04: committed LEASED frame never forwarded to Worker; original node SIGKILL confirmed; natural lease expiry preserved unauthorized history; new node completed original task with exactly one handler call and two budgeted attempts")
+}