orbit/sdk/go/dispatch_process_integration_test.go

package orbit

import (
	"context"
	"encoding/json"
	"sync/atomic"
	"testing"
	"time"
)

// A04：数据库已提交派发，但真实WS帧被网关截留，业务Worker从未收到该尝试。
func TestIntegrationWorkerProcessUndeliveredAssignment(t *testing.T) {
	a := startOrbitProcess(t)
	_, _, token := integrationWorkerClient(t)
	api := processClient(t, a, token)
	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Second)
	defer cancel()
	name := "dispatch.crash." + workerTestID(t)
	queue, err := api.CreateQueue(ctx, name, CreateQueueRequest{Name: name, MaxConcurrency: 1, RatePerSecond: 20, MaxUnfinishedTasks: 2})
	if err != nil {
		t.Fatal(err)
	}
	in := SubmitTaskRequest{QueueID: queue.Queue.ID, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`), TaskTimeoutMS: 45000, Retry: RetryPolicy{MaxAttempts: 2, AttemptTimeoutMS: 5000, InitialDelayMS: 100, MaxDelayMS: 100}, RecoverySafety: RecoveryReconcileOnly}
	key := workerTestID(t)
	accepted, err := api.SubmitTask(ctx, key, in)
	if err != nil {
		t.Fatal(err)
	}
	g := newWorkerFaultGateway(t, a.url)
	g.holdAssignment.Store(true)
	client, err := NewClient(g.server.URL, Options{Token: token, HTTPClient: g.server.Client()})
	if err != nil {
		t.Fatal(err)
	}
	var calls atomic.Int32
	w, err := NewWorker(client, WorkerOptions{QueueID: queue.Queue.ID, Capacity: 1, ReconnectMaxDelay: time.Second, Handlers: []WorkerHandler{runtimeHandler(name, func(_ context.Context, e WorkerExecution) (json.RawMessage, error) {
		calls.Add(1)
		return e.Task.Payload, nil
	})}})
	if err != nil {
		t.Fatal(err)
	}
	run, stop := context.WithCancel(ctx)
	defer stop()
	done := make(chan error, 1)
	go func() { done <- w.Run(run) }()
	var held WorkerAssignment
	select {
	case held = <-g.assignmentsHeld:
	case err := <-done:
		t.Fatal("Worker stopped before assignment fault", err)
	case <-ctx.Done():
		t.Fatal("assignment fault was not reached")
	}
	if held.Task.ID != accepted.Task.ID || calls.Load() != 0 || w.Stats().Started != 0 {
		t.Fatal("fault was not before Worker execution")
	}
	leased, err := api.GetTask(ctx, held.Task.ID)
	if err != nil || leased.State != TaskLeased || leased.AttemptCount != 1 {
		t.Fatal("dispatch was not durably committed", err)
	}
	first, err := api.GetTaskAttempt(ctx, held.Task.ID, held.AttemptID)
	if err != nil || first.State != AttemptLeased || first.AuthorizedAt != nil || first.EndedAt != nil {
		t.Fatal("intercepted assignment already authorized", err)
	}
	a.kill(t)
	close(g.releaseAssignment)
	b := startOrbitProcess(t)
	if a.nodeID == b.nodeID {
		t.Fatal("node identity reused")
	}
	api = processClient(t, b, token)
	receipt, err := api.SubmitTask(ctx, key, in)
	if err != nil || !receipt.Deduplicated || receipt.Task.ID != accepted.Task.ID || !receipt.Task.DeadlineAt.Equal(accepted.Task.DeadlineAt) {
		t.Fatal("dispatch crash lost admission receipt", err)
	}
	// 保持Worker离线，等真实原租约自然过期；不使用SQL改时钟或伪造恢复结果。
	recovered := waitRuntimeState(t, ctx, api, held.Task.ID, TaskWaiting)
	old, err := api.GetTaskAttempt(ctx, held.Task.ID, held.AttemptID)
	if err != nil || old.State != AttemptExpired || old.AuthorizedAt != nil || old.EndedAt == nil || old.EndedAt.Before(first.LeaseUntil) || recovered.EffectUncertain || recovered.AttemptCount != 1 || calls.Load() != 0 {
		t.Fatal("unreceived assignment recovery fabricated an execution", err)
	}
	g.target.Store(b.url)
	g.offline.Store(false)
	finished := waitRuntimeState(t, ctx, api, held.Task.ID, TaskSucceeded)
	for w.Stats().ReportsConfirmed != 1 {
		select {
		case err := <-done:
			t.Fatal(err)
		case <-ctx.Done():
			t.Fatal("result receipt not recovered")
		case <-time.After(20 * time.Millisecond):
		}
	}
	stop()
	select {
	case err := <-done:
		if err != nil {
			t.Fatal(err)
		}
	case <-ctx.Done():
		t.Fatal("Worker did not drain")
	}
	history, err := api.ListTaskAttempts(ctx, held.Task.ID, HistoryPageRequest{})
	if err != nil || len(history.Items) != 2 || history.NextCursor != "" || history.Items[1].ID != held.AttemptID || history.Items[1].AuthorizedAt != nil || history.Items[1].State != AttemptExpired || history.Items[0].State != AttemptSucceeded || history.Items[0].Epoch != held.Epoch+1 {
		t.Fatal("dispatch fault duplicated or replaced history", err)
	}
	if calls.Load() != 1 || w.Stats().Started != 1 || finished.AttemptCount != 2 || finished.EffectUncertain || !finished.DeadlineAt.Equal(accepted.Task.DeadlineAt) || finished.Retry != accepted.Task.Retry || string(finished.Result) != string(in.Payload) {
		t.Fatal("dispatch recovery reset budget or executed twice")
	}
	resources, err := api.GetQueue(ctx, queue.Queue.ID)
	if err != nil || resources.UnfinishedTasks != 0 {
		t.Fatal("dispatch recovery leaked unfinished task quota", err)
	}
	t.Log("A04: committed LEASED frame never forwarded to Worker; original node SIGKILL confirmed; natural lease expiry preserved unauthorized history; new node completed original task with exactly one handler call and two budgeted attempts")
}