feat: add audited task review and linked manual retries

sdk/go/recovery_integration_test.go

@@ -0,0 +1,175 @@
+package orbit
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"net/url"
+	"reflect"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestIntegrationTaskRecoverySDK(t *testing.T) {
+	api, queue, token := integrationWorkerClient(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	handler := "recovery." + workerTestID(t)
+	accepted, err := api.SubmitTask(ctx, workerTestID(t), SubmitTaskRequest{QueueID: queue, Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1", BusinessKey: "object:recovery-example", Payload: json.RawMessage(`{"sequence":9007199254740993}`), TaskTimeoutMS: 60000, Retry: RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: 10000}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	reg, err := NewWorkerRegistration(queue, 1, []WorkerCapability{{Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	w, err := api.DialWorker(ctx, reg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer w.Close()
+	if _, err = w.Capacity(ctx, workerTestID(t), WorkerCapacity{Sequence: 1, Capacity: 1}); err != nil {
+		t.Fatal(err)
+	}
+	a, err := w.NextAssignment(ctx)
+	if err != nil || a.Task.ID != accepted.Task.ID {
+		t.Fatal(a, err)
+	}
+	if _, err = w.Start(ctx, workerTestID(t), a.Attempt()); err != nil {
+		t.Fatal(err)
+	}
+	if _, err = w.Finish(ctx, workerTestID(t), WorkerFinish{Ref: a.Attempt(), Outcome: WorkerUnknown, Result: json.RawMessage(`{"status":"unknown"}`), ErrorCode: "EXAMPLE_UNCERTAIN"}); err != nil {
+		t.Fatal(err)
+	}
+	unknown := waitRuntimeState(t, ctx, api, a.Task.ID, TaskNeedsReview)
+	oldAttempt, err := api.GetTaskAttempt(ctx, a.Task.ID, a.AttemptID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// 后端完整提交后主动丢失HTTP响应。禁用连接复用，排除net/http对复用连接的隐式重发。
+	target, _ := url.Parse(api.baseURL)
+	proxy := httputil.NewSingleHostReverseProxy(target)
+	var drop atomic.Bool
+	var calls atomic.Int32
+	var committed atomic.Value
+	proxy.ModifyResponse = func(r *http.Response) error {
+		calls.Add(1)
+		if !drop.CompareAndSwap(true, false) {
+			return nil
+		}
+		raw, e := io.ReadAll(r.Body)
+		r.Body.Close()
+		if e != nil {
+			return e
+		}
+		if r.StatusCode != 200 && r.StatusCode != 201 {
+			t.Errorf("unexpected response before drop: %s", raw)
+		}
+		committed.Store(raw)
+		return io.ErrUnexpectedEOF
+	}
+	proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
+		conn, _, e := w.(http.Hijacker).Hijack()
+		if e == nil {
+			conn.Close()
+		}
+	}
+	server := httptest.NewServer(proxy)
+	defer server.Close()
+	transport := &http.Transport{DisableKeepAlives: true}
+	defer transport.CloseIdleConnections()
+	lost, err := NewClient(server.URL, Options{Token: token, HTTPClient: &http.Client{Transport: transport}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	review := ReviewTaskRequest{ExpectedRevision: unknown.Revision, Reason: "确认模拟执行已结束且未完成", EvidenceRef: "evidence:demo-reconciliation", ExecutionSettled: true, Outcome: TaskFailed, Result: json.RawMessage(`{"confirmed":true,"sequence":9007199254740993}`)}
+	reviewKey := workerTestID(t)
+	drop.Store(true)
+	if _, err = lost.ReviewTask(ctx, unknown.ID, reviewKey, review); err == nil || calls.Load() != 1 {
+		t.Fatal("review response loss was hidden", err)
+	}
+	var reviewEnvelope struct {
+		Data ReviewTaskResult `json:"data"`
+	}
+	if err = json.Unmarshal(committed.Load().([]byte), &reviewEnvelope); err != nil {
+		t.Fatal(err)
+	}
+	reviewed, err := api.ReviewTask(ctx, unknown.ID, reviewKey, review)
+	if err != nil || !reviewed.Deduplicated || reviewed.AuditID != reviewEnvelope.Data.AuditID || reviewed.Task.State != TaskFailed || reviewed.Task.EffectUncertain {
+		t.Fatal(reviewed, err)
+	}
+	attempt, err := api.GetTaskAttempt(ctx, a.Task.ID, a.AttemptID)
+	if err != nil || !reflect.DeepEqual(attempt, oldAttempt) {
+		t.Fatal("review changed immutable attempt", err)
+	}
+	audit, err := api.GetTaskOperation(ctx, unknown.ID, reviewed.AuditID)
+	if err != nil || audit.Action != "task.review" || audit.EvidenceRef != review.EvidenceRef || audit.Reason != review.Reason {
+		t.Fatal(audit, err)
+	}
+	retry := RetryTaskRequest{ExpectedRevision: reviewed.Task.Revision, Reason: "已核实模拟旧执行不会再产生效果，允许重新执行", EvidenceRef: "evidence:demo-safe-retry", SafeToRetry: true, TaskTimeoutMS: 60000}
+	retryKey := workerTestID(t)
+	drop.Store(true)
+	if _, err = lost.RetryTask(ctx, unknown.ID, retryKey, retry); err == nil || calls.Load() != 2 {
+		t.Fatal("retry response loss was hidden", err)
+	}
+	var retryEnvelope struct {
+		Data RetryTaskResult `json:"data"`
+	}
+	if err = json.Unmarshal(committed.Load().([]byte), &retryEnvelope); err != nil {
+		t.Fatal(err)
+	}
+	child, err := api.RetryTask(ctx, unknown.ID, retryKey, retry)
+	if err != nil || !child.Deduplicated || child.Task.ID != retryEnvelope.Data.Task.ID || child.AuditID != retryEnvelope.Data.AuditID || child.Task.RetryOf == nil || *child.Task.RetryOf != unknown.ID {
+		t.Fatal(child, err)
+	}
+	assigned, err := w.NextAssignment(ctx)
+	if err != nil || assigned.Task.ID != child.Task.ID || assigned.Task.RetryOf == nil || *assigned.Task.RetryOf != unknown.ID {
+		t.Fatal("wire lost retry lineage", assigned, err)
+	}
+	if _, err = w.Start(ctx, workerTestID(t), assigned.Attempt()); err != nil {
+		t.Fatal(err)
+	}
+	if _, err = w.Finish(ctx, workerTestID(t), WorkerFinish{Ref: assigned.Attempt(), Outcome: WorkerSucceeded, Result: json.RawMessage(`{"object_ref":"examples/output"}`)}); err != nil {
+		t.Fatal(err)
+	}
+	waitRuntimeState(t, ctx, api, child.Task.ID, TaskSucceeded)
+	late, err := w.Finish(ctx, workerTestID(t), WorkerFinish{Ref: a.Attempt(), Outcome: WorkerSucceeded, Result: json.RawMessage(`{"late":true}`)})
+	if err != nil || late.Receipt.Disposition != "EVIDENCE_ONLY" {
+		t.Fatal(late, err)
+	}
+	source, err := api.GetTask(ctx, unknown.ID)
+	if err != nil || !reflect.DeepEqual(source, reviewed.Task) {
+		t.Fatal("late source rewrote review", err)
+	}
+	seen := map[string]bool{}
+	actions := map[string]int{}
+	cursor := ""
+	for {
+		page, e := api.ListTaskOperations(ctx, unknown.ID, HistoryPageRequest{Limit: 1, Cursor: cursor})
+		if e != nil {
+			t.Fatal(e)
+		}
+		for _, v := range page.Items {
+			if seen[v.ID] || v.TaskID != unknown.ID {
+				t.Fatal("duplicate/unscoped audit")
+			}
+			seen[v.ID] = true
+			actions[v.Action]++
+		}
+		if page.NextCursor == "" {
+			break
+		}
+		cursor = page.NextCursor
+	}
+	if actions["task.review"] != 1 || actions["task.retry"] != 1 {
+		t.Fatal("response loss created duplicate operations", actions)
+	}
+	if _, err = w.Capacity(ctx, workerTestID(t), WorkerCapacity{Sequence: 2, Drain: true}); err != nil {
+		t.Fatal(err)
+	}
+	t.Log("real UNKNOWN -> reviewed FAILED -> linked new task SUCCEEDED; both committed HTTP receipts lost/recovered; old attempt and audit preserved; late report evidence only")
+}