feat: add audited task review and linked manual retries
This commit is contained in:
182
sdk/go/examples/recovery/main.go
Normal file
182
sdk/go/examples/recovery/main.go
Normal file
@@ -0,0 +1,182 @@
|
||||
// recovery 演示未知结果、人工核查、关联新任务和不可变审计;只处理本地模拟输入。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
func run() error {
|
||||
token, queue := os.Getenv("ORBIT_TOKEN"), os.Getenv("ORBIT_QUEUE_ID")
|
||||
if path := os.Getenv("ORBIT_CREDENTIALS_FILE"); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read private credentials: %w", err)
|
||||
}
|
||||
var c struct {
|
||||
Token string `json:"token"`
|
||||
QueueID string `json:"queue_id"`
|
||||
}
|
||||
if json.Unmarshal(raw, &c) != nil {
|
||||
return errors.New("invalid credentials file")
|
||||
}
|
||||
if token == "" {
|
||||
token = c.Token
|
||||
}
|
||||
if queue == "" {
|
||||
queue = c.QueueID
|
||||
}
|
||||
}
|
||||
client, err := orbit.NewClient(os.Getenv("ORBIT_URL"), orbit.Options{Token: token})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
op, err := orbit.NewWorkerMessageID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
handler := "recovery.inspect." + op
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
accepted, err := client.SubmitTask(ctx, "recovery-"+op, orbit.SubmitTaskRequest{QueueID: queue, Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"object_ref":"examples/input"}`), TaskTimeoutMS: 60000, Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: 10000}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
registration, err := orbit.NewWorkerRegistration(queue, 1, []orbit.WorkerCapability{{Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1"}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w, err := client.DialWorker(ctx, registration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer w.Close()
|
||||
nextID := func() (string, error) { return orbit.NewWorkerMessageID() }
|
||||
id, err := nextID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.Capacity(ctx, id, orbit.WorkerCapacity{Sequence: 1, Capacity: 1}); err != nil {
|
||||
return err
|
||||
}
|
||||
a, err := w.NextAssignment(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if a.Task.ID != accepted.Task.ID {
|
||||
return errors.New("unexpected task")
|
||||
}
|
||||
id, err = nextID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
started, err := w.Start(ctx, id, a.Attempt())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, active := started.Lease.ExecutionDeadline(); !active {
|
||||
return errors.New("missing current execution authority")
|
||||
}
|
||||
// 本例不操作外部系统:模拟“旧执行已结束,但最初无法判断结果”的场景。
|
||||
id, err = nextID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.Finish(ctx, id, orbit.WorkerFinish{Ref: a.Attempt(), Outcome: orbit.WorkerUnknown, Result: json.RawMessage(`{"status":"unknown"}`), ErrorCode: "EXAMPLE_UNCERTAIN"}); err != nil {
|
||||
return err
|
||||
}
|
||||
current, err := client.GetTask(ctx, a.Task.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current.State != orbit.TaskNeedsReview {
|
||||
return errors.New("expected manual review")
|
||||
}
|
||||
// 生产中先核查旧执行和外部效果,保存真实证据,再由专门管理凭据提交此结论。
|
||||
// 每次操作将幂等键与完整请求持久化;响应丢失时保持原键、版本、结论与证据重报。
|
||||
reviewKey := "review-" + op
|
||||
reviewInput := orbit.ReviewTaskRequest{ExpectedRevision: current.Revision, Reason: "模拟旧执行已结束,确认操作未完成", EvidenceRef: "evidence:example-reconciliation", ExecutionSettled: true, Outcome: orbit.TaskFailed, Result: json.RawMessage(`{"confirmed":true,"sequence":9007199254740993}`)}
|
||||
reviewed, err := client.ReviewTask(ctx, current.ID, reviewKey, reviewInput)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
replay, err := client.ReviewTask(ctx, current.ID, reviewKey, reviewInput)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !replay.Deduplicated || replay.AuditID != reviewed.AuditID {
|
||||
return errors.New("review identity changed")
|
||||
}
|
||||
// 人工重试保留原任务,另建同业务身份的新任务,并显式给出新的总时间预算。
|
||||
retryKey := "retry-" + op
|
||||
retryInput := orbit.RetryTaskRequest{ExpectedRevision: reviewed.Task.Revision, Reason: "已核实模拟任务可安全重新执行", EvidenceRef: "evidence:example-safe-retry", SafeToRetry: true, TaskTimeoutMS: 60000}
|
||||
retried, err := client.RetryTask(ctx, current.ID, retryKey, retryInput)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a, err = w.NextAssignment(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if a.Task.ID != retried.Task.ID || a.Task.RetryOf == nil || *a.Task.RetryOf != current.ID {
|
||||
return errors.New("missing retry lineage")
|
||||
}
|
||||
id, err = nextID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
started, err = w.Start(ctx, id, a.Attempt())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, active := started.Lease.ExecutionDeadline(); !active {
|
||||
return errors.New("missing retry execution authority")
|
||||
}
|
||||
id, err = nextID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.Finish(ctx, id, orbit.WorkerFinish{Ref: a.Attempt(), Outcome: orbit.WorkerSucceeded, Result: json.RawMessage(`{"object_ref":"examples/output"}`)}); err != nil {
|
||||
return err
|
||||
}
|
||||
audit, err := client.GetTaskOperation(ctx, current.ID, retried.AuditID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
page, err := client.ListTaskOperations(ctx, current.ID, orbit.HistoryPageRequest{Limit: 20})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
original, err := client.GetTask(ctx, current.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
child, err := client.GetTask(ctx, retried.Task.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if original.State != orbit.TaskFailed || child.State != orbit.TaskSucceeded || audit.Action != "task.retry" {
|
||||
return errors.New("unexpected recovery result")
|
||||
}
|
||||
id, err = nextID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.Capacity(ctx, id, orbit.WorkerCapacity{Sequence: 2, Drain: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("source=%s state=%s retry=%s state=%s audit=%s operations=%d\n", original.ID, original.State, child.ID, child.State, audit.ID, len(page.Items))
|
||||
return nil
|
||||
}
|
||||
143
sdk/go/recovery.go
Normal file
143
sdk/go/recovery.go
Normal file
@@ -0,0 +1,143 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ReviewTaskRequest 人工核查未知效果;声明与证据由获授权的核查主体负责。
|
||||
type ReviewTaskRequest struct {
|
||||
// ExpectedRevision 当前待核查任务版本,非零;重报保留原版本。
|
||||
ExpectedRevision uint64 `json:"expected_revision,string"`
|
||||
// Reason 核查原因与结论说明,最多1024个Unicode字符。
|
||||
Reason string `json:"reason"`
|
||||
// EvidenceRef 受控证据引用,1至512个无空白/控制字符的Unicode字符;平台不请求该地址。
|
||||
EvidenceRef string `json:"evidence_ref"`
|
||||
// ExecutionSettled 必须显式为true,声明已核实全部历史执行已结束且不会继续产生效果。
|
||||
ExecutionSettled bool `json:"execution_settled"`
|
||||
// Outcome 核查结论,仅SUCCEEDED或FAILED;不能直接重开旧任务。
|
||||
Outcome TaskState `json:"outcome"`
|
||||
// Result 非空JSON结论或结果引用,最大256 KiB,保留精确数值。
|
||||
Result json.RawMessage `json:"result"`
|
||||
}
|
||||
|
||||
type ReviewTaskResult struct {
|
||||
// Task 核查提交时的任务快照;当前状态另行查询。
|
||||
Task Task `json:"task"`
|
||||
// AuditID 不可变核查审计记录标识,可通过任务操作历史读取。
|
||||
AuditID string `json:"audit_id"`
|
||||
// Deduplicated 是否返回原核查操作回执。
|
||||
Deduplicated bool `json:"deduplicated"`
|
||||
}
|
||||
|
||||
// RetryTaskRequest 为已确认失败/取消的独立任务建立新预算,不修改原任务历史。
|
||||
type RetryTaskRequest struct {
|
||||
// ExpectedRevision 原任务的当前版本,重报保持原值。
|
||||
ExpectedRevision uint64 `json:"expected_revision,string"`
|
||||
// Reason 人工重新执行的原因,最多1024个Unicode字符。
|
||||
Reason string `json:"reason"`
|
||||
// EvidenceRef 重新执行的安全依据引用,最多512个字符,不包含秘密。
|
||||
EvidenceRef string `json:"evidence_ref"`
|
||||
// SafeToRetry 必须显式为true,声明旧执行已结束且按原业务身份重新执行安全。
|
||||
SafeToRetry bool `json:"safe_to_retry"`
|
||||
// TaskTimeoutMS 新任务从受理起的总时限,必须显式指定,最多30天。
|
||||
TaskTimeoutMS uint64 `json:"task_timeout_ms"`
|
||||
// AvailableAt 新任务最早开始时间,空表示立即;不复用旧任务的过期绝对时间。
|
||||
AvailableAt *time.Time `json:"available_at,omitempty"`
|
||||
// StartDeadlineAt 新任务最晚开始时间,空表示使用新的总截止时间。
|
||||
StartDeadlineAt *time.Time `json:"start_deadline_at,omitempty"`
|
||||
}
|
||||
|
||||
type RetryTaskResult struct {
|
||||
// SourceTaskID 被人工重新发起的原任务标识,其终态与尝试历史保持不变。
|
||||
SourceTaskID string `json:"source_task_id"`
|
||||
// Task 新任务的原始受理快照,RetryOf指回原任务。
|
||||
Task Task `json:"task"`
|
||||
// AuditID 原任务上的人工重试审计标识,包含新任务关联。
|
||||
AuditID string `json:"audit_id"`
|
||||
// Deduplicated 是否返回原人工重试回执。
|
||||
Deduplicated bool `json:"deduplicated"`
|
||||
}
|
||||
|
||||
type TaskOperationSummary struct {
|
||||
// ID 不可改写的审计记录标识。
|
||||
ID string `json:"id"`
|
||||
// TaskID 本页授权任务标识。
|
||||
TaskID string `json:"task_id"`
|
||||
// OperationID 服务端管理操作身份,不是调用方原始幂等键。
|
||||
OperationID string `json:"operation_id"`
|
||||
// ActorID 实际认证主体标识,不接受请求体自报。
|
||||
ActorID string `json:"actor_id"`
|
||||
// Action 管理动作,例如task.review、task.retry、task.cancel、task.result。
|
||||
Action string `json:"action"`
|
||||
// CreatedAt 不可变审计创建时间,UTC微秒精度。
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type TaskOperation struct {
|
||||
TaskOperationSummary
|
||||
// Reason 实际提交的操作原因。
|
||||
Reason string `json:"reason"`
|
||||
// EvidenceRef 操作证据引用;非核查类操作可能为空。
|
||||
EvidenceRef string `json:"evidence_ref,omitempty"`
|
||||
// Before 操作前的受控状态摘要,不包含完整业务输入。
|
||||
Before json.RawMessage `json:"before"`
|
||||
// After 操作后摘要、核查声明或关联新任务,不包含凭据秘密。
|
||||
After json.RawMessage `json:"after"`
|
||||
}
|
||||
|
||||
type TaskOperationPage struct {
|
||||
// Items 按创建时间及审计ID倒序排列,不返回原因与摘要正文。
|
||||
Items []TaskOperationSummary `json:"items"`
|
||||
// NextCursor 必须沿相同任务继续使用,空表示本次已无下一页。
|
||||
NextCursor string `json:"next_cursor,omitempty"`
|
||||
}
|
||||
|
||||
// ReviewTask 需要显式核查权限;超时重报必须保留原键、原版本、结论与证据。
|
||||
func (c *Client) ReviewTask(ctx context.Context, id, key string, in ReviewTaskRequest) (ReviewTaskResult, error) {
|
||||
var out ReviewTaskResult
|
||||
if !validID(id) {
|
||||
return out, errors.New("invalid task id")
|
||||
}
|
||||
err := c.call(ctx, "POST", "/v1/tasks/"+id+":review", key, in, &out)
|
||||
if err == nil && (!validID(out.AuditID) || out.Task.ID != id || out.Task.State != in.Outcome || out.Task.EffectUncertain) {
|
||||
return ReviewTaskResult{}, errors.New("invalid task review receipt; reconcile using original identity")
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
|
||||
// RetryTask 创建与原任务关联的新任务;不会修改原任务的终态或重置原尝试预算。
|
||||
func (c *Client) RetryTask(ctx context.Context, id, key string, in RetryTaskRequest) (RetryTaskResult, error) {
|
||||
var out RetryTaskResult
|
||||
if !validID(id) {
|
||||
return out, errors.New("invalid task id")
|
||||
}
|
||||
err := c.call(ctx, "POST", "/v1/tasks/"+id+":retry", key, in, &out)
|
||||
if err == nil && (!validID(out.AuditID) || out.SourceTaskID != id || !validID(out.Task.ID) || out.Task.ID == id || out.Task.RetryOf == nil || *out.Task.RetryOf != id) {
|
||||
return RetryTaskResult{}, errors.New("invalid task retry receipt; reconcile using original identity")
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (c *Client) ListTaskOperations(ctx context.Context, id string, in HistoryPageRequest) (TaskOperationPage, error) {
|
||||
var out TaskOperationPage
|
||||
if !validID(id) {
|
||||
return out, errors.New("invalid task id")
|
||||
}
|
||||
query, err := historyPageQuery(in)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
err = c.call(ctx, "GET", "/v1/tasks/"+id+"/operations"+query, "", nil, &out)
|
||||
return out, err
|
||||
}
|
||||
func (c *Client) GetTaskOperation(ctx context.Context, id, operation string) (TaskOperation, error) {
|
||||
var out TaskOperation
|
||||
if !validID(id) || !validID(operation) {
|
||||
return out, errors.New("invalid task operation id")
|
||||
}
|
||||
err := c.call(ctx, "GET", "/v1/tasks/"+id+"/operations/"+operation, "", nil, &out)
|
||||
return out, err
|
||||
}
|
||||
175
sdk/go/recovery_integration_test.go
Normal file
175
sdk/go/recovery_integration_test.go
Normal file
@@ -0,0 +1,175 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIntegrationTaskRecoverySDK(t *testing.T) {
|
||||
api, queue, token := integrationWorkerClient(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
handler := "recovery." + workerTestID(t)
|
||||
accepted, err := api.SubmitTask(ctx, workerTestID(t), SubmitTaskRequest{QueueID: queue, Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1", BusinessKey: "object:recovery-example", Payload: json.RawMessage(`{"sequence":9007199254740993}`), TaskTimeoutMS: 60000, Retry: RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: 10000}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reg, err := NewWorkerRegistration(queue, 1, []WorkerCapability{{Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w, err := api.DialWorker(ctx, reg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer w.Close()
|
||||
if _, err = w.Capacity(ctx, workerTestID(t), WorkerCapacity{Sequence: 1, Capacity: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, err := w.NextAssignment(ctx)
|
||||
if err != nil || a.Task.ID != accepted.Task.ID {
|
||||
t.Fatal(a, err)
|
||||
}
|
||||
if _, err = w.Start(ctx, workerTestID(t), a.Attempt()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = w.Finish(ctx, workerTestID(t), WorkerFinish{Ref: a.Attempt(), Outcome: WorkerUnknown, Result: json.RawMessage(`{"status":"unknown"}`), ErrorCode: "EXAMPLE_UNCERTAIN"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unknown := waitRuntimeState(t, ctx, api, a.Task.ID, TaskNeedsReview)
|
||||
oldAttempt, err := api.GetTaskAttempt(ctx, a.Task.ID, a.AttemptID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 后端完整提交后主动丢失HTTP响应。禁用连接复用,排除net/http对复用连接的隐式重发。
|
||||
target, _ := url.Parse(api.baseURL)
|
||||
proxy := httputil.NewSingleHostReverseProxy(target)
|
||||
var drop atomic.Bool
|
||||
var calls atomic.Int32
|
||||
var committed atomic.Value
|
||||
proxy.ModifyResponse = func(r *http.Response) error {
|
||||
calls.Add(1)
|
||||
if !drop.CompareAndSwap(true, false) {
|
||||
return nil
|
||||
}
|
||||
raw, e := io.ReadAll(r.Body)
|
||||
r.Body.Close()
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if r.StatusCode != 200 && r.StatusCode != 201 {
|
||||
t.Errorf("unexpected response before drop: %s", raw)
|
||||
}
|
||||
committed.Store(raw)
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
|
||||
conn, _, e := w.(http.Hijacker).Hijack()
|
||||
if e == nil {
|
||||
conn.Close()
|
||||
}
|
||||
}
|
||||
server := httptest.NewServer(proxy)
|
||||
defer server.Close()
|
||||
transport := &http.Transport{DisableKeepAlives: true}
|
||||
defer transport.CloseIdleConnections()
|
||||
lost, err := NewClient(server.URL, Options{Token: token, HTTPClient: &http.Client{Transport: transport}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
review := ReviewTaskRequest{ExpectedRevision: unknown.Revision, Reason: "确认模拟执行已结束且未完成", EvidenceRef: "evidence:demo-reconciliation", ExecutionSettled: true, Outcome: TaskFailed, Result: json.RawMessage(`{"confirmed":true,"sequence":9007199254740993}`)}
|
||||
reviewKey := workerTestID(t)
|
||||
drop.Store(true)
|
||||
if _, err = lost.ReviewTask(ctx, unknown.ID, reviewKey, review); err == nil || calls.Load() != 1 {
|
||||
t.Fatal("review response loss was hidden", err)
|
||||
}
|
||||
var reviewEnvelope struct {
|
||||
Data ReviewTaskResult `json:"data"`
|
||||
}
|
||||
if err = json.Unmarshal(committed.Load().([]byte), &reviewEnvelope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reviewed, err := api.ReviewTask(ctx, unknown.ID, reviewKey, review)
|
||||
if err != nil || !reviewed.Deduplicated || reviewed.AuditID != reviewEnvelope.Data.AuditID || reviewed.Task.State != TaskFailed || reviewed.Task.EffectUncertain {
|
||||
t.Fatal(reviewed, err)
|
||||
}
|
||||
attempt, err := api.GetTaskAttempt(ctx, a.Task.ID, a.AttemptID)
|
||||
if err != nil || !reflect.DeepEqual(attempt, oldAttempt) {
|
||||
t.Fatal("review changed immutable attempt", err)
|
||||
}
|
||||
audit, err := api.GetTaskOperation(ctx, unknown.ID, reviewed.AuditID)
|
||||
if err != nil || audit.Action != "task.review" || audit.EvidenceRef != review.EvidenceRef || audit.Reason != review.Reason {
|
||||
t.Fatal(audit, err)
|
||||
}
|
||||
retry := RetryTaskRequest{ExpectedRevision: reviewed.Task.Revision, Reason: "已核实模拟旧执行不会再产生效果,允许重新执行", EvidenceRef: "evidence:demo-safe-retry", SafeToRetry: true, TaskTimeoutMS: 60000}
|
||||
retryKey := workerTestID(t)
|
||||
drop.Store(true)
|
||||
if _, err = lost.RetryTask(ctx, unknown.ID, retryKey, retry); err == nil || calls.Load() != 2 {
|
||||
t.Fatal("retry response loss was hidden", err)
|
||||
}
|
||||
var retryEnvelope struct {
|
||||
Data RetryTaskResult `json:"data"`
|
||||
}
|
||||
if err = json.Unmarshal(committed.Load().([]byte), &retryEnvelope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
child, err := api.RetryTask(ctx, unknown.ID, retryKey, retry)
|
||||
if err != nil || !child.Deduplicated || child.Task.ID != retryEnvelope.Data.Task.ID || child.AuditID != retryEnvelope.Data.AuditID || child.Task.RetryOf == nil || *child.Task.RetryOf != unknown.ID {
|
||||
t.Fatal(child, err)
|
||||
}
|
||||
assigned, err := w.NextAssignment(ctx)
|
||||
if err != nil || assigned.Task.ID != child.Task.ID || assigned.Task.RetryOf == nil || *assigned.Task.RetryOf != unknown.ID {
|
||||
t.Fatal("wire lost retry lineage", assigned, err)
|
||||
}
|
||||
if _, err = w.Start(ctx, workerTestID(t), assigned.Attempt()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = w.Finish(ctx, workerTestID(t), WorkerFinish{Ref: assigned.Attempt(), Outcome: WorkerSucceeded, Result: json.RawMessage(`{"object_ref":"examples/output"}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitRuntimeState(t, ctx, api, child.Task.ID, TaskSucceeded)
|
||||
late, err := w.Finish(ctx, workerTestID(t), WorkerFinish{Ref: a.Attempt(), Outcome: WorkerSucceeded, Result: json.RawMessage(`{"late":true}`)})
|
||||
if err != nil || late.Receipt.Disposition != "EVIDENCE_ONLY" {
|
||||
t.Fatal(late, err)
|
||||
}
|
||||
source, err := api.GetTask(ctx, unknown.ID)
|
||||
if err != nil || !reflect.DeepEqual(source, reviewed.Task) {
|
||||
t.Fatal("late source rewrote review", err)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
actions := map[string]int{}
|
||||
cursor := ""
|
||||
for {
|
||||
page, e := api.ListTaskOperations(ctx, unknown.ID, HistoryPageRequest{Limit: 1, Cursor: cursor})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, v := range page.Items {
|
||||
if seen[v.ID] || v.TaskID != unknown.ID {
|
||||
t.Fatal("duplicate/unscoped audit")
|
||||
}
|
||||
seen[v.ID] = true
|
||||
actions[v.Action]++
|
||||
}
|
||||
if page.NextCursor == "" {
|
||||
break
|
||||
}
|
||||
cursor = page.NextCursor
|
||||
}
|
||||
if actions["task.review"] != 1 || actions["task.retry"] != 1 {
|
||||
t.Fatal("response loss created duplicate operations", actions)
|
||||
}
|
||||
if _, err = w.Capacity(ctx, workerTestID(t), WorkerCapacity{Sequence: 2, Drain: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("real UNKNOWN -> reviewed FAILED -> linked new task SUCCEEDED; both committed HTTP receipts lost/recovered; old attempt and audit preserved; late report evidence only")
|
||||
}
|
||||
105
sdk/go/recovery_test.go
Normal file
105
sdk/go/recovery_test.go
Normal file
@@ -0,0 +1,105 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRecoveryReceiptAndAuditBoundary(t *testing.T) {
|
||||
const task = "11111111111111111111111111111111"
|
||||
const child = "22222222222222222222222222222222"
|
||||
const audit = "33333333333333333333333333333333"
|
||||
var calls atomic.Int32
|
||||
var malformed atomic.Bool
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
if r.Header.Get("Authorization") != "Bearer example-token" {
|
||||
t.Error("missing token")
|
||||
}
|
||||
var data any
|
||||
switch r.URL.Path {
|
||||
case "/v1/tasks/" + task + ":review":
|
||||
if r.Method != "POST" || r.Header.Get("Idempotency-Key") != "stable-review" {
|
||||
t.Error("review identity")
|
||||
}
|
||||
var in ReviewTaskRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&in); err != nil || in.ExpectedRevision != 9007199254740993 || !in.ExecutionSettled {
|
||||
t.Error("lost review fields", err)
|
||||
}
|
||||
out := ReviewTaskResult{AuditID: audit, Task: Task{ID: task, State: TaskFailed, Revision: 9007199254740994}}
|
||||
if malformed.Load() {
|
||||
out.Task.EffectUncertain = true
|
||||
}
|
||||
data = out
|
||||
case "/v1/tasks/" + task + ":retry":
|
||||
if r.Method != "POST" || r.Header.Get("Idempotency-Key") != "stable-retry" {
|
||||
t.Error("retry identity")
|
||||
}
|
||||
parent := task
|
||||
out := RetryTaskResult{AuditID: audit, SourceTaskID: task, Task: Task{ID: child, RetryOf: &parent}}
|
||||
if malformed.Load() {
|
||||
out.Task.RetryOf = nil
|
||||
}
|
||||
data = out
|
||||
case "/v1/tasks/" + task + "/operations":
|
||||
if r.Method != "GET" || r.Header.Get("Idempotency-Key") != "" || r.URL.Query().Get("cursor") != "opaque+cursor/=" {
|
||||
t.Error("audit page request")
|
||||
}
|
||||
data = TaskOperationPage{Items: []TaskOperationSummary{{ID: audit, TaskID: task, Action: "task.review"}}}
|
||||
case "/v1/tasks/" + task + "/operations/" + audit:
|
||||
data = TaskOperation{TaskOperationSummary: TaskOperationSummary{ID: audit, TaskID: task}, After: json.RawMessage(`{"revision":"9007199254740994"}`)}
|
||||
default:
|
||||
t.Error("unexpected path", r.URL.Path)
|
||||
}
|
||||
json.NewEncoder(w).Encode(map[string]any{"request_id": audit, "data": data})
|
||||
}))
|
||||
defer server.Close()
|
||||
c, _ := NewClient(server.URL, Options{Token: "example-token"})
|
||||
ctx := context.Background()
|
||||
in := ReviewTaskRequest{ExpectedRevision: 9007199254740993, ExecutionSettled: true, Outcome: TaskFailed}
|
||||
review, err := c.ReviewTask(ctx, task, "stable-review", in)
|
||||
if err != nil || review.Task.Revision != 9007199254740994 {
|
||||
t.Fatal(review, err)
|
||||
}
|
||||
retry, err := c.RetryTask(ctx, task, "stable-retry", RetryTaskRequest{})
|
||||
if err != nil || retry.Task.ID != child {
|
||||
t.Fatal(retry, err)
|
||||
}
|
||||
page, err := c.ListTaskOperations(ctx, task, HistoryPageRequest{Cursor: "opaque+cursor/="})
|
||||
if err != nil || len(page.Items) != 1 {
|
||||
t.Fatal(page, err)
|
||||
}
|
||||
detail, err := c.GetTaskOperation(ctx, task, audit)
|
||||
if err != nil || string(detail.After) != `{"revision":"9007199254740994"}` {
|
||||
t.Fatal(detail, err)
|
||||
}
|
||||
malformed.Store(true)
|
||||
if _, err = c.ReviewTask(ctx, task, "stable-review", in); err == nil {
|
||||
t.Fatal("inconsistent review accepted")
|
||||
}
|
||||
if _, err = c.RetryTask(ctx, task, "stable-retry", RetryTaskRequest{}); err == nil {
|
||||
t.Fatal("missing retry parent accepted")
|
||||
}
|
||||
for _, id := range []string{"../tasks", strings.Repeat("A", 32), "id?token=secret"} {
|
||||
if _, err = c.ReviewTask(ctx, id, "key", in); err == nil {
|
||||
t.Fatal("unsafe path")
|
||||
}
|
||||
if _, err = c.RetryTask(ctx, id, "key", RetryTaskRequest{}); err == nil {
|
||||
t.Fatal("unsafe path")
|
||||
}
|
||||
if _, err = c.GetTaskOperation(ctx, task, id); err == nil {
|
||||
t.Fatal("unsafe audit path")
|
||||
}
|
||||
}
|
||||
if _, err = c.ListTaskOperations(ctx, task, HistoryPageRequest{Limit: 101}); err == nil {
|
||||
t.Fatal("unbounded page")
|
||||
}
|
||||
if calls.Load() != 6 {
|
||||
t.Fatal("invalid path reached transport")
|
||||
}
|
||||
}
|
||||
@@ -38,6 +38,8 @@ type SubmitTaskRequest struct {
|
||||
|
||||
// Task 是对外任务快照,不包含 Worker 令牌或凭据摘要。
|
||||
type Task struct {
|
||||
// RetryOf 人工重新发起时的原任务标识;为空表示不是人工派生任务。
|
||||
RetryOf *string `json:"retry_of,omitempty"`
|
||||
// ID 资源标识,32 位小写十六进制字符串。
|
||||
ID string `json:"id"`
|
||||
// NamespaceID 命名空间标识,由凭据确定隔离范围。
|
||||
|
||||
@@ -399,6 +399,9 @@ func (w *WorkerConnection) command(ctx context.Context, op, id string, ref Worke
|
||||
}
|
||||
|
||||
func (w *WorkerConnection) validAssignment(a WorkerAssignment) bool {
|
||||
if a.Task.RetryOf != nil && (!validID(*a.Task.RetryOf) || *a.Task.RetryOf == a.Task.ID) {
|
||||
return false
|
||||
}
|
||||
if a.Task.Priority < 0 || a.Task.Priority > 9 {
|
||||
return false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user