feat: add durable safe retries with uncertainty tracking and SDK examples

This commit is contained in:
2026-09-17 15:43:44 +08:00
parent 9e3d4be5cc
commit d65cf96ea7
44 changed files with 1300 additions and 77 deletions

View File

@@ -0,0 +1,13 @@
-- 安全重试保留跨尝试的不确定效果,旧迁移及旧操作摘要保持不变。
ALTER TABLE pulse_tasks
ADD COLUMN effect_uncertain BOOLEAN NOT NULL DEFAULT FALSE COMMENT '曾有已授权执行的效果尚未核实,重试与耗尽不清除,仅权威成功或核查可解除',
ADD CONSTRAINT ck_task_effect_uncertain CHECK (effect_uncertain IN (0,1) AND (effect_uncertain=FALSE OR state NOT IN ('SUCCEEDED','FAILED','CANCELLED')));
UPDATE pulse_tasks SET effect_uncertain=TRUE WHERE state='NEEDS_REVIEW';
ALTER TABLE pulse_attempts
ADD COLUMN retryable BOOLEAN NOT NULL DEFAULT FALSE COMMENT 'Worker 明确失败时声明可安全重试,不用于未知效果或证明实际已安排重试',
ADD CONSTRAINT ck_attempt_retryable CHECK (retryable IN (0,1) AND (retryable=FALSE OR state='FAILED'));
ALTER TABLE pulse_attempt_evidence
ADD COLUMN retryable BOOLEAN NOT NULL DEFAULT FALSE COMMENT '原失败报告的安全重试声明,证据本身不触发重试',
ADD CONSTRAINT ck_evidence_retryable CHECK (retryable IN (0,1) AND (retryable=FALSE OR outcome='FAILED'));
ALTER TABLE pulse_attempt_receipts
ADD KEY ix_result_receipt(namespace_id,attempt_id,operation,request_digest) COMMENT '同一结果换消息身份重报时读取原始已应用回执';

View File

@@ -0,0 +1,116 @@
package mysql
import (
"context"
"os"
"testing"
"time"
core "git.yuntuo.hk/k2safe/orbit/model"
mysqldriver "github.com/go-sql-driver/mysql"
)
// 在独立库中重建真实 v4,再走正式迁移入口;不改变主测试库的结构或迁移记录。
func TestIntegrationSafeRetryUpgrade(t *testing.T) {
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("requires disposable MySQL with CREATE DATABASE permission")
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
admin, err := Open(ctx, dsn, 2, 1)
if err != nil {
t.Fatal(err)
}
defer admin.Close()
name := "orbit_retry_upgrade_" + string(executionID(t))
if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+name+"`"); err != nil {
t.Fatal(err)
}
defer func() {
cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if _, err := admin.ExecContext(cleanup, "DROP DATABASE `"+name+"`"); err != nil {
t.Error(err)
}
}()
cfg, err := mysqldriver.ParseDSN(dsn)
if err != nil {
t.Fatal("invalid test DSN")
}
cfg.DBName = name
db, err := Open(ctx, cfg.FormatDSN(), 4, 2)
if err != nil {
t.Fatal(err)
}
defer db.Close()
exec := func(q string, args ...any) {
t.Helper()
if _, err := db.ExecContext(ctx, q, args...); err != nil {
t.Fatal(err)
}
}
exec(`CREATE TABLE orbit_schema_migrations (version BIGINT UNSIGNED PRIMARY KEY,name VARCHAR(191) NOT NULL,checksum CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,dirty BOOLEAN NOT NULL,applied_at DATETIME(6) NULL) ENGINE=InnoDB`)
migrations, err := Migrations()
if err != nil {
t.Fatal(err)
}
for _, m := range migrations {
if m.Version > 4 {
break
}
statements, err := splitSQL(m.SQL)
if err != nil {
t.Fatal(err)
}
for _, statement := range statements {
exec(statement)
}
exec(`INSERT INTO orbit_schema_migrations(version,name,checksum,dirty,applied_at) VALUES (?,?,?,FALSE,UTC_TIMESTAMP(6))`, m.Version, m.Name, m.Checksum)
}
app, ns, queue := executionID(t), executionID(t), executionID(t)
exec(`INSERT INTO orbit_applications(id,name) VALUES (?,?)`, app, string(app))
exec(`INSERT INTO orbit_namespaces(id,application_id,name,environment) VALUES (?,?,?,'test')`, ns, app, string(ns))
exec(`INSERT INTO pulse_queues(namespace_id,id,name,max_concurrency,rate_per_second,policy) VALUES (?,?,?,4,10,'{}')`, ns, queue, string(queue))
policy := `{"retry":{"max_attempts":3,"initial_delay_ms":1000,"max_delay_ms":30000,"attempt_timeout_ms":5000}}`
ids := map[string]core.ID{}
for _, state := range []string{"NEEDS_REVIEW", "WAITING", "SUCCEEDED", "FAILED", "CANCELLED"} {
id := executionID(t)
ids[state] = id
var completed *time.Time
if state == "SUCCEEDED" || state == "FAILED" || state == "CANCELLED" {
now := time.Now().UTC()
completed = &now
}
exec(`INSERT INTO pulse_tasks(namespace_id,id,queue_id,state,handler,handler_version,payload_version,payload,policy_snapshot,available_at,deadline_at,attempt_timeout_ms,max_attempts,completed_at) VALUES (?,?,?,?,'object.prepare','v1','v1','{}',?,UTC_TIMESTAMP(6),DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 1 HOUR),5000,3,?)`, ns, id, queue, state, policy, completed)
}
// MySQL 原生 JSON 会规范化文本,以迁移前实际存储值核对冻结策略。
if err := db.QueryRowContext(ctx, `SELECT policy_snapshot FROM pulse_tasks WHERE namespace_id=? AND id=?`, ns, ids["WAITING"]).Scan(&policy); err != nil {
t.Fatal(err)
}
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
if err := CheckSchema(ctx, db); err != nil {
t.Fatal(err)
}
for state, id := range ids {
var gotState, gotPolicy string
var uncertain bool
var revision uint64
if err := db.QueryRowContext(ctx, `SELECT state,effect_uncertain,policy_snapshot,revision FROM pulse_tasks WHERE namespace_id=? AND id=?`, ns, id).Scan(&gotState, &uncertain, &gotPolicy, &revision); err != nil {
t.Fatal(err)
}
if gotState != state || uncertain != (state == "NEEDS_REVIEW") || gotPolicy != policy || revision != 1 {
t.Fatalf("upgrade changed frozen data or lost uncertainty for %s", state)
}
}
// 再次运行不重复修改回填或策略。
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
var count int
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_tasks WHERE effect_uncertain=TRUE`).Scan(&count); err != nil || count != 1 {
t.Fatalf("upgrade is not idempotent: %d %v", count, err)
}
}

View File

@@ -91,11 +91,11 @@ func historyAttemptExists(ctx context.Context, tx *sql.Tx, ns, task, attempt cor
return err
}
const attemptHistoryColumns = `a.id,a.task_id,a.epoch,a.state,a.revision,a.lease_until,a.authorized_at,a.ended_at,COALESCE(a.error_code,''),a.created_at,a.updated_at`
const attemptHistoryColumns = `a.id,a.task_id,a.epoch,a.state,a.revision,a.lease_until,a.authorized_at,a.ended_at,COALESCE(a.error_code,''),a.retryable,a.created_at,a.updated_at`
func scanAttemptSummary(row interface{ Scan(...any) error }) (service.AttemptSummary, error) {
var a service.AttemptSummary
err := row.Scan(&a.ID, &a.TaskID, &a.Epoch, &a.State, &a.Revision, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt)
err := row.Scan(&a.ID, &a.TaskID, &a.Epoch, &a.State, &a.Revision, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Retryable, &a.CreatedAt, &a.UpdatedAt)
return a, err
}
func (s *TaskStore) ListTaskAttempts(ctx context.Context, t auth.Token, task core.ID, in service.HistoryPageInput) (service.AttemptPage, error) {
@@ -148,7 +148,7 @@ func (s *TaskStore) GetTaskAttempt(ctx context.Context, t auth.Token, task, atte
return err
}
var result []byte
err := tx.QueryRowContext(ctx, `SELECT `+attemptHistoryColumns+`,a.session_id,w.worker_instance_id,a.result FROM pulse_attempts a JOIN pulse_worker_sessions w ON w.namespace_id=a.namespace_id AND w.id=a.session_id WHERE a.namespace_id=? AND a.task_id=? AND a.id=?`, t.NamespaceID, task, attempt).Scan(&out.ID, &out.TaskID, &out.Epoch, &out.State, &out.Revision, &out.LeaseUntil, &out.AuthorizedAt, &out.EndedAt, &out.ErrorCode, &out.CreatedAt, &out.UpdatedAt, &out.SessionID, &out.WorkerInstanceID, &result)
err := tx.QueryRowContext(ctx, `SELECT `+attemptHistoryColumns+`,a.session_id,w.worker_instance_id,a.result FROM pulse_attempts a JOIN pulse_worker_sessions w ON w.namespace_id=a.namespace_id AND w.id=a.session_id WHERE a.namespace_id=? AND a.task_id=? AND a.id=?`, t.NamespaceID, task, attempt).Scan(&out.ID, &out.TaskID, &out.Epoch, &out.State, &out.Revision, &out.LeaseUntil, &out.AuthorizedAt, &out.EndedAt, &out.ErrorCode, &out.Retryable, &out.CreatedAt, &out.UpdatedAt, &out.SessionID, &out.WorkerInstanceID, &result)
if errors.Is(err, sql.ErrNoRows) {
return fault.NotFound
}
@@ -166,7 +166,7 @@ func (s *TaskStore) ListAttemptEvidence(ctx context.Context, t auth.Token, task,
if err := historyAttemptExists(ctx, tx, t.NamespaceID, task, attempt); err != nil {
return err
}
q := `SELECT id,attempt_id,outcome,COALESCE(error_code,''),reason,created_at FROM pulse_attempt_evidence WHERE namespace_id=? AND attempt_id=?`
q := `SELECT id,attempt_id,outcome,COALESCE(error_code,''),retryable,reason,created_at FROM pulse_attempt_evidence WHERE namespace_id=? AND attempt_id=?`
args := []any{t.NamespaceID, attempt}
if in.Cursor != "" {
c, err := decodeHistoryCursor(in.Cursor, t.NamespaceID, task, attempt)
@@ -185,7 +185,7 @@ func (s *TaskStore) ListAttemptEvidence(ctx context.Context, t auth.Token, task,
defer rows.Close()
for rows.Next() {
e := service.EvidenceSummary{TaskID: task, Disposition: "EVIDENCE_ONLY"}
if err := rows.Scan(&e.ID, &e.AttemptID, &e.Outcome, &e.ErrorCode, &e.Reason, &e.CreatedAt); err != nil {
if err := rows.Scan(&e.ID, &e.AttemptID, &e.Outcome, &e.ErrorCode, &e.Retryable, &e.Reason, &e.CreatedAt); err != nil {
return err
}
out.Items = append(out.Items, e)
@@ -213,7 +213,7 @@ func (s *TaskStore) GetAttemptEvidence(ctx context.Context, t auth.Token, task,
return err
}
var payload []byte
err := tx.QueryRowContext(ctx, `SELECT id,attempt_id,outcome,COALESCE(error_code,''),reason,created_at,message_id,payload FROM pulse_attempt_evidence WHERE namespace_id=? AND attempt_id=? AND id=?`, t.NamespaceID, attempt, evidence).Scan(&out.ID, &out.AttemptID, &out.Outcome, &out.ErrorCode, &out.Reason, &out.CreatedAt, &out.MessageID, &payload)
err := tx.QueryRowContext(ctx, `SELECT id,attempt_id,outcome,COALESCE(error_code,''),retryable,reason,created_at,message_id,payload FROM pulse_attempt_evidence WHERE namespace_id=? AND attempt_id=? AND id=?`, t.NamespaceID, attempt, evidence).Scan(&out.ID, &out.AttemptID, &out.Outcome, &out.ErrorCode, &out.Retryable, &out.Reason, &out.CreatedAt, &out.MessageID, &payload)
if errors.Is(err, sql.ErrNoRows) {
return fault.NotFound
}

View File

@@ -216,14 +216,14 @@ func (s *TaskStore) Submit(ctx context.Context, token auth.Token, key string, in
return out, err
}
const taskColumns = `id,namespace_id,queue_id,state,revision,handler,handler_version,payload_version,payload,result,COALESCE(business_key,''),available_at,start_deadline_at,deadline_at,recovery_safety,policy_snapshot,attempt_count,cancel_requested_at,completed_at,created_at,updated_at`
const taskColumns = `id,namespace_id,queue_id,state,revision,handler,handler_version,payload_version,payload,result,COALESCE(business_key,''),available_at,start_deadline_at,deadline_at,recovery_safety,policy_snapshot,attempt_count,cancel_requested_at,completed_at,created_at,updated_at,effect_uncertain`
type scanner interface{ Scan(...any) error }
func scanTask(row scanner) (service.TaskView, error) {
var v service.TaskView
var policy, payload, result []byte
err := row.Scan(&v.ID, &v.NamespaceID, &v.QueueID, &v.State, &v.Revision, &v.Handler, &v.HandlerVersion, &v.PayloadVersion, &payload, &result, &v.BusinessKey, &v.AvailableAt, &v.StartDeadlineAt, &v.DeadlineAt, &v.RecoverySafety, &policy, &v.AttemptCount, &v.CancelRequestedAt, &v.CompletedAt, &v.CreatedAt, &v.UpdatedAt)
err := row.Scan(&v.ID, &v.NamespaceID, &v.QueueID, &v.State, &v.Revision, &v.Handler, &v.HandlerVersion, &v.PayloadVersion, &payload, &result, &v.BusinessKey, &v.AvailableAt, &v.StartDeadlineAt, &v.DeadlineAt, &v.RecoverySafety, &policy, &v.AttemptCount, &v.CancelRequestedAt, &v.CompletedAt, &v.CreatedAt, &v.UpdatedAt, &v.EffectUncertain)
if err != nil {
return v, err
}
@@ -273,7 +273,7 @@ func (s *TaskStore) ListTasks(ctx context.Context, token auth.Token, in service.
if !p.AllowsQueue(in.QueueID) {
return fault.Forbidden
}
q := `SELECT id,queue_id,state,revision,handler,handler_version,COALESCE(business_key,''),available_at,deadline_at,attempt_count,created_at,updated_at FROM pulse_tasks WHERE namespace_id=? AND queue_id=?`
q := `SELECT id,queue_id,state,revision,handler,handler_version,COALESCE(business_key,''),available_at,deadline_at,attempt_count,created_at,updated_at,effect_uncertain FROM pulse_tasks WHERE namespace_id=? AND queue_id=?`
args := []any{token.NamespaceID, in.QueueID}
if in.State != "" {
q += " AND state=?"
@@ -297,7 +297,7 @@ func (s *TaskStore) ListTasks(ctx context.Context, token auth.Token, in service.
defer rows.Close()
for rows.Next() {
var v service.TaskSummary
if err := rows.Scan(&v.ID, &v.QueueID, &v.State, &v.Revision, &v.Handler, &v.HandlerVersion, &v.BusinessKey, &v.AvailableAt, &v.DeadlineAt, &v.AttemptCount, &v.CreatedAt, &v.UpdatedAt); err != nil {
if err := rows.Scan(&v.ID, &v.QueueID, &v.State, &v.Revision, &v.Handler, &v.HandlerVersion, &v.BusinessKey, &v.AvailableAt, &v.DeadlineAt, &v.AttemptCount, &v.CreatedAt, &v.UpdatedAt, &v.EffectUncertain); err != nil {
return err
}
out.Items = append(out.Items, v)
@@ -357,7 +357,17 @@ func (s *TaskStore) CancelTask(ctx context.Context, token auth.Token, id core.ID
if owner.Valid {
return fault.Conflict
}
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='CANCELLED',cancel_requested_at=UTC_TIMESTAMP(6),completed_at=UTC_TIMESTAMP(6),updated_at=UTC_TIMESTAMP(6),revision=revision+1 WHERE namespace_id=? AND id=? AND revision=? AND state='WAITING'`, token.NamespaceID, id, in.ExpectedRevision)
now, err := dbNow(ctx, tx)
if err != nil {
return err
}
next := pulse.TaskCancelled
var completed *time.Time = &now
if task.EffectUncertain {
next = pulse.TaskNeedsReview
completed = nil
}
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,cancel_requested_at=?,completed_at=?,updated_at=?,revision=revision+1 WHERE namespace_id=? AND id=? AND revision=? AND state='WAITING'`, next, now, completed, now, token.NamespaceID, id, in.ExpectedRevision)
if err != nil {
return err
}
@@ -372,7 +382,7 @@ func (s *TaskStore) CancelTask(ctx context.Context, token auth.Token, id core.ID
if err != nil {
return err
}
if err := emitEvent(ctx, tx, token.NamespaceID, id, out.Task.Revision, "task", "task.cancelled", map[string]any{"task_id": id, "state": out.Task.State}); err != nil {
if err := emitEvent(ctx, tx, token.NamespaceID, id, out.Task.Revision, "task", resultEvent(out.Task.State), map[string]any{"task_id": id, "state": out.Task.State, "reason": "CANCEL_REQUESTED", "effect_uncertain": out.Task.EffectUncertain}); err != nil {
return err
}
if err := appendAudit(ctx, tx, token, identity, id, "task", "task.cancel", in.Reason, map[string]any{"state": task.State, "revision": fmt.Sprint(task.Revision)}, map[string]any{"state": out.Task.State, "revision": fmt.Sprint(out.Task.Revision)}); err != nil {

View File

@@ -309,7 +309,7 @@ func (s *ExecutionStore) lockExecution(ctx context.Context, tx *sql.Tx, token au
func readAttempt(ctx context.Context, tx *sql.Tx, ns, id core.ID) (pulse.Attempt, error) {
var a pulse.Attempt
a.NamespaceID = ns
err := tx.QueryRowContext(ctx, `SELECT id,revision,task_id,epoch,session_id,token_digest,state,lease_until,authorized_at,ended_at,error_code,result,result_digest FROM pulse_attempts WHERE namespace_id=? AND id=? FOR UPDATE`, ns, id).Scan(&a.ID, &a.Revision, &a.TaskID, &a.Epoch, &a.SessionID, &a.TokenDigest, &a.State, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Result, &a.ResultDigest)
err := tx.QueryRowContext(ctx, `SELECT id,revision,task_id,epoch,session_id,token_digest,state,lease_until,authorized_at,ended_at,error_code,retryable,result,result_digest FROM pulse_attempts WHERE namespace_id=? AND id=? FOR UPDATE`, ns, id).Scan(&a.ID, &a.Revision, &a.TaskID, &a.Epoch, &a.SessionID, &a.TokenDigest, &a.State, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Retryable, &a.Result, &a.ResultDigest)
if errors.Is(err, sql.ErrNoRows) {
err = fault.NotFound
}

View File

@@ -10,11 +10,12 @@ import (
"git.yuntuo.hk/k2safe/orbit/internal/fault"
core "git.yuntuo.hk/k2safe/orbit/model"
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
"git.yuntuo.hk/k2safe/orbit/pulse/retry"
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
)
// Recover 是受控后台扫描入口,每批最多 100 项、每项一个短事务,可由多个节点竞争。
// 当前已授权执行的失联一律转核查,自动安全重试在 P3 接入冻结策略后实现。
// 已授权执行的失联保留效果未知,只有冻结的幂等恢复声明与剩余预算允许自动重试。
func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.RecoveryStats, error) {
var out worker.RecoveryStats
if limit < 1 || limit > 100 {
@@ -88,11 +89,16 @@ func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.Recover
return nil
}
changed = pulse.TaskFailed
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='FAILED',completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, now, now, c.ns, c.id, task.view.Revision)
var completed *time.Time = &now
if task.view.EffectUncertain {
changed = pulse.TaskNeedsReview
completed = nil
}
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, changed, completed, now, c.ns, c.id, task.view.Revision)
if err := exactlyOne(result, err); err != nil {
return err
}
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", "task.failed", map[string]any{"reason": "START_DEADLINE_EXCEEDED", "node_id": s.node})
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", resultEvent(changed), map[string]any{"reason": "START_DEADLINE_EXCEEDED", "node_id": s.node, "effect_uncertain": task.view.EffectUncertain})
}
if task.view.State != pulse.TaskLeased && task.view.State != pulse.TaskRunning {
return nil
@@ -118,29 +124,31 @@ func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.Recover
return errors.New("active task references ended attempt")
}
attemptState := pulse.AttemptExpired
next := pulse.TaskFailed
outcome := "NOT_STARTED"
reason := "LEASE_EXPIRED_BEFORE_START"
var completed *time.Time = &now
available := task.view.AvailableAt
if attempt.AuthorizedAt != nil {
attemptState = pulse.AttemptUnknown
next = pulse.TaskNeedsReview
outcome = "UNKNOWN"
reason = "EXECUTION_OUTCOME_UNKNOWN"
completed = nil
} else if task.view.AttemptCount < task.maxAttempts {
// 未批准开始的尝试可以有界重派,次数不重置;开始截止与总截止仍有效。
nextAt := now.Add(time.Second)
if nextAt.Before(task.view.DeadlineAt) && (task.view.StartDeadlineAt == nil || nextAt.Before(*task.view.StartDeadlineAt)) {
next = pulse.TaskWaiting
available = nextAt
completed = nil
}
}
decision, err := retry.Decide(retryFacts(task, attempt.ID, now, outcome, false))
if err != nil {
return err
}
next := decision.State
var completed *time.Time
if next.Terminal() {
completed = &now
}
available := task.view.AvailableAt
if decision.NextAt != nil {
available = *decision.NextAt
}
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, reason, now, c.ns, attempt.ID, attempt.Revision)
if err := exactlyOne(result, err); err != nil {
return err
}
result, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,lease_until=NULL,available_at=?,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, next, available, completed, now, c.ns, c.id, task.view.Revision)
result, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,effect_uncertain=?,lease_until=NULL,available_at=?,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, next, decision.Uncertain, available, completed, now, c.ns, c.id, task.view.Revision)
if err := exactlyOne(result, err); err != nil {
return err
}
@@ -148,7 +156,7 @@ func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.Recover
return err
}
changed = next
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", resultEvent(next), map[string]any{"attempt_id": attempt.ID, "epoch": fmt.Sprint(attempt.Epoch), "reason": reason, "node_id": s.node})
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", resultEvent(next), map[string]any{"attempt_id": attempt.ID, "epoch": fmt.Sprint(attempt.Epoch), "reason": reason, "node_id": s.node, "effect_uncertain": decision.Uncertain, "decision": decision.Reason, "next_available_at": decision.NextAt, "delay_ms": decision.DelayMS, "attempt_count": task.view.AttemptCount, "max_attempts": task.maxAttempts})
})
if err != nil {
return out, err

View File

@@ -3,12 +3,14 @@ package mysql
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"time"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/platform/auth"
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
"git.yuntuo.hk/k2safe/orbit/pulse/retry"
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
)
@@ -19,7 +21,8 @@ func (s *ExecutionStore) Finish(ctx context.Context, token auth.Token, b worker.
Outcome string
Result core.JSON
ErrorCode string
}{attemptIdentity(in.Ref), in.Outcome, core.JSON(in.Result), in.ErrorCode})
Retryable bool `json:"Retryable,omitempty"`
}{attemptIdentity(in.Ref), in.Outcome, core.JSON(in.Result), in.ErrorCode, in.Retryable})
if err != nil {
return out, err
}
@@ -45,6 +48,14 @@ func (s *ExecutionStore) Finish(ctx context.Context, token auth.Token, b worker.
receipt := worker.Receipt{Operation: "finish", TaskState: task.view.State, AttemptState: a.State, TaskRevision: task.view.Revision, LeaseUntil: a.LeaseUntil, Disposition: "APPLIED"}
if a.ResultDigest != nil && *a.ResultDigest == digest {
// 新消息 ID 重报已保存的同一结果,也不能重复事件、许可释放或状态推进。
// 重试可能已推进其他尝试,必须返回原操作回执,不能套用当前任务状态。
var raw []byte
if err := tx.QueryRowContext(ctx, `SELECT response FROM pulse_attempt_receipts WHERE namespace_id=? AND attempt_id=? AND operation='finish' AND request_digest=? AND JSON_UNQUOTE(JSON_EXTRACT(response,'$.disposition'))='APPLIED' LIMIT 1`, token.NamespaceID, a.ID, digest).Scan(&raw); err != nil {
return err
}
if err := json.Unmarshal(raw, &receipt); err != nil {
return err
}
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil {
return err
}
@@ -70,24 +81,34 @@ func (s *ExecutionStore) Finish(ctx context.Context, token auth.Token, b worker.
out = worker.CommandResult{Receipt: receipt, Lease: status}
return nil
}
attemptState, taskState := pulse.AttemptState(in.Outcome), pulse.TaskState(in.Outcome)
var completed *time.Time = &now
if in.Outcome == "UNKNOWN" {
taskState = pulse.TaskNeedsReview
completed = nil
attemptState := pulse.AttemptState(in.Outcome)
decision, err := retry.Decide(retryFacts(task, a.ID, now, in.Outcome, in.Retryable))
if err != nil {
return err
}
changed, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=NULLIF(?,''),result=?,result_digest=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, in.ErrorCode, string(in.Result), digest, now, token.NamespaceID, a.ID, a.Revision)
taskState := decision.State
var completed *time.Time
if taskState.Terminal() {
completed = &now
}
available := task.view.AvailableAt
var result any = string(in.Result)
if decision.NextAt != nil {
available = *decision.NextAt
result = nil
}
changed, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=NULLIF(?,''),retryable=?,result=?,result_digest=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, in.ErrorCode, in.Retryable, string(in.Result), digest, now, token.NamespaceID, a.ID, a.Revision)
if err := exactlyOne(changed, err); err != nil {
return err
}
changed, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,result=?,lease_until=NULL,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, taskState, string(in.Result), completed, now, token.NamespaceID, in.Ref.TaskID, task.view.Revision)
changed, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,result=?,effect_uncertain=?,available_at=?,lease_until=NULL,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, taskState, result, decision.Uncertain, available, completed, now, token.NamespaceID, in.Ref.TaskID, task.view.Revision)
if err := exactlyOne(changed, err); err != nil {
return err
}
if err := releasePermits(ctx, tx, token.NamespaceID, a.ID, now); err != nil {
return err
}
event := map[string]any{"attempt_id": a.ID, "epoch": fmt.Sprint(a.Epoch), "state": taskState, "outcome": in.Outcome, "error_code": in.ErrorCode}
event := map[string]any{"attempt_id": a.ID, "epoch": fmt.Sprint(a.Epoch), "state": taskState, "outcome": in.Outcome, "error_code": in.ErrorCode, "retryable": in.Retryable, "effect_uncertain": decision.Uncertain, "decision": decision.Reason, "next_available_at": decision.NextAt, "delay_ms": decision.DelayMS, "attempt_count": task.view.AttemptCount, "max_attempts": task.maxAttempts}
if err := emitEvent(ctx, tx, token.NamespaceID, in.Ref.TaskID, task.view.Revision+1, "task", resultEvent(taskState), event); err != nil {
return err
}
@@ -124,9 +145,16 @@ func saveEvidence(ctx context.Context, tx *sql.Tx, ns, message core.ID, in worke
if err != nil {
return err
}
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempt_evidence(namespace_id,id,attempt_id,message_id,outcome,error_code,payload,request_digest,reason) VALUES (?,?,?,?,?,NULLIF(?,''),?,?,?)`, ns, id, in.Ref.AttemptID, message, in.Outcome, in.ErrorCode, string(in.Result), digest, reason)
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempt_evidence(namespace_id,id,attempt_id,message_id,outcome,error_code,retryable,payload,request_digest,reason) VALUES (?,?,?,?,?,NULLIF(?,''),?,?,?,?)`, ns, id, in.Ref.AttemptID, message, in.Outcome, in.ErrorCode, in.Retryable, string(in.Result), digest, reason)
if err != nil {
return err
}
return emitEvent(ctx, tx, ns, id, 1, "attempt_evidence", "attempt.evidence_received", map[string]any{"attempt_id": in.Ref.AttemptID, "task_id": in.Ref.TaskID, "reason": reason, "outcome": in.Outcome})
}
// retryFacts 以持久化预算为准,JSON 快照只提供冻结的等待策略。
func retryFacts(task executionTask, attempt core.ID, now time.Time, outcome string, retryable bool) retry.Input {
policy := task.view.Retry
policy.MaxAttempts = task.maxAttempts
return retry.Input{Policy: policy, Safety: task.view.RecoverySafety, Consumed: task.view.AttemptCount, Outcome: outcome, Retryable: retryable, Uncertain: task.view.EffectUncertain, CancelRequested: task.view.CancelRequestedAt != nil, Now: now, Deadline: task.view.DeadlineAt, StartDeadline: task.view.StartDeadlineAt, AttemptID: attempt}
}

View File

@@ -0,0 +1,307 @@
package mysql
import (
"context"
"encoding/json"
"errors"
"os"
"sync"
"testing"
"time"
core "git.yuntuo.hk/k2safe/orbit/model"
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
"git.yuntuo.hk/k2safe/orbit/pulse/service"
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
mysqldriver "github.com/go-sql-driver/mysql"
)
func retryTask(t *testing.T, f *executionFixture, max uint32, safety pulse.RecoverySafety) service.TaskView {
t.Helper()
out, err := f.tasks.Submit(f.ctx, f.token, string(executionID(t)), service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`), RecoverySafety: safety, Retry: pulse.RetryPolicy{MaxAttempts: max, InitialDelayMS: 1000, MaxDelayMS: 5000, AttemptTimeoutMS: 30000}})
if err != nil {
t.Fatal(err)
}
return out.Task
}
func retryStart(t *testing.T, f *executionFixture) worker.AttemptRef {
t.Helper()
_, ref := f.claim()
if _, err := f.a.Start(f.ctx, f.token, f.binding, executionID(t), ref); err != nil {
t.Fatal(err)
}
return ref
}
func retryFinish(t *testing.T, f *executionFixture, ref worker.AttemptRef, outcome string, retryable bool) worker.CommandResult {
t.Helper()
in := worker.FinishInput{Ref: ref, Outcome: outcome, Retryable: retryable, Result: json.RawMessage(`{"n":9007199254740993}`)}
if outcome != "SUCCEEDED" {
in.ErrorCode = "CONTROLLED_RESULT"
}
out, err := f.a.Finish(f.ctx, f.token, f.binding, executionID(t), in)
if err != nil {
t.Fatal(err)
}
return out
}
func makeRetryDue(f *executionFixture, id core.ID) {
f.exec(`UPDATE pulse_tasks SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, id)
}
func TestIntegrationSafeRetry(t *testing.T) {
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("requires disposable MySQL")
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
db, err := Open(ctx, dsn, 32, 8)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
t.Run("original_applied_receipt_excludes_early_evidence", func(t *testing.T) {
f := newExecutionFixture(t, ctx, db)
retryTask(t, f, 2, pulse.RecoveryReconcileOnly)
_, ref := f.claim()
in := worker.FinishInput{Ref: ref, Outcome: "FAILED", Retryable: true, ErrorCode: "TEMPORARY_REJECTION", Result: json.RawMessage(`{}`)}
early, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), in)
if err != nil || early.Receipt.Disposition != "EVIDENCE_ONLY" {
t.Fatal("early report was applied")
}
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
t.Fatal(err)
}
applied, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), in)
if err != nil || applied.Receipt.Disposition != "APPLIED" || applied.Receipt.TaskState != pulse.TaskWaiting {
t.Fatal("authorized result not applied")
}
duplicate, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), in)
if err != nil || !duplicate.Deduplicated || duplicate.Receipt != applied.Receipt {
t.Fatal("replay selected an evidence receipt")
}
})
t.Run("legacy_digest_and_database_retry_constraints", func(t *testing.T) {
f := newExecutionFixture(t, ctx, db)
task := retryTask(t, f, 3, pulse.RecoveryReconcileOnly)
ref := retryStart(t, f)
reject := func(q string, args ...any) {
t.Helper()
_, err := db.ExecContext(ctx, q, args...)
var failure *mysqldriver.MySQLError
if !errors.As(err, &failure) || failure.Number != 3819 {
t.Fatalf("wanted CHECK rejection, got %v", err)
}
}
reject(`UPDATE pulse_attempts SET retryable=TRUE WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, ref.AttemptID)
in := worker.FinishInput{Ref: ref, Outcome: "FAILED", ErrorCode: "REJECTED", Result: json.RawMessage(`{"value":9007199254740993}`)}
if _, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), in); err != nil {
t.Fatal(err)
}
// v4 的摘要结构没有 Retryable;省略或 false 必须保持旧身份不变。
legacy, err := requestDigest(struct {
Identity any
Outcome string
Result core.JSON
ErrorCode string
}{attemptIdentity(ref), in.Outcome, core.JSON(in.Result), in.ErrorCode})
if err != nil {
t.Fatal(err)
}
var saved core.Digest
if err := db.QueryRowContext(ctx, `SELECT result_digest FROM pulse_attempts WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, ref.AttemptID).Scan(&saved); err != nil || saved != legacy {
t.Fatal("legacy finish digest changed")
}
reject(`UPDATE pulse_tasks SET effect_uncertain=TRUE WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, task.ID)
reject(`INSERT INTO pulse_attempt_evidence(namespace_id,id,attempt_id,message_id,outcome,payload,request_digest,reason,retryable) VALUES (?,?,?,?,'UNKNOWN','{}',?,'RESULT_CONFLICT',TRUE)`, f.resources.NamespaceID, executionID(t), ref.AttemptID, executionID(t), legacy)
})
t.Run("known_retry_duplicate_receipts_and_history", func(t *testing.T) {
f := newExecutionFixture(t, ctx, db)
task := retryTask(t, f, 2, pulse.RecoveryReconcileOnly)
ref := retryStart(t, f)
in := worker.FinishInput{Ref: ref, Outcome: "FAILED", Retryable: true, ErrorCode: "TEMPORARY_REJECTION", Result: json.RawMessage(`{"n":9007199254740993}`)}
message := executionID(t)
first, err := f.a.Finish(ctx, f.token, f.binding, message, in)
if err != nil {
t.Fatal(err)
}
pending := f.get(task.ID)
if first.Receipt.TaskState != pulse.TaskWaiting || first.Lease.Active || pending.EffectUncertain || len(pending.Result) != 0 || pending.CompletedAt != nil || f.used() != 0 || pending.AttemptCount != 1 {
t.Fatal("retry did not preserve task/attempt separation")
}
if delay := pending.AvailableAt.Sub(first.Lease.ServerTime); delay < 500*time.Millisecond || delay > time.Second {
t.Fatal("retry time is not bounded")
}
if a, err := f.a.Claim(ctx, f.token, f.binding); err != nil || a != nil {
t.Fatal("retry dispatched before due")
}
duplicate, err := f.a.Finish(ctx, f.token, f.binding, message, in)
if err != nil || !duplicate.Deduplicated || duplicate.Receipt != first.Receipt || !f.get(task.ID).AvailableAt.Equal(pending.AvailableAt) {
t.Fatal("same-message replay changed retry timing")
}
in.Retryable = false
_, err = f.a.Finish(ctx, f.token, f.binding, message, in)
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
in.Retryable = true
makeRetryDue(f, task.ID)
second := retryStart(t, f)
if second.Epoch != 2 || second.AttemptID == ref.AttemptID {
t.Fatal("retry reused execution identity")
}
retryFinish(t, f, second, "SUCCEEDED", false)
final := f.get(task.ID)
// 在新尝试完成后用新消息身份重报旧失败,仍返回第一次的 WAITING 回执。
duplicate, err = f.a.Finish(ctx, f.token, f.binding, executionID(t), in)
if err != nil || !duplicate.Deduplicated || duplicate.Receipt != first.Receipt || duplicate.Lease.Active || f.get(task.ID).Revision != final.Revision {
t.Fatal("old result replay borrowed newer task state")
}
in.Result = json.RawMessage(`{"n":2}`)
late, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), in)
if err != nil || late.Receipt.Disposition != "EVIDENCE_ONLY" {
t.Fatal("conflicting retry report changed authority")
}
attempts, err := f.tasks.ListTaskAttempts(ctx, f.token, task.ID, service.HistoryPageInput{})
if err != nil || len(attempts.Items) != 2 || !attempts.Items[1].Retryable || attempts.Items[0].State != pulse.AttemptSucceeded {
t.Fatal("retry history lost classification")
}
evidence, err := f.tasks.ListAttemptEvidence(ctx, f.token, task.ID, ref.AttemptID, service.HistoryPageInput{})
if err != nil || len(evidence.Items) != 1 || !evidence.Items[0].Retryable {
t.Fatal("evidence lost original retry declaration")
}
if f.count("orbit_events", "aggregate_id=? AND type='task.requeued'", task.ID) != 1 || f.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 1 || f.used() != 0 {
t.Fatal("retry duplicated events or permits")
}
})
t.Run("failure_classification_and_budget", func(t *testing.T) {
for _, retryable := range []bool{false, true} {
f := newExecutionFixture(t, ctx, db)
task := retryTask(t, f, 2, pulse.RecoveryReconcileOnly)
ref := retryStart(t, f)
retryFinish(t, f, ref, "FAILED", retryable)
count := uint32(1)
if retryable {
if f.get(task.ID).State != pulse.TaskWaiting {
t.Fatal("retryable failure not retried")
}
makeRetryDue(f, task.ID)
retryFinish(t, f, retryStart(t, f), "FAILED", true)
count = 2
}
got := f.get(task.ID)
if got.State != pulse.TaskFailed || got.AttemptCount != count || got.EffectUncertain || got.CompletedAt == nil {
t.Fatal("failure budget or classification ignored")
}
}
})
t.Run("unknown_history_remains_uncertain_until_authoritative_success", func(t *testing.T) {
for _, last := range []string{"SUCCEEDED", "FAILED", "UNKNOWN", "NOT_STARTED", "CANCEL", "DEADLINE"} {
t.Run(last, func(t *testing.T) {
f := newExecutionFixture(t, ctx, db)
task := retryTask(t, f, 2, pulse.RecoveryIdempotent)
ref := retryStart(t, f)
retryFinish(t, f, ref, "UNKNOWN", false)
pending := f.get(task.ID)
if pending.State != pulse.TaskWaiting || !pending.EffectUncertain || len(pending.Result) != 0 {
t.Fatal("unknown retry lost uncertainty")
}
switch last {
case "CANCEL":
out, err := f.tasks.CancelTask(ctx, f.token, task.ID, string(executionID(t)), service.CancelInput{ExpectedRevision: pending.Revision, Reason: "stop remaining retries"})
if err != nil || out.Task.CancelRequestedAt == nil {
t.Fatal(err)
}
case "DEADLINE":
f.exec(`UPDATE pulse_tasks SET available_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 2 SECOND),deadline_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, task.ID)
if _, err := f.storeB.Recover(ctx, 100); err != nil {
t.Fatal(err)
}
case "NOT_STARTED":
makeRetryDue(f, task.ID)
_, next := f.claim()
f.expire(next)
if _, err := f.storeB.Recover(ctx, 100); err != nil {
t.Fatal(err)
}
default:
makeRetryDue(f, task.ID)
retryFinish(t, f, retryStart(t, f), last, last == "FAILED")
}
got := f.get(task.ID)
if last == "SUCCEEDED" {
if got.State != pulse.TaskSucceeded || got.EffectUncertain {
t.Fatal("success did not resolve uncertainty")
}
} else if got.State != pulse.TaskNeedsReview || !got.EffectUncertain || got.CompletedAt != nil {
t.Fatal("unresolved effects became false terminal failure/cancellation")
}
if f.used() != 0 {
t.Fatal("retry leaked permit")
}
})
}
})
t.Run("unknown_requires_explicit_idempotent_contract", func(t *testing.T) {
f := newExecutionFixture(t, ctx, db)
task := retryTask(t, f, 3, pulse.RecoveryReconcileOnly)
retryFinish(t, f, retryStart(t, f), "UNKNOWN", false)
got := f.get(task.ID)
if got.State != pulse.TaskNeedsReview || !got.EffectUncertain || got.AttemptCount != 1 {
t.Fatal("unknown replayed without idempotence")
}
})
t.Run("recovery_race_pause_and_late_success", func(t *testing.T) {
f := newExecutionFixture(t, ctx, db)
task := retryTask(t, f, 2, pulse.RecoveryIdempotent)
ref := retryStart(t, f)
f.expire(ref)
var wg sync.WaitGroup
errs := make(chan error, 2)
for _, store := range []*ExecutionStore{f.storeA, f.storeB} {
wg.Add(1)
go func(s *ExecutionStore) { defer wg.Done(); _, err := s.Recover(ctx, 100); errs <- err }(store)
}
wg.Wait()
close(errs)
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
pending := f.get(task.ID)
if pending.State != pulse.TaskWaiting || !pending.EffectUncertain || f.used() != 0 || f.count("orbit_events", "aggregate_id=? AND type='task.requeued'", task.ID) != 1 {
t.Fatal("competing recovery did not make one retry decision")
}
makeRetryDue(f, task.ID)
f.exec(`UPDATE pulse_queues SET paused=TRUE WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, f.resources.QueueID)
if a, err := f.a.Claim(ctx, f.token, f.binding); err != nil || a != nil {
t.Fatal("retry bypassed pause")
}
late := retryFinish(t, f, ref, "SUCCEEDED", false)
if late.Receipt.Disposition != "EVIDENCE_ONLY" || !f.get(task.ID).EffectUncertain {
t.Fatal("late success cleared unresolved authority")
}
f.exec(`UPDATE pulse_queues SET paused=FALSE WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, f.resources.QueueID)
retryFinish(t, f, retryStart(t, f), "SUCCEEDED", false)
if f.get(task.ID).State != pulse.TaskSucceeded {
t.Fatal("resumed retry failed")
}
})
t.Run("retry_transaction_rolls_back_with_audit", func(t *testing.T) {
f := newExecutionFixture(t, ctx, db)
task := retryTask(t, f, 2, pulse.RecoveryReconcileOnly)
ref := retryStart(t, f)
f.exec(`CREATE TRIGGER orbit_test_retry_audit BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='retry audit failure'`)
in := worker.FinishInput{Ref: ref, Outcome: "FAILED", Retryable: true, ErrorCode: "TEMPORARY_REJECTION", Result: json.RawMessage(`{}`)}
message := executionID(t)
_, err := f.a.Finish(ctx, f.token, f.binding, message, in)
f.exec(`DROP TRIGGER orbit_test_retry_audit`)
if err == nil || f.get(task.ID).State != pulse.TaskRunning || f.used() != 1 || f.count("orbit_events", "aggregate_id=? AND type='task.requeued'", task.ID) != 0 {
t.Fatal("partial retry transition committed")
}
if _, err := f.a.Finish(ctx, f.token, f.binding, message, in); err != nil {
t.Fatal(err)
}
})
}

View File

@@ -107,6 +107,7 @@ func TestWorkerPublishedContract(t *testing.T) {
{"task.finish", worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"n":9007199254740993}`)}},
{"task.finish", worker.FinishInput{Ref: ref, Outcome: "FAILED", ErrorCode: "OBJECT_REJECTED", Result: json.RawMessage(`{}`)}},
{"task.finish", worker.FinishInput{Ref: ref, Outcome: "UNKNOWN", ErrorCode: "LEASE_LOST", Result: json.RawMessage(`{}`)}},
{"task.finish", worker.FinishInput{Ref: ref, Outcome: "FAILED", Retryable: true, ErrorCode: "TEMPORARY_REJECTION", Result: json.RawMessage(`{}`)}},
}
var valid [][]byte
for _, tc := range cases {
@@ -145,6 +146,7 @@ func TestWorkerPublishedContract(t *testing.T) {
{"null_result", strings.Replace(string(valid[5]), `{"n":9007199254740993}`, `null`, 1)},
{"failure_missing_code", strings.Replace(string(valid[6]), `,"error_code":"OBJECT_REJECTED"`, ``, 1)},
{"success_with_code", strings.Replace(string(valid[6]), `"FAILED"`, `"SUCCEEDED"`, 1)},
{"unknown_retry_flag", strings.Replace(string(valid[8]), `"FAILED"`, `"UNKNOWN"`, 1)},
}
for _, tc := range invalid {
t.Run(tc.name, func(t *testing.T) {