feat: add durable safe retries with uncertainty tracking and SDK examples
This commit is contained in:
13
internal/storage/mysql/migrations/000005_safe_retry.up.sql
Normal file
13
internal/storage/mysql/migrations/000005_safe_retry.up.sql
Normal file
@@ -0,0 +1,13 @@
|
||||
-- 安全重试保留跨尝试的不确定效果,旧迁移及旧操作摘要保持不变。
|
||||
ALTER TABLE pulse_tasks
|
||||
ADD COLUMN effect_uncertain BOOLEAN NOT NULL DEFAULT FALSE COMMENT '曾有已授权执行的效果尚未核实,重试与耗尽不清除,仅权威成功或核查可解除',
|
||||
ADD CONSTRAINT ck_task_effect_uncertain CHECK (effect_uncertain IN (0,1) AND (effect_uncertain=FALSE OR state NOT IN ('SUCCEEDED','FAILED','CANCELLED')));
|
||||
UPDATE pulse_tasks SET effect_uncertain=TRUE WHERE state='NEEDS_REVIEW';
|
||||
ALTER TABLE pulse_attempts
|
||||
ADD COLUMN retryable BOOLEAN NOT NULL DEFAULT FALSE COMMENT 'Worker 明确失败时声明可安全重试,不用于未知效果或证明实际已安排重试',
|
||||
ADD CONSTRAINT ck_attempt_retryable CHECK (retryable IN (0,1) AND (retryable=FALSE OR state='FAILED'));
|
||||
ALTER TABLE pulse_attempt_evidence
|
||||
ADD COLUMN retryable BOOLEAN NOT NULL DEFAULT FALSE COMMENT '原失败报告的安全重试声明,证据本身不触发重试',
|
||||
ADD CONSTRAINT ck_evidence_retryable CHECK (retryable IN (0,1) AND (retryable=FALSE OR outcome='FAILED'));
|
||||
ALTER TABLE pulse_attempt_receipts
|
||||
ADD KEY ix_result_receipt(namespace_id,attempt_id,operation,request_digest) COMMENT '同一结果换消息身份重报时读取原始已应用回执';
|
||||
116
internal/storage/mysql/retry_upgrade_integration_test.go
Normal file
116
internal/storage/mysql/retry_upgrade_integration_test.go
Normal file
@@ -0,0 +1,116 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
mysqldriver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// 在独立库中重建真实 v4,再走正式迁移入口;不改变主测试库的结构或迁移记录。
|
||||
func TestIntegrationSafeRetryUpgrade(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL with CREATE DATABASE permission")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
admin, err := Open(ctx, dsn, 2, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer admin.Close()
|
||||
name := "orbit_retry_upgrade_" + string(executionID(t))
|
||||
if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+name+"`"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := admin.ExecContext(cleanup, "DROP DATABASE `"+name+"`"); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
cfg, err := mysqldriver.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
t.Fatal("invalid test DSN")
|
||||
}
|
||||
cfg.DBName = name
|
||||
db, err := Open(ctx, cfg.FormatDSN(), 4, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
exec := func(q string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := db.ExecContext(ctx, q, args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
exec(`CREATE TABLE orbit_schema_migrations (version BIGINT UNSIGNED PRIMARY KEY,name VARCHAR(191) NOT NULL,checksum CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,dirty BOOLEAN NOT NULL,applied_at DATETIME(6) NULL) ENGINE=InnoDB`)
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range migrations {
|
||||
if m.Version > 4 {
|
||||
break
|
||||
}
|
||||
statements, err := splitSQL(m.SQL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, statement := range statements {
|
||||
exec(statement)
|
||||
}
|
||||
exec(`INSERT INTO orbit_schema_migrations(version,name,checksum,dirty,applied_at) VALUES (?,?,?,FALSE,UTC_TIMESTAMP(6))`, m.Version, m.Name, m.Checksum)
|
||||
}
|
||||
app, ns, queue := executionID(t), executionID(t), executionID(t)
|
||||
exec(`INSERT INTO orbit_applications(id,name) VALUES (?,?)`, app, string(app))
|
||||
exec(`INSERT INTO orbit_namespaces(id,application_id,name,environment) VALUES (?,?,?,'test')`, ns, app, string(ns))
|
||||
exec(`INSERT INTO pulse_queues(namespace_id,id,name,max_concurrency,rate_per_second,policy) VALUES (?,?,?,4,10,'{}')`, ns, queue, string(queue))
|
||||
policy := `{"retry":{"max_attempts":3,"initial_delay_ms":1000,"max_delay_ms":30000,"attempt_timeout_ms":5000}}`
|
||||
ids := map[string]core.ID{}
|
||||
for _, state := range []string{"NEEDS_REVIEW", "WAITING", "SUCCEEDED", "FAILED", "CANCELLED"} {
|
||||
id := executionID(t)
|
||||
ids[state] = id
|
||||
var completed *time.Time
|
||||
if state == "SUCCEEDED" || state == "FAILED" || state == "CANCELLED" {
|
||||
now := time.Now().UTC()
|
||||
completed = &now
|
||||
}
|
||||
exec(`INSERT INTO pulse_tasks(namespace_id,id,queue_id,state,handler,handler_version,payload_version,payload,policy_snapshot,available_at,deadline_at,attempt_timeout_ms,max_attempts,completed_at) VALUES (?,?,?,?,'object.prepare','v1','v1','{}',?,UTC_TIMESTAMP(6),DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 1 HOUR),5000,3,?)`, ns, id, queue, state, policy, completed)
|
||||
}
|
||||
// MySQL 原生 JSON 会规范化文本,以迁移前实际存储值核对冻结策略。
|
||||
if err := db.QueryRowContext(ctx, `SELECT policy_snapshot FROM pulse_tasks WHERE namespace_id=? AND id=?`, ns, ids["WAITING"]).Scan(&policy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := CheckSchema(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for state, id := range ids {
|
||||
var gotState, gotPolicy string
|
||||
var uncertain bool
|
||||
var revision uint64
|
||||
if err := db.QueryRowContext(ctx, `SELECT state,effect_uncertain,policy_snapshot,revision FROM pulse_tasks WHERE namespace_id=? AND id=?`, ns, id).Scan(&gotState, &uncertain, &gotPolicy, &revision); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gotState != state || uncertain != (state == "NEEDS_REVIEW") || gotPolicy != policy || revision != 1 {
|
||||
t.Fatalf("upgrade changed frozen data or lost uncertainty for %s", state)
|
||||
}
|
||||
}
|
||||
// 再次运行不重复修改回填或策略。
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var count int
|
||||
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_tasks WHERE effect_uncertain=TRUE`).Scan(&count); err != nil || count != 1 {
|
||||
t.Fatalf("upgrade is not idempotent: %d %v", count, err)
|
||||
}
|
||||
}
|
||||
@@ -91,11 +91,11 @@ func historyAttemptExists(ctx context.Context, tx *sql.Tx, ns, task, attempt cor
|
||||
return err
|
||||
}
|
||||
|
||||
const attemptHistoryColumns = `a.id,a.task_id,a.epoch,a.state,a.revision,a.lease_until,a.authorized_at,a.ended_at,COALESCE(a.error_code,''),a.created_at,a.updated_at`
|
||||
const attemptHistoryColumns = `a.id,a.task_id,a.epoch,a.state,a.revision,a.lease_until,a.authorized_at,a.ended_at,COALESCE(a.error_code,''),a.retryable,a.created_at,a.updated_at`
|
||||
|
||||
func scanAttemptSummary(row interface{ Scan(...any) error }) (service.AttemptSummary, error) {
|
||||
var a service.AttemptSummary
|
||||
err := row.Scan(&a.ID, &a.TaskID, &a.Epoch, &a.State, &a.Revision, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt)
|
||||
err := row.Scan(&a.ID, &a.TaskID, &a.Epoch, &a.State, &a.Revision, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Retryable, &a.CreatedAt, &a.UpdatedAt)
|
||||
return a, err
|
||||
}
|
||||
func (s *TaskStore) ListTaskAttempts(ctx context.Context, t auth.Token, task core.ID, in service.HistoryPageInput) (service.AttemptPage, error) {
|
||||
@@ -148,7 +148,7 @@ func (s *TaskStore) GetTaskAttempt(ctx context.Context, t auth.Token, task, atte
|
||||
return err
|
||||
}
|
||||
var result []byte
|
||||
err := tx.QueryRowContext(ctx, `SELECT `+attemptHistoryColumns+`,a.session_id,w.worker_instance_id,a.result FROM pulse_attempts a JOIN pulse_worker_sessions w ON w.namespace_id=a.namespace_id AND w.id=a.session_id WHERE a.namespace_id=? AND a.task_id=? AND a.id=?`, t.NamespaceID, task, attempt).Scan(&out.ID, &out.TaskID, &out.Epoch, &out.State, &out.Revision, &out.LeaseUntil, &out.AuthorizedAt, &out.EndedAt, &out.ErrorCode, &out.CreatedAt, &out.UpdatedAt, &out.SessionID, &out.WorkerInstanceID, &result)
|
||||
err := tx.QueryRowContext(ctx, `SELECT `+attemptHistoryColumns+`,a.session_id,w.worker_instance_id,a.result FROM pulse_attempts a JOIN pulse_worker_sessions w ON w.namespace_id=a.namespace_id AND w.id=a.session_id WHERE a.namespace_id=? AND a.task_id=? AND a.id=?`, t.NamespaceID, task, attempt).Scan(&out.ID, &out.TaskID, &out.Epoch, &out.State, &out.Revision, &out.LeaseUntil, &out.AuthorizedAt, &out.EndedAt, &out.ErrorCode, &out.Retryable, &out.CreatedAt, &out.UpdatedAt, &out.SessionID, &out.WorkerInstanceID, &result)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return fault.NotFound
|
||||
}
|
||||
@@ -166,7 +166,7 @@ func (s *TaskStore) ListAttemptEvidence(ctx context.Context, t auth.Token, task,
|
||||
if err := historyAttemptExists(ctx, tx, t.NamespaceID, task, attempt); err != nil {
|
||||
return err
|
||||
}
|
||||
q := `SELECT id,attempt_id,outcome,COALESCE(error_code,''),reason,created_at FROM pulse_attempt_evidence WHERE namespace_id=? AND attempt_id=?`
|
||||
q := `SELECT id,attempt_id,outcome,COALESCE(error_code,''),retryable,reason,created_at FROM pulse_attempt_evidence WHERE namespace_id=? AND attempt_id=?`
|
||||
args := []any{t.NamespaceID, attempt}
|
||||
if in.Cursor != "" {
|
||||
c, err := decodeHistoryCursor(in.Cursor, t.NamespaceID, task, attempt)
|
||||
@@ -185,7 +185,7 @@ func (s *TaskStore) ListAttemptEvidence(ctx context.Context, t auth.Token, task,
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
e := service.EvidenceSummary{TaskID: task, Disposition: "EVIDENCE_ONLY"}
|
||||
if err := rows.Scan(&e.ID, &e.AttemptID, &e.Outcome, &e.ErrorCode, &e.Reason, &e.CreatedAt); err != nil {
|
||||
if err := rows.Scan(&e.ID, &e.AttemptID, &e.Outcome, &e.ErrorCode, &e.Retryable, &e.Reason, &e.CreatedAt); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Items = append(out.Items, e)
|
||||
@@ -213,7 +213,7 @@ func (s *TaskStore) GetAttemptEvidence(ctx context.Context, t auth.Token, task,
|
||||
return err
|
||||
}
|
||||
var payload []byte
|
||||
err := tx.QueryRowContext(ctx, `SELECT id,attempt_id,outcome,COALESCE(error_code,''),reason,created_at,message_id,payload FROM pulse_attempt_evidence WHERE namespace_id=? AND attempt_id=? AND id=?`, t.NamespaceID, attempt, evidence).Scan(&out.ID, &out.AttemptID, &out.Outcome, &out.ErrorCode, &out.Reason, &out.CreatedAt, &out.MessageID, &payload)
|
||||
err := tx.QueryRowContext(ctx, `SELECT id,attempt_id,outcome,COALESCE(error_code,''),retryable,reason,created_at,message_id,payload FROM pulse_attempt_evidence WHERE namespace_id=? AND attempt_id=? AND id=?`, t.NamespaceID, attempt, evidence).Scan(&out.ID, &out.AttemptID, &out.Outcome, &out.ErrorCode, &out.Retryable, &out.Reason, &out.CreatedAt, &out.MessageID, &payload)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return fault.NotFound
|
||||
}
|
||||
|
||||
@@ -216,14 +216,14 @@ func (s *TaskStore) Submit(ctx context.Context, token auth.Token, key string, in
|
||||
return out, err
|
||||
}
|
||||
|
||||
const taskColumns = `id,namespace_id,queue_id,state,revision,handler,handler_version,payload_version,payload,result,COALESCE(business_key,''),available_at,start_deadline_at,deadline_at,recovery_safety,policy_snapshot,attempt_count,cancel_requested_at,completed_at,created_at,updated_at`
|
||||
const taskColumns = `id,namespace_id,queue_id,state,revision,handler,handler_version,payload_version,payload,result,COALESCE(business_key,''),available_at,start_deadline_at,deadline_at,recovery_safety,policy_snapshot,attempt_count,cancel_requested_at,completed_at,created_at,updated_at,effect_uncertain`
|
||||
|
||||
type scanner interface{ Scan(...any) error }
|
||||
|
||||
func scanTask(row scanner) (service.TaskView, error) {
|
||||
var v service.TaskView
|
||||
var policy, payload, result []byte
|
||||
err := row.Scan(&v.ID, &v.NamespaceID, &v.QueueID, &v.State, &v.Revision, &v.Handler, &v.HandlerVersion, &v.PayloadVersion, &payload, &result, &v.BusinessKey, &v.AvailableAt, &v.StartDeadlineAt, &v.DeadlineAt, &v.RecoverySafety, &policy, &v.AttemptCount, &v.CancelRequestedAt, &v.CompletedAt, &v.CreatedAt, &v.UpdatedAt)
|
||||
err := row.Scan(&v.ID, &v.NamespaceID, &v.QueueID, &v.State, &v.Revision, &v.Handler, &v.HandlerVersion, &v.PayloadVersion, &payload, &result, &v.BusinessKey, &v.AvailableAt, &v.StartDeadlineAt, &v.DeadlineAt, &v.RecoverySafety, &policy, &v.AttemptCount, &v.CancelRequestedAt, &v.CompletedAt, &v.CreatedAt, &v.UpdatedAt, &v.EffectUncertain)
|
||||
if err != nil {
|
||||
return v, err
|
||||
}
|
||||
@@ -273,7 +273,7 @@ func (s *TaskStore) ListTasks(ctx context.Context, token auth.Token, in service.
|
||||
if !p.AllowsQueue(in.QueueID) {
|
||||
return fault.Forbidden
|
||||
}
|
||||
q := `SELECT id,queue_id,state,revision,handler,handler_version,COALESCE(business_key,''),available_at,deadline_at,attempt_count,created_at,updated_at FROM pulse_tasks WHERE namespace_id=? AND queue_id=?`
|
||||
q := `SELECT id,queue_id,state,revision,handler,handler_version,COALESCE(business_key,''),available_at,deadline_at,attempt_count,created_at,updated_at,effect_uncertain FROM pulse_tasks WHERE namespace_id=? AND queue_id=?`
|
||||
args := []any{token.NamespaceID, in.QueueID}
|
||||
if in.State != "" {
|
||||
q += " AND state=?"
|
||||
@@ -297,7 +297,7 @@ func (s *TaskStore) ListTasks(ctx context.Context, token auth.Token, in service.
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var v service.TaskSummary
|
||||
if err := rows.Scan(&v.ID, &v.QueueID, &v.State, &v.Revision, &v.Handler, &v.HandlerVersion, &v.BusinessKey, &v.AvailableAt, &v.DeadlineAt, &v.AttemptCount, &v.CreatedAt, &v.UpdatedAt); err != nil {
|
||||
if err := rows.Scan(&v.ID, &v.QueueID, &v.State, &v.Revision, &v.Handler, &v.HandlerVersion, &v.BusinessKey, &v.AvailableAt, &v.DeadlineAt, &v.AttemptCount, &v.CreatedAt, &v.UpdatedAt, &v.EffectUncertain); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Items = append(out.Items, v)
|
||||
@@ -357,7 +357,17 @@ func (s *TaskStore) CancelTask(ctx context.Context, token auth.Token, id core.ID
|
||||
if owner.Valid {
|
||||
return fault.Conflict
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='CANCELLED',cancel_requested_at=UTC_TIMESTAMP(6),completed_at=UTC_TIMESTAMP(6),updated_at=UTC_TIMESTAMP(6),revision=revision+1 WHERE namespace_id=? AND id=? AND revision=? AND state='WAITING'`, token.NamespaceID, id, in.ExpectedRevision)
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := pulse.TaskCancelled
|
||||
var completed *time.Time = &now
|
||||
if task.EffectUncertain {
|
||||
next = pulse.TaskNeedsReview
|
||||
completed = nil
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,cancel_requested_at=?,completed_at=?,updated_at=?,revision=revision+1 WHERE namespace_id=? AND id=? AND revision=? AND state='WAITING'`, next, now, completed, now, token.NamespaceID, id, in.ExpectedRevision)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -372,7 +382,7 @@ func (s *TaskStore) CancelTask(ctx context.Context, token auth.Token, id core.ID
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, token.NamespaceID, id, out.Task.Revision, "task", "task.cancelled", map[string]any{"task_id": id, "state": out.Task.State}); err != nil {
|
||||
if err := emitEvent(ctx, tx, token.NamespaceID, id, out.Task.Revision, "task", resultEvent(out.Task.State), map[string]any{"task_id": id, "state": out.Task.State, "reason": "CANCEL_REQUESTED", "effect_uncertain": out.Task.EffectUncertain}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := appendAudit(ctx, tx, token, identity, id, "task", "task.cancel", in.Reason, map[string]any{"state": task.State, "revision": fmt.Sprint(task.Revision)}, map[string]any{"state": out.Task.State, "revision": fmt.Sprint(out.Task.Revision)}); err != nil {
|
||||
|
||||
@@ -309,7 +309,7 @@ func (s *ExecutionStore) lockExecution(ctx context.Context, tx *sql.Tx, token au
|
||||
func readAttempt(ctx context.Context, tx *sql.Tx, ns, id core.ID) (pulse.Attempt, error) {
|
||||
var a pulse.Attempt
|
||||
a.NamespaceID = ns
|
||||
err := tx.QueryRowContext(ctx, `SELECT id,revision,task_id,epoch,session_id,token_digest,state,lease_until,authorized_at,ended_at,error_code,result,result_digest FROM pulse_attempts WHERE namespace_id=? AND id=? FOR UPDATE`, ns, id).Scan(&a.ID, &a.Revision, &a.TaskID, &a.Epoch, &a.SessionID, &a.TokenDigest, &a.State, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Result, &a.ResultDigest)
|
||||
err := tx.QueryRowContext(ctx, `SELECT id,revision,task_id,epoch,session_id,token_digest,state,lease_until,authorized_at,ended_at,error_code,retryable,result,result_digest FROM pulse_attempts WHERE namespace_id=? AND id=? FOR UPDATE`, ns, id).Scan(&a.ID, &a.Revision, &a.TaskID, &a.Epoch, &a.SessionID, &a.TokenDigest, &a.State, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Retryable, &a.Result, &a.ResultDigest)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
err = fault.NotFound
|
||||
}
|
||||
|
||||
@@ -10,11 +10,12 @@ import (
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/retry"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
// Recover 是受控后台扫描入口,每批最多 100 项、每项一个短事务,可由多个节点竞争。
|
||||
// 当前已授权执行的失联一律转核查,自动安全重试在 P3 接入冻结策略后实现。
|
||||
// 已授权执行的失联保留效果未知,只有冻结的幂等恢复声明与剩余预算允许自动重试。
|
||||
func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.RecoveryStats, error) {
|
||||
var out worker.RecoveryStats
|
||||
if limit < 1 || limit > 100 {
|
||||
@@ -88,11 +89,16 @@ func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.Recover
|
||||
return nil
|
||||
}
|
||||
changed = pulse.TaskFailed
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='FAILED',completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, now, now, c.ns, c.id, task.view.Revision)
|
||||
var completed *time.Time = &now
|
||||
if task.view.EffectUncertain {
|
||||
changed = pulse.TaskNeedsReview
|
||||
completed = nil
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, changed, completed, now, c.ns, c.id, task.view.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", "task.failed", map[string]any{"reason": "START_DEADLINE_EXCEEDED", "node_id": s.node})
|
||||
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", resultEvent(changed), map[string]any{"reason": "START_DEADLINE_EXCEEDED", "node_id": s.node, "effect_uncertain": task.view.EffectUncertain})
|
||||
}
|
||||
if task.view.State != pulse.TaskLeased && task.view.State != pulse.TaskRunning {
|
||||
return nil
|
||||
@@ -118,29 +124,31 @@ func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.Recover
|
||||
return errors.New("active task references ended attempt")
|
||||
}
|
||||
attemptState := pulse.AttemptExpired
|
||||
next := pulse.TaskFailed
|
||||
outcome := "NOT_STARTED"
|
||||
reason := "LEASE_EXPIRED_BEFORE_START"
|
||||
var completed *time.Time = &now
|
||||
available := task.view.AvailableAt
|
||||
if attempt.AuthorizedAt != nil {
|
||||
attemptState = pulse.AttemptUnknown
|
||||
next = pulse.TaskNeedsReview
|
||||
outcome = "UNKNOWN"
|
||||
reason = "EXECUTION_OUTCOME_UNKNOWN"
|
||||
completed = nil
|
||||
} else if task.view.AttemptCount < task.maxAttempts {
|
||||
// 未批准开始的尝试可以有界重派,次数不重置;开始截止与总截止仍有效。
|
||||
nextAt := now.Add(time.Second)
|
||||
if nextAt.Before(task.view.DeadlineAt) && (task.view.StartDeadlineAt == nil || nextAt.Before(*task.view.StartDeadlineAt)) {
|
||||
next = pulse.TaskWaiting
|
||||
available = nextAt
|
||||
completed = nil
|
||||
}
|
||||
}
|
||||
decision, err := retry.Decide(retryFacts(task, attempt.ID, now, outcome, false))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := decision.State
|
||||
var completed *time.Time
|
||||
if next.Terminal() {
|
||||
completed = &now
|
||||
}
|
||||
available := task.view.AvailableAt
|
||||
if decision.NextAt != nil {
|
||||
available = *decision.NextAt
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, reason, now, c.ns, attempt.ID, attempt.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,lease_until=NULL,available_at=?,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, next, available, completed, now, c.ns, c.id, task.view.Revision)
|
||||
result, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,effect_uncertain=?,lease_until=NULL,available_at=?,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, next, decision.Uncertain, available, completed, now, c.ns, c.id, task.view.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -148,7 +156,7 @@ func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.Recover
|
||||
return err
|
||||
}
|
||||
changed = next
|
||||
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", resultEvent(next), map[string]any{"attempt_id": attempt.ID, "epoch": fmt.Sprint(attempt.Epoch), "reason": reason, "node_id": s.node})
|
||||
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", resultEvent(next), map[string]any{"attempt_id": attempt.ID, "epoch": fmt.Sprint(attempt.Epoch), "reason": reason, "node_id": s.node, "effect_uncertain": decision.Uncertain, "decision": decision.Reason, "next_available_at": decision.NextAt, "delay_ms": decision.DelayMS, "attempt_count": task.view.AttemptCount, "max_attempts": task.maxAttempts})
|
||||
})
|
||||
if err != nil {
|
||||
return out, err
|
||||
|
||||
@@ -3,12 +3,14 @@ package mysql
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/retry"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
@@ -19,7 +21,8 @@ func (s *ExecutionStore) Finish(ctx context.Context, token auth.Token, b worker.
|
||||
Outcome string
|
||||
Result core.JSON
|
||||
ErrorCode string
|
||||
}{attemptIdentity(in.Ref), in.Outcome, core.JSON(in.Result), in.ErrorCode})
|
||||
Retryable bool `json:"Retryable,omitempty"`
|
||||
}{attemptIdentity(in.Ref), in.Outcome, core.JSON(in.Result), in.ErrorCode, in.Retryable})
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
@@ -45,6 +48,14 @@ func (s *ExecutionStore) Finish(ctx context.Context, token auth.Token, b worker.
|
||||
receipt := worker.Receipt{Operation: "finish", TaskState: task.view.State, AttemptState: a.State, TaskRevision: task.view.Revision, LeaseUntil: a.LeaseUntil, Disposition: "APPLIED"}
|
||||
if a.ResultDigest != nil && *a.ResultDigest == digest {
|
||||
// 新消息 ID 重报已保存的同一结果,也不能重复事件、许可释放或状态推进。
|
||||
// 重试可能已推进其他尝试,必须返回原操作回执,不能套用当前任务状态。
|
||||
var raw []byte
|
||||
if err := tx.QueryRowContext(ctx, `SELECT response FROM pulse_attempt_receipts WHERE namespace_id=? AND attempt_id=? AND operation='finish' AND request_digest=? AND JSON_UNQUOTE(JSON_EXTRACT(response,'$.disposition'))='APPLIED' LIMIT 1`, token.NamespaceID, a.ID, digest).Scan(&raw); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(raw, &receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -70,24 +81,34 @@ func (s *ExecutionStore) Finish(ctx context.Context, token auth.Token, b worker.
|
||||
out = worker.CommandResult{Receipt: receipt, Lease: status}
|
||||
return nil
|
||||
}
|
||||
attemptState, taskState := pulse.AttemptState(in.Outcome), pulse.TaskState(in.Outcome)
|
||||
var completed *time.Time = &now
|
||||
if in.Outcome == "UNKNOWN" {
|
||||
taskState = pulse.TaskNeedsReview
|
||||
completed = nil
|
||||
attemptState := pulse.AttemptState(in.Outcome)
|
||||
decision, err := retry.Decide(retryFacts(task, a.ID, now, in.Outcome, in.Retryable))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=NULLIF(?,''),result=?,result_digest=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, in.ErrorCode, string(in.Result), digest, now, token.NamespaceID, a.ID, a.Revision)
|
||||
taskState := decision.State
|
||||
var completed *time.Time
|
||||
if taskState.Terminal() {
|
||||
completed = &now
|
||||
}
|
||||
available := task.view.AvailableAt
|
||||
var result any = string(in.Result)
|
||||
if decision.NextAt != nil {
|
||||
available = *decision.NextAt
|
||||
result = nil
|
||||
}
|
||||
changed, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=NULLIF(?,''),retryable=?,result=?,result_digest=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, in.ErrorCode, in.Retryable, string(in.Result), digest, now, token.NamespaceID, a.ID, a.Revision)
|
||||
if err := exactlyOne(changed, err); err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,result=?,lease_until=NULL,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, taskState, string(in.Result), completed, now, token.NamespaceID, in.Ref.TaskID, task.view.Revision)
|
||||
changed, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,result=?,effect_uncertain=?,available_at=?,lease_until=NULL,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, taskState, result, decision.Uncertain, available, completed, now, token.NamespaceID, in.Ref.TaskID, task.view.Revision)
|
||||
if err := exactlyOne(changed, err); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := releasePermits(ctx, tx, token.NamespaceID, a.ID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
event := map[string]any{"attempt_id": a.ID, "epoch": fmt.Sprint(a.Epoch), "state": taskState, "outcome": in.Outcome, "error_code": in.ErrorCode}
|
||||
event := map[string]any{"attempt_id": a.ID, "epoch": fmt.Sprint(a.Epoch), "state": taskState, "outcome": in.Outcome, "error_code": in.ErrorCode, "retryable": in.Retryable, "effect_uncertain": decision.Uncertain, "decision": decision.Reason, "next_available_at": decision.NextAt, "delay_ms": decision.DelayMS, "attempt_count": task.view.AttemptCount, "max_attempts": task.maxAttempts}
|
||||
if err := emitEvent(ctx, tx, token.NamespaceID, in.Ref.TaskID, task.view.Revision+1, "task", resultEvent(taskState), event); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -124,9 +145,16 @@ func saveEvidence(ctx context.Context, tx *sql.Tx, ns, message core.ID, in worke
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempt_evidence(namespace_id,id,attempt_id,message_id,outcome,error_code,payload,request_digest,reason) VALUES (?,?,?,?,?,NULLIF(?,''),?,?,?)`, ns, id, in.Ref.AttemptID, message, in.Outcome, in.ErrorCode, string(in.Result), digest, reason)
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempt_evidence(namespace_id,id,attempt_id,message_id,outcome,error_code,retryable,payload,request_digest,reason) VALUES (?,?,?,?,?,NULLIF(?,''),?,?,?,?)`, ns, id, in.Ref.AttemptID, message, in.Outcome, in.ErrorCode, in.Retryable, string(in.Result), digest, reason)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return emitEvent(ctx, tx, ns, id, 1, "attempt_evidence", "attempt.evidence_received", map[string]any{"attempt_id": in.Ref.AttemptID, "task_id": in.Ref.TaskID, "reason": reason, "outcome": in.Outcome})
|
||||
}
|
||||
|
||||
// retryFacts 以持久化预算为准,JSON 快照只提供冻结的等待策略。
|
||||
func retryFacts(task executionTask, attempt core.ID, now time.Time, outcome string, retryable bool) retry.Input {
|
||||
policy := task.view.Retry
|
||||
policy.MaxAttempts = task.maxAttempts
|
||||
return retry.Input{Policy: policy, Safety: task.view.RecoverySafety, Consumed: task.view.AttemptCount, Outcome: outcome, Retryable: retryable, Uncertain: task.view.EffectUncertain, CancelRequested: task.view.CancelRequestedAt != nil, Now: now, Deadline: task.view.DeadlineAt, StartDeadline: task.view.StartDeadlineAt, AttemptID: attempt}
|
||||
}
|
||||
|
||||
307
internal/storage/mysql/worker_retry_integration_test.go
Normal file
307
internal/storage/mysql/worker_retry_integration_test.go
Normal file
@@ -0,0 +1,307 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
mysqldriver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func retryTask(t *testing.T, f *executionFixture, max uint32, safety pulse.RecoverySafety) service.TaskView {
|
||||
t.Helper()
|
||||
out, err := f.tasks.Submit(f.ctx, f.token, string(executionID(t)), service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`), RecoverySafety: safety, Retry: pulse.RetryPolicy{MaxAttempts: max, InitialDelayMS: 1000, MaxDelayMS: 5000, AttemptTimeoutMS: 30000}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.Task
|
||||
}
|
||||
func retryStart(t *testing.T, f *executionFixture) worker.AttemptRef {
|
||||
t.Helper()
|
||||
_, ref := f.claim()
|
||||
if _, err := f.a.Start(f.ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return ref
|
||||
}
|
||||
func retryFinish(t *testing.T, f *executionFixture, ref worker.AttemptRef, outcome string, retryable bool) worker.CommandResult {
|
||||
t.Helper()
|
||||
in := worker.FinishInput{Ref: ref, Outcome: outcome, Retryable: retryable, Result: json.RawMessage(`{"n":9007199254740993}`)}
|
||||
if outcome != "SUCCEEDED" {
|
||||
in.ErrorCode = "CONTROLLED_RESULT"
|
||||
}
|
||||
out, err := f.a.Finish(f.ctx, f.token, f.binding, executionID(t), in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func makeRetryDue(f *executionFixture, id core.ID) {
|
||||
f.exec(`UPDATE pulse_tasks SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, id)
|
||||
}
|
||||
func TestIntegrationSafeRetry(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
db, err := Open(ctx, dsn, 32, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Run("original_applied_receipt_excludes_early_evidence", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
retryTask(t, f, 2, pulse.RecoveryReconcileOnly)
|
||||
_, ref := f.claim()
|
||||
in := worker.FinishInput{Ref: ref, Outcome: "FAILED", Retryable: true, ErrorCode: "TEMPORARY_REJECTION", Result: json.RawMessage(`{}`)}
|
||||
early, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), in)
|
||||
if err != nil || early.Receipt.Disposition != "EVIDENCE_ONLY" {
|
||||
t.Fatal("early report was applied")
|
||||
}
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
applied, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), in)
|
||||
if err != nil || applied.Receipt.Disposition != "APPLIED" || applied.Receipt.TaskState != pulse.TaskWaiting {
|
||||
t.Fatal("authorized result not applied")
|
||||
}
|
||||
duplicate, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), in)
|
||||
if err != nil || !duplicate.Deduplicated || duplicate.Receipt != applied.Receipt {
|
||||
t.Fatal("replay selected an evidence receipt")
|
||||
}
|
||||
})
|
||||
t.Run("legacy_digest_and_database_retry_constraints", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := retryTask(t, f, 3, pulse.RecoveryReconcileOnly)
|
||||
ref := retryStart(t, f)
|
||||
reject := func(q string, args ...any) {
|
||||
t.Helper()
|
||||
_, err := db.ExecContext(ctx, q, args...)
|
||||
var failure *mysqldriver.MySQLError
|
||||
if !errors.As(err, &failure) || failure.Number != 3819 {
|
||||
t.Fatalf("wanted CHECK rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
reject(`UPDATE pulse_attempts SET retryable=TRUE WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, ref.AttemptID)
|
||||
in := worker.FinishInput{Ref: ref, Outcome: "FAILED", ErrorCode: "REJECTED", Result: json.RawMessage(`{"value":9007199254740993}`)}
|
||||
if _, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// v4 的摘要结构没有 Retryable;省略或 false 必须保持旧身份不变。
|
||||
legacy, err := requestDigest(struct {
|
||||
Identity any
|
||||
Outcome string
|
||||
Result core.JSON
|
||||
ErrorCode string
|
||||
}{attemptIdentity(ref), in.Outcome, core.JSON(in.Result), in.ErrorCode})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var saved core.Digest
|
||||
if err := db.QueryRowContext(ctx, `SELECT result_digest FROM pulse_attempts WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, ref.AttemptID).Scan(&saved); err != nil || saved != legacy {
|
||||
t.Fatal("legacy finish digest changed")
|
||||
}
|
||||
reject(`UPDATE pulse_tasks SET effect_uncertain=TRUE WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, task.ID)
|
||||
reject(`INSERT INTO pulse_attempt_evidence(namespace_id,id,attempt_id,message_id,outcome,payload,request_digest,reason,retryable) VALUES (?,?,?,?,'UNKNOWN','{}',?,'RESULT_CONFLICT',TRUE)`, f.resources.NamespaceID, executionID(t), ref.AttemptID, executionID(t), legacy)
|
||||
})
|
||||
t.Run("known_retry_duplicate_receipts_and_history", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := retryTask(t, f, 2, pulse.RecoveryReconcileOnly)
|
||||
ref := retryStart(t, f)
|
||||
in := worker.FinishInput{Ref: ref, Outcome: "FAILED", Retryable: true, ErrorCode: "TEMPORARY_REJECTION", Result: json.RawMessage(`{"n":9007199254740993}`)}
|
||||
message := executionID(t)
|
||||
first, err := f.a.Finish(ctx, f.token, f.binding, message, in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending := f.get(task.ID)
|
||||
if first.Receipt.TaskState != pulse.TaskWaiting || first.Lease.Active || pending.EffectUncertain || len(pending.Result) != 0 || pending.CompletedAt != nil || f.used() != 0 || pending.AttemptCount != 1 {
|
||||
t.Fatal("retry did not preserve task/attempt separation")
|
||||
}
|
||||
if delay := pending.AvailableAt.Sub(first.Lease.ServerTime); delay < 500*time.Millisecond || delay > time.Second {
|
||||
t.Fatal("retry time is not bounded")
|
||||
}
|
||||
if a, err := f.a.Claim(ctx, f.token, f.binding); err != nil || a != nil {
|
||||
t.Fatal("retry dispatched before due")
|
||||
}
|
||||
duplicate, err := f.a.Finish(ctx, f.token, f.binding, message, in)
|
||||
if err != nil || !duplicate.Deduplicated || duplicate.Receipt != first.Receipt || !f.get(task.ID).AvailableAt.Equal(pending.AvailableAt) {
|
||||
t.Fatal("same-message replay changed retry timing")
|
||||
}
|
||||
in.Retryable = false
|
||||
_, err = f.a.Finish(ctx, f.token, f.binding, message, in)
|
||||
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
|
||||
in.Retryable = true
|
||||
makeRetryDue(f, task.ID)
|
||||
second := retryStart(t, f)
|
||||
if second.Epoch != 2 || second.AttemptID == ref.AttemptID {
|
||||
t.Fatal("retry reused execution identity")
|
||||
}
|
||||
retryFinish(t, f, second, "SUCCEEDED", false)
|
||||
final := f.get(task.ID)
|
||||
// 在新尝试完成后用新消息身份重报旧失败,仍返回第一次的 WAITING 回执。
|
||||
duplicate, err = f.a.Finish(ctx, f.token, f.binding, executionID(t), in)
|
||||
if err != nil || !duplicate.Deduplicated || duplicate.Receipt != first.Receipt || duplicate.Lease.Active || f.get(task.ID).Revision != final.Revision {
|
||||
t.Fatal("old result replay borrowed newer task state")
|
||||
}
|
||||
in.Result = json.RawMessage(`{"n":2}`)
|
||||
late, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), in)
|
||||
if err != nil || late.Receipt.Disposition != "EVIDENCE_ONLY" {
|
||||
t.Fatal("conflicting retry report changed authority")
|
||||
}
|
||||
attempts, err := f.tasks.ListTaskAttempts(ctx, f.token, task.ID, service.HistoryPageInput{})
|
||||
if err != nil || len(attempts.Items) != 2 || !attempts.Items[1].Retryable || attempts.Items[0].State != pulse.AttemptSucceeded {
|
||||
t.Fatal("retry history lost classification")
|
||||
}
|
||||
evidence, err := f.tasks.ListAttemptEvidence(ctx, f.token, task.ID, ref.AttemptID, service.HistoryPageInput{})
|
||||
if err != nil || len(evidence.Items) != 1 || !evidence.Items[0].Retryable {
|
||||
t.Fatal("evidence lost original retry declaration")
|
||||
}
|
||||
if f.count("orbit_events", "aggregate_id=? AND type='task.requeued'", task.ID) != 1 || f.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 1 || f.used() != 0 {
|
||||
t.Fatal("retry duplicated events or permits")
|
||||
}
|
||||
})
|
||||
t.Run("failure_classification_and_budget", func(t *testing.T) {
|
||||
for _, retryable := range []bool{false, true} {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := retryTask(t, f, 2, pulse.RecoveryReconcileOnly)
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "FAILED", retryable)
|
||||
count := uint32(1)
|
||||
if retryable {
|
||||
if f.get(task.ID).State != pulse.TaskWaiting {
|
||||
t.Fatal("retryable failure not retried")
|
||||
}
|
||||
makeRetryDue(f, task.ID)
|
||||
retryFinish(t, f, retryStart(t, f), "FAILED", true)
|
||||
count = 2
|
||||
}
|
||||
got := f.get(task.ID)
|
||||
if got.State != pulse.TaskFailed || got.AttemptCount != count || got.EffectUncertain || got.CompletedAt == nil {
|
||||
t.Fatal("failure budget or classification ignored")
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("unknown_history_remains_uncertain_until_authoritative_success", func(t *testing.T) {
|
||||
for _, last := range []string{"SUCCEEDED", "FAILED", "UNKNOWN", "NOT_STARTED", "CANCEL", "DEADLINE"} {
|
||||
t.Run(last, func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := retryTask(t, f, 2, pulse.RecoveryIdempotent)
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "UNKNOWN", false)
|
||||
pending := f.get(task.ID)
|
||||
if pending.State != pulse.TaskWaiting || !pending.EffectUncertain || len(pending.Result) != 0 {
|
||||
t.Fatal("unknown retry lost uncertainty")
|
||||
}
|
||||
switch last {
|
||||
case "CANCEL":
|
||||
out, err := f.tasks.CancelTask(ctx, f.token, task.ID, string(executionID(t)), service.CancelInput{ExpectedRevision: pending.Revision, Reason: "stop remaining retries"})
|
||||
if err != nil || out.Task.CancelRequestedAt == nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "DEADLINE":
|
||||
f.exec(`UPDATE pulse_tasks SET available_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 2 SECOND),deadline_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, task.ID)
|
||||
if _, err := f.storeB.Recover(ctx, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "NOT_STARTED":
|
||||
makeRetryDue(f, task.ID)
|
||||
_, next := f.claim()
|
||||
f.expire(next)
|
||||
if _, err := f.storeB.Recover(ctx, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
default:
|
||||
makeRetryDue(f, task.ID)
|
||||
retryFinish(t, f, retryStart(t, f), last, last == "FAILED")
|
||||
}
|
||||
got := f.get(task.ID)
|
||||
if last == "SUCCEEDED" {
|
||||
if got.State != pulse.TaskSucceeded || got.EffectUncertain {
|
||||
t.Fatal("success did not resolve uncertainty")
|
||||
}
|
||||
} else if got.State != pulse.TaskNeedsReview || !got.EffectUncertain || got.CompletedAt != nil {
|
||||
t.Fatal("unresolved effects became false terminal failure/cancellation")
|
||||
}
|
||||
if f.used() != 0 {
|
||||
t.Fatal("retry leaked permit")
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
t.Run("unknown_requires_explicit_idempotent_contract", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := retryTask(t, f, 3, pulse.RecoveryReconcileOnly)
|
||||
retryFinish(t, f, retryStart(t, f), "UNKNOWN", false)
|
||||
got := f.get(task.ID)
|
||||
if got.State != pulse.TaskNeedsReview || !got.EffectUncertain || got.AttemptCount != 1 {
|
||||
t.Fatal("unknown replayed without idempotence")
|
||||
}
|
||||
})
|
||||
t.Run("recovery_race_pause_and_late_success", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := retryTask(t, f, 2, pulse.RecoveryIdempotent)
|
||||
ref := retryStart(t, f)
|
||||
f.expire(ref)
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 2)
|
||||
for _, store := range []*ExecutionStore{f.storeA, f.storeB} {
|
||||
wg.Add(1)
|
||||
go func(s *ExecutionStore) { defer wg.Done(); _, err := s.Recover(ctx, 100); errs <- err }(store)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
pending := f.get(task.ID)
|
||||
if pending.State != pulse.TaskWaiting || !pending.EffectUncertain || f.used() != 0 || f.count("orbit_events", "aggregate_id=? AND type='task.requeued'", task.ID) != 1 {
|
||||
t.Fatal("competing recovery did not make one retry decision")
|
||||
}
|
||||
makeRetryDue(f, task.ID)
|
||||
f.exec(`UPDATE pulse_queues SET paused=TRUE WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, f.resources.QueueID)
|
||||
if a, err := f.a.Claim(ctx, f.token, f.binding); err != nil || a != nil {
|
||||
t.Fatal("retry bypassed pause")
|
||||
}
|
||||
late := retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
if late.Receipt.Disposition != "EVIDENCE_ONLY" || !f.get(task.ID).EffectUncertain {
|
||||
t.Fatal("late success cleared unresolved authority")
|
||||
}
|
||||
f.exec(`UPDATE pulse_queues SET paused=FALSE WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, f.resources.QueueID)
|
||||
retryFinish(t, f, retryStart(t, f), "SUCCEEDED", false)
|
||||
if f.get(task.ID).State != pulse.TaskSucceeded {
|
||||
t.Fatal("resumed retry failed")
|
||||
}
|
||||
})
|
||||
t.Run("retry_transaction_rolls_back_with_audit", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := retryTask(t, f, 2, pulse.RecoveryReconcileOnly)
|
||||
ref := retryStart(t, f)
|
||||
f.exec(`CREATE TRIGGER orbit_test_retry_audit BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='retry audit failure'`)
|
||||
in := worker.FinishInput{Ref: ref, Outcome: "FAILED", Retryable: true, ErrorCode: "TEMPORARY_REJECTION", Result: json.RawMessage(`{}`)}
|
||||
message := executionID(t)
|
||||
_, err := f.a.Finish(ctx, f.token, f.binding, message, in)
|
||||
f.exec(`DROP TRIGGER orbit_test_retry_audit`)
|
||||
if err == nil || f.get(task.ID).State != pulse.TaskRunning || f.used() != 1 || f.count("orbit_events", "aggregate_id=? AND type='task.requeued'", task.ID) != 0 {
|
||||
t.Fatal("partial retry transition committed")
|
||||
}
|
||||
if _, err := f.a.Finish(ctx, f.token, f.binding, message, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user