feat: add durable safe retries with uncertainty tracking and SDK examples
This commit is contained in:
@@ -29,6 +29,8 @@ type Attempt struct {
|
||||
EndedAt *time.Time `db:"ended_at"`
|
||||
// ErrorCode 稳定错误分类，供恢复策略判断；不得写入敏感原始错误；可为空，表示尚未设置该关联或值。
|
||||
ErrorCode *string `db:"error_code"`
|
||||
// Retryable Worker 对明确失败声明可安全重试；只用于 FAILED，不证明未知效果安全。
|
||||
Retryable bool `db:"retryable"`
|
||||
// Result 已保存的结果或结果引用 JSON；未取得结果时为空。
|
||||
Result core.JSON `db:"result"`
|
||||
// ResultDigest 结果语义摘要，用于识别确认重报与冲突结果；可为空，表示尚未设置该关联或值。
|
||||
|
||||
@@ -14,6 +14,8 @@ type AttemptEvidence struct {
|
||||
Outcome string `db:"outcome"`
|
||||
// ErrorCode 可选稳定错误分类；无错误分类时为空。
|
||||
ErrorCode *string `db:"error_code"`
|
||||
// Retryable 原失败报告的安全重试声明；证据本身不触发重试。
|
||||
Retryable bool `db:"retryable"`
|
||||
// Payload 结果或证据引用 JSON 文本，保留数字精度。
|
||||
Payload core.JSON `db:"payload"`
|
||||
// RequestDigest 结果语义摘要，排除会话身份与原始秘密。
|
||||
|
||||
@@ -22,9 +22,9 @@ func (s TaskState) Terminal() bool {
|
||||
func (s TaskState) CanTransitionTo(next TaskState) bool {
|
||||
switch s {
|
||||
case TaskWaiting:
|
||||
return next == TaskLeased || next == TaskCancelled || next == TaskFailed
|
||||
return next == TaskLeased || next == TaskCancelled || next == TaskFailed || next == TaskNeedsReview
|
||||
case TaskLeased:
|
||||
return next == TaskRunning || next == TaskWaiting || next == TaskCancelled || next == TaskFailed
|
||||
return next == TaskRunning || next == TaskWaiting || next == TaskCancelled || next == TaskFailed || next == TaskNeedsReview
|
||||
case TaskRunning:
|
||||
return next == TaskSucceeded || next == TaskWaiting || next == TaskFailed || next == TaskNeedsReview || next == TaskCancelled
|
||||
case TaskNeedsReview:
|
||||
|
||||
@@ -27,6 +27,8 @@ type Task struct {
|
||||
Payload core.JSON `db:"payload"`
|
||||
// Result 已保存的结果或结果引用 JSON；未取得结果时为空。
|
||||
Result core.JSON `db:"result"`
|
||||
// EffectUncertain 曾有已授权执行的效果尚未核实；重试/超时不能清除，仅权威成功或后续核查可解除。
|
||||
EffectUncertain bool `db:"effect_uncertain"`
|
||||
// PolicySnapshot 受理时冻结的实际生效策略 JSON。
|
||||
PolicySnapshot core.JSON `db:"policy_snapshot"`
|
||||
// RecoverySafety 恢复安全等级；未知值按需要核查处理，不能默认自动重跑。
|
||||
|
||||
115
pulse/retry/decision.go
Normal file
115
pulse/retry/decision.go
Normal file
@@ -0,0 +1,115 @@
|
||||
// Package retry 以冻结策略和持久化预算决定下一状态，不执行任务或维护内存定时器。
|
||||
package retry
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
)
|
||||
|
||||
// Input 描述本次尝试结束时数据库已锁定的事实，不能从客户端覆盖预算和时间。
|
||||
type Input struct {
|
||||
Policy pulse.RetryPolicy // Policy 受理时冻结的有限策略。
|
||||
Safety pulse.RecoverySafety // Safety 仅 IDEMPOTENT 允许未知效果自动恢复。
|
||||
Consumed uint32 // Consumed 已领取次数，包含本次及未获批开始的尝试。
|
||||
Outcome string // Outcome SUCCEEDED、FAILED、UNKNOWN 或 NOT_STARTED。
|
||||
Retryable bool // Retryable 仅对明确失败有效，Worker 声明可安全重试。
|
||||
Uncertain bool // Uncertain 之前仍未核实的外部效果，失败或超时不能抹掉。
|
||||
CancelRequested bool // CancelRequested 已保存取消意图时禁止后续尝试。
|
||||
Now time.Time // Now 完成资源锁定后读取的数据库时间。
|
||||
Deadline time.Time // Deadline 任务总截止时间。
|
||||
StartDeadline *time.Time // StartDeadline 所有新尝试的最晚开始时间，可为空。
|
||||
AttemptID core.ID // AttemptID 持久化尝试身份，用作可重现抖动的种子。
|
||||
}
|
||||
|
||||
// Decision 必须与尝试结果、许可释放、事件及操作回执在同一事务提交。
|
||||
type Decision struct {
|
||||
State pulse.TaskState // State 下一任务状态，WAITING 不是业务终态。
|
||||
Uncertain bool // Uncertain 跨尝试保留的不确定性；权威成功才自动清除。
|
||||
NextAt *time.Time // NextAt 仅 WAITING 有值，写入 available_at。
|
||||
Reason string // Reason 稳定决策分类，用于事件和审计。
|
||||
DelayMS uint64 // DelayMS 实际等待毫秒数，非重试时为零。
|
||||
}
|
||||
|
||||
func Decide(in Input) (Decision, error) {
|
||||
out := Decision{State: pulse.TaskFailed, Uncertain: in.Uncertain, Reason: "NON_RETRYABLE_FAILURE"}
|
||||
if err := in.Policy.Validate(); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if in.Consumed == 0 || in.Consumed > in.Policy.MaxAttempts || in.AttemptID.Validate() != nil || in.Now.IsZero() || in.Deadline.IsZero() {
|
||||
return out, errors.New("invalid retry facts")
|
||||
}
|
||||
canRetry := false
|
||||
switch in.Outcome {
|
||||
case "SUCCEEDED":
|
||||
return Decision{State: pulse.TaskSucceeded, Reason: "SUCCEEDED"}, nil
|
||||
case "FAILED":
|
||||
canRetry = in.Retryable
|
||||
if canRetry {
|
||||
out.Reason = "RETRYABLE_FAILURE"
|
||||
}
|
||||
case "UNKNOWN":
|
||||
out.Uncertain = true
|
||||
canRetry = in.Safety.CanRecoverAutomatically()
|
||||
out.Reason = "RECONCILIATION_REQUIRED"
|
||||
if canRetry {
|
||||
out.Reason = "IDEMPOTENT_UNKNOWN"
|
||||
}
|
||||
case "NOT_STARTED":
|
||||
canRetry = true
|
||||
out.Reason = "UNSTARTED_LEASE_EXPIRED"
|
||||
default:
|
||||
return out, errors.New("invalid retry outcome")
|
||||
}
|
||||
if out.Uncertain {
|
||||
out.State = pulse.TaskNeedsReview
|
||||
if !in.Safety.CanRecoverAutomatically() {
|
||||
canRetry = false
|
||||
out.Reason = "RECONCILIATION_REQUIRED"
|
||||
}
|
||||
}
|
||||
if in.CancelRequested {
|
||||
out.Reason = "CANCEL_REQUESTED"
|
||||
return out, nil
|
||||
}
|
||||
if !canRetry {
|
||||
return out, nil
|
||||
}
|
||||
if in.Consumed >= in.Policy.MaxAttempts {
|
||||
out.Reason = "ATTEMPT_LIMIT"
|
||||
return out, nil
|
||||
}
|
||||
delay := DelayMS(in.Policy, in.Consumed, in.AttemptID)
|
||||
next := in.Now.Add(time.Duration(delay) * time.Millisecond)
|
||||
if !next.Before(in.Deadline) || (in.StartDeadline != nil && !next.Before(*in.StartDeadline)) {
|
||||
out.Reason = "DEADLINE_EXCEEDED"
|
||||
return out, nil
|
||||
}
|
||||
out.State = pulse.TaskWaiting
|
||||
out.DelayMS = delay
|
||||
out.NextAt = &next
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DelayMS 使用有上限的指数退避和 [ceil(base/2),base] 确定性抖动；策略须先 Validate。
|
||||
// 相同尝试重复计算不改变结果，倍增先截断，避免次数或 Duration 溢出。
|
||||
func DelayMS(policy pulse.RetryPolicy, consumed uint32, attempt core.ID) uint64 {
|
||||
base := policy.InitialDelayMS
|
||||
for n := uint32(1); n < consumed && base < policy.MaxDelayMS; n++ {
|
||||
if base > policy.MaxDelayMS/2 {
|
||||
base = policy.MaxDelayMS
|
||||
break
|
||||
}
|
||||
base *= 2
|
||||
}
|
||||
if base > policy.MaxDelayMS {
|
||||
base = policy.MaxDelayMS
|
||||
}
|
||||
lower := (base + 1) / 2
|
||||
sum := sha256.Sum256([]byte("orbit.retry.v1:" + string(attempt)))
|
||||
return lower + binary.BigEndian.Uint64(sum[:8])%(base-lower+1)
|
||||
}
|
||||
102
pulse/retry/decision_test.go
Normal file
102
pulse/retry/decision_test.go
Normal file
@@ -0,0 +1,102 @@
|
||||
package retry
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
)
|
||||
|
||||
func TestDecisionPreservesSafetyBudgetAndDeadline(t *testing.T) {
|
||||
now := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC)
|
||||
base := Input{Policy: pulse.RetryPolicy{MaxAttempts: 3, InitialDelayMS: 1000, MaxDelayMS: 5000, AttemptTimeoutMS: 10000}, Consumed: 1, Outcome: "FAILED", Safety: pulse.RecoveryReconcileOnly, Now: now, Deadline: now.Add(time.Hour), AttemptID: core.ID("11111111111111111111111111111111")}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
change func(*Input)
|
||||
state pulse.TaskState
|
||||
uncertain bool
|
||||
reason string
|
||||
}{
|
||||
{"explicit_failure_default", func(*Input) {}, pulse.TaskFailed, false, "NON_RETRYABLE_FAILURE"},
|
||||
{"safe_failure", func(v *Input) { v.Retryable = true }, pulse.TaskWaiting, false, "RETRYABLE_FAILURE"},
|
||||
{"unknown_default", func(v *Input) { v.Outcome = "UNKNOWN" }, pulse.TaskNeedsReview, true, "RECONCILIATION_REQUIRED"},
|
||||
{"idempotent_unknown", func(v *Input) { v.Outcome = "UNKNOWN"; v.Safety = pulse.RecoveryIdempotent }, pulse.TaskWaiting, true, "IDEMPOTENT_UNKNOWN"},
|
||||
{"known_exhaustion", func(v *Input) { v.Retryable = true; v.Consumed = 3 }, pulse.TaskFailed, false, "ATTEMPT_LIMIT"},
|
||||
{"unknown_exhaustion", func(v *Input) { v.Outcome = "UNKNOWN"; v.Safety = pulse.RecoveryIdempotent; v.Consumed = 3 }, pulse.TaskNeedsReview, true, "ATTEMPT_LIMIT"},
|
||||
{"old_uncertainty_is_sticky", func(v *Input) { v.Uncertain = true; v.Safety = pulse.RecoveryIdempotent }, pulse.TaskNeedsReview, true, "NON_RETRYABLE_FAILURE"},
|
||||
{"prior_uncertainty_and_prestart_exhaustion", func(v *Input) {
|
||||
v.Uncertain = true
|
||||
v.Safety = pulse.RecoveryIdempotent
|
||||
v.Consumed = 3
|
||||
v.Outcome = "NOT_STARTED"
|
||||
}, pulse.TaskNeedsReview, true, "ATTEMPT_LIMIT"},
|
||||
{"known_deadline", func(v *Input) { v.Retryable = true; v.Deadline = now.Add(time.Millisecond) }, pulse.TaskFailed, false, "DEADLINE_EXCEEDED"},
|
||||
{"unknown_start_deadline", func(v *Input) {
|
||||
v.Outcome = "UNKNOWN"
|
||||
v.Safety = pulse.RecoveryIdempotent
|
||||
until := now.Add(time.Millisecond)
|
||||
v.StartDeadline = &until
|
||||
}, pulse.TaskNeedsReview, true, "DEADLINE_EXCEEDED"},
|
||||
{"cancel_prevents_retry", func(v *Input) { v.Retryable = true; v.CancelRequested = true }, pulse.TaskFailed, false, "CANCEL_REQUESTED"},
|
||||
{"success_resolves_uncertainty", func(v *Input) { v.Uncertain = true; v.Outcome = "SUCCEEDED" }, pulse.TaskSucceeded, false, "SUCCEEDED"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
input := base
|
||||
tc.change(&input)
|
||||
got, err := Decide(input)
|
||||
if err != nil || got.State != tc.state || got.Uncertain != tc.uncertain || got.Reason != tc.reason {
|
||||
t.Fatalf("unexpected decision: %+v %v", got, err)
|
||||
}
|
||||
if (got.NextAt != nil) != (got.State == pulse.TaskWaiting) {
|
||||
t.Fatal("retry time does not match state")
|
||||
}
|
||||
if got.NextAt != nil && (got.DelayMS < 500 || got.DelayMS > 1000 || !got.NextAt.Equal(now.Add(time.Duration(got.DelayMS)*time.Millisecond))) {
|
||||
t.Fatal("first retry escaped bounded jitter")
|
||||
}
|
||||
})
|
||||
}
|
||||
// 截止恰好等于选定下一次时间也不能授予新尝试。
|
||||
base.Retryable = true
|
||||
out, err := Decide(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base.Deadline = *out.NextAt
|
||||
if out, err = Decide(base); err != nil || out.State != pulse.TaskFailed {
|
||||
t.Fatal("inclusive deadline allowed retry")
|
||||
}
|
||||
}
|
||||
func TestBackoffBoundedDeterministicAndNoOverflow(t *testing.T) {
|
||||
policy := pulse.RetryPolicy{MaxAttempts: 100, InitialDelayMS: 1000, MaxDelayMS: 5000, AttemptTimeoutMS: 1}
|
||||
distinct := map[uint64]bool{}
|
||||
for n := 1; n <= 256; n++ {
|
||||
id := core.ID(fmt.Sprintf("%032x", n))
|
||||
first := DelayMS(policy, 1, id)
|
||||
distinct[first] = true
|
||||
if first != DelayMS(policy, 1, id) || first < 500 || first > 1000 {
|
||||
t.Fatal("unstable first delay")
|
||||
}
|
||||
if next := DelayMS(policy, 2, id); next < 1000 || next > 2000 {
|
||||
t.Fatal("exponential delay lost bound")
|
||||
}
|
||||
if last := DelayMS(policy, 100, id); last < 2500 || last > 5000 {
|
||||
t.Fatal("delay did not saturate")
|
||||
}
|
||||
}
|
||||
if len(distinct) < 100 {
|
||||
t.Fatal("jitter failed to distribute identities")
|
||||
}
|
||||
policy.InitialDelayMS = 1
|
||||
policy.MaxDelayMS = 1
|
||||
if DelayMS(policy, 100, core.ID(fmt.Sprintf("%032x", 1))) != 1 {
|
||||
t.Fatal("positive delay became zero")
|
||||
}
|
||||
policy.InitialDelayMS = 9223372036853
|
||||
policy.MaxDelayMS = 9223372036854
|
||||
n := DelayMS(policy, 100, core.ID(fmt.Sprintf("%032x", 1)))
|
||||
if n < 4611686018427 || n > 9223372036854 || time.Duration(n)*time.Millisecond <= 0 {
|
||||
t.Fatal("overflowing retry duration")
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,7 @@ type AttemptSummary struct {
|
||||
LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 数据库记录的租约期限；查询不能授权业务执行。
|
||||
AuthorizedAt *time.Time `json:"authorized_at,omitempty"` // AuthorizedAt 开始批准时间；空表示未批准，不证明副作用已经发生。
|
||||
EndedAt *time.Time `json:"ended_at,omitempty"` // EndedAt 尝试结束时间；仍在执行时为空。
|
||||
Retryable bool `json:"retryable,omitempty"` // Retryable 该明确失败报告是否声明可安全重试；不代表平台必定安排重试。
|
||||
ErrorCode string `json:"error_code,omitempty"` // ErrorCode 稳定错误分类；无错误时为空。
|
||||
CreatedAt time.Time `json:"created_at"` // CreatedAt 创建时间，UTC 微秒精度。
|
||||
UpdatedAt time.Time `json:"updated_at"` // UpdatedAt 最新变更时间，不能代替版本校验。
|
||||
@@ -53,6 +54,7 @@ type EvidenceSummary struct {
|
||||
AttemptID core.ID `json:"attempt_id"` // AttemptID 所属执行尝试。
|
||||
Outcome string `json:"outcome"` // Outcome Worker 声明的 SUCCEEDED、FAILED 或 UNKNOWN，不代表平台认可。
|
||||
ErrorCode string `json:"error_code,omitempty"` // ErrorCode 原报告的稳定分类，可为空。
|
||||
Retryable bool `json:"retryable,omitempty"` // Retryable 原报告的重试声明；证据不触发重试。
|
||||
Reason string `json:"reason"` // Reason LEASE_LOST、NOT_STARTED 或 RESULT_CONFLICT。
|
||||
Disposition string `json:"disposition"` // Disposition 固定为 EVIDENCE_ONLY，表示采集时仅保存证据。
|
||||
CreatedAt time.Time `json:"created_at"` // CreatedAt 首次收到时间，重复报告不改变它。
|
||||
|
||||
@@ -59,6 +59,8 @@ type TaskView struct {
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
// Result 业务 JSON 结果；未取得结果时为空并省略。
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
// EffectUncertain 曾有执行效果尚未核实；重试与预算耗尽不会把未知伪装成明确失败。
|
||||
EffectUncertain bool `json:"effect_uncertain,omitempty"`
|
||||
// BusinessKey 可选业务关联键，不自动提供业务效果幂等。
|
||||
BusinessKey string `json:"business_key,omitempty"`
|
||||
// AvailableAt 最早可执行时间，UTC。
|
||||
@@ -125,6 +127,8 @@ type TaskSummary struct {
|
||||
Handler string `json:"handler"`
|
||||
// HandlerVersion 受理时固定的处理器版本。
|
||||
HandlerVersion string `json:"handler_version"`
|
||||
// EffectUncertain 曾有执行效果尚未核实；重试与预算耗尽不会把未知伪装成明确失败。
|
||||
EffectUncertain bool `json:"effect_uncertain,omitempty"`
|
||||
// BusinessKey 可选业务关联键，不自动提供业务效果幂等。
|
||||
BusinessKey string `json:"business_key,omitempty"`
|
||||
// AvailableAt 最早可执行时间，UTC。
|
||||
|
||||
@@ -120,6 +120,9 @@ func (s *Service) Finish(ctx context.Context, t auth.Token, b Binding, message c
|
||||
if in.Outcome != "SUCCEEDED" && in.Outcome != "FAILED" && in.Outcome != "UNKNOWN" {
|
||||
return CommandResult{}, fault.Invalid("invalid result outcome")
|
||||
}
|
||||
if in.Retryable && in.Outcome != "FAILED" {
|
||||
return CommandResult{}, fault.Invalid("retryable is only valid for known failure")
|
||||
}
|
||||
if len(in.Result) == 0 || len(in.Result) > 256<<10 || bytes.Equal(bytes.TrimSpace(in.Result), []byte("null")) {
|
||||
return CommandResult{}, fault.Invalid("result must be non-null JSON up to 256 KiB")
|
||||
}
|
||||
|
||||
@@ -77,6 +77,7 @@ type FinishInput struct {
|
||||
Outcome string `json:"outcome"` // Outcome 为 SUCCEEDED、FAILED 或 UNKNOWN。
|
||||
Result json.RawMessage `json:"result"` // Result 非 null JSON 结果或证据引用，最多 256 KiB。
|
||||
ErrorCode string `json:"error_code,omitempty"` // ErrorCode FAILED/UNKNOWN 必填稳定分类，不包含原始异常正文。
|
||||
Retryable bool `json:"retryable,omitempty"` // Retryable 仅 FAILED 可为真，声明该明确失败可安全重试；默认假。
|
||||
}
|
||||
|
||||
// Receipt 是不可变的操作回执，不代表当前仍有执行权。
|
||||
|
||||
Reference in New Issue
Block a user