feat: add durable safe retries with uncertainty tracking and SDK examples
This commit is contained in:
115
pulse/retry/decision.go
Normal file
115
pulse/retry/decision.go
Normal file
@@ -0,0 +1,115 @@
|
||||
// Package retry 以冻结策略和持久化预算决定下一状态，不执行任务或维护内存定时器。
|
||||
package retry
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
)
|
||||
|
||||
// Input 描述本次尝试结束时数据库已锁定的事实，不能从客户端覆盖预算和时间。
|
||||
type Input struct {
|
||||
Policy pulse.RetryPolicy // Policy 受理时冻结的有限策略。
|
||||
Safety pulse.RecoverySafety // Safety 仅 IDEMPOTENT 允许未知效果自动恢复。
|
||||
Consumed uint32 // Consumed 已领取次数，包含本次及未获批开始的尝试。
|
||||
Outcome string // Outcome SUCCEEDED、FAILED、UNKNOWN 或 NOT_STARTED。
|
||||
Retryable bool // Retryable 仅对明确失败有效，Worker 声明可安全重试。
|
||||
Uncertain bool // Uncertain 之前仍未核实的外部效果，失败或超时不能抹掉。
|
||||
CancelRequested bool // CancelRequested 已保存取消意图时禁止后续尝试。
|
||||
Now time.Time // Now 完成资源锁定后读取的数据库时间。
|
||||
Deadline time.Time // Deadline 任务总截止时间。
|
||||
StartDeadline *time.Time // StartDeadline 所有新尝试的最晚开始时间，可为空。
|
||||
AttemptID core.ID // AttemptID 持久化尝试身份，用作可重现抖动的种子。
|
||||
}
|
||||
|
||||
// Decision 必须与尝试结果、许可释放、事件及操作回执在同一事务提交。
|
||||
type Decision struct {
|
||||
State pulse.TaskState // State 下一任务状态，WAITING 不是业务终态。
|
||||
Uncertain bool // Uncertain 跨尝试保留的不确定性；权威成功才自动清除。
|
||||
NextAt *time.Time // NextAt 仅 WAITING 有值，写入 available_at。
|
||||
Reason string // Reason 稳定决策分类，用于事件和审计。
|
||||
DelayMS uint64 // DelayMS 实际等待毫秒数，非重试时为零。
|
||||
}
|
||||
|
||||
func Decide(in Input) (Decision, error) {
|
||||
out := Decision{State: pulse.TaskFailed, Uncertain: in.Uncertain, Reason: "NON_RETRYABLE_FAILURE"}
|
||||
if err := in.Policy.Validate(); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if in.Consumed == 0 || in.Consumed > in.Policy.MaxAttempts || in.AttemptID.Validate() != nil || in.Now.IsZero() || in.Deadline.IsZero() {
|
||||
return out, errors.New("invalid retry facts")
|
||||
}
|
||||
canRetry := false
|
||||
switch in.Outcome {
|
||||
case "SUCCEEDED":
|
||||
return Decision{State: pulse.TaskSucceeded, Reason: "SUCCEEDED"}, nil
|
||||
case "FAILED":
|
||||
canRetry = in.Retryable
|
||||
if canRetry {
|
||||
out.Reason = "RETRYABLE_FAILURE"
|
||||
}
|
||||
case "UNKNOWN":
|
||||
out.Uncertain = true
|
||||
canRetry = in.Safety.CanRecoverAutomatically()
|
||||
out.Reason = "RECONCILIATION_REQUIRED"
|
||||
if canRetry {
|
||||
out.Reason = "IDEMPOTENT_UNKNOWN"
|
||||
}
|
||||
case "NOT_STARTED":
|
||||
canRetry = true
|
||||
out.Reason = "UNSTARTED_LEASE_EXPIRED"
|
||||
default:
|
||||
return out, errors.New("invalid retry outcome")
|
||||
}
|
||||
if out.Uncertain {
|
||||
out.State = pulse.TaskNeedsReview
|
||||
if !in.Safety.CanRecoverAutomatically() {
|
||||
canRetry = false
|
||||
out.Reason = "RECONCILIATION_REQUIRED"
|
||||
}
|
||||
}
|
||||
if in.CancelRequested {
|
||||
out.Reason = "CANCEL_REQUESTED"
|
||||
return out, nil
|
||||
}
|
||||
if !canRetry {
|
||||
return out, nil
|
||||
}
|
||||
if in.Consumed >= in.Policy.MaxAttempts {
|
||||
out.Reason = "ATTEMPT_LIMIT"
|
||||
return out, nil
|
||||
}
|
||||
delay := DelayMS(in.Policy, in.Consumed, in.AttemptID)
|
||||
next := in.Now.Add(time.Duration(delay) * time.Millisecond)
|
||||
if !next.Before(in.Deadline) || (in.StartDeadline != nil && !next.Before(*in.StartDeadline)) {
|
||||
out.Reason = "DEADLINE_EXCEEDED"
|
||||
return out, nil
|
||||
}
|
||||
out.State = pulse.TaskWaiting
|
||||
out.DelayMS = delay
|
||||
out.NextAt = &next
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DelayMS 使用有上限的指数退避和 [ceil(base/2),base] 确定性抖动；策略须先 Validate。
|
||||
// 相同尝试重复计算不改变结果，倍增先截断，避免次数或 Duration 溢出。
|
||||
func DelayMS(policy pulse.RetryPolicy, consumed uint32, attempt core.ID) uint64 {
|
||||
base := policy.InitialDelayMS
|
||||
for n := uint32(1); n < consumed && base < policy.MaxDelayMS; n++ {
|
||||
if base > policy.MaxDelayMS/2 {
|
||||
base = policy.MaxDelayMS
|
||||
break
|
||||
}
|
||||
base *= 2
|
||||
}
|
||||
if base > policy.MaxDelayMS {
|
||||
base = policy.MaxDelayMS
|
||||
}
|
||||
lower := (base + 1) / 2
|
||||
sum := sha256.Sum256([]byte("orbit.retry.v1:" + string(attempt)))
|
||||
return lower + binary.BigEndian.Uint64(sum[:8])%(base-lower+1)
|
||||
}
|
||||
102
pulse/retry/decision_test.go
Normal file
102
pulse/retry/decision_test.go
Normal file
@@ -0,0 +1,102 @@
|
||||
package retry
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
)
|
||||
|
||||
func TestDecisionPreservesSafetyBudgetAndDeadline(t *testing.T) {
|
||||
now := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC)
|
||||
base := Input{Policy: pulse.RetryPolicy{MaxAttempts: 3, InitialDelayMS: 1000, MaxDelayMS: 5000, AttemptTimeoutMS: 10000}, Consumed: 1, Outcome: "FAILED", Safety: pulse.RecoveryReconcileOnly, Now: now, Deadline: now.Add(time.Hour), AttemptID: core.ID("11111111111111111111111111111111")}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
change func(*Input)
|
||||
state pulse.TaskState
|
||||
uncertain bool
|
||||
reason string
|
||||
}{
|
||||
{"explicit_failure_default", func(*Input) {}, pulse.TaskFailed, false, "NON_RETRYABLE_FAILURE"},
|
||||
{"safe_failure", func(v *Input) { v.Retryable = true }, pulse.TaskWaiting, false, "RETRYABLE_FAILURE"},
|
||||
{"unknown_default", func(v *Input) { v.Outcome = "UNKNOWN" }, pulse.TaskNeedsReview, true, "RECONCILIATION_REQUIRED"},
|
||||
{"idempotent_unknown", func(v *Input) { v.Outcome = "UNKNOWN"; v.Safety = pulse.RecoveryIdempotent }, pulse.TaskWaiting, true, "IDEMPOTENT_UNKNOWN"},
|
||||
{"known_exhaustion", func(v *Input) { v.Retryable = true; v.Consumed = 3 }, pulse.TaskFailed, false, "ATTEMPT_LIMIT"},
|
||||
{"unknown_exhaustion", func(v *Input) { v.Outcome = "UNKNOWN"; v.Safety = pulse.RecoveryIdempotent; v.Consumed = 3 }, pulse.TaskNeedsReview, true, "ATTEMPT_LIMIT"},
|
||||
{"old_uncertainty_is_sticky", func(v *Input) { v.Uncertain = true; v.Safety = pulse.RecoveryIdempotent }, pulse.TaskNeedsReview, true, "NON_RETRYABLE_FAILURE"},
|
||||
{"prior_uncertainty_and_prestart_exhaustion", func(v *Input) {
|
||||
v.Uncertain = true
|
||||
v.Safety = pulse.RecoveryIdempotent
|
||||
v.Consumed = 3
|
||||
v.Outcome = "NOT_STARTED"
|
||||
}, pulse.TaskNeedsReview, true, "ATTEMPT_LIMIT"},
|
||||
{"known_deadline", func(v *Input) { v.Retryable = true; v.Deadline = now.Add(time.Millisecond) }, pulse.TaskFailed, false, "DEADLINE_EXCEEDED"},
|
||||
{"unknown_start_deadline", func(v *Input) {
|
||||
v.Outcome = "UNKNOWN"
|
||||
v.Safety = pulse.RecoveryIdempotent
|
||||
until := now.Add(time.Millisecond)
|
||||
v.StartDeadline = &until
|
||||
}, pulse.TaskNeedsReview, true, "DEADLINE_EXCEEDED"},
|
||||
{"cancel_prevents_retry", func(v *Input) { v.Retryable = true; v.CancelRequested = true }, pulse.TaskFailed, false, "CANCEL_REQUESTED"},
|
||||
{"success_resolves_uncertainty", func(v *Input) { v.Uncertain = true; v.Outcome = "SUCCEEDED" }, pulse.TaskSucceeded, false, "SUCCEEDED"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
input := base
|
||||
tc.change(&input)
|
||||
got, err := Decide(input)
|
||||
if err != nil || got.State != tc.state || got.Uncertain != tc.uncertain || got.Reason != tc.reason {
|
||||
t.Fatalf("unexpected decision: %+v %v", got, err)
|
||||
}
|
||||
if (got.NextAt != nil) != (got.State == pulse.TaskWaiting) {
|
||||
t.Fatal("retry time does not match state")
|
||||
}
|
||||
if got.NextAt != nil && (got.DelayMS < 500 || got.DelayMS > 1000 || !got.NextAt.Equal(now.Add(time.Duration(got.DelayMS)*time.Millisecond))) {
|
||||
t.Fatal("first retry escaped bounded jitter")
|
||||
}
|
||||
})
|
||||
}
|
||||
// 截止恰好等于选定下一次时间也不能授予新尝试。
|
||||
base.Retryable = true
|
||||
out, err := Decide(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base.Deadline = *out.NextAt
|
||||
if out, err = Decide(base); err != nil || out.State != pulse.TaskFailed {
|
||||
t.Fatal("inclusive deadline allowed retry")
|
||||
}
|
||||
}
|
||||
func TestBackoffBoundedDeterministicAndNoOverflow(t *testing.T) {
|
||||
policy := pulse.RetryPolicy{MaxAttempts: 100, InitialDelayMS: 1000, MaxDelayMS: 5000, AttemptTimeoutMS: 1}
|
||||
distinct := map[uint64]bool{}
|
||||
for n := 1; n <= 256; n++ {
|
||||
id := core.ID(fmt.Sprintf("%032x", n))
|
||||
first := DelayMS(policy, 1, id)
|
||||
distinct[first] = true
|
||||
if first != DelayMS(policy, 1, id) || first < 500 || first > 1000 {
|
||||
t.Fatal("unstable first delay")
|
||||
}
|
||||
if next := DelayMS(policy, 2, id); next < 1000 || next > 2000 {
|
||||
t.Fatal("exponential delay lost bound")
|
||||
}
|
||||
if last := DelayMS(policy, 100, id); last < 2500 || last > 5000 {
|
||||
t.Fatal("delay did not saturate")
|
||||
}
|
||||
}
|
||||
if len(distinct) < 100 {
|
||||
t.Fatal("jitter failed to distribute identities")
|
||||
}
|
||||
policy.InitialDelayMS = 1
|
||||
policy.MaxDelayMS = 1
|
||||
if DelayMS(policy, 100, core.ID(fmt.Sprintf("%032x", 1))) != 1 {
|
||||
t.Fatal("positive delay became zero")
|
||||
}
|
||||
policy.InitialDelayMS = 9223372036853
|
||||
policy.MaxDelayMS = 9223372036854
|
||||
n := DelayMS(policy, 100, core.ID(fmt.Sprintf("%032x", 1)))
|
||||
if n < 4611686018427 || n > 9223372036854 || time.Duration(n)*time.Millisecond <= 0 {
|
||||
t.Fatal("overflowing retry duration")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user