103 lines
4.5 KiB
Go
103 lines
4.5 KiB
Go
package retry
|
|
|
|
import (
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
|
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
|
)
|
|
|
|
func TestDecisionPreservesSafetyBudgetAndDeadline(t *testing.T) {
|
|
now := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC)
|
|
base := Input{Policy: pulse.RetryPolicy{MaxAttempts: 3, InitialDelayMS: 1000, MaxDelayMS: 5000, AttemptTimeoutMS: 10000}, Consumed: 1, Outcome: "FAILED", Safety: pulse.RecoveryReconcileOnly, Now: now, Deadline: now.Add(time.Hour), AttemptID: core.ID("11111111111111111111111111111111")}
|
|
for _, tc := range []struct {
|
|
name string
|
|
change func(*Input)
|
|
state pulse.TaskState
|
|
uncertain bool
|
|
reason string
|
|
}{
|
|
{"explicit_failure_default", func(*Input) {}, pulse.TaskFailed, false, "NON_RETRYABLE_FAILURE"},
|
|
{"safe_failure", func(v *Input) { v.Retryable = true }, pulse.TaskWaiting, false, "RETRYABLE_FAILURE"},
|
|
{"unknown_default", func(v *Input) { v.Outcome = "UNKNOWN" }, pulse.TaskNeedsReview, true, "RECONCILIATION_REQUIRED"},
|
|
{"idempotent_unknown", func(v *Input) { v.Outcome = "UNKNOWN"; v.Safety = pulse.RecoveryIdempotent }, pulse.TaskWaiting, true, "IDEMPOTENT_UNKNOWN"},
|
|
{"known_exhaustion", func(v *Input) { v.Retryable = true; v.Consumed = 3 }, pulse.TaskFailed, false, "ATTEMPT_LIMIT"},
|
|
{"unknown_exhaustion", func(v *Input) { v.Outcome = "UNKNOWN"; v.Safety = pulse.RecoveryIdempotent; v.Consumed = 3 }, pulse.TaskNeedsReview, true, "ATTEMPT_LIMIT"},
|
|
{"old_uncertainty_is_sticky", func(v *Input) { v.Uncertain = true; v.Safety = pulse.RecoveryIdempotent }, pulse.TaskNeedsReview, true, "NON_RETRYABLE_FAILURE"},
|
|
{"prior_uncertainty_and_prestart_exhaustion", func(v *Input) {
|
|
v.Uncertain = true
|
|
v.Safety = pulse.RecoveryIdempotent
|
|
v.Consumed = 3
|
|
v.Outcome = "NOT_STARTED"
|
|
}, pulse.TaskNeedsReview, true, "ATTEMPT_LIMIT"},
|
|
{"known_deadline", func(v *Input) { v.Retryable = true; v.Deadline = now.Add(time.Millisecond) }, pulse.TaskFailed, false, "DEADLINE_EXCEEDED"},
|
|
{"unknown_start_deadline", func(v *Input) {
|
|
v.Outcome = "UNKNOWN"
|
|
v.Safety = pulse.RecoveryIdempotent
|
|
until := now.Add(time.Millisecond)
|
|
v.StartDeadline = &until
|
|
}, pulse.TaskNeedsReview, true, "DEADLINE_EXCEEDED"},
|
|
{"cancel_prevents_retry", func(v *Input) { v.Retryable = true; v.CancelRequested = true }, pulse.TaskFailed, false, "CANCEL_REQUESTED"},
|
|
{"success_resolves_uncertainty", func(v *Input) { v.Uncertain = true; v.Outcome = "SUCCEEDED" }, pulse.TaskSucceeded, false, "SUCCEEDED"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
input := base
|
|
tc.change(&input)
|
|
got, err := Decide(input)
|
|
if err != nil || got.State != tc.state || got.Uncertain != tc.uncertain || got.Reason != tc.reason {
|
|
t.Fatalf("unexpected decision: %+v %v", got, err)
|
|
}
|
|
if (got.NextAt != nil) != (got.State == pulse.TaskWaiting) {
|
|
t.Fatal("retry time does not match state")
|
|
}
|
|
if got.NextAt != nil && (got.DelayMS < 500 || got.DelayMS > 1000 || !got.NextAt.Equal(now.Add(time.Duration(got.DelayMS)*time.Millisecond))) {
|
|
t.Fatal("first retry escaped bounded jitter")
|
|
}
|
|
})
|
|
}
|
|
// 截止恰好等于选定下一次时间也不能授予新尝试。
|
|
base.Retryable = true
|
|
out, err := Decide(base)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
base.Deadline = *out.NextAt
|
|
if out, err = Decide(base); err != nil || out.State != pulse.TaskFailed {
|
|
t.Fatal("inclusive deadline allowed retry")
|
|
}
|
|
}
|
|
func TestBackoffBoundedDeterministicAndNoOverflow(t *testing.T) {
|
|
policy := pulse.RetryPolicy{MaxAttempts: 100, InitialDelayMS: 1000, MaxDelayMS: 5000, AttemptTimeoutMS: 1}
|
|
distinct := map[uint64]bool{}
|
|
for n := 1; n <= 256; n++ {
|
|
id := core.ID(fmt.Sprintf("%032x", n))
|
|
first := DelayMS(policy, 1, id)
|
|
distinct[first] = true
|
|
if first != DelayMS(policy, 1, id) || first < 500 || first > 1000 {
|
|
t.Fatal("unstable first delay")
|
|
}
|
|
if next := DelayMS(policy, 2, id); next < 1000 || next > 2000 {
|
|
t.Fatal("exponential delay lost bound")
|
|
}
|
|
if last := DelayMS(policy, 100, id); last < 2500 || last > 5000 {
|
|
t.Fatal("delay did not saturate")
|
|
}
|
|
}
|
|
if len(distinct) < 100 {
|
|
t.Fatal("jitter failed to distribute identities")
|
|
}
|
|
policy.InitialDelayMS = 1
|
|
policy.MaxDelayMS = 1
|
|
if DelayMS(policy, 100, core.ID(fmt.Sprintf("%032x", 1))) != 1 {
|
|
t.Fatal("positive delay became zero")
|
|
}
|
|
policy.InitialDelayMS = 9223372036853
|
|
policy.MaxDelayMS = 9223372036854
|
|
n := DelayMS(policy, 100, core.ID(fmt.Sprintf("%032x", 1)))
|
|
if n < 4611686018427 || n > 9223372036854 || time.Duration(n)*time.Millisecond <= 0 {
|
|
t.Fatal("overflowing retry duration")
|
|
}
|
|
}
|