feat: add durable safe retries with uncertainty tracking and SDK examples

pulse/retry/decision_test.go

@@ -0,0 +1,102 @@
+package retry
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	core "git.yuntuo.hk/k2safe/orbit/model"
+	pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
+)
+
+func TestDecisionPreservesSafetyBudgetAndDeadline(t *testing.T) {
+	now := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC)
+	base := Input{Policy: pulse.RetryPolicy{MaxAttempts: 3, InitialDelayMS: 1000, MaxDelayMS: 5000, AttemptTimeoutMS: 10000}, Consumed: 1, Outcome: "FAILED", Safety: pulse.RecoveryReconcileOnly, Now: now, Deadline: now.Add(time.Hour), AttemptID: core.ID("11111111111111111111111111111111")}
+	for _, tc := range []struct {
+		name      string
+		change    func(*Input)
+		state     pulse.TaskState
+		uncertain bool
+		reason    string
+	}{
+		{"explicit_failure_default", func(*Input) {}, pulse.TaskFailed, false, "NON_RETRYABLE_FAILURE"},
+		{"safe_failure", func(v *Input) { v.Retryable = true }, pulse.TaskWaiting, false, "RETRYABLE_FAILURE"},
+		{"unknown_default", func(v *Input) { v.Outcome = "UNKNOWN" }, pulse.TaskNeedsReview, true, "RECONCILIATION_REQUIRED"},
+		{"idempotent_unknown", func(v *Input) { v.Outcome = "UNKNOWN"; v.Safety = pulse.RecoveryIdempotent }, pulse.TaskWaiting, true, "IDEMPOTENT_UNKNOWN"},
+		{"known_exhaustion", func(v *Input) { v.Retryable = true; v.Consumed = 3 }, pulse.TaskFailed, false, "ATTEMPT_LIMIT"},
+		{"unknown_exhaustion", func(v *Input) { v.Outcome = "UNKNOWN"; v.Safety = pulse.RecoveryIdempotent; v.Consumed = 3 }, pulse.TaskNeedsReview, true, "ATTEMPT_LIMIT"},
+		{"old_uncertainty_is_sticky", func(v *Input) { v.Uncertain = true; v.Safety = pulse.RecoveryIdempotent }, pulse.TaskNeedsReview, true, "NON_RETRYABLE_FAILURE"},
+		{"prior_uncertainty_and_prestart_exhaustion", func(v *Input) {
+			v.Uncertain = true
+			v.Safety = pulse.RecoveryIdempotent
+			v.Consumed = 3
+			v.Outcome = "NOT_STARTED"
+		}, pulse.TaskNeedsReview, true, "ATTEMPT_LIMIT"},
+		{"known_deadline", func(v *Input) { v.Retryable = true; v.Deadline = now.Add(time.Millisecond) }, pulse.TaskFailed, false, "DEADLINE_EXCEEDED"},
+		{"unknown_start_deadline", func(v *Input) {
+			v.Outcome = "UNKNOWN"
+			v.Safety = pulse.RecoveryIdempotent
+			until := now.Add(time.Millisecond)
+			v.StartDeadline = &until
+		}, pulse.TaskNeedsReview, true, "DEADLINE_EXCEEDED"},
+		{"cancel_prevents_retry", func(v *Input) { v.Retryable = true; v.CancelRequested = true }, pulse.TaskFailed, false, "CANCEL_REQUESTED"},
+		{"success_resolves_uncertainty", func(v *Input) { v.Uncertain = true; v.Outcome = "SUCCEEDED" }, pulse.TaskSucceeded, false, "SUCCEEDED"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			input := base
+			tc.change(&input)
+			got, err := Decide(input)
+			if err != nil || got.State != tc.state || got.Uncertain != tc.uncertain || got.Reason != tc.reason {
+				t.Fatalf("unexpected decision: %+v %v", got, err)
+			}
+			if (got.NextAt != nil) != (got.State == pulse.TaskWaiting) {
+				t.Fatal("retry time does not match state")
+			}
+			if got.NextAt != nil && (got.DelayMS < 500 || got.DelayMS > 1000 || !got.NextAt.Equal(now.Add(time.Duration(got.DelayMS)*time.Millisecond))) {
+				t.Fatal("first retry escaped bounded jitter")
+			}
+		})
+	}
+	// 截止恰好等于选定下一次时间也不能授予新尝试。
+	base.Retryable = true
+	out, err := Decide(base)
+	if err != nil {
+		t.Fatal(err)
+	}
+	base.Deadline = *out.NextAt
+	if out, err = Decide(base); err != nil || out.State != pulse.TaskFailed {
+		t.Fatal("inclusive deadline allowed retry")
+	}
+}
+func TestBackoffBoundedDeterministicAndNoOverflow(t *testing.T) {
+	policy := pulse.RetryPolicy{MaxAttempts: 100, InitialDelayMS: 1000, MaxDelayMS: 5000, AttemptTimeoutMS: 1}
+	distinct := map[uint64]bool{}
+	for n := 1; n <= 256; n++ {
+		id := core.ID(fmt.Sprintf("%032x", n))
+		first := DelayMS(policy, 1, id)
+		distinct[first] = true
+		if first != DelayMS(policy, 1, id) || first < 500 || first > 1000 {
+			t.Fatal("unstable first delay")
+		}
+		if next := DelayMS(policy, 2, id); next < 1000 || next > 2000 {
+			t.Fatal("exponential delay lost bound")
+		}
+		if last := DelayMS(policy, 100, id); last < 2500 || last > 5000 {
+			t.Fatal("delay did not saturate")
+		}
+	}
+	if len(distinct) < 100 {
+		t.Fatal("jitter failed to distribute identities")
+	}
+	policy.InitialDelayMS = 1
+	policy.MaxDelayMS = 1
+	if DelayMS(policy, 100, core.ID(fmt.Sprintf("%032x", 1))) != 1 {
+		t.Fatal("positive delay became zero")
+	}
+	policy.InitialDelayMS = 9223372036853
+	policy.MaxDelayMS = 9223372036854
+	n := DelayMS(policy, 100, core.ID(fmt.Sprintf("%032x", 1)))
+	if n < 4611686018427 || n > 9223372036854 || time.Duration(n)*time.Millisecond <= 0 {
+		t.Fatal("overflowing retry duration")
+	}
+}