k2safe/orbit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add durable safe retries with uncertainty tracking and SDK examples

Browse Source
This commit is contained in:
k2safe
2026-09-17 15:43:44 +08:00
parent 9e3d4be5cc
commit d65cf96ea7
44 changed files with 1300 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
internal/storage/mysql/worker_recovery.go
View File

@@ -10,11 +10,12 @@ import (
"git.yuntuo.hk/k2safe/orbit/internal/fault"
core "git.yuntuo.hk/k2safe/orbit/model"
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
"git.yuntuo.hk/k2safe/orbit/pulse/retry"
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
)
// Recover 是受控后台扫描入口，每批最多 100 项、每项一个短事务，可由多个节点竞争。
// 当前已授权执行的失联一律转核查，自动安全重试在 P3 接入冻结策略后实现。
// 已授权执行的失联保留效果未知，只有冻结的幂等恢复声明与剩余预算允许自动重试。
func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.RecoveryStats, error) {
var out worker.RecoveryStats
if limit < 1 || limit > 100 {
@@ -88,11 +89,16 @@ func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.Recover
return nil
}
changed = pulse.TaskFailed
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='FAILED',completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, now, now, c.ns, c.id, task.view.Revision)
var completed *time.Time = &now
if task.view.EffectUncertain {
changed = pulse.TaskNeedsReview
completed = nil
}
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, changed, completed, now, c.ns, c.id, task.view.Revision)
if err := exactlyOne(result, err); err != nil {
return err
}
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", "task.failed", map[string]any{"reason": "START_DEADLINE_EXCEEDED", "node_id": s.node})
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", resultEvent(changed), map[string]any{"reason": "START_DEADLINE_EXCEEDED", "node_id": s.node, "effect_uncertain": task.view.EffectUncertain})
}
if task.view.State != pulse.TaskLeased && task.view.State != pulse.TaskRunning {
return nil
@@ -118,29 +124,31 @@ func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.Recover
return errors.New("active task references ended attempt")
}
attemptState := pulse.AttemptExpired
next := pulse.TaskFailed
outcome := "NOT_STARTED"
reason := "LEASE_EXPIRED_BEFORE_START"
var completed *time.Time = &now
available := task.view.AvailableAt
if attempt.AuthorizedAt != nil {
attemptState = pulse.AttemptUnknown
next = pulse.TaskNeedsReview
outcome = "UNKNOWN"
reason = "EXECUTION_OUTCOME_UNKNOWN"
completed = nil
} else if task.view.AttemptCount < task.maxAttempts {
// 未批准开始的尝试可以有界重派，次数不重置；开始截止与总截止仍有效。
nextAt := now.Add(time.Second)
if nextAt.Before(task.view.DeadlineAt) && (task.view.StartDeadlineAt == nil || nextAt.Before(*task.view.StartDeadlineAt)) {
next = pulse.TaskWaiting
available = nextAt
completed = nil
}
}
decision, err := retry.Decide(retryFacts(task, attempt.ID, now, outcome, false))
if err != nil {
return err
}
next := decision.State
var completed *time.Time
if next.Terminal() {
completed = &now
}
available := task.view.AvailableAt
if decision.NextAt != nil {
available = *decision.NextAt
}
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, reason, now, c.ns, attempt.ID, attempt.Revision)
if err := exactlyOne(result, err); err != nil {
return err
}
result, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,lease_until=NULL,available_at=?,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, next, available, completed, now, c.ns, c.id, task.view.Revision)
result, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,effect_uncertain=?,lease_until=NULL,available_at=?,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, next, decision.Uncertain, available, completed, now, c.ns, c.id, task.view.Revision)
if err := exactlyOne(result, err); err != nil {
return err
}
@@ -148,7 +156,7 @@ func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.Recover
return err
}
changed = next
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", resultEvent(next), map[string]any{"attempt_id": attempt.ID, "epoch": fmt.Sprint(attempt.Epoch), "reason": reason, "node_id": s.node})
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", resultEvent(next), map[string]any{"attempt_id": attempt.ID, "epoch": fmt.Sprint(attempt.Epoch), "reason": reason, "node_id": s.node, "effect_uncertain": decision.Uncertain, "decision": decision.Reason, "next_available_at": decision.NextAt, "delay_ms": decision.DelayMS, "attempt_count": task.view.AttemptCount, "max_attempts": task.maxAttempts})
})
if err != nil {
return out, err