feat: recover poll activations within original round budgets

This commit is contained in:
2026-09-18 05:40:08 +08:00
parent 1d1a441167
commit 56b9a2ccfe
32 changed files with 1051 additions and 65 deletions

View File

@@ -64,6 +64,7 @@ func TestIntegrationApplicationResourcesUpgrade(t *testing.T) {
retryFinish(t, g, finished, "SUCCEEDED", false)
assertApplicationCounts(t, f, 4, 2, 1)
// 还原后续新增表和字段,得到真实 v9 结构;现有任务内容保持不变。
f.exec(`ALTER TABLE weave_poll_states DROP CHECK ck_poll_recovery_budget, DROP COLUMN first_round, DROP COLUMN first_round_attempts_used`)
f.exec(`ALTER TABLE pulse_tasks DROP INDEX ix_task_workflow_owner`)
f.exec(`ALTER TABLE weave_instances DROP CHECK ck_workflow_stop_intent, DROP COLUMN stop_kind, DROP COLUMN stop_requested_at`)
f.exec(`ALTER TABLE weave_signals DROP CHECK ck_signal_version_sequence, DROP CHECK ck_signal_pending_identity, DROP INDEX ix_signal_expiry, DROP INDEX ix_signal_activation, DROP COLUMN payload_version, DROP COLUMN target_activation_sequence, DROP COLUMN evidence_ref, DROP COLUMN disposition, DROP COLUMN available_at, MODIFY COLUMN type VARCHAR(64) NOT NULL COMMENT '事件或信号的稳定类型标识', MODIFY COLUMN correlation_key VARCHAR(191) NOT NULL COMMENT '外部业务操作关联键,防止旧信号误用于新工作'`)

View File

@@ -0,0 +1,8 @@
-- 查询恢复保留业务轮次编号及同轮历史尝试,不引入外键或重新计算截止。
ALTER TABLE weave_poll_states
ADD COLUMN first_round INT UNSIGNED NOT NULL DEFAULT 1 COMMENT '本激活保存的首个业务查询轮次;人工恢复沿用原轮次,不从1重新计数',
ADD COLUMN first_round_attempts_used INT UNSIGNED NOT NULL DEFAULT 0 COMMENT '首个业务轮次在旧激活中已消耗的执行尝试数;新任务只获得原单轮预算的剩余部分',
ADD CONSTRAINT ck_poll_recovery_budget CHECK (first_round BETWEEN 1 AND max_rounds AND first_round_attempts_used < max_attempts_per_round AND ((used_rounds=0 AND first_round=1 AND first_round_attempts_used=0) OR first_round<=used_rounds));
ALTER TABLE weave_poll_rounds
MODIFY COLUMN `round` INT UNSIGNED NOT NULL COMMENT '业务查询轮次编号,从首次激活起连续计数;人工恢复同一轮时保留编号,由激活身份区分对应任务';

View File

@@ -0,0 +1,37 @@
package mysql
import (
"context"
"database/sql"
"encoding/json"
"errors"
"time"
core "git.yuntuo.hk/k2safe/orbit/model"
)
type workflowActivationBase struct {
id core.ID
previous *core.ID
sequence uint32
started time.Time
input json.RawMessage
}
// workflowActivationOrigin 按同一逻辑步骤的连续激活序号定位首次预算;新激活创建时间不成为新的计时起点。
func workflowActivationOrigin(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep) (workflowActivationBase, error) {
var base workflowActivationBase
var raw []byte
err := tx.QueryRowContext(ctx, `SELECT a.sequence,o.id,o.created_at,o.input,p.id FROM weave_activations a
JOIN weave_activations o ON o.namespace_id=a.namespace_id AND o.workflow_id=a.workflow_id AND o.step_id=a.step_id AND o.sequence=1
LEFT JOIN weave_activations p ON p.namespace_id=a.namespace_id AND p.workflow_id=a.workflow_id AND p.step_id=a.step_id AND p.sequence=a.sequence-1
WHERE a.namespace_id=? AND a.workflow_id=? AND a.step_id=? AND a.id=?`, g.view.NamespaceID, g.view.ID, row.view.ID, row.view.ActivationID).Scan(&base.sequence, &base.id, &base.started, &raw, &base.previous)
if err != nil {
return base, err
}
base.input = raw
if base.sequence == 0 || base.sequence > 10000 || base.sequence == 1 && (row.view.ActivationID == nil || *row.view.ActivationID != base.id || base.previous != nil) || base.sequence > 1 && base.previous == nil || !sameWorkflowJSON(row.input, base.input) {
return base, errors.New("workflow activation origin invalid")
}
return base, nil
}

View File

@@ -79,7 +79,7 @@ func (r *WorkflowRuntime) advanceClock(ctx context.Context, tx *sql.Tx, g *workf
return changed, nil
}
}
return r.advancePoll(ctx, tx, g, row, c, activated, allow, now)
return r.advancePoll(ctx, tx, g, row, c, allow, now)
}
if checkpoint.Kind == "SIGNAL" {
return r.advanceSignalClock(ctx, tx, g, row, c, allow, now)

View File

@@ -92,7 +92,7 @@ func readWorkflowResult(ctx context.Context, tx *sql.Tx, g *workflowGraph, c wor
if node.Poll != nil {
taskDefinition = &node.Poll.Task
var round uint32
err = tx.QueryRowContext(ctx, `SELECT r.round FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=? AND r.workflow_id=? AND r.activation_id=? AND r.task_id=? AND r.round BETWEEN 1 AND p.used_rounds`, c.ns, c.workflow, source.activation, source.task).Scan(&round)
err = tx.QueryRowContext(ctx, `SELECT r.round FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=? AND r.workflow_id=? AND r.activation_id=? AND r.task_id=? AND r.round BETWEEN p.first_round AND p.used_rounds`, c.ns, c.workflow, source.activation, source.task).Scan(&round)
if errors.Is(err, sql.ErrNoRows) || direct != nil {
return nil, workflowSourceError("RESULT_POLL_OWNER")
}
@@ -167,14 +167,13 @@ func (r *WorkflowRuntime) consume(ctx context.Context, tx *sql.Tx, g *workflowGr
}
node, _ := g.program.Step(row.view.Key)
if node.Poll != nil {
var used, max uint32
var deadline time.Time
var current *core.ID
var resolved *time.Time
if err := tx.QueryRowContext(ctx, `SELECT used_rounds,max_rounds,deadline_at,current_task_id,resolved_at FROM weave_poll_states WHERE namespace_id=? AND workflow_id=? AND activation_id=? FOR UPDATE`, c.ns, c.workflow, source.activation).Scan(&used, &max, &deadline, &current, &resolved); err != nil {
budget, lastTask, err := readWorkflowPollBudget(ctx, tx, g, row)
if err != nil {
return false, err
}
if resolved != nil || current == nil || *current != source.task {
used, max, deadline := budget.UsedRounds, budget.MaxRounds, budget.DeadlineAt
current := budget.CurrentTaskID
if budget.ResolvedAt != nil || current == nil || *current != source.task || lastTask != source.task {
return true, r.confirmWorkflowResult(ctx, tx, g, c, "PROCESSED", "STALE_POLL_ROUND", now)
}
if node.Poll.FinalSignal != nil {

View File

@@ -61,7 +61,7 @@ func appendWorkflowResult(ctx context.Context, tx *sql.Tx, ns, eventID, taskID c
return errors.New("poll activation cannot own a direct task")
}
var round uint32
err = tx.QueryRowContext(ctx, `SELECT r.round FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=? AND r.workflow_id=? AND r.activation_id=? AND r.task_id=? AND r.round BETWEEN 1 AND p.used_rounds`, ns, workflow, ownerID, taskID).Scan(&round)
err = tx.QueryRowContext(ctx, `SELECT r.round FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=? AND r.workflow_id=? AND r.activation_id=? AND r.task_id=? AND r.round BETWEEN p.first_round AND p.used_rounds`, ns, workflow, ownerID, taskID).Scan(&round)
if errors.Is(err, sql.ErrNoRows) {
return errors.New("workflow poll task association invalid")
}

View File

@@ -17,36 +17,66 @@ import (
)
// readWorkflowPollState 校验当前激活的完整轮次与原预算;时钟和最终信号共用相同检查。
func readWorkflowPollState(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, activated time.Time) (weave.PollState, core.ID, error) {
func readWorkflowPollState(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep) (weave.PollState, core.ID, error) {
p, last, err := readWorkflowPollBudget(ctx, tx, g, row)
if err == nil && p.ResolvedAt != nil {
err = errors.New("unsettled poll step has resolved budget")
}
return p, last, err
}
// readWorkflowPollBudget 也允许读取已结清预算,供有证据的人工恢复核对;不重新计时。
func readWorkflowPollBudget(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep) (weave.PollState, core.ID, error) {
var p weave.PollState
err := tx.QueryRowContext(ctx, `SELECT id,revision,max_rounds,used_rounds,max_attempts_per_round,interval_ms,deadline_at,next_poll_at,current_task_id,resolved_at FROM weave_poll_states WHERE namespace_id=? AND workflow_id=? AND activation_id=? FOR UPDATE`, g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(&p.ID, &p.Revision, &p.MaxRounds, &p.UsedRounds, &p.MaxAttemptsPerRound, &p.IntervalMS, &p.DeadlineAt, &p.NextPollAt, &p.CurrentTaskID, &p.ResolvedAt)
err := tx.QueryRowContext(ctx, `SELECT id,revision,max_rounds,used_rounds,first_round,first_round_attempts_used,max_attempts_per_round,interval_ms,deadline_at,next_poll_at,current_task_id,resolved_at FROM weave_poll_states WHERE namespace_id=? AND workflow_id=? AND activation_id=? FOR UPDATE`, g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(&p.ID, &p.Revision, &p.MaxRounds, &p.UsedRounds, &p.FirstRound, &p.FirstRoundAttemptsUsed, &p.MaxAttemptsPerRound, &p.IntervalMS, &p.DeadlineAt, &p.NextPollAt, &p.CurrentTaskID, &p.ResolvedAt)
if err != nil {
return p, "", err
}
base, err := workflowActivationOrigin(ctx, tx, g, row)
if err != nil {
return p, "", err
}
node, _ := g.program.Step(row.view.Key)
policy := node.Poll
if policy == nil || p.MaxRounds != policy.MaxRounds || p.MaxAttemptsPerRound != policy.Task.Retry.MaxAttempts || p.IntervalMS != policy.IntervalMS || p.UsedRounds < 1 || !p.DeadlineAt.Equal(minimumTime(g.view.DeadlineAt, activated.Add(time.Duration(policy.TimeoutMS)*time.Millisecond))) {
if policy == nil || p.MaxRounds != policy.MaxRounds || p.MaxAttemptsPerRound != policy.Task.Retry.MaxAttempts || p.IntervalMS != policy.IntervalMS || p.UsedRounds < 1 || p.UsedRounds > p.MaxRounds || p.FirstRound < 1 || p.FirstRound > p.UsedRounds || p.FirstRoundAttemptsUsed >= p.MaxAttemptsPerRound || !p.DeadlineAt.Equal(minimumTime(g.view.DeadlineAt, base.started.Add(time.Duration(policy.TimeoutMS)*time.Millisecond))) {
return p, "", errors.New("workflow poll policy mismatch")
}
if p.ResolvedAt != nil {
return p, "", errors.New("unsettled poll step has resolved budget")
if base.sequence == 1 && (p.FirstRound != 1 || p.FirstRoundAttemptsUsed != 0) {
return p, "", errors.New("initial poll recovery budget invalid")
}
if base.previous != nil {
var used uint32
var deadline time.Time
var settled *time.Time
if err := tx.QueryRowContext(ctx, `SELECT used_rounds,deadline_at,resolved_at FROM weave_poll_states WHERE namespace_id=? AND workflow_id=? AND activation_id=?`, g.view.NamespaceID, g.view.ID, base.previous).Scan(&used, &deadline, &settled); err != nil {
return p, "", err
}
if used != p.FirstRound || !deadline.Equal(p.DeadlineAt) || settled == nil {
return p, "", errors.New("poll recovery parent budget invalid")
}
if _, err := readWorkflowPollRoundHistory(ctx, tx, g, row, p, p.FirstRound); err != nil {
return p, "", err
}
}
var count, minRound, maxRound uint32
if err := tx.QueryRowContext(ctx, "SELECT COUNT(*),COALESCE(MIN(`round`),0),COALESCE(MAX(`round`),0) FROM weave_poll_rounds WHERE namespace_id=? AND workflow_id=? AND activation_id=?", g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(&count, &minRound, &maxRound); err != nil {
return p, "", err
}
if count != p.UsedRounds || minRound != 1 || maxRound != p.UsedRounds {
if count != p.UsedRounds-p.FirstRound+1 || minRound != p.FirstRound || maxRound != p.UsedRounds {
return p, "", errors.New("workflow poll rounds incomplete")
}
var lastTask core.ID
if err := tx.QueryRowContext(ctx, "SELECT task_id FROM weave_poll_rounds WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND `round`=?", g.view.NamespaceID, g.view.ID, row.view.ActivationID, p.UsedRounds).Scan(&lastTask); err != nil {
return p, "", err
}
if p.CurrentTaskID != nil && (*p.CurrentTaskID != lastTask || p.NextPollAt != nil) || p.ResolvedAt != nil && (p.CurrentTaskID != nil || p.NextPollAt != nil) {
return p, "", errors.New("poll budget execution pointers invalid")
}
return p, lastTask, nil
}
func (r *WorkflowRuntime) advancePoll(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, c workflowCandidate, activated time.Time, allow bool, now time.Time) (bool, error) {
p, lastTask, err := readWorkflowPollState(ctx, tx, g, row, activated)
func (r *WorkflowRuntime) advancePoll(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, c workflowCandidate, allow bool, now time.Time) (bool, error) {
p, lastTask, err := readWorkflowPollState(ctx, tx, g, row)
if err != nil {
return false, err
}
@@ -196,11 +226,17 @@ func validatePollTaskOwner(ctx context.Context, tx *sql.Tx, g *workflowGraph, ro
// pollTaskInput 复用首轮已经解析并冻结的输入、业务键及互斥键;不重算依赖可能已改变的表达式。
func pollTaskInput(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, spec definition.Task) (service.SubmitInput, error) {
var first core.ID
if err := tx.QueryRowContext(ctx, "SELECT task_id FROM weave_poll_rounds WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND `round`=1", g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(&first); err != nil {
origin, err := workflowActivationOrigin(ctx, tx, g, row)
if err != nil {
return service.SubmitInput{}, err
}
if err := validatePollTaskOwner(ctx, tx, g, row, first); err != nil {
originRow := *row
originRow.view.ActivationID = &origin.id
var first core.ID
if err := tx.QueryRowContext(ctx, "SELECT task_id FROM weave_poll_rounds WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND `round`=1", g.view.NamespaceID, g.view.ID, origin.id).Scan(&first); err != nil {
return service.SubmitInput{}, err
}
if err := validatePollTaskOwner(ctx, tx, g, &originRow, first); err != nil {
return service.SubmitInput{}, err
}
base, err := readTask(ctx, tx, g.view.NamespaceID, first, false)

View File

@@ -0,0 +1,95 @@
package mysql
import (
"context"
"database/sql"
"errors"
"time"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/pulse/service"
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
)
type workflowPollRoundHistory struct {
first, last service.TaskView
used, count uint32
}
// readWorkflowPollRoundHistory 跨激活核验同一业务轮的请求链,计数来自真实尝试行;新业务轮不继承上一轮尝试。
func readWorkflowPollRoundHistory(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, p weave.PollState, round uint32) (workflowPollRoundHistory, error) {
var out workflowPollRoundHistory
node, ok := g.program.Step(row.view.Key)
if !ok || node.Poll == nil {
return out, errors.New("poll history requires fixed poll definition")
}
frozen, err := pollTaskInput(ctx, tx, g, row, node.Poll.Task)
if err != nil {
return out, err
}
rows, err := tx.QueryContext(ctx, "SELECT a.id,a.sequence,a.input,r.task_id,p.first_round,p.first_round_attempts_used,p.used_rounds,p.deadline_at,(SELECT COUNT(*) FROM pulse_attempts x WHERE x.namespace_id=r.namespace_id AND x.task_id=r.task_id) FROM weave_activations a JOIN weave_poll_states p ON p.namespace_id=a.namespace_id AND p.workflow_id=a.workflow_id AND p.activation_id=a.id JOIN weave_poll_rounds r ON r.namespace_id=a.namespace_id AND r.workflow_id=a.workflow_id AND r.activation_id=a.id AND r.`round`=? WHERE a.namespace_id=? AND a.workflow_id=? AND a.step_id=? ORDER BY a.sequence LIMIT 101", round, g.view.NamespaceID, g.view.ID, row.view.ID)
if err != nil {
return out, err
}
type prior struct {
activation, task core.ID
sequence, first, carried, used, attempts uint32
deadline time.Time
input []byte
}
var history []prior
for rows.Next() {
var v prior
if err := rows.Scan(&v.activation, &v.sequence, &v.input, &v.task, &v.first, &v.carried, &v.used, &v.deadline, &v.attempts); err != nil {
rows.Close()
return out, err
}
history = append(history, v)
}
err = rows.Err()
rows.Close()
if err != nil {
return out, err
}
if len(history) == 0 || len(history) > int(p.MaxAttemptsPerRound) || history[len(history)-1].activation != *row.view.ActivationID {
return out, errors.New("poll round recovery history incomplete")
}
for i, prior := range history {
if prior.first > round || prior.used < round || !prior.deadline.Equal(p.DeadlineAt) || !sameWorkflowJSON(prior.input, row.input) || i > 0 && (prior.sequence != history[i-1].sequence+1 || prior.first != round) || i < len(history)-1 && prior.used != round || prior.first == round && prior.carried != out.used {
return out, errors.New("poll round recovery budget mismatch")
}
task, err := readTask(ctx, tx, g.view.NamespaceID, prior.task, false)
if err != nil {
return out, err
}
owner := *row
owner.view.ActivationID = &prior.activation
if err := validatePollTaskOwner(ctx, tx, g, &owner, task.ID); err != nil {
return out, err
}
if i == 0 {
out.first = task
if !task.DeadlineAt.Equal(minimumTime(task.CreatedAt.Add(time.Duration(frozen.TaskTimeoutMS)*time.Millisecond), p.DeadlineAt)) {
return out, errors.New("poll round original deadline invalid")
}
}
expected := frozen.Retry
if out.used >= expected.MaxAttempts {
return out, errors.New("poll round history exceeds attempts")
}
expected.MaxAttempts -= out.used
if task.Retry != expected || task.AttemptCount > task.Retry.MaxAttempts || prior.attempts != task.AttemptCount || task.QueueID != frozen.QueueID || task.Handler != frozen.Handler || task.HandlerVersion != frozen.HandlerVersion || task.PayloadVersion != frozen.PayloadVersion || task.Priority != frozen.Priority || task.RecoverySafety != frozen.RecoverySafety || !sameWorkflowJSON(task.Payload, frozen.Payload) || task.BusinessKey != frozen.BusinessKey || task.ConcurrencyKey != frozen.ConcurrencyKey || !task.DeadlineAt.Equal(out.first.DeadlineAt) || task.StartDeadlineAt != nil {
return out, errors.New("poll round frozen task mismatch")
}
if i == 0 && task.RetryOf != nil || i > 0 && (task.RetryOf == nil || *task.RetryOf != out.last.ID) {
return out, errors.New("poll round task ancestry invalid")
}
if i < len(history)-1 && (!task.State.Terminal() || task.EffectUncertain) {
return out, errors.New("poll round previous execution unresolved")
}
out.last = task
out.used += task.AttemptCount
}
out.count = uint32(len(history))
return out, nil
}

View File

@@ -0,0 +1,383 @@
package mysql
import (
"context"
"database/sql"
"os"
"sync"
"testing"
"time"
"git.yuntuo.hk/k2safe/orbit/internal/fault"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/pulse/service"
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
"git.yuntuo.hk/k2safe/orbit/weave/definition"
"git.yuntuo.hk/k2safe/orbit/weave/instance"
"git.yuntuo.hk/k2safe/orbit/weave/recovery"
)
func recoveryPollSpec(queue core.ID, final bool) definition.Specification {
s := pollingSpec(queue)
if final {
s = finalSignalSpec(queue)
}
p := s.Body.Steps[0].Poll
p.TimeoutMS = 60000
p.Task.TimeoutMS = 30000
p.Task.Retry.MaxAttempts = 3
p.Task.Retry.InitialDelayMS = 1000
p.Task.Retry.MaxDelayMS = 1000
if final {
p.FinalSignal.TimeoutMS = p.TimeoutMS
}
return s
}
func retryPollStep(t *testing.T, f *executionFixture, id core.ID) recovery.Result {
t.Helper()
out, err := recovery.New(NewWorkflowRecoveryStore(f.db)).RetryStep(f.ctx, f.token, id, string(executionID(t)), recovery.RetryStepInput{StepDecision: recoveryStep(t, f, id, "query"), SafeToRetry: true})
if err != nil {
t.Fatal(err)
}
return out
}
func failedPoll(t *testing.T, ctx context.Context, db *sql.DB, final bool) (*executionFixture, instance.View, worker.AttemptRef) {
t.Helper()
f := newClockFixture(t, ctx, db)
v := runtimeStart(t, f, recoveryPollSpec(f.resources.QueueID, final))
grantWorkflowRecovery(f)
ref := retryStart(t, f)
retryFinish(t, f, ref, "FAILED", false)
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
return f, v, ref
}
func TestIntegrationWorkflowPollRecovery(t *testing.T) {
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("requires disposable MySQL")
}
ctx, stop := context.WithTimeout(context.Background(), 150*time.Second)
defer stop()
db, err := Open(ctx, dsn, 32, 8)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
api := recovery.New(NewWorkflowRecoveryStore(db))
t.Run("retry_same_round_then_next_round_and_retry_again", func(t *testing.T) {
f, v, ref := failedPoll(t, ctx, db, false)
first := f.get(ref.TaskID)
original := recoveryStep(t, f, v.ID, "query")
out := retryPollStep(t, f, v.ID)
if out.Task.Retry.MaxAttempts != 2 || !out.Task.DeadlineAt.Equal(first.DeadlineAt) || *out.Task.RetryOf != first.ID || *out.Step.ActivationID == original.ActivationID || out.Step.TaskID != nil || !sameWorkflowJSON(out.Task.Payload, first.Payload) {
t.Fatal("recovery reset original identity, task deadline or budget")
}
if f.count("weave_poll_states", "first_round=1 AND first_round_attempts_used=1 AND used_rounds=1 AND resolved_at IS NULL") != 1 || f.count("weave_activations", "id=? AND state='NEEDS_REVIEW'", original.ActivationID) != 1 {
t.Fatal("old activation or round changed")
}
f.exec(`UPDATE weave_inbox SET state='PENDING',processed_at=NULL WHERE namespace_id=? AND source_task_id=?`, f.resources.NamespaceID, first.ID)
runtimeConsume(t, f, newRuntime(t, db), first.ID)
if f.count("weave_inbox", "source_task_id=? AND error_code='STALE_ACTIVATION'", first.ID) != 1 {
t.Fatal("old result applied to new activation")
}
continueWorkflow(t, f, v.ID)
ref = retryStart(t, f)
finishPoll(t, f, ref, "PENDING")
c := clockCandidate(t, f, v.ID, "query")
waitClockDue(t, f, c)
runClock(t, f, c)
ref = retryStart(t, f)
second := f.get(ref.TaskID)
if second.Retry.MaxAttempts != 3 || second.RetryOf != nil || second.BusinessKey != first.BusinessKey || second.ConcurrencyKey != first.ConcurrencyKey {
t.Fatal("next business round did not receive independent original policy")
}
retryFinish(t, f, ref, "FAILED", false)
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
out = retryPollStep(t, f, v.ID)
if out.Task.Retry.MaxAttempts != 2 || !out.Task.DeadlineAt.Equal(second.DeadlineAt) || *out.Task.RetryOf != second.ID || f.count("weave_poll_states", "first_round=2 AND used_rounds=2 AND first_round_attempts_used=1") != 1 {
t.Fatal("second round recovery reset logical round")
}
continueWorkflow(t, f, v.ID)
ref = retryStart(t, f)
finishPoll(t, f, ref, "PENDING")
c = clockCandidate(t, f, v.ID, "query")
waitClockDue(t, f, c)
runClock(t, f, c)
ref = retryStart(t, f)
finishPoll(t, f, ref, "DONE")
runtimeView(t, f, v.ID, "SUCCEEDED")
if f.count("pulse_tasks", "1=1") != 5 || f.count("weave_poll_rounds", "`round`=2") != 2 || f.count("weave_poll_states", "first_round=2 AND used_rounds=3 AND resolved_at IS NOT NULL") != 1 || f.used() != 0 {
t.Fatal("extra query, lost round or leaked permits")
}
})
t.Run("repeated_same_round_exhausts_original_attempts", func(t *testing.T) {
f, v, _ := failedPoll(t, ctx, db, false)
for remaining := uint32(2); remaining > 0; remaining-- {
out := retryPollStep(t, f, v.ID)
if out.Task.Retry.MaxAttempts != remaining {
t.Fatal("attempt budget reset")
}
continueWorkflow(t, f, v.ID)
ref := retryStart(t, f)
retryFinish(t, f, ref, "FAILED", false)
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
}
_, err := api.RetryStep(ctx, f.token, v.ID, string(executionID(t)), recovery.RetryStepInput{StepDecision: recoveryStep(t, f, v.ID, "query"), SafeToRetry: true})
if fault.Code(err) != "STATE_CONFLICT" || f.count("pulse_tasks", "1=1") != 3 || f.count("weave_poll_rounds", "`round`=1") != 3 {
t.Fatal("original round attempts exceeded", err)
}
})
t.Run("successful_pending_cannot_bypass_business_round_limit", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
spec := recoveryPollSpec(f.resources.QueueID, false)
spec.Body.Steps[0].Poll.MaxRounds = 1
v := runtimeStart(t, f, spec)
grantWorkflowRecovery(f)
ref := retryStart(t, f)
finishPoll(t, f, ref, "PENDING")
_, err := api.RetryStep(ctx, f.token, v.ID, string(executionID(t)), recovery.RetryStepInput{StepDecision: recoveryStep(t, f, v.ID, "query"), SafeToRetry: true})
if fault.Code(err) != "STATE_CONFLICT" || f.count("pulse_tasks", "1=1") != 1 {
t.Fatal("PENDING restarted original business round", err)
}
})
t.Run("same_key_concurrency_receipt_and_permission", func(t *testing.T) {
f, v, _ := failedPoll(t, ctx, db, false)
in := recovery.RetryStepInput{StepDecision: recoveryStep(t, f, v.ID, "query"), SafeToRetry: true}
key := string(executionID(t))
var wg sync.WaitGroup
results := make(chan recovery.Result, 10)
errs := make(chan error, 10)
for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
defer wg.Done()
out, err := api.RetryStep(ctx, f.token, v.ID, key, in)
results <- out
errs <- err
}()
}
wg.Wait()
close(results)
close(errs)
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
var task core.ID
fresh := 0
for out := range results {
if !out.Deduplicated {
fresh++
}
if task != "" && task != out.Task.ID {
t.Fatal("duplicate task")
}
task = out.Task.ID
}
if fresh != 1 || f.count("pulse_tasks", "1=1") != 2 || f.count("orbit_audit_events", "action='workflow.step.retry'") != 1 {
t.Fatal("duplicate recovery effects")
}
stale := in
stale.ExpectedRevision = recoveryDecision(t, f, v.ID).ExpectedRevision
if _, err := api.RetryStep(ctx, f.token, v.ID, string(executionID(t)), stale); fault.Code(err) != "STATE_CONFLICT" {
t.Fatal("old activation accepted", err)
}
f.exec(`UPDATE orbit_credentials SET permissions=JSON_SET(permissions,'$.queues',JSON_ARRAY(?)) WHERE namespace_id=? AND id=?`, string(executionID(t)), f.resources.NamespaceID, f.resources.CredentialID)
if _, err := api.RetryStep(ctx, f.token, v.ID, key, in); fault.Code(err) != "NOT_FOUND" {
t.Fatal("receipt bypassed current permissions", err)
}
})
t.Run("audit_failure_rolls_back_and_same_key_retries", func(t *testing.T) {
f, v, _ := failedPoll(t, ctx, db, false)
in := recovery.RetryStepInput{StepDecision: recoveryStep(t, f, v.ID, "query"), SafeToRetry: true}
key := string(executionID(t))
trigger := "reject_poll_recovery_" + string(executionID(t))
f.exec("CREATE TRIGGER " + trigger + " BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='controlled recovery audit failure'")
_, err := api.RetryStep(ctx, f.token, v.ID, key, in)
f.exec("DROP TRIGGER " + trigger)
if err == nil || f.count("pulse_tasks", "1=1") != 1 || f.count("weave_poll_states", "1=1") != 1 || f.count("weave_pending_advances", "1=1") != 0 || recoveryDecision(t, f, v.ID).ExpectedRevision != in.ExpectedRevision {
t.Fatal("partial recovery committed", err)
}
assertApplicationCounts(t, f, 0, 0, 0)
if _, err := api.RetryStep(ctx, f.token, v.ID, key, in); err != nil {
t.Fatal(err)
}
assertApplicationCounts(t, f, 1, 0, 0)
})
t.Run("unknown_requires_execution_review_before_retry", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
v := runtimeStart(t, f, recoveryPollSpec(f.resources.QueueID, false))
grantWorkflowRecovery(f)
ref := retryStart(t, f)
retryFinish(t, f, ref, "UNKNOWN", false)
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
_, err := api.RetryStep(ctx, f.token, v.ID, string(executionID(t)), recovery.RetryStepInput{StepDecision: recoveryStep(t, f, v.ID, "query"), SafeToRetry: true})
if fault.Code(err) != "STATE_CONFLICT" {
t.Fatal("unknown execution retried", err)
}
reviewWorkflowTask(t, f, v.ID, ref.TaskID, "FAILED")
_, err = api.RetryStep(ctx, f.token, v.ID, string(executionID(t)), recovery.RetryStepInput{StepDecision: recoveryStep(t, f, v.ID, "query"), SafeToRetry: true})
if fault.Code(err) != "STATE_CONFLICT" {
t.Fatal("pending review result skipped", err)
}
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
out := retryPollStep(t, f, v.ID)
if out.Task.Retry.MaxAttempts != 2 || f.count("pulse_attempts", "state='UNKNOWN'") != 1 {
t.Fatal("unknown evidence lost or budget reset")
}
})
t.Run("final_signal_new_generation_keeps_deadline_and_old_signals_stale", func(t *testing.T) {
f, v, _ := failedPoll(t, ctx, db, true)
grantWorkflowSignal(f)
var deadline time.Time
if err := db.QueryRowContext(ctx, `SELECT deadline_at FROM weave_waits WHERE namespace_id=?`, f.resources.NamespaceID).Scan(&deadline); err != nil {
t.Fatal(err)
}
out := retryPollStep(t, f, v.ID)
if f.count("weave_waits", "state='WAITING' AND deadline_at=? AND winner_signal_id IS NULL", deadline) != 1 {
t.Fatal("new wait reset deadline or reused winner")
}
old := sendSignal(t, f, v, pollSignalInput(v))
if old.Signal.Disposition != "STALE_ACTIVATION" || old.Signal.State != "REJECTED" {
t.Fatal("old generation accepted", old)
}
continueWorkflow(t, f, v.ID)
ref := retryStart(t, f)
if ref.TaskID != out.Task.ID {
t.Fatal("wrong query activated")
}
finishPoll(t, f, ref, "PENDING")
in := pollSignalInput(v)
in.TargetActivationSequence = 2
sendSignal(t, f, v, in)
runClock(t, f, clockCandidate(t, f, v.ID, "query"))
runtimeView(t, f, v.ID, "SUCCEEDED")
if f.count("pulse_tasks", "1=1") != 2 || f.count("weave_waits", "state='SATISFIED' AND winner_signal_id IS NOT NULL") != 1 {
t.Fatal("final signal lost or extra round issued")
}
})
t.Run("original_task_deadline_not_extended", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
spec := recoveryPollSpec(f.resources.QueueID, false)
spec.Body.Steps[0].Poll.Task.TimeoutMS = 1200
spec.Body.Steps[0].Poll.Task.Retry.AttemptTimeoutMS = 1000
v := runtimeStart(t, f, spec)
grantWorkflowRecovery(f)
ref := retryStart(t, f)
retryFinish(t, f, ref, "FAILED", false)
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
time.Sleep(time.Until(f.get(ref.TaskID).DeadlineAt.Add(20 * time.Millisecond)))
_, err := api.RetryStep(ctx, f.token, v.ID, string(executionID(t)), recovery.RetryStepInput{StepDecision: recoveryStep(t, f, v.ID, "query"), SafeToRetry: true})
if fault.Code(err) != "STATE_CONFLICT" || f.count("pulse_tasks", "1=1") != 1 {
t.Fatal("original request deadline restarted", err)
}
})
t.Run("corrupt_prior_attempt_count_is_rejected", func(t *testing.T) {
f, v, ref := failedPoll(t, ctx, db, false)
f.exec(`UPDATE pulse_tasks SET attempt_count=0 WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, ref.TaskID)
_, err := api.RetryStep(ctx, f.token, v.ID, string(executionID(t)), recovery.RetryStepInput{StepDecision: recoveryStep(t, f, v.ID, "query"), SafeToRetry: true})
f.exec(`UPDATE pulse_tasks SET attempt_count=1 WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, ref.TaskID)
if err == nil || f.count("pulse_tasks", "1=1") != 1 {
t.Fatal("attempt row proof bypassed")
}
})
t.Run("zero_attempt_recoveries_are_also_bounded", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
v := runtimeStart(t, f, recoveryPollSpec(f.resources.QueueID, false))
grantWorkflowRecovery(f)
for i := 0; i < 3; i++ {
var task core.ID
if err := db.QueryRowContext(ctx, `SELECT current_task_id FROM weave_poll_states WHERE namespace_id=? AND resolved_at IS NULL`, f.resources.NamespaceID).Scan(&task); err != nil {
t.Fatal(err)
}
// 在隔离夹具中直接使用所有者内部取消原语,制造未开始即已知取消的完整事件;公共任务入口仍禁止绕过流程。
err := WithTx(ctx, db, func(tx *sql.Tx) error {
if _, _, err := lockQueue(ctx, tx, f.resources.NamespaceID, f.resources.QueueID); err != nil {
return err
}
current, err := readExecutionTask(ctx, tx, f.resources.NamespaceID, task)
if err != nil {
return err
}
now, err := dbNow(ctx, tx)
if err != nil {
return err
}
_, err = cancelExecutionTask(ctx, tx, f.resources.NamespaceID, current, now, "TEST_OWNER_CANCEL_BEFORE_START")
return err
})
if err != nil {
t.Fatal(err)
}
runtimeConsume(t, f, newRuntime(t, db), task)
in := recovery.RetryStepInput{StepDecision: recoveryStep(t, f, v.ID, "query"), SafeToRetry: true}
out, err := api.RetryStep(ctx, f.token, v.ID, string(executionID(t)), in)
if i == 2 {
if fault.Code(err) != "STATE_CONFLICT" {
t.Fatal("unlimited zero-attempt activations", err)
}
} else {
if err != nil {
t.Fatal(err)
}
if out.Task.Retry.MaxAttempts != 3 {
t.Fatal("unstarted recovery charged execution attempt")
}
continueWorkflow(t, f, v.ID)
}
}
if f.count("pulse_tasks", "1=1") != 3 || f.count("pulse_attempts", "1=1") != 0 {
t.Fatal("zero-attempt bound failed")
}
})
t.Run("old_final_signal_winner_is_history_not_new_winner", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
v := runtimeStart(t, f, recoveryPollSpec(f.resources.QueueID, true))
grantWorkflowRecovery(f)
grantWorkflowSignal(f)
ref := retryStart(t, f)
oldSignal := sendSignal(t, f, v, pollSignalInput(v))
runClock(t, f, clockCandidate(t, f, v.ID, "query"))
finishQueryOnly(t, f, ref, "FAILED")
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
retryPollStep(t, f, v.ID)
if f.count("weave_signals", "id=? AND state='CONSUMED'", oldSignal.Signal.ID) != 1 || f.count("weave_waits", "state='WAITING' AND winner_signal_id IS NULL") != 1 || f.count("weave_waits", "winner_signal_id=?", oldSignal.Signal.ID) != 1 {
t.Fatal("old winner reused or evidence lost")
}
continueWorkflow(t, f, v.ID)
f.exec(`UPDATE weave_pending_advances SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, v.ID)
runClock(t, f, clockCandidate(t, f, v.ID, "query"))
if f.count("weave_steps", "workflow_id=? AND kind='POLL' AND state='ACTIVE'", v.ID) != 1 || f.count("pulse_tasks", "state='WAITING'") != 1 {
t.Fatal("old signal completed or cancelled new generation")
}
ref = retryStart(t, f)
finishPoll(t, f, ref, "DONE")
runtimeView(t, f, v.ID, "SUCCEEDED")
})
t.Run("queue_full_and_cancel_intent_do_not_create_recovery", func(t *testing.T) {
f, v, _ := failedPoll(t, ctx, db, false)
in := recovery.RetryStepInput{StepDecision: recoveryStep(t, f, v.ID, "query"), SafeToRetry: true}
key := string(executionID(t))
f.exec(`UPDATE pulse_queues SET max_unfinished_tasks=1 WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, f.resources.QueueID)
blocker := f.submit(1)
if _, err := api.RetryStep(ctx, f.token, v.ID, key, in); fault.Code(err) != "QUEUE_FULL" {
t.Fatal("recovery bypassed capacity", err)
}
if f.count("weave_poll_states", "1=1") != 1 || recoveryDecision(t, f, v.ID).ExpectedRevision != in.ExpectedRevision {
t.Fatal("full queue left partial recovery")
}
if _, err := f.tasks.CancelTask(ctx, f.token, blocker.ID, string(executionID(t)), service.CancelInput{ExpectedRevision: blocker.Revision, Reason: "release test capacity"}); err != nil {
t.Fatal(err)
}
applyControl(t, f, runtimeView(t, f, v.ID, "NEEDS_REVIEW"), "CANCEL")
in.StepDecision = recoveryStep(t, f, v.ID, "query")
if _, err := api.RetryStep(ctx, f.token, v.ID, key, in); fault.Code(err) != "STATE_CONFLICT" {
t.Fatal("recovery revoked stop intent", err)
}
})
}

View File

@@ -0,0 +1,117 @@
package mysql
import (
"context"
"os"
"strings"
"testing"
"time"
driver "github.com/go-sql-driver/mysql"
)
// 用真实v21结构验证轮次预算升级;数据夹具用于验证迁移保留,不冒充运行器实例。
func TestIntegrationWorkflowPollRecoveryUpgrade(t *testing.T) {
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("requires disposable MySQL")
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
admin, err := Open(ctx, dsn, 2, 1)
if err != nil {
t.Fatal(err)
}
defer admin.Close()
name := "orbit_poll_recovery_upgrade_" + string(executionID(t))
if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+name+"`"); err != nil {
t.Fatal(err)
}
defer func() {
cleanup, stop := context.WithTimeout(context.Background(), 10*time.Second)
defer stop()
if _, err := admin.ExecContext(cleanup, "DROP DATABASE `"+name+"`"); err != nil {
t.Error(err)
}
}()
cfg, err := driver.ParseDSN(dsn)
if err != nil {
t.Fatal("invalid test DSN")
}
cfg.DBName = name
db, err := Open(ctx, cfg.FormatDSN(), 4, 2)
if err != nil {
t.Fatal(err)
}
defer db.Close()
exec := func(q string, args ...any) {
t.Helper()
if _, err := db.ExecContext(ctx, q, args...); err != nil {
t.Fatal(err)
}
}
exec(`CREATE TABLE orbit_schema_migrations (version BIGINT UNSIGNED PRIMARY KEY,name VARCHAR(191) NOT NULL,checksum CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,dirty BOOLEAN NOT NULL,applied_at DATETIME(6) NULL) ENGINE=InnoDB`)
migrations, err := Migrations()
if err != nil {
t.Fatal(err)
}
for _, m := range migrations {
if m.Version > 21 {
break
}
statements, err := splitSQL(m.SQL)
if err != nil {
t.Fatal(err)
}
for _, q := range statements {
exec(q)
}
exec(`INSERT INTO orbit_schema_migrations(version,name,checksum,dirty,applied_at) VALUES (?,?,?,FALSE,UTC_TIMESTAMP(6))`, m.Version, m.Name, m.Checksum)
}
var now time.Time
if err := db.QueryRowContext(ctx, `SELECT UTC_TIMESTAMP(6)`).Scan(&now); err != nil {
t.Fatal(err)
}
ns, workflow, activation, id, task := executionID(t), executionID(t), executionID(t), executionID(t), executionID(t)
deadline := now.Add(time.Hour)
exec(`INSERT INTO weave_poll_states(namespace_id,id,workflow_id,activation_id,max_rounds,used_rounds,max_attempts_per_round,interval_ms,deadline_at,current_task_id,revision,created_at,updated_at) VALUES (?,?,?,?,3,2,4,1000,?,?,7,?,?)`, ns, id, workflow, activation, deadline, task, now, now)
exec("INSERT INTO weave_poll_rounds(namespace_id,id,workflow_id,activation_id,`round`,task_id) VALUES (?,?,?,?,1,?),(?,?,?,?,2,?)", ns, executionID(t), workflow, activation, executionID(t), ns, executionID(t), workflow, activation, task)
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
if err := CheckSchema(ctx, db); err != nil {
t.Fatal(err)
}
var first, carried, used, max, attempts, revision uint32
var gotDeadline, created, updated time.Time
if err := db.QueryRowContext(ctx, `SELECT first_round,first_round_attempts_used,used_rounds,max_rounds,max_attempts_per_round,revision,deadline_at,created_at,updated_at FROM weave_poll_states WHERE namespace_id=? AND id=?`, ns, id).Scan(&first, &carried, &used, &max, &attempts, &revision, &gotDeadline, &created, &updated); err != nil {
t.Fatal(err)
}
if first != 1 || carried != 0 || used != 2 || max != 3 || attempts != 4 || revision != 7 || !gotDeadline.Equal(deadline) || !created.Equal(now) || !updated.Equal(now) {
t.Fatal("migration changed original policy, revision or timestamps")
}
var rounds string
if err := db.QueryRowContext(ctx, "SELECT GROUP_CONCAT(`round` ORDER BY `round`) FROM weave_poll_rounds WHERE namespace_id=? AND activation_id=?", ns, activation).Scan(&rounds); err != nil || rounds != "1,2" {
t.Fatal("original rounds changed", rounds, err)
}
for _, clause := range []string{"first_round=0", "first_round=3", "first_round_attempts_used=4"} {
if _, err := db.ExecContext(ctx, "UPDATE weave_poll_states SET "+clause+" WHERE namespace_id=? AND id=?", ns, id); err == nil {
t.Fatal("invalid recovery budget accepted", clause)
}
}
var count int
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&count); err != nil || count != 0 {
t.Fatal("foreign keys introduced", err)
}
for _, column := range []string{"first_round", "first_round_attempts_used"} {
var comment string
if err := db.QueryRowContext(ctx, `SELECT COLUMN_COMMENT FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='weave_poll_states' AND COLUMN_NAME=?`, column).Scan(&comment); err != nil || !strings.Contains(comment, "业务") {
t.Fatal("missing Chinese comment", column, comment, err)
}
}
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
t.Log("real v21 upgrade preserves original rounds, budget, deadline and revisions; default first_round=1/carry=0; invalid recovery budget rejected; zero foreign keys")
}

View File

@@ -0,0 +1,144 @@
package mysql
import (
"context"
"database/sql"
"errors"
"time"
"git.yuntuo.hk/k2safe/orbit/internal/fault"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/pulse/service"
"git.yuntuo.hk/k2safe/orbit/weave/engine"
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
"git.yuntuo.hk/k2safe/orbit/weave/recovery"
)
// retryWorkflowPoll 重新执行当前业务轮;新激活保留旧轮次、同轮尝试、精确输入及两个绝对截止。
func retryWorkflowPoll(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, in recovery.RetryStepInput, now time.Time, out *recovery.Result) (map[string]any, error) {
p, last, err := readWorkflowPollBudget(ctx, tx, g, row)
if err != nil {
return nil, err
}
origin, err := workflowActivationOrigin(ctx, tx, g, row)
if err != nil {
return nil, err
}
if !now.Before(p.DeadlineAt) || origin.sequence >= p.MaxRounds*p.MaxAttemptsPerRound {
return nil, fault.Conflict
}
h, err := readWorkflowPollRoundHistory(ctx, tx, g, row, p, p.UsedRounds)
if err != nil {
return nil, err
}
if h.last.ID != last || !h.last.State.Terminal() || h.last.EffectUncertain || h.used >= p.MaxAttemptsPerRound || h.count >= p.MaxAttemptsPerRound || !now.Before(h.first.DeadlineAt) {
return nil, fault.Conflict
}
// 已确认的PENDING应按间隔和业务轮次预算推进,不能借人工恢复再查同一轮。
if h.last.State == "SUCCEEDED" {
kind, err := g.program.ClassifyPoll(row.view.Key, h.last.Result)
if err == nil && kind == engine.PollPending {
return nil, fault.Conflict
}
}
node, _ := g.program.Step(row.view.Key)
var wait *workflowSignalWait
if node.Poll.FinalSignal != nil {
w, err := readWorkflowSignalWait(ctx, tx, g, row)
if err != nil {
return nil, err
}
if !now.Before(w.deadline) {
return nil, fault.Conflict
}
var pending int
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_signals WHERE namespace_id=? AND workflow_id=? AND target_step_key=? AND target_activation_sequence=? AND state='PENDING'`, g.view.NamespaceID, g.view.ID, row.view.Key, w.sequence).Scan(&pending); err != nil {
return nil, err
}
if pending != 0 {
return nil, fault.Conflict
}
if w.winner != nil {
if _, err := selectedWorkflowSignal(ctx, tx, g, row, w, now); err != nil {
return nil, err
}
}
wait = &w
}
policy := h.first.Retry
policy.MaxAttempts = p.MaxAttemptsPerRound - h.used
input, err := service.NormalizeSubmit(service.SubmitInput{QueueID: h.first.QueueID, Handler: h.first.Handler, HandlerVersion: h.first.HandlerVersion, PayloadVersion: h.first.PayloadVersion, Payload: h.first.Payload, BusinessKey: h.first.BusinessKey, ConcurrencyKey: h.first.ConcurrencyKey, Priority: h.first.Priority, RecoverySafety: h.first.RecoverySafety, Retry: policy, TaskTimeoutMS: node.Poll.Task.TimeoutMS})
if err != nil {
return nil, err
}
activation, err := core.NewID()
if err != nil {
return nil, err
}
taskID, err := core.NewID()
if err != nil {
return nil, err
}
task, err := insertAcceptedTaskUntil(ctx, tx, g.view.NamespaceID, taskID, input, &last, &h.first.DeadlineAt)
if err != nil {
return nil, err
}
if _, err := tx.ExecContext(ctx, `INSERT INTO weave_activations(namespace_id,id,workflow_id,step_id,sequence,state,input,created_at,updated_at) VALUES (?,?,?,?,?,'ACTIVE',?,?,?)`, g.view.NamespaceID, activation, g.view.ID, row.view.ID, origin.sequence+1, string(row.input), now, now); err != nil {
return nil, err
}
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET owner_kind=?,owner_id=?,gate_id=? WHERE namespace_id=? AND id=? AND owner_kind IS NULL`, engine.TaskOwnerKind, activation, g.gate, g.view.NamespaceID, taskID)
if err := exactlyOne(result, err); err != nil {
return nil, err
}
// 旧预算仅结清检查点;旧任务、旧激活输出、轮次及已匹配信号不改写。
if _, err := tx.ExecContext(ctx, `UPDATE weave_poll_states SET current_task_id=NULL,next_poll_at=NULL,resolved_at=COALESCE(resolved_at,?),revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, now, now, g.view.NamespaceID, p.ID); err != nil {
return nil, err
}
pollID, err := core.NewID()
if err != nil {
return nil, err
}
if _, err := tx.ExecContext(ctx, `INSERT INTO weave_poll_states(namespace_id,id,workflow_id,activation_id,max_rounds,used_rounds,first_round,first_round_attempts_used,max_attempts_per_round,interval_ms,deadline_at,current_task_id,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, g.view.NamespaceID, pollID, g.view.ID, activation, p.MaxRounds, p.UsedRounds, p.UsedRounds, h.used, p.MaxAttemptsPerRound, p.IntervalMS, p.DeadlineAt, taskID, now, now); err != nil {
return nil, err
}
roundID, err := core.NewID()
if err != nil {
return nil, err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO weave_poll_rounds(namespace_id,id,workflow_id,activation_id,`round`,task_id) VALUES (?,?,?,?,?,?)", g.view.NamespaceID, roundID, g.view.ID, activation, p.UsedRounds, taskID); err != nil {
return nil, err
}
if wait != nil {
if _, err := tx.ExecContext(ctx, `UPDATE weave_waits SET state='CANCELLED',resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND state='WAITING'`, now, now, g.view.NamespaceID, wait.id); err != nil {
return nil, err
}
id, err := core.NewID()
if err != nil {
return nil, err
}
if _, err := tx.ExecContext(ctx, `INSERT INTO weave_waits(namespace_id,id,workflow_id,activation_id,kind,correlation_key,deadline_at,signal_name,payload_version,created_at,updated_at) VALUES (?,?,?,?,'POLL',?,?,?,?,?,?)`, g.view.NamespaceID, id, g.view.ID, activation, []byte(wait.correlation), wait.deadline, wait.contract.Name, wait.contract.PayloadVersion, now, now); err != nil {
return nil, err
}
}
result, err = tx.ExecContext(ctx, `UPDATE weave_steps SET state='ACTIVE',current_activation_id=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=? AND current_activation_id=?`, activation, now, g.view.NamespaceID, row.view.ID, row.view.Revision, row.view.ActivationID)
if err := exactlyOne(result, err); err != nil {
return nil, err
}
if row.view.TaskID != nil {
return nil, errors.New("poll cannot have direct task reference")
}
row.view.State, row.view.ActivationID = weave.StepActive, &activation
row.view.Revision++
row.output = nil
if err := saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision, p.DeadlineAt); err != nil {
return nil, err
}
if err := emitEvent(ctx, tx, g.view.NamespaceID, taskID, task.Revision, "task", "task.accepted", map[string]any{"task_id": taskID, "state": task.State, "workflow_id": g.view.ID, "activation_id": activation, "retry_of": last, "round": p.UsedRounds}); err != nil {
return nil, err
}
if err := emitEvent(ctx, tx, g.view.NamespaceID, row.view.ID, row.view.Revision, "workflow_step", "workflow.step.changed", map[string]any{"workflow_id": g.view.ID, "activation_id": activation, "state": row.view.State}); err != nil {
return nil, err
}
out.Step, out.Task = &row.view, &task
return map[string]any{"step_id": row.view.ID, "previous_activation_id": in.ActivationID, "activation_id": activation, "sequence": origin.sequence + 1, "retry_of": last, "task_id": taskID, "safe_to_retry": true, "round": p.UsedRounds, "max_rounds": p.MaxRounds, "used_attempts": h.used, "remaining_attempts": policy.MaxAttempts, "deadline_at": p.DeadlineAt, "task_deadline_at": h.first.DeadlineAt}, nil
}

View File

@@ -22,7 +22,7 @@ func TestIntegrationWorkflowRuntimeProcess(t *testing.T) {
if dsn == "" || binary == "" {
t.Skip("requires disposable MySQL and built Orbit binary")
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
admin, err := Open(ctx, dsn, 2, 1)
if err != nil {
@@ -375,4 +375,37 @@ func TestIntegrationWorkflowRuntimeProcess(t *testing.T) {
t.Fatal("manual recovery duplicated work or rewrote old activation")
}
t.Log("manual execution review, explicit workflow release and new TASK activation survived all-stop and two fresh controllers; original cancellation, task deadline and remaining attempt budget preserved")
for _, p := range processes {
stop(p)
}
// 所有控制进程退出后人工重查同轮,再让两台全新节点推进下一业务轮。
pollRetry := runtimeStart(t, f, recoveryPollSpec(f.resources.QueueID, false))
grantWorkflowRecovery(f)
ref = retryStart(t, f)
retryFinish(t, f, ref, "FAILED", false)
start("poll-review-a")
start("poll-review-b")
wait(func() bool { return f.count("weave_instances", "id=? AND state='NEEDS_REVIEW'", pollRetry.ID) == 1 }, "poll enters review")
for _, p := range processes {
stop(p)
}
old = f.get(ref.TaskID)
recovered := retryPollStep(t, f, pollRetry.ID)
continueWorkflow(t, f, pollRetry.ID)
if recovered.Task.Retry.MaxAttempts != 2 || !recovered.Task.DeadlineAt.Equal(old.DeadlineAt) {
t.Fatal("poll recovery restarted budget")
}
start("poll-retry-a")
start("poll-retry-b")
ref = retryStart(t, f)
finishQueryOnly(t, f, ref, "PENDING")
wait(func() bool { return f.count("pulse_tasks", "1=1") == 16 }, "new round after recovered request")
ref = retryStart(t, f)
finishQueryOnly(t, f, ref, "DONE")
wait(func() bool { return f.count("weave_instances", "id=? AND state='SUCCEEDED'", pollRetry.ID) == 1 }, "poll recovery final completion")
if f.count("weave_poll_states", "workflow_id=? AND first_round=1 AND used_rounds=2 AND first_round_attempts_used=1 AND resolved_at IS NOT NULL", pollRetry.ID) != 1 || f.count("pulse_tasks", "1=1") != 16 || f.used() != 0 {
t.Fatal("recovered query duplicated, reset budget or leaked permit")
}
t.Log("POLL manual same-round recovery survived all-stop and two fresh controllers; reduced request attempts, original deadline, old activation and subsequent business-round progression preserved")
}

View File

@@ -250,11 +250,7 @@ func cancelWorkflowWait(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *
}
}
if row.view.Kind == weave.NodePoll {
var activated time.Time
if err := tx.QueryRowContext(ctx, `SELECT created_at FROM weave_activations WHERE namespace_id=? AND workflow_id=? AND step_id=? AND id=?`, g.view.NamespaceID, g.view.ID, row.view.ID, row.view.ActivationID).Scan(&activated); err != nil {
return err
}
p, _, err := readWorkflowPollState(ctx, tx, g, row, activated)
p, _, err := readWorkflowPollState(ctx, tx, g, row)
if err != nil {
return err
}
@@ -300,7 +296,7 @@ func validateWorkflowStopTask(ctx context.Context, tx *sql.Tx, g *workflowGraph,
return errors.New("stopping query association invalid")
}
var count int
if err := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=? AND r.workflow_id=? AND r.activation_id=? AND r.task_id=? AND r.`round` BETWEEN 1 AND p.used_rounds", g.view.NamespaceID, g.view.ID, owner, id).Scan(&count); err != nil {
if err := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=? AND r.workflow_id=? AND r.activation_id=? AND r.task_id=? AND r.`round` BETWEEN p.first_round AND p.used_rounds", g.view.NamespaceID, g.view.ID, owner, id).Scan(&count); err != nil {
return err
}
if count != 1 {
@@ -326,7 +322,7 @@ func verifyWorkflowStopWork(ctx context.Context, tx *sql.Tx, g *workflowGraph) (
}
}
// 一次读取关联元数据并通过索引EXISTS核对查询轮次,避免持有应用锁期间逐任务发出SQL。
rows, err := tx.QueryContext(ctx, "SELECT t.id,t.state,t.effect_uncertain,t.owner_kind,t.owner_id,t.queue_id,a.workflow_id,a.step_id,a.task_id,s.`key`,EXISTS(SELECT 1 FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=t.namespace_id AND r.workflow_id=a.workflow_id AND r.activation_id=a.id AND r.task_id=t.id AND r.`round` BETWEEN 1 AND p.used_rounds) FROM pulse_tasks t LEFT JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id LEFT JOIN weave_steps s ON s.namespace_id=a.namespace_id AND s.workflow_id=a.workflow_id AND s.id=a.step_id WHERE t.namespace_id=? AND t.gate_id=? ORDER BY t.id LIMIT 10001", g.view.NamespaceID, g.gate)
rows, err := tx.QueryContext(ctx, "SELECT t.id,t.state,t.effect_uncertain,t.owner_kind,t.owner_id,t.queue_id,a.workflow_id,a.step_id,a.task_id,s.`key`,EXISTS(SELECT 1 FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=t.namespace_id AND r.workflow_id=a.workflow_id AND r.activation_id=a.id AND r.task_id=t.id AND r.`round` BETWEEN p.first_round AND p.used_rounds) FROM pulse_tasks t LEFT JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id LEFT JOIN weave_steps s ON s.namespace_id=a.namespace_id AND s.workflow_id=a.workflow_id AND s.id=a.step_id WHERE t.namespace_id=? AND t.gate_id=? ORDER BY t.id LIMIT 10001", g.view.NamespaceID, g.gate)
if err != nil {
return false, false, err
}

View File

@@ -71,7 +71,7 @@ func (r *WorkflowRuntime) applyWorkflowSignal(ctx context.Context, tx *sql.Tx, g
var poll weave.PollState
if node.Poll != nil {
var last core.ID
poll, last, err = readWorkflowPollState(ctx, tx, g, row, w.activated)
poll, last, err = readWorkflowPollState(ctx, tx, g, row)
if err != nil {
return false, false, "", err
}

View File

@@ -44,6 +44,23 @@ func readWorkflowSignalWait(ctx context.Context, tx *sql.Tx, g *workflowGraph, r
if err != nil {
return w, err
}
base, err := workflowActivationOrigin(ctx, tx, g, row)
if err != nil {
return w, err
}
w.activated = base.started
if base.sequence > 1 {
original := *row
original.view.ActivationID = &base.id
original.input = base.input
initial, err := readWorkflowSignalWait(ctx, tx, g, &original)
if err != nil {
return w, err
}
if initial.correlation != w.correlation {
return w, workflowSourceError("SIGNAL_RECOVERY_CORRELATION_INVALID")
}
}
if name == nil || version == nil {
return w, workflowSourceError("SIGNAL_WAIT_UNVERIFIED")
}

View File

@@ -16,8 +16,7 @@ import (
"git.yuntuo.hk/k2safe/orbit/weave/recovery"
)
// RetryStep 为TASK建立新的执行激活;保留旧激活、原输入和绝对截止,跨激活扣除已用尝试。
// POLL恢复需要同时保留原轮次身份,不在此处降级成新查询轮次。
// RetryStep 为TASK或POLL建立新的执行激活;保留原输入、绝对截止和跨激活尝试预算。
func (s *WorkflowRecoveryStore) RetryStep(ctx context.Context, t auth.Token, id core.ID, key string, in recovery.RetryStepInput) (recovery.Result, error) {
return s.mutate(ctx, t, id, key, "workflow.step.retry", in, in.Decision, func(tx *sql.Tx, g *workflowGraph, _ core.ID, now time.Time, out *recovery.Result) (map[string]any, error) {
row, err := workflowRecoveryStep(g, in.StepDecision)
@@ -28,12 +27,15 @@ func (s *WorkflowRecoveryStore) RetryStep(ctx context.Context, t auth.Token, id
return nil, fault.Conflict
}
node, ok := g.program.Step(row.view.Key)
if !ok || node.Task == nil {
return nil, fault.Invalid("this retry operation currently supports TASK steps only")
if !ok || node.Task == nil && node.Poll == nil {
return nil, fault.Invalid("retry currently supports TASK and POLL steps")
}
if err := workflowStepSettled(ctx, tx, g, row); err != nil {
return nil, err
}
if node.Poll != nil {
return retryWorkflowPoll(ctx, tx, g, row, in, now, out)
}
var sequence uint32
if err := tx.QueryRowContext(ctx, `SELECT sequence FROM weave_activations WHERE namespace_id=? AND id=?`, t.NamespaceID, row.view.ActivationID).Scan(&sequence); err != nil {
return nil, err