k2safe/orbit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: persist workflow timers and bounded poll rounds

Browse Source
This commit is contained in:
k2safe
2026-09-18 03:03:31 +08:00
parent f602a5ee05
commit e2baffbe37
21 changed files with 1201 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
internal/storage/mysql/migrations/000019_workflow_clock_checkpoints.up.sql Normal file
View File

@@ -0,0 +1,20 @@
-- 为已有定时等待和查询补齐持久化唤醒；检查点只产生候选，运行器仍核验固定版本和全部归属。
-- 复用既有到期索引及步骤唯一键，不引入外键，也不重算原截止或重置轮次。
INSERT INTO weave_pending_advances(namespace_id,id,workflow_id,step_id,expected_step_revision,checkpoint,available_at)
SELECT s.namespace_id,RANDOM_BYTES(16),s.workflow_id,s.id,s.revision,
JSON_OBJECT('schema_version',1,'kind','TIMER','activation_id',LOWER(HEX(a.id))),w.deadline_at
FROM weave_steps s JOIN weave_activations a ON a.namespace_id=s.namespace_id AND a.id=s.current_activation_id AND a.workflow_id=s.workflow_id AND a.step_id=s.id
JOIN weave_instances i ON i.namespace_id=s.namespace_id AND i.id=s.workflow_id
JOIN weave_waits w ON w.namespace_id=s.namespace_id AND w.workflow_id=s.workflow_id AND w.activation_id=a.id
WHERE s.kind='TIMER' AND s.state='WAITING' AND a.state='WAITING' AND w.kind='TIMER' AND w.state='WAITING' AND i.state IN ('RUNNING','WAITING','CANCELLING','NEEDS_REVIEW')
ON DUPLICATE KEY UPDATE id=weave_pending_advances.id;
INSERT INTO weave_pending_advances(namespace_id,id,workflow_id,step_id,expected_step_revision,checkpoint,available_at)
SELECT s.namespace_id,RANDOM_BYTES(16),s.workflow_id,s.id,s.revision,
JSON_OBJECT('schema_version',1,'kind','POLL','activation_id',LOWER(HEX(a.id))),
CASE WHEN p.current_task_id IS NOT NULL THEN p.deadline_at ELSE COALESCE(p.next_poll_at,p.deadline_at) END
FROM weave_steps s JOIN weave_activations a ON a.namespace_id=s.namespace_id AND a.id=s.current_activation_id AND a.workflow_id=s.workflow_id AND a.step_id=s.id
JOIN weave_instances i ON i.namespace_id=s.namespace_id AND i.id=s.workflow_id
JOIN weave_poll_states p ON p.namespace_id=s.namespace_id AND p.workflow_id=s.workflow_id AND p.activation_id=a.id
WHERE s.kind='POLL' AND s.state IN ('ACTIVE','WAITING') AND a.state=s.state AND p.resolved_at IS NULL AND i.state IN ('RUNNING','WAITING','CANCELLING','NEEDS_REVIEW')
ON DUPLICATE KEY UPDATE id=weave_pending_advances.id;

8
internal/storage/mysql/workflow_apply.go
View File

@@ -251,6 +251,11 @@ func activateWorkflowStep(ctx context.Context, tx *sql.Tx, g *workflowGraph, row
if _, err := tx.ExecContext(ctx, `INSERT INTO weave_waits(namespace_id,id,workflow_id,activation_id,kind,correlation_key,deadline_at,signal_name,payload_version,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)`, g.view.NamespaceID, waitID, g.view.ID, id, c.Wait.Kind, []byte(c.Wait.Correlation), until, name, version, now, now); err != nil {
return err
}
if c.Wait.Kind == "TIMER" {
if err := saveWorkflowClock(ctx, tx, g, row, "TIMER", row.view.Revision+1, until); err != nil {
return err
}
}
}
if c.Task != nil {
if taskID == nil {
@@ -268,5 +273,8 @@ func activateWorkflowStep(ctx context.Context, tx *sql.Tx, g *workflowGraph, row
return err
}
}
if c.Kind == "POLL" {
return saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision+1, deadline)
}
return nil
}

114
internal/storage/mysql/workflow_clock.go Normal file
View File

@@ -0,0 +1,114 @@
package mysql
import (
"context"
"database/sql"
"encoding/json"
"errors"
"time"
core "git.yuntuo.hk/k2safe/orbit/model"
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
)
// workflowCheckpoint 是待续表中版本化的内部唤醒身份；不保存内存执行栈或新的业务预算。
type workflowCheckpoint struct {
SchemaVersion uint32 `json:"schema_version"` // SchemaVersion 当前结构版本为1，未知版本禁止推进。
Kind string `json:"kind"` // Kind 为ROOT、TIMER或POLL，决定实际权威记录的读取方式。
ActivationID *core.ID `json:"activation_id,omitempty"` // ActivationID 固定等待激活；ROOT不使用该字段。
}
func saveWorkflowClock(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, kind string, expected uint64, due time.Time) error {
id, err := core.NewID()
if err != nil {
return err
}
if row.view.ActivationID == nil || kind != "TIMER" && kind != "POLL" {
return errors.New("workflow clock identity invalid")
}
raw, err := json.Marshal(workflowCheckpoint{1, kind, row.view.ActivationID})
if err != nil {
return err
}
_, err = tx.ExecContext(ctx, `INSERT INTO weave_pending_advances(namespace_id,id,workflow_id,step_id,expected_step_revision,checkpoint,available_at) VALUES (?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE expected_step_revision=VALUES(expected_step_revision),checkpoint=VALUES(checkpoint),available_at=VALUES(available_at),revision=revision+1,updated_at=UTC_TIMESTAMP(6)`, g.view.NamespaceID, id, g.view.ID, row.view.ID, expected, string(raw), due)
return err
}
func deleteWorkflowClock(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep) error {
_, err := tx.ExecContext(ctx, `DELETE FROM weave_pending_advances WHERE namespace_id=? AND workflow_id=? AND step_id=?`, g.view.NamespaceID, g.view.ID, row.view.ID)
return err
}
func (r *WorkflowRuntime) advanceClock(ctx context.Context, tx *sql.Tx, g *workflowGraph, c workflowCandidate, step core.ID, expected uint64, checkpoint workflowCheckpoint, allow bool, now time.Time) (bool, error) {
var row *workflowStep
for _, candidate := range g.steps {
if candidate.view.ID == step {
row = candidate
break
}
}
if row == nil || checkpoint.ActivationID == nil || string(row.view.Kind) != checkpoint.Kind || expected > row.view.Revision {
return false, errors.New("workflow clock checkpoint mismatch")
}
if row.view.ActivationID == nil || *row.view.ActivationID != *checkpoint.ActivationID || g.view.State.Terminal() {
return false, deleteWorkflowClock(ctx, tx, g, row)
}
if row.view.State != weave.StepActive && row.view.State != weave.StepWaiting {
return false, deleteWorkflowClock(ctx, tx, g, row)
}
var activated time.Time
if err := tx.QueryRowContext(ctx, `SELECT created_at FROM weave_activations WHERE namespace_id=? AND id=? AND workflow_id=? AND step_id=?`, c.ns, checkpoint.ActivationID, c.workflow, step).Scan(&activated); err != nil {
return false, err
}
if checkpoint.Kind == "POLL" {
return r.advancePoll(ctx, tx, g, row, c, activated, allow, now)
}
if checkpoint.Kind != "TIMER" || row.view.State != weave.StepWaiting {
return false, errors.New("workflow timer state invalid")
}
var wait core.ID
var deadline time.Time
var kind, state string
var name, version sql.NullString
var correlation []byte
var count int
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_waits WHERE namespace_id=? AND workflow_id=? AND activation_id=?`, c.ns, c.workflow, checkpoint.ActivationID).Scan(&count); err != nil {
return false, err
}
if count != 1 {
return false, errors.New("workflow timer wait set invalid")
}
if err := tx.QueryRowContext(ctx, `SELECT id,kind,state,deadline_at,signal_name,payload_version,correlation_key FROM weave_waits WHERE namespace_id=? AND workflow_id=? AND activation_id=? FOR UPDATE`, c.ns, c.workflow, checkpoint.ActivationID).Scan(&wait, &kind, &state, &deadline, &name, &version, &correlation); err != nil {
return false, err
}
node, _ := g.program.Step(row.view.Key)
intended := activated.Add(time.Duration(node.Timer.DelayMS) * time.Millisecond)
if kind != "TIMER" || state != "WAITING" || name.Valid || version.Valid || len(correlation) != 0 || !deadline.Equal(minimumTime(intended, g.view.DeadlineAt)) {
return false, errors.New("workflow timer wait invalid")
}
if now.Before(deadline) {
return false, saveWorkflowClock(ctx, tx, g, row, "TIMER", row.view.Revision, deadline)
}
state, reason, stepState := "SATISFIED", "TIMER_ELAPSED", weave.StepSucceeded
output := json.RawMessage(`null`)
if intended.After(g.view.DeadlineAt) {
state, reason, stepState, output = "EXPIRED", "TIMER_WORKFLOW_DEADLINE", weave.StepNeedsReview, nil
}
res, err := tx.ExecContext(ctx, `UPDATE weave_waits SET state=?,resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND state='WAITING'`, state, now, now, c.ns, wait)
if err := exactlyOne(res, err); err != nil {
return false, err
}
if err := saveWorkflowResultStep(ctx, tx, g, row, stepState, output, now); err != nil {
return false, err
}
if err := deleteWorkflowClock(ctx, tx, g, row); err != nil {
return false, err
}
if stepState == weave.StepNeedsReview {
if err := holdWorkflow(ctx, tx, g, weave.InstanceNeedsReview, reason, now); err != nil {
return false, err
}
} else if err := r.progress(ctx, tx, g, allow, now); err != nil {
return false, err
}
return true, r.audit(ctx, tx, g, c.id, reason, now)
}

454
internal/storage/mysql/workflow_clock_integration_test.go Normal file
View File

@@ -0,0 +1,454 @@
package mysql
import (
"context"
"database/sql"
"encoding/json"
"os"
"strings"
"sync"
"testing"
"time"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
"git.yuntuo.hk/k2safe/orbit/weave/definition"
)
func newClockFixture(t *testing.T, ctx context.Context, db *sql.DB) *executionFixture {
t.Helper()
f := newExecutionFixture(t, ctx, db)
return keyedWorker(t, f, f.resources.QueueID, f.a)
}
func clockCandidate(t *testing.T, f *executionFixture, workflow core.ID, key string) workflowCandidate {
t.Helper()
c := workflowCandidate{kind: "ROOT", ns: f.resources.NamespaceID, workflow: workflow}
if err := f.db.QueryRowContext(f.ctx, "SELECT p.id,p.revision FROM weave_pending_advances p JOIN weave_steps s ON s.namespace_id=p.namespace_id AND s.id=p.step_id AND s.workflow_id=p.workflow_id WHERE p.namespace_id=? AND p.workflow_id=? AND s.`key`=?", c.ns, workflow, key).Scan(&c.id, &c.revision); err != nil {
t.Fatal(err)
}
return c
}
func waitClockDue(t *testing.T, f *executionFixture, c workflowCandidate) {
t.Helper()
deadline := time.Now().Add(8 * time.Second)
for time.Now().Before(deadline) {
var due bool
if err := f.db.QueryRowContext(f.ctx, `SELECT available_at<=UTC_TIMESTAMP(6) FROM weave_pending_advances WHERE namespace_id=? AND id=?`, c.ns, c.id).Scan(&due); err != nil {
t.Fatal(err)
}
if due {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatal("clock did not become due")
}
func runClock(t *testing.T, f *executionFixture, c workflowCandidate) bool {
t.Helper()
done, err := newRuntime(t, f.db).process(f.ctx, c, false)
if err != nil {
t.Fatal(err)
}
return done
}
func timerSpec(queue core.ID, delay uint64, tail bool) definition.Specification {
s := runtimeSpec(queue, 1)
step := s.Body.Steps[0]
s.Body.Steps = []definition.Step{{Key: "timer", Kind: "TIMER", Timer: &definition.Timer{DelayMS: delay}}}
s.Body.Output = definition.Value{Kind: "REFERENCE", Reference: &definition.Reference{Source: "timer"}}
s.Body.OutputType = definition.Type{Kind: "NULL"}
if tail {
s.Body.Steps = append(s.Body.Steps, step)
s.Body.Output = definition.Value{Kind: "REFERENCE", Reference: &definition.Reference{Source: step.Key}}
s.Body.OutputType = s.InputType
}
return s
}
func pollingSpec(queue core.ID) definition.Specification {
s := runtimeSpec(queue, 1)
task := *s.Body.Steps[0].Task
task.TimeoutMS = 6000
task.Retry.AttemptTimeoutMS = 3000
task.ResultType = definition.Type{Kind: "STRING", Enum: []string{"PENDING", "DONE"}}
key := definition.Value{Kind: "REFERENCE", Reference: &definition.Reference{Source: "workflow", Path: []string{"business_key"}}}
concurrencyKey := definition.Value{Kind: "REFERENCE", Reference: &definition.Reference{Source: "workflow", Path: []string{"id"}}}
task.BusinessKey = &key
task.ConcurrencyKey = &concurrencyKey
p := definition.Poll{Task: task, MaxRounds: 3, TimeoutMS: 12000, IntervalMS: 1000, Pending: []string{"PENDING"}, Succeeded: []string{"DONE"}}
s.Body = definition.Block{Steps: []definition.Step{{Key: "query", Kind: "POLL", Poll: &p}}, OutputType: task.ResultType, Output: definition.Value{Kind: "REFERENCE", Reference: &definition.Reference{Source: "query"}}}
return s
}
func finishPoll(t *testing.T, f *executionFixture, ref worker.AttemptRef, result string) {
t.Helper()
raw, _ := json.Marshal(result)
if _, err := f.a.Finish(f.ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: raw}); err != nil {
t.Fatal(err)
}
runtimeConsume(t, f, newRuntime(t, f.db), ref.TaskID)
}
func TestIntegrationWorkflowClock(t *testing.T) {
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("requires disposable MySQL")
}
ctx, stop := context.WithTimeout(context.Background(), 180*time.Second)
defer stop()
db, err := Open(ctx, dsn, 32, 8)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
t.Run("timer_due_duplicate_consumers_and_task_successor", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
v := runtimeStart(t, f, timerSpec(f.resources.QueueID, 1000, true))
c := clockCandidate(t, f, v.ID, "timer")
if runClock(t, f, c) || f.count("pulse_tasks", "1=1") != 0 {
t.Fatal("timer executed before persisted due time")
}
waitClockDue(t, f, c)
var wg sync.WaitGroup
outcomes := make(chan bool, 12)
errs := make(chan error, 12)
for i := 0; i < 12; i++ {
r := newRuntime(t, db)
wg.Add(1)
go func() { defer wg.Done(); done, err := r.process(ctx, c, false); outcomes <- done; errs <- err }()
}
wg.Wait()
close(outcomes)
close(errs)
done := 0
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
for ok := range outcomes {
if ok {
done++
}
}
if done != 1 || f.count("pulse_tasks", "1=1") != 1 || f.count("weave_waits", "state='SATISFIED'") != 1 {
t.Fatal("timer duplicated successor", done)
}
ref := retryStart(t, f)
retryFinish(t, f, ref, "SUCCEEDED", false)
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
runtimeView(t, f, v.ID, "SUCCEEDED")
})
t.Run("timer_pause_collects_and_keeps_root_continuation", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
v := runtimeStart(t, f, timerSpec(f.resources.QueueID, 30, true))
f.exec(`UPDATE weave_instances SET paused=TRUE WHERE namespace_id=?`, f.resources.NamespaceID)
f.exec(`UPDATE pulse_execution_gates SET paused=TRUE WHERE namespace_id=?`, f.resources.NamespaceID)
c := clockCandidate(t, f, v.ID, "timer")
waitClockDue(t, f, c)
if !runClock(t, f, c) || f.count("pulse_tasks", "1=1") != 0 || f.count("weave_waits", "state='SATISFIED'") != 1 {
t.Fatal("paused timer not collected")
}
f.exec(`UPDATE weave_instances SET paused=FALSE WHERE namespace_id=?`, f.resources.NamespaceID)
f.exec(`UPDATE pulse_execution_gates SET paused=FALSE WHERE namespace_id=?`, f.resources.NamespaceID)
runtimeAdvance(t, f, newRuntime(t, db), v.ID)
if f.count("pulse_tasks", "1=1") != 1 {
t.Fatal("timer successor not resumed")
}
})
t.Run("timer_audit_failure_rolls_back", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
v := runtimeStart(t, f, timerSpec(f.resources.QueueID, 20, true))
c := clockCandidate(t, f, v.ID, "timer")
waitClockDue(t, f, c)
f.exec(`CREATE TRIGGER test_clock_audit_failure BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected clock audit failure'`)
_, err := newRuntime(t, db).process(ctx, c, false)
f.exec(`DROP TRIGGER test_clock_audit_failure`)
if err == nil || f.count("pulse_tasks", "1=1") != 0 || f.count("weave_waits", "state='WAITING'") != 1 || f.count("weave_pending_advances", "1=1") != 1 {
t.Fatal("clock failure partially committed")
}
if !runClock(t, f, c) {
t.Fatal("clock did not recover")
}
})
t.Run("timer_late_processing_preserves_original_elapsed_time", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
spec := timerSpec(f.resources.QueueID, 20, false)
spec.TimeoutMS = 250
v := runtimeStart(t, f, spec)
c := clockCandidate(t, f, v.ID, "timer")
time.Sleep(300 * time.Millisecond)
if !runClock(t, f, c) {
t.Fatal("elapsed timer did not complete")
}
runtimeView(t, f, v.ID, "SUCCEEDED")
})
t.Run("timer_clamped_by_instance_deadline_is_not_early_success", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
spec := runtimeSpec(f.resources.QueueID, 1)
spec.TimeoutMS = 2000
spec.Body.Steps[0].Task.TimeoutMS = 1000
spec.Body.Steps[0].Task.Retry.AttemptTimeoutMS = 1000
spec.Body.Steps = append(spec.Body.Steps, definition.Step{Key: "timer", Kind: "TIMER", Timer: &definition.Timer{DelayMS: 1800}})
spec.Body.Output = definition.Value{Kind: "REFERENCE", Reference: &definition.Reference{Source: "timer"}}
spec.Body.OutputType = definition.Type{Kind: "NULL"}
v := runtimeStart(t, f, spec)
ref := retryStart(t, f)
time.Sleep(250 * time.Millisecond)
retryFinish(t, f, ref, "SUCCEEDED", false)
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
c := clockCandidate(t, f, v.ID, "timer")
waitClockDue(t, f, c)
if !runClock(t, f, c) {
t.Fatal("clamped timer not handled")
}
runtimeView(t, f, v.ID, "NEEDS_REVIEW")
if f.count("weave_waits", "state='EXPIRED'") != 1 {
t.Fatal("clamped duration treated as complete")
}
})
t.Run("poll_rounds_fixed_payload_concurrency_and_budget", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
v := runtimeStart(t, f, pollingSpec(f.resources.QueueID))
first := retryStart(t, f)
finishPoll(t, f, first, "PENDING")
c := clockCandidate(t, f, v.ID, "query")
if runClock(t, f, c) {
t.Fatal("new round before interval")
}
waitClockDue(t, f, c)
var wg sync.WaitGroup
errs := make(chan error, 8)
for i := 0; i < 8; i++ {
r := newRuntime(t, db)
wg.Add(1)
go func() { defer wg.Done(); _, err := r.process(ctx, c, false); errs <- err }()
}
wg.Wait()
close(errs)
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
if f.count("pulse_tasks", "1=1") != 2 || f.count("weave_poll_states", "used_rounds=2 AND current_task_id IS NOT NULL") != 1 || f.count("weave_poll_rounds", "1=1") != 2 {
t.Fatal("poll round duplicated or budget reset")
}
second := retryStart(t, f)
task := f.get(second.TaskID)
if task.BusinessKey != v.BusinessKey || task.ConcurrencyKey != string(v.ID) || !strings.Contains(string(task.Payload), "9007199254740993") || task.Retry.MaxAttempts != 1 {
t.Fatal("poll parameters changed")
}
finishPoll(t, f, second, "PENDING")
c = clockCandidate(t, f, v.ID, "query")
waitClockDue(t, f, c)
if !runClock(t, f, c) {
t.Fatal("third round absent")
}
third := retryStart(t, f)
finishPoll(t, f, third, "PENDING")
runtimeView(t, f, v.ID, "NEEDS_REVIEW")
if f.count("pulse_tasks", "1=1") != 3 || f.count("weave_pending_advances", "1=1") != 0 || f.count("weave_poll_states", "used_rounds=3 AND resolved_at IS NOT NULL") != 1 {
t.Fatal("exhaustion reset loop")
}
})
t.Run("poll_success_and_failure_rollback_of_round_creation", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
v := runtimeStart(t, f, pollingSpec(f.resources.QueueID))
first := retryStart(t, f)
finishPoll(t, f, first, "PENDING")
c := clockCandidate(t, f, v.ID, "query")
waitClockDue(t, f, c)
f.exec(`CREATE TRIGGER test_poll_round_failure BEFORE INSERT ON weave_poll_rounds FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected round failure'`)
_, err := newRuntime(t, db).process(ctx, c, false)
f.exec(`DROP TRIGGER test_poll_round_failure`)
if err == nil || f.count("pulse_tasks", "1=1") != 1 || f.count("weave_poll_states", "used_rounds=1 AND current_task_id IS NULL") != 1 || instanceCount(f, `SELECT unfinished_tasks FROM pulse_queues WHERE namespace_id=?`, f.resources.NamespaceID) != 0 {
t.Fatal("partial poll task or budget committed")
}
if !runClock(t, f, c) {
t.Fatal("poll failed to resume after rollback")
}
second := retryStart(t, f)
finishPoll(t, f, second, "DONE")
runtimeView(t, f, v.ID, "SUCCEEDED")
if f.count("weave_pending_advances", "1=1") != 0 {
t.Fatal("resolved poll retained clock")
}
})
t.Run("poll_pause_and_backpressure_keep_round_and_deadline", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
v := runtimeStart(t, f, pollingSpec(f.resources.QueueID))
ref := retryStart(t, f)
finishPoll(t, f, ref, "PENDING")
var original time.Time
if err := db.QueryRowContext(ctx, `SELECT deadline_at FROM weave_poll_states WHERE namespace_id=?`, f.resources.NamespaceID).Scan(&original); err != nil {
t.Fatal(err)
}
c := clockCandidate(t, f, v.ID, "query")
waitClockDue(t, f, c)
f.exec(`UPDATE weave_instances SET paused=TRUE WHERE namespace_id=?`, f.resources.NamespaceID)
f.exec(`UPDATE pulse_execution_gates SET paused=TRUE WHERE namespace_id=?`, f.resources.NamespaceID)
if runClock(t, f, c) || f.count("pulse_tasks", "1=1") != 1 {
t.Fatal("paused poll started round")
}
f.exec(`UPDATE weave_instances SET paused=FALSE WHERE namespace_id=?`, f.resources.NamespaceID)
f.exec(`UPDATE pulse_execution_gates SET paused=FALSE WHERE namespace_id=?`, f.resources.NamespaceID)
f.exec(`UPDATE pulse_queues SET max_unfinished_tasks=1 WHERE namespace_id=?`, f.resources.NamespaceID)
blocker := f.submit(1)
waitClockDue(t, f, c)
if runClock(t, f, c) || f.count("weave_poll_states", "used_rounds=1 AND current_task_id IS NULL") != 1 {
t.Fatal("quota consumed poll budget")
}
f.exec(`UPDATE pulse_queues SET max_unfinished_tasks=2 WHERE namespace_id=?`, f.resources.NamespaceID)
waitClockDue(t, f, c)
if !runClock(t, f, c) {
t.Fatal("backpressure did not resume")
}
var after time.Time
if err := db.QueryRowContext(ctx, `SELECT deadline_at FROM weave_poll_states WHERE namespace_id=?`, f.resources.NamespaceID).Scan(&after); err != nil {
t.Fatal(err)
}
if !after.Equal(original) || f.count("pulse_tasks", "id<>?", blocker.ID) != 2 {
t.Fatal("recovery extended deadline or duplicated round")
}
})
t.Run("expired_inflight_poll_collects_existing_result_first", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
spec := pollingSpec(f.resources.QueueID)
spec.Body.Steps[0].Poll.TimeoutMS = 2000
spec.Body.Steps[0].Poll.Task.TimeoutMS = 1500
spec.Body.Steps[0].Poll.Task.Retry.AttemptTimeoutMS = 1000
v := runtimeStart(t, f, spec)
ref := retryStart(t, f)
if _, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`"DONE"`)}); err != nil {
t.Fatal(err)
}
c := clockCandidate(t, f, v.ID, "query")
waitClockDue(t, f, c)
if runClock(t, f, c) {
t.Fatal("deadline scanner preempted committed result")
}
runtimeView(t, f, v.ID, "RUNNING")
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
runtimeView(t, f, v.ID, "SUCCEEDED")
})
t.Run("poll_deadline_does_not_grant_extra_round", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
spec := pollingSpec(f.resources.QueueID)
spec.Body.Steps[0].Poll.TimeoutMS = 2000
spec.Body.Steps[0].Poll.Task.TimeoutMS = 1500
spec.Body.Steps[0].Poll.Task.Retry.AttemptTimeoutMS = 1000
v := runtimeStart(t, f, spec)
ref := retryStart(t, f)
finishPoll(t, f, ref, "PENDING")
c := clockCandidate(t, f, v.ID, "query")
time.Sleep(2100 * time.Millisecond)
if !runClock(t, f, c) {
t.Fatal("poll timeout absent")
}
runtimeView(t, f, v.ID, "NEEDS_REVIEW")
if f.count("pulse_tasks", "1=1") != 1 || f.count("weave_poll_states", "used_rounds=1 AND resolved_at IS NOT NULL") != 1 {
t.Fatal("late poll started extra work")
}
})
t.Run("request_retry_stays_in_same_business_round", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
spec := pollingSpec(f.resources.QueueID)
p := spec.Body.Steps[0].Poll
p.Task.Retry.MaxAttempts = 2
p.Task.Retry.InitialDelayMS = 1000
p.Task.Retry.MaxDelayMS = 1000
v := runtimeStart(t, f, spec)
first := retryStart(t, f)
retryFinish(t, f, first, "FAILED", true)
if f.count("weave_inbox", "1=1") != 0 || f.count("weave_poll_states", "used_rounds=1") != 1 {
t.Fatal("network retry advanced business round")
}
c := clockCandidate(t, f, v.ID, "query")
if runClock(t, f, c) {
t.Fatal("in-flight round overlapped")
}
time.Sleep(1050 * time.Millisecond)
retry := retryStart(t, f)
if retry.TaskID != first.TaskID {
t.Fatal("request retry created a new task")
}
finishPoll(t, f, retry, "PENDING")
c = clockCandidate(t, f, v.ID, "query")
waitClockDue(t, f, c)
if !runClock(t, f, c) {
t.Fatal("business round did not advance after successful pending result")
}
if f.count("weave_poll_states", "used_rounds=2 AND max_attempts_per_round=2") != 1 || f.count("pulse_tasks", "attempt_count=2") != 1 || f.count("pulse_tasks", "attempt_count=0 AND max_attempts=2") != 1 {
t.Fatal("round/attempt budgets were mixed")
}
})
t.Run("poll_deadline_keeps_inflight_evidence_until_pulse_recovery", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
spec := pollingSpec(f.resources.QueueID)
spec.Body.Steps[0].Poll.TimeoutMS = 2000
spec.Body.Steps[0].Poll.Task.TimeoutMS = 1500
spec.Body.Steps[0].Poll.Task.Retry.AttemptTimeoutMS = 1000
v := runtimeStart(t, f, spec)
ref := retryStart(t, f)
permits := f.used()
if permits == 0 {
t.Fatal("started query did not reserve permits")
}
c := clockCandidate(t, f, v.ID, "query")
waitClockDue(t, f, c)
if !runClock(t, f, c) {
t.Fatal("in-flight deadline did not close workflow gate")
}
runtimeView(t, f, v.ID, "NEEDS_REVIEW")
if f.count("weave_steps", "kind='POLL' AND state='ACTIVE'") != 1 || f.count("weave_poll_states", "current_task_id=? AND resolved_at IS NULL", ref.TaskID) != 1 || f.count("pulse_tasks", "state='RUNNING'") != 1 || f.used() != permits {
t.Fatal("clock falsely settled in-flight work or released its permit")
}
if _, err := f.storeB.Recover(ctx, 100); err != nil {
t.Fatal(err)
}
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
runtimeView(t, f, v.ID, "NEEDS_REVIEW")
if f.count("weave_steps", "kind='POLL' AND state='NEEDS_REVIEW'") != 1 || f.count("weave_poll_states", "current_task_id IS NULL AND used_rounds=1 AND resolved_at IS NOT NULL") != 1 || f.count("weave_pending_advances", "1=1") != 0 || f.used() != 0 {
t.Fatal("Pulse unknown result was lost or created another round")
}
})
t.Run("poll_continuation_rejects_corrupt_frozen_task", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
v := runtimeStart(t, f, pollingSpec(f.resources.QueueID))
ref := retryStart(t, f)
finishPoll(t, f, ref, "PENDING")
c := clockCandidate(t, f, v.ID, "query")
waitClockDue(t, f, c)
original := f.get(ref.TaskID).Payload
defer f.exec(`UPDATE pulse_tasks SET payload=? WHERE namespace_id=? AND id=?`, string(original), f.resources.NamespaceID, ref.TaskID)
f.exec(`UPDATE pulse_tasks SET payload='{"n":1}' WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, ref.TaskID)
if _, err := newRuntime(t, db).process(ctx, c, false); err == nil {
t.Fatal("corrupt first-round parameters granted new work")
}
if f.count("pulse_tasks", "1=1") != 1 || f.count("weave_poll_states", "used_rounds=1 AND current_task_id IS NULL") != 1 || f.count("weave_pending_advances", "1=1") != 1 {
t.Fatal("rejected frozen parameters changed budget")
}
})
t.Run("poll_clock_validates_owner_without_foreign_keys", func(t *testing.T) {
f := newClockFixture(t, ctx, db)
v := runtimeStart(t, f, pollingSpec(f.resources.QueueID))
ref := retryStart(t, f)
c := clockCandidate(t, f, v.ID, "query")
var original core.ID
if err := db.QueryRowContext(ctx, `SELECT owner_id FROM pulse_tasks WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, ref.TaskID).Scan(&original); err != nil {
t.Fatal(err)
}
defer f.exec(`UPDATE pulse_tasks SET owner_id=? WHERE namespace_id=? AND id=?`, original, f.resources.NamespaceID, ref.TaskID)
f.exec(`UPDATE pulse_tasks SET owner_id=? WHERE namespace_id=? AND id=?`, executionID(t), f.resources.NamespaceID, ref.TaskID)
// 提前形成到期候选不能绕过实际父记录与所有者校验。
f.exec(`UPDATE weave_pending_advances SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, c.ns, c.id)
if _, err := newRuntime(t, db).process(ctx, c, false); err == nil {
t.Fatal("wrong activation owner accepted without a foreign key")
}
if f.count("weave_poll_states", "used_rounds=1 AND current_task_id=?", ref.TaskID) != 1 || f.count("pulse_tasks", "1=1") != 1 {
t.Fatal("corrupt owner changed round state")
}
})
}

162
internal/storage/mysql/workflow_clock_upgrade_integration_test.go Normal file
View File

@@ -0,0 +1,162 @@
package mysql
import (
"context"
"encoding/json"
"os"
"testing"
"time"
core "git.yuntuo.hk/k2safe/orbit/model"
driver "github.com/go-sql-driver/mysql"
)
// 用真实v18结构和直接写入的历史等待记录验证数据回填，不冒充完整实例的执行回归。
func TestIntegrationWorkflowClockUpgrade(t *testing.T) {
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("requires disposable MySQL")
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
admin, err := Open(ctx, dsn, 2, 1)
if err != nil {
t.Fatal(err)
}
defer admin.Close()
name := "orbit_clock_upgrade_" + string(executionID(t))
if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+name+"`"); err != nil {
t.Fatal(err)
}
defer func() {
cleanup, stop := context.WithTimeout(context.Background(), 10*time.Second)
defer stop()
if _, err := admin.ExecContext(cleanup, "DROP DATABASE `"+name+"`"); err != nil {
t.Error(err)
}
}()
cfg, err := driver.ParseDSN(dsn)
if err != nil {
t.Fatal("invalid test DSN")
}
cfg.DBName = name
db, err := Open(ctx, cfg.FormatDSN(), 4, 2)
if err != nil {
t.Fatal(err)
}
defer db.Close()
exec := func(q string, args ...any) {
t.Helper()
if _, err := db.ExecContext(ctx, q, args...); err != nil {
t.Fatal(err)
}
}
exec(`CREATE TABLE orbit_schema_migrations (version BIGINT UNSIGNED PRIMARY KEY,name VARCHAR(191) NOT NULL,checksum CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,dirty BOOLEAN NOT NULL,applied_at DATETIME(6) NULL) ENGINE=InnoDB`)
migrations, err := Migrations()
if err != nil {
t.Fatal(err)
}
for _, m := range migrations {
if m.Version > 18 {
break
}
statements, err := splitSQL(m.SQL)
if err != nil {
t.Fatal(err)
}
for _, q := range statements {
exec(q)
}
exec(`INSERT INTO orbit_schema_migrations(version,name,checksum,dirty,applied_at) VALUES (?,?,?,FALSE,UTC_TIMESTAMP(6))`, m.Version, m.Name, m.Checksum)
}
var now time.Time
if err := db.QueryRowContext(ctx, `SELECT UTC_TIMESTAMP(6)`).Scan(&now); err != nil {
t.Fatal(err)
}
ns := executionID(t)
deadline, next := now.Add(time.Hour), now.Add(time.Minute)
type fixture struct {
workflow, step, activation core.ID
kind string
due time.Time
}
seed := func(key, kind, state string, terminal bool) fixture {
t.Helper()
f := fixture{executionID(t), executionID(t), executionID(t), kind, deadline}
instanceState := "WAITING"
var completed *time.Time
if terminal {
instanceState, completed = "CANCELLED", &now
}
exec(`INSERT INTO weave_instances(namespace_id,id,definition_id,definition_version_id,gate_id,state,input,policy_snapshot,deadline_at,completed_at) VALUES (?,?,?,?,?,?,'{}','{}',?,?)`, ns, f.workflow, executionID(t), executionID(t), executionID(t), instanceState, deadline, completed)
exec("INSERT INTO weave_steps(namespace_id,id,workflow_id,`key`,kind,state,current_activation_id,revision) VALUES (?,?,?,?,?,?,?,7)", ns, f.step, f.workflow, key, kind, state, f.activation)
exec(`INSERT INTO weave_activations(namespace_id,id,workflow_id,step_id,sequence,state,input) VALUES (?,?,?,?,1,?,'null')`, ns, f.activation, f.workflow, f.step, state)
if kind == "TIMER" {
exec(`INSERT INTO weave_waits(namespace_id,id,workflow_id,activation_id,kind,correlation_key,deadline_at) VALUES (?,?,?,?,'TIMER','',?)`, ns, executionID(t), f.workflow, f.activation, deadline)
} else {
var current *core.ID
var nextPoll *time.Time
if state == "ACTIVE" {
id := executionID(t)
current = &id
} else {
nextPoll, f.due = &next, next
}
exec(`INSERT INTO weave_poll_states(namespace_id,id,workflow_id,activation_id,max_rounds,used_rounds,max_attempts_per_round,interval_ms,deadline_at,next_poll_at,current_task_id) VALUES (?,?,?,?,10,3,2,1000,?,?,?)`, ns, executionID(t), f.workflow, f.activation, deadline, nextPoll, current)
}
return f
}
timer := seed("timer", "TIMER", "WAITING", false)
poll := seed("query", "POLL", "WAITING", false)
inflight := seed("query", "POLL", "ACTIVE", false)
seed("terminal", "TIMER", "WAITING", true)
stale := seed("stale", "TIMER", "WAITING", false)
exec(`UPDATE weave_steps SET current_activation_id=? WHERE namespace_id=? AND id=?`, executionID(t), ns, stale.step)
// 已有时钟与根待续均不能被数据回填重写；模拟提前部署已产生的检查点。
existing := seed("existing", "TIMER", "WAITING", false)
existingID, rootID, rootStep := executionID(t), executionID(t), executionID(t)
original := `{"schema_version":1,"kind":"TIMER","activation_id":"` + string(existing.activation) + `"}`
exec(`INSERT INTO weave_pending_advances(namespace_id,id,workflow_id,step_id,expected_step_revision,checkpoint,available_at,revision) VALUES (?,?,?,?,7,?,?,11)`, ns, existingID, existing.workflow, existing.step, original, next)
exec(`INSERT INTO weave_pending_advances(namespace_id,id,workflow_id,step_id,expected_step_revision,checkpoint,available_at) VALUES (?,?,?,?,4,'{"schema_version":1,"kind":"ROOT"}',?)`, ns, rootID, timer.workflow, rootStep, now)
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
if err := CheckSchema(ctx, db); err != nil {
t.Fatal(err)
}
for _, f := range []fixture{timer, poll, inflight} {
var raw []byte
var due time.Time
var expected uint64
if err := db.QueryRowContext(ctx, `SELECT checkpoint,available_at,expected_step_revision FROM weave_pending_advances WHERE namespace_id=? AND workflow_id=? AND step_id=?`, ns, f.workflow, f.step).Scan(&raw, &due, &expected); err != nil {
t.Fatal(err)
}
var checkpoint workflowCheckpoint
if err := json.Unmarshal(raw, &checkpoint); err != nil || checkpoint.SchemaVersion != 1 || checkpoint.Kind != f.kind || checkpoint.ActivationID == nil || *checkpoint.ActivationID != f.activation || expected != 7 || !due.Equal(f.due) {
t.Fatalf("incorrect v18 clock backfill: %s %v", raw, err)
}
}
var count int
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_pending_advances`).Scan(&count); err != nil || count != 5 {
t.Fatal("missing/duplicate/terminal/stale checkpoint", count, err)
}
var raw string
var revision uint64
var due time.Time
if err := db.QueryRowContext(ctx, `SELECT checkpoint,revision,available_at FROM weave_pending_advances WHERE namespace_id=? AND id=?`, ns, existingID).Scan(&raw, &revision, &due); err != nil || raw != original || revision != 11 || !due.Equal(next) {
t.Fatal("existing clock overwritten", err)
}
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_pending_advances WHERE namespace_id=? AND id=? AND checkpoint='{"schema_version":1,"kind":"ROOT"}' AND expected_step_revision=4 AND available_at=?`, ns, rootID, now).Scan(&count); err != nil || count != 1 {
t.Fatal("root checkpoint overwritten", err)
}
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_poll_states WHERE max_rounds=10 AND used_rounds=3 AND max_attempts_per_round=2 AND interval_ms=1000 AND deadline_at=? AND resolved_at IS NULL`, deadline).Scan(&count); err != nil || count != 2 {
t.Fatal("poll budget or deadline changed", err)
}
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_pending_advances`).Scan(&count); err != nil || count != 5 {
t.Fatal("repeated migration duplicated checkpoints", err)
}
t.Log("v18 timer/pending-query/in-flight-query checkpoints backfilled at original time; existing ROOT/clock preserved; terminal and stale activations excluded; budgets unchanged; zero foreign keys")
}

7
internal/storage/mysql/workflow_consume.go
View File

@@ -208,6 +208,13 @@ func (r *WorkflowRuntime) consume(ctx context.Context, tx *sql.Tx, g *workflowGr
if _, err := tx.ExecContext(ctx, `UPDATE weave_poll_states SET current_task_id=NULL,next_poll_at=?,resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND activation_id=?`, next, settled, now, c.ns, c.workflow, source.activation); err != nil {
return false, err
}
if next != nil {
if err := saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision+1, *next); err != nil {
return false, err
}
} else if err := deleteWorkflowClock(ctx, tx, g, row); err != nil {
return false, err
}
if settled != nil {
waitState := "CANCELLED"
if state == weave.StepSucceeded {

211
internal/storage/mysql/workflow_poll_clock.go Normal file
View File

@@ -0,0 +1,211 @@
package mysql
import (
"bytes"
"context"
"database/sql"
"errors"
"time"
"git.yuntuo.hk/k2safe/orbit/internal/fault"
"git.yuntuo.hk/k2safe/orbit/internal/jsonvalue"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/pulse/service"
"git.yuntuo.hk/k2safe/orbit/weave/definition"
"git.yuntuo.hk/k2safe/orbit/weave/engine"
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
)
func (r *WorkflowRuntime) advancePoll(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, c workflowCandidate, activated time.Time, allow bool, now time.Time) (bool, error) {
var p weave.PollState
err := tx.QueryRowContext(ctx, `SELECT id,revision,max_rounds,used_rounds,max_attempts_per_round,interval_ms,deadline_at,next_poll_at,current_task_id,resolved_at FROM weave_poll_states WHERE namespace_id=? AND workflow_id=? AND activation_id=? FOR UPDATE`, c.ns, c.workflow, row.view.ActivationID).Scan(&p.ID, &p.Revision, &p.MaxRounds, &p.UsedRounds, &p.MaxAttemptsPerRound, &p.IntervalMS, &p.DeadlineAt, &p.NextPollAt, &p.CurrentTaskID, &p.ResolvedAt)
if err != nil {
return false, err
}
node, _ := g.program.Step(row.view.Key)
policy := node.Poll
if policy == nil || p.MaxRounds != policy.MaxRounds || p.MaxAttemptsPerRound != policy.Task.Retry.MaxAttempts || p.IntervalMS != policy.IntervalMS || p.UsedRounds < 1 || !p.DeadlineAt.Equal(minimumTime(g.view.DeadlineAt, activated.Add(time.Duration(policy.TimeoutMS)*time.Millisecond))) {
return false, errors.New("workflow poll policy mismatch")
}
if p.ResolvedAt != nil {
return false, errors.New("unsettled poll step has resolved budget")
}
var count, minRound, maxRound uint32
if err := tx.QueryRowContext(ctx, "SELECT COUNT(*),COALESCE(MIN(`round`),0),COALESCE(MAX(`round`),0) FROM weave_poll_rounds WHERE namespace_id=? AND workflow_id=? AND activation_id=?", c.ns, c.workflow, row.view.ActivationID).Scan(&count, &minRound, &maxRound); err != nil {
return false, err
}
if count != p.UsedRounds || minRound != 1 || maxRound != p.UsedRounds {
return false, errors.New("workflow poll rounds incomplete")
}
var lastTask core.ID
if err := tx.QueryRowContext(ctx, "SELECT task_id FROM weave_poll_rounds WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND `round`=?", c.ns, c.workflow, row.view.ActivationID, p.UsedRounds).Scan(&lastTask); err != nil {
return false, err
}
if p.CurrentTaskID != nil {
if *p.CurrentTaskID != lastTask || p.NextPollAt != nil || row.view.State != weave.StepActive {
return false, errors.New("workflow in-flight poll mismatch")
}
if err := validatePollTaskOwner(ctx, tx, g, row, lastTask); err != nil {
return false, err
}
if now.Before(p.DeadlineAt) {
return false, saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision, p.DeadlineAt)
}
task, err := readTask(ctx, tx, c.ns, lastTask, false)
if err != nil {
return false, err
}
if task.QueueID != policy.Task.QueueID {
return false, errors.New("workflow in-flight poll queue mismatch")
}
if task.State.Terminal() || task.State == "NEEDS_REVIEW" {
// 原结果已可靠落库时，先让Inbox按事件时间裁决，不能用扫描器的当前时间覆盖它。
var available time.Time
err := tx.QueryRowContext(ctx, `SELECT available_at FROM weave_inbox WHERE namespace_id=? AND workflow_id=? AND source_task_id=? AND source_activation_id=? AND state='PENDING' ORDER BY source_revision DESC LIMIT 1`, c.ns, c.workflow, lastTask, row.view.ActivationID).Scan(&available)
if err != nil {
return false, err
}
if available.Before(now.Add(time.Second)) {
available = now.Add(time.Second)
}
return false, saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision, available)
}
// 到期但仍在途时先关闭门控，保留当前轮次和ACTIVE步骤继续接收Pulse结清结果。
changed := g.view.State != weave.InstanceNeedsReview
if changed {
if err := holdWorkflow(ctx, tx, g, weave.InstanceNeedsReview, "POLL_INFLIGHT_DEADLINE", now); err != nil {
return false, err
}
if err := r.audit(ctx, tx, g, c.id, "POLL_INFLIGHT_DEADLINE", now); err != nil {
return false, err
}
}
return changed, saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision, now.Add(time.Second))
}
if row.view.State != weave.StepWaiting || p.NextPollAt == nil {
return false, errors.New("workflow poll continuation missing")
}
if !now.Before(p.DeadlineAt) || p.UsedRounds >= p.MaxRounds {
reason := "POLL_DEADLINE"
if p.UsedRounds >= p.MaxRounds {
reason = "POLL_ROUNDS_EXHAUSTED"
}
if _, err := tx.ExecContext(ctx, `UPDATE weave_poll_states SET resolved_at=?,next_poll_at=NULL,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, now, now, c.ns, p.ID); err != nil {
return false, err
}
if _, err := tx.ExecContext(ctx, `UPDATE weave_waits SET state='EXPIRED',resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND state='WAITING'`, now, now, c.ns, c.workflow, row.view.ActivationID); err != nil {
return false, err
}
if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepNeedsReview, nil, now); err != nil {
return false, err
}
if err := deleteWorkflowClock(ctx, tx, g, row); err != nil {
return false, err
}
if err := holdWorkflow(ctx, tx, g, weave.InstanceNeedsReview, reason, now); err != nil {
return false, err
}
return true, r.audit(ctx, tx, g, c.id, reason, now)
}
if now.Before(*p.NextPollAt) {
return false, saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision, *p.NextPollAt)
}
if !allow {
return false, saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision, minimumTime(p.DeadlineAt, now.Add(time.Second)))
}
round, err := p.NextRound(now)
if err != nil {
return false, err
}
// 只有上一轮成功查询并已分类为PENDING才可以消耗新轮次，不能用时间推进绕过失败预算。
var disposition string
if err := tx.QueryRowContext(ctx, `SELECT error_code FROM weave_inbox WHERE namespace_id=? AND workflow_id=? AND source_task_id=? AND source_activation_id=? AND state='PROCESSED' ORDER BY source_revision DESC LIMIT 1`, c.ns, c.workflow, lastTask, row.view.ActivationID).Scan(&disposition); err != nil {
return false, err
}
if disposition != "POLL_PENDING" {
return false, errors.New("poll previous round was not pending")
}
in, err := pollTaskInput(ctx, tx, g, row, policy.Task)
if err != nil {
return false, err
}
taskID, err := core.NewID()
if err != nil {
return false, err
}
task, err := insertAcceptedTaskUntil(ctx, tx, c.ns, taskID, in, nil, &p.DeadlineAt)
if errors.Is(err, fault.QueueFull) || errors.Is(err, fault.ApplicationFull) {
return false, saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision, minimumTime(p.DeadlineAt, now.Add(time.Second)))
}
if err != nil {
return false, err
}
res, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET owner_kind=?,owner_id=?,gate_id=? WHERE namespace_id=? AND id=? AND owner_kind IS NULL`, engine.TaskOwnerKind, row.view.ActivationID, g.gate, c.ns, taskID)
if err := exactlyOne(res, err); err != nil {
return false, err
}
roundID, err := core.NewID()
if err != nil {
return false, err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO weave_poll_rounds(namespace_id,id,workflow_id,activation_id,`round`,task_id) VALUES (?,?,?,?,?,?)", c.ns, roundID, c.workflow, row.view.ActivationID, round, taskID); err != nil {
return false, err
}
res, err = tx.ExecContext(ctx, `UPDATE weave_poll_states SET used_rounds=?,current_task_id=?,next_poll_at=NULL,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=? AND current_task_id IS NULL`, round, taskID, now, c.ns, p.ID, p.Revision)
if err := exactlyOne(res, err); err != nil {
return false, err
}
if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepActive, nil, now); err != nil {
return false, err
}
if err := saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision, p.DeadlineAt); err != nil {
return false, err
}
if err := emitEvent(ctx, tx, c.ns, taskID, task.Revision, "task", "task.accepted", map[string]any{"task_id": taskID, "state": task.State, "workflow_id": c.workflow, "activation_id": row.view.ActivationID, "round": round}); err != nil {
return false, err
}
// 新查询已持久化，实例从纯等待恢复RUNNING；组合推进仍受同一个32步预算。
if err := r.progress(ctx, tx, g, allow, now); err != nil {
return false, err
}
return true, r.audit(ctx, tx, g, c.id, "POLL_ROUND_ACCEPTED", now)
}
func validatePollTaskOwner(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, id core.ID) error {
var owner, gate *core.ID
var kind sql.NullString
if err := tx.QueryRowContext(ctx, `SELECT owner_kind,owner_id,gate_id FROM pulse_tasks WHERE namespace_id=? AND id=?`, g.view.NamespaceID, id).Scan(&kind, &owner, &gate); err != nil {
return err
}
if kind.String != engine.TaskOwnerKind || owner == nil || row.view.ActivationID == nil || *owner != *row.view.ActivationID || gate == nil || *gate != g.gate {
return errors.New("poll task ownership invalid")
}
return nil
}
// pollTaskInput 复用首轮已经解析并冻结的输入、业务键及互斥键；不重算依赖可能已改变的表达式。
func pollTaskInput(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, spec definition.Task) (service.SubmitInput, error) {
var first core.ID
if err := tx.QueryRowContext(ctx, "SELECT task_id FROM weave_poll_rounds WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND `round`=1", g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(&first); err != nil {
return service.SubmitInput{}, err
}
if err := validatePollTaskOwner(ctx, tx, g, row, first); err != nil {
return service.SubmitInput{}, err
}
base, err := readTask(ctx, tx, g.view.NamespaceID, first, false)
if err != nil {
return service.SubmitInput{}, err
}
input, err := jsonvalue.Canonical(base.Payload)
if err != nil {
return service.SubmitInput{}, err
}
activationInput, err := jsonvalue.Canonical(row.input)
if err != nil {
return service.SubmitInput{}, err
}
if !bytes.Equal(input, activationInput) || definition.ValidateData(spec.InputType, base.Payload) != nil || base.QueueID != spec.QueueID || base.Handler != spec.Handler || base.HandlerVersion != spec.HandlerVersion || base.PayloadVersion != spec.PayloadVersion || base.Retry != spec.Retry || base.Priority != spec.Priority || base.RecoverySafety != spec.RecoverySafety {
return service.SubmitInput{}, errors.New("poll frozen task parameters mismatch")
}
return service.NormalizeSubmit(service.SubmitInput{QueueID: base.QueueID, Handler: base.Handler, HandlerVersion: base.HandlerVersion, PayloadVersion: base.PayloadVersion, Payload: base.Payload, BusinessKey: base.BusinessKey, ConcurrencyKey: base.ConcurrencyKey, Priority: base.Priority, TaskTimeoutMS: spec.TimeoutMS, Retry: base.Retry, RecoverySafety: base.RecoverySafety})
}

16
internal/storage/mysql/workflow_runtime.go
View File

@@ -13,7 +13,7 @@ import (
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
)
// WorkflowRuntime 消费持久化结果与根推进检查点；不持有业务执行租约或调用业务系统。
// WorkflowRuntime 消费持久化结果、根推进与等待时钟检查点；不持有业务执行租约或调用业务系统。
// 每个候选独立短事务，数据库行锁决定唯一推进权，节点内存不保存恢复游标。
type WorkflowRuntime struct {
db *sql.DB
@@ -30,7 +30,7 @@ func NewWorkflowRuntime(db *sql.DB, node core.ID) (*WorkflowRuntime, error) {
type WorkflowRuntimeStats struct {
Examined int // Examined 本批取出的持久化候选数，包含被其他节点先处理或跳过的记录。
Consumed int // Consumed 已提交确认或隔离的内部结果数，事务回滚不计入。
Advanced int // Advanced 已提交推进的根检查点数，不等于业务任务执行次数。
Advanced int // Advanced 已提交推进的根或等待检查点数，不等于业务任务执行次数。
}
type workflowCandidate struct {
@@ -129,12 +129,16 @@ func (r *WorkflowRuntime) process(ctx context.Context, c workflowCandidate, skip
if due.After(now) {
return nil
}
var checkpoint struct {
SchemaVersion uint32 `json:"schema_version"`
Kind string `json:"kind"`
var checkpoint workflowCheckpoint
if json.Unmarshal(raw, &checkpoint) != nil || checkpoint.SchemaVersion != 1 {
return errors.New("workflow checkpoint invalid")
}
if checkpoint.Kind == "TIMER" || checkpoint.Kind == "POLL" {
done, err = r.advanceClock(ctx, tx, g, c, step, expected, checkpoint, allow, now)
return err
}
root := g.steps[workflowRootKey]
if json.Unmarshal(raw, &checkpoint) != nil || checkpoint.SchemaVersion != 1 || checkpoint.Kind != "ROOT" || step != root.view.ID || expected > root.view.Revision {
if checkpoint.Kind != "ROOT" || checkpoint.ActivationID != nil || step != root.view.ID || expected > root.view.Revision {
return errors.New("workflow checkpoint invalid")
}
if g.view.State.Terminal() {

66
internal/storage/mysql/workflow_runtime_process_integration_test.go
View File

@@ -2,12 +2,14 @@ package mysql
import (
"context"
"encoding/json"
"os"
"os/exec"
"path/filepath"
"testing"
"time"
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
"git.yuntuo.hk/k2safe/orbit/weave/instance"
mysqldriver "github.com/go-sql-driver/mysql"
)
@@ -145,4 +147,68 @@ func TestIntegrationWorkflowRuntimeProcess(t *testing.T) {
t.Fatal("process restart duplicated work or lost confirmation")
}
t.Log("sole Orbit process SIGKILL confirmed; result committed with all controllers stopped; two new Orbit processes completed exactly three tasks and three result confirmations")
for _, p := range processes {
stop(p)
}
timer := runtimeStart(t, f, timerSpec(f.resources.QueueID, 100, true))
var originalWake time.Time
if err := db.QueryRowContext(ctx, `SELECT deadline_at FROM weave_waits WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, timer.ID).Scan(&originalWake); err != nil {
t.Fatal(err)
}
time.Sleep(150 * time.Millisecond) // 全部控制进程保持退出，定时等待自然到期。
if f.count("pulse_tasks", "1=1") != 3 {
t.Fatal("timer advanced with every controller stopped")
}
start("timer-replacement-a")
start("timer-replacement-b")
wait(func() bool { return f.count("pulse_tasks", "1=1") == 4 }, "expired timer takeover")
ref = retryStart(t, f)
if f.get(ref.TaskID).CreatedAt.Before(originalWake) {
t.Fatal("timer successor accepted before original wake")
}
retryFinish(t, f, ref, "SUCCEEDED", false)
wait(func() bool {
current, err := api.Get(ctx, f.token, timer.ID)
return err == nil && current.State == "SUCCEEDED"
}, "timer workflow completion")
var afterWake time.Time
if err := db.QueryRowContext(ctx, `SELECT deadline_at FROM weave_waits WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, timer.ID).Scan(&afterWake); err != nil || !afterWake.Equal(originalWake) {
t.Fatal("timer wake was reset on restart", err)
}
for _, p := range processes {
stop(p)
}
f = keyedWorker(t, f, f.resources.QueueID, f.a)
poll := runtimeStart(t, f, pollingSpec(f.resources.QueueID))
var originalDeadline time.Time
if err := db.QueryRowContext(ctx, `SELECT deadline_at FROM weave_poll_states WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, poll.ID).Scan(&originalDeadline); err != nil {
t.Fatal(err)
}
ref = retryStart(t, f)
if _, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`"PENDING"`)}); err != nil {
t.Fatal(err)
}
time.Sleep(1100 * time.Millisecond)
if f.count("pulse_tasks", "1=1") != 5 || f.count("weave_poll_states", "used_rounds=1") != 1 {
t.Fatal("poll continued while all controllers stopped")
}
start("poll-replacement-a")
start("poll-replacement-b")
wait(func() bool { return f.count("pulse_tasks", "1=1") == 6 }, "next poll round after all-stop")
ref = retryStart(t, f)
if _, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`"DONE"`)}); err != nil {
t.Fatal(err)
}
wait(func() bool {
current, err := api.Get(ctx, f.token, poll.ID)
return err == nil && current.State == "SUCCEEDED"
}, "poll workflow completion")
var afterDeadline time.Time
if err := db.QueryRowContext(ctx, `SELECT deadline_at FROM weave_poll_states WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, poll.ID).Scan(&afterDeadline); err != nil || !afterDeadline.Equal(originalDeadline) {
t.Fatal("poll deadline reset", err)
}
if f.count("weave_poll_states", "used_rounds=2 AND resolved_at IS NOT NULL") != 1 || f.count("weave_poll_rounds", "1=1") != 2 || f.count("weave_pending_advances", "1=1") != 0 {
t.Fatal("poll takeover lost budget or duplicated round")
}
t.Log("all controllers exited before timer expiry and poll PENDING delivery; two fresh processes preserved original wake/deadline and completed timer plus exactly two query rounds")
}