feat: persist workflow controls and settle stopped branches

This commit is contained in:
2026-09-18 04:32:02 +08:00
parent 7d5ebfcb09
commit 7eac8c365a
35 changed files with 1539 additions and 36 deletions

View File

@@ -64,6 +64,8 @@ func TestIntegrationApplicationResourcesUpgrade(t *testing.T) {
retryFinish(t, g, finished, "SUCCEEDED", false)
assertApplicationCounts(t, f, 4, 2, 1)
// 还原后续新增表和字段,得到真实 v9 结构;现有任务内容保持不变。
f.exec(`ALTER TABLE pulse_tasks DROP INDEX ix_task_workflow_owner`)
f.exec(`ALTER TABLE weave_instances DROP CHECK ck_workflow_stop_intent, DROP COLUMN stop_kind, DROP COLUMN stop_requested_at`)
f.exec(`ALTER TABLE weave_signals DROP CHECK ck_signal_version_sequence, DROP CHECK ck_signal_pending_identity, DROP INDEX ix_signal_expiry, DROP INDEX ix_signal_activation, DROP COLUMN payload_version, DROP COLUMN target_activation_sequence, DROP COLUMN evidence_ref, DROP COLUMN disposition, DROP COLUMN available_at, MODIFY COLUMN type VARCHAR(64) NOT NULL COMMENT '事件或信号的稳定类型标识', MODIFY COLUMN correlation_key VARCHAR(191) NOT NULL COMMENT '外部业务操作关联键,防止旧信号误用于新工作'`)
f.exec(`ALTER TABLE weave_waits DROP INDEX uq_wait_winner_signal, DROP COLUMN winner_signal_id`)
f.exec(`ALTER TABLE weave_waits DROP COLUMN signal_name, DROP COLUMN payload_version, MODIFY COLUMN correlation_key VARCHAR(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_as_cs NOT NULL COMMENT '外部业务操作关联键,防止旧信号误用于新工作'`)

View File

@@ -0,0 +1,30 @@
-- 独立保存首次停止意图,避免进入核查后丢失用户取消/失败结清的目标;不增加外键。
ALTER TABLE weave_instances
ADD COLUMN stop_kind VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NULL COMMENT '首次停止原因:CANCEL为用户取消,FAILURE为固定失败策略;为空表示尚未要求结清',
ADD COLUMN stop_requested_at DATETIME(6) NULL COMMENT '首次停止意图的数据库时间,重报、进入核查或后来取消均不覆盖';
-- 历史取消保留原意图时间;旧失败结清及失败策略触发的核查保留原更新时间作为兼容边界。
UPDATE weave_instances i SET stop_kind=CASE WHEN cancel_requested_at IS NOT NULL THEN 'CANCEL' ELSE 'FAILURE' END,
stop_requested_at=COALESCE(cancel_requested_at,updated_at)
WHERE state NOT IN ('SUCCEEDED','FAILED','CANCELLED') AND
(cancel_requested_at IS NOT NULL OR state='CANCELLING' OR
(JSON_UNQUOTE(JSON_EXTRACT(policy_snapshot,'$.failure_policy'))='FAIL_AFTER_SETTLED' AND EXISTS
(SELECT 1 FROM weave_steps s WHERE s.namespace_id=i.namespace_id AND s.workflow_id=i.id AND s.`key`<>'$root' AND s.state IN ('FAILED','CANCELLED','NEEDS_REVIEW'))));
ALTER TABLE weave_instances
ADD CONSTRAINT ck_workflow_stop_intent CHECK ((stop_kind IS NULL AND stop_requested_at IS NULL) OR (stop_kind IS NOT NULL AND stop_kind IN ('CANCEL','FAILURE') AND stop_requested_at IS NOT NULL));
-- 结清复用根检查点,已有根只提前检查,不更改固定版本、原预算和截止。
INSERT INTO weave_pending_advances(namespace_id,id,workflow_id,step_id,expected_step_revision,checkpoint,available_at)
SELECT i.namespace_id,RANDOM_BYTES(16),i.id,s.id,s.revision,JSON_OBJECT('schema_version',1,'kind','ROOT'),UTC_TIMESTAMP(6)
FROM weave_instances i JOIN weave_steps s ON s.namespace_id=i.namespace_id AND s.workflow_id=i.id AND s.`key`='$root'
JOIN weave_activations a ON a.namespace_id=s.namespace_id AND a.workflow_id=s.workflow_id AND a.step_id=s.id AND a.id=s.current_activation_id
WHERE i.stop_kind IS NOT NULL AND i.state NOT IN ('SUCCEEDED','FAILED','CANCELLED') AND s.state='ACTIVE' AND a.state='ACTIVE'
ON DUPLICATE KEY UPDATE available_at=LEAST(weave_pending_advances.available_at,VALUES(available_at));
UPDATE pulse_execution_gates g JOIN weave_instances i ON i.namespace_id=g.namespace_id AND i.gate_id=g.id
SET g.paused=TRUE,g.revision=g.revision+1,g.updated_at=UTC_TIMESTAMP(6)
WHERE i.stop_kind IS NOT NULL AND i.state NOT IN ('SUCCEEDED','FAILED','CANCELLED') AND g.paused=FALSE;
-- 反向核对流程所有者,防止无外键时遗漏门控被误改的历史任务。
ALTER TABLE pulse_tasks ADD KEY ix_task_workflow_owner (namespace_id,owner_kind,owner_id,gate_id,id);

View File

@@ -89,7 +89,7 @@ func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
indexes := func() []string {
t.Helper()
// v15替换retry_of索引、v17增加内部结果唯一索引、v20增加信号索引,下方单独验证定义。
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result','ix_signal_expiry','ix_signal_activation','uq_wait_winner_signal') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result','ix_signal_expiry','ix_signal_activation','uq_wait_winner_signal','ix_task_workflow_owner') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
if err != nil {
t.Fatal(err)
}
@@ -179,6 +179,7 @@ func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
}
var collation string
for _, index := range []struct{ table, name, want string }{
{"pulse_tasks", "ix_task_workflow_owner", "namespace_id:1,owner_kind:1,owner_id:1,gate_id:1,id:1"},
{"weave_signals", "ix_signal_expiry", "state:1,available_at:1,namespace_id:1,id:1"},
{"weave_signals", "ix_signal_activation", "namespace_id:1,workflow_id:1,target_step_key:1,target_activation_sequence:1,state:1,accepted_at:1,id:1"},
{"weave_waits", "uq_wait_winner_signal", "namespace_id:0,winner_signal_id:0"},

View File

@@ -55,6 +55,12 @@ func (r *WorkflowRuntime) advanceClock(ctx context.Context, tx *sql.Tx, g *workf
if row.view.State != weave.StepActive && row.view.State != weave.StepWaiting {
return false, deleteWorkflowClock(ctx, tx, g, row)
}
if g.view.StopKind != nil {
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
return false, err
}
return false, deleteWorkflowClock(ctx, tx, g, row)
}
var activated time.Time
if err := tx.QueryRowContext(ctx, `SELECT created_at FROM weave_activations WHERE namespace_id=? AND id=? AND workflow_id=? AND step_id=?`, c.ns, checkpoint.ActivationID, c.workflow, step).Scan(&activated); err != nil {
return false, err

View File

@@ -187,7 +187,9 @@ func (r *WorkflowRuntime) consume(ctx context.Context, tx *sql.Tx, g *workflowGr
}
}
// 单轮请求失败不能通过新建业务轮次绕过Pulse预算。
if state != weave.StepSucceeded {
if g.view.StopKind != nil && (state == weave.StepCancelled || state == weave.StepFailed) {
reason = "POLL_STOP_REQUEST_SETTLED"
} else if state != weave.StepSucceeded {
state, reason = weave.StepNeedsReview, "POLL_REQUEST_REVIEW"
} else if !source.occurred.Before(deadline) {
state, output, reason = weave.StepNeedsReview, nil, "POLL_DEADLINE"
@@ -198,7 +200,9 @@ func (r *WorkflowRuntime) consume(ctx context.Context, tx *sql.Tx, g *workflowGr
state, output, reason = weave.StepNeedsReview, nil, "POLL_RESULT_INVALID"
case outcome == engine.PollPending:
state, output, reason = weave.StepWaiting, nil, "POLL_PENDING"
if used >= max {
if g.view.StopKind != nil {
state, reason = weave.StepCancelled, "POLL_STOP_PENDING_SETTLED"
} else if used >= max {
state, reason = weave.StepNeedsReview, "POLL_ROUNDS_EXHAUSTED"
}
case outcome == engine.PollFailed:
@@ -250,7 +254,7 @@ func (r *WorkflowRuntime) consume(ctx context.Context, tx *sql.Tx, g *workflowGr
}
if state == weave.StepNeedsReview || state == weave.StepFailed || state == weave.StepCancelled {
next := weave.InstanceNeedsReview
if (state == weave.StepFailed || state == weave.StepCancelled) && (g.version.Specification.FailurePolicy == "FAIL_AFTER_SETTLED" || g.view.CancelRequestedAt != nil) {
if (state == weave.StepFailed || state == weave.StepCancelled) && (g.view.StopKind != nil || g.version.Specification.FailurePolicy == "FAIL_AFTER_SETTLED" || g.view.CancelRequestedAt != nil) {
next = weave.InstanceCancelling
}
if err := holdWorkflow(ctx, tx, g, next, reason, now); err != nil {

View File

@@ -0,0 +1,136 @@
package mysql
import (
"context"
"database/sql"
"encoding/json"
"errors"
"time"
"git.yuntuo.hk/k2safe/orbit/internal/fault"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/platform/auth"
"git.yuntuo.hk/k2safe/orbit/weave/control"
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
)
// WorkflowControlStore 控制决定和门控同事务,不在请求中等待外部执行停止。
type WorkflowControlStore struct{ *TaskStore }
func NewWorkflowControlStore(db *sql.DB) *WorkflowControlStore {
return &WorkflowControlStore{NewTaskStore(db)}
}
var _ control.Store = (*WorkflowControlStore)(nil)
func (s *WorkflowControlStore) Apply(ctx context.Context, t auth.Token, id core.ID, key string, in control.Input) (control.Result, error) {
var out control.Result
err := s.transaction(ctx, t, control.Action, func(tx *sql.Tx, p auth.Permissions) error {
if _, err := lockApplicationResources(ctx, tx, t.NamespaceID, false); err != nil {
return err
}
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "workflow.control", string(id), key, "workflow", id, in)
if err != nil {
return err
}
g, _, err := lockWorkflowGraph(ctx, tx, t.NamespaceID, id, false)
if err != nil {
return err
}
for _, q := range g.version.Analysis.Queues {
if !p.AllowsQueue(q) {
return fault.NotFound
}
}
if cached != nil {
if err := json.Unmarshal(cached, &out); err != nil {
return err
}
if out.Instance.NamespaceID != t.NamespaceID || out.Instance.ID != id || out.Instance.DefinitionVersionID != g.view.DefinitionVersionID {
return errors.New("workflow control receipt mismatch")
}
out.Deduplicated = true
return nil
}
if g.view.State.Terminal() || g.view.ControlRevision != in.ExpectedControlRevision {
return fault.Conflict
}
before := g.view
now, err := dbNow(ctx, tx)
if err != nil {
return err
}
switch in.Operation {
case "PAUSE":
if g.view.Paused {
return fault.Conflict
}
g.view.Paused = true
case "RESUME":
if !g.view.Paused {
return fault.Conflict
}
g.view.Paused = false
case "CANCEL":
if g.view.CancelRequestedAt != nil {
return fault.Conflict
}
g.view.CancelRequestedAt = &now
if err := recordWorkflowStop(ctx, tx, g, "CANCEL", now); err != nil {
return err
}
if g.view.State != weave.InstanceNeedsReview {
g.view.State = weave.InstanceCancelling
}
default:
return fault.Invalid("unsupported workflow control operation")
}
res, err := tx.ExecContext(ctx, `UPDATE weave_instances SET paused=?,state=?,cancel_requested_at=?,control_revision=control_revision+1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND control_revision=? AND revision=?`, g.view.Paused, g.view.State, g.view.CancelRequestedAt, now, t.NamespaceID, id, in.ExpectedControlRevision, g.view.Revision)
if err := exactlyOne(res, err); err != nil {
return err
}
g.view.ControlRevision++
g.view.Revision++
gatePaused := g.view.Paused || g.view.StopKind != nil || g.view.State == weave.InstanceNeedsReview || g.view.State == weave.InstanceCancelling
if _, err := tx.ExecContext(ctx, `UPDATE pulse_execution_gates SET paused=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, gatePaused, now, t.NamespaceID, g.gate); err != nil {
return err
}
if in.Operation == "CANCEL" || in.Operation == "RESUME" {
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
return err
}
}
meta := map[string]any{"operation": in.Operation, "paused": g.view.Paused, "state": g.view.State, "control_revision": g.view.ControlRevision, "stop_kind": g.view.StopKind, "stop_requested_at": g.view.StopRequestedAt, "cancel_requested_at": g.view.CancelRequestedAt}
if err := emitEvent(ctx, tx, t.NamespaceID, id, g.view.Revision, "workflow", "workflow.controlled", meta); err != nil {
return err
}
if err := appendAudit(ctx, tx, t, identity, id, "workflow", "workflow.control", in.Reason, map[string]any{"paused": before.Paused, "state": before.State, "control_revision": before.ControlRevision, "stop_kind": before.StopKind}, meta); err != nil {
return err
}
out.Instance, _, err = readWorkflowInstance(ctx, tx, t.NamespaceID, id, false)
if err != nil {
return err
}
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
})
return out, err
}
// recordWorkflowStop 冻结首次停止原因;后来的用户取消不能抹去已启动的失败结清。
// 调用方同时关闭门控、保存根检查点和审计,不能将此函数单独提交。
func recordWorkflowStop(ctx context.Context, tx *sql.Tx, g *workflowGraph, kind string, now time.Time) error {
if g.view.StopKind != nil {
return nil
}
if kind != "CANCEL" && kind != "FAILURE" {
return errors.New("invalid workflow stop kind")
}
res, err := tx.ExecContext(ctx, `UPDATE weave_instances SET stop_kind=?,stop_requested_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=? AND stop_kind IS NULL`, kind, now, now, g.view.NamespaceID, g.view.ID, g.view.Revision)
if err := exactlyOne(res, err); err != nil {
return err
}
g.view.StopKind = &kind
g.view.StopRequestedAt = &now
g.view.Revision++
return nil
}

View File

@@ -0,0 +1,484 @@
package mysql
import (
"context"
"database/sql"
"os"
"sync"
"testing"
"time"
"git.yuntuo.hk/k2safe/orbit/internal/fault"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/weave/control"
"git.yuntuo.hk/k2safe/orbit/weave/definition"
"git.yuntuo.hk/k2safe/orbit/weave/instance"
)
func grantWorkflowControl(f *executionFixture) {
f.exec(`UPDATE orbit_credentials SET permissions=JSON_ARRAY_APPEND(permissions,'$.actions','workflows:control') WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, f.resources.CredentialID)
}
func applyControl(t *testing.T, f *executionFixture, v instance.View, operation string) control.Result {
t.Helper()
out, err := control.New(NewWorkflowControlStore(f.db)).Apply(f.ctx, f.token, v.ID, string(executionID(t)), control.Input{Operation: operation, ExpectedControlRevision: v.ControlRevision, Reason: "controlled integration action"})
if err != nil {
t.Fatal(err)
}
return out
}
func stopStep(t *testing.T, f *executionFixture, id core.ID) bool {
t.Helper()
f.exec(`UPDATE weave_pending_advances SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, id)
rows, err := f.db.QueryContext(f.ctx, `SELECT 'INBOX',id,revision FROM weave_inbox WHERE namespace_id=? AND workflow_id=? AND state='PENDING' UNION ALL SELECT 'ROOT',id,revision FROM weave_pending_advances WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, id, f.resources.NamespaceID, id)
if err != nil {
t.Fatal(err)
}
var candidates []workflowCandidate
for rows.Next() {
c := workflowCandidate{ns: f.resources.NamespaceID, workflow: id}
if err := rows.Scan(&c.kind, &c.id, &c.revision); err != nil {
rows.Close()
t.Fatal(err)
}
candidates = append(candidates, c)
}
err = rows.Err()
rows.Close()
if err != nil {
t.Fatal(err)
}
for _, c := range candidates {
if _, err := newRuntime(t, f.db).process(f.ctx, c, false); err != nil {
t.Fatal(err)
}
}
return len(candidates) > 0
}
func drainStop(t *testing.T, f *executionFixture, id core.ID, state string) {
t.Helper()
for i := 0; i < 40; i++ {
if !stopStep(t, f, id) {
runtimeView(t, f, id, state)
return
}
}
t.Fatal("stop did not become quiescent")
}
func controlFixture(t *testing.T, ctx context.Context, db *sql.DB, count int) (*executionFixture, instance.View) {
t.Helper()
f := newExecutionFixture(t, ctx, db)
v := runtimeStart(t, f, runtimeSpec(f.resources.QueueID, count))
grantWorkflowControl(f)
return f, v
}
func TestIntegrationWorkflowControls(t *testing.T) {
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("requires disposable MySQL")
}
ctx, stop := context.WithTimeout(context.Background(), 150*time.Second)
defer stop()
db, err := Open(ctx, dsn, 32, 8)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
t.Run("pause_collect_resume_uses_control_revision", func(t *testing.T) {
f, v := controlFixture(t, ctx, db, 2)
ref := retryStart(t, f)
paused := applyControl(t, f, v, "PAUSE").Instance
if !paused.Paused || paused.ControlRevision != 2 || f.used() != 1 {
t.Fatal("pause stopped running work or failed version")
}
retryFinish(t, f, ref, "SUCCEEDED", false)
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
after := runtimeView(t, f, v.ID, "RUNNING")
if after.Revision <= paused.Revision || after.ControlRevision != paused.ControlRevision || f.count("pulse_tasks", "1=1") != 1 {
t.Fatal("background result corrupted control version")
}
resumed := applyControl(t, f, paused, "RESUME").Instance
if resumed.Paused || resumed.ControlRevision != 3 || !resumed.DeadlineAt.Equal(v.DeadlineAt) {
t.Fatal("resume reset state or deadline")
}
runtimeAdvance(t, f, newRuntime(t, db), v.ID)
if f.count("pulse_tasks", "1=1") != 2 {
t.Fatal("resume lost successor")
}
})
t.Run("cancel_waiting_cascades_in_application_and_preserves_receipt", func(t *testing.T) {
f, v := controlFixture(t, ctx, db, 3)
api := control.New(NewWorkflowControlStore(db))
key := string(executionID(t))
in := control.Input{Operation: "CANCEL", ExpectedControlRevision: 1, Reason: "stop all work"}
out, err := api.Apply(ctx, f.token, v.ID, key, in)
if err != nil {
t.Fatal(err)
}
if out.Instance.StopKind == nil || *out.Instance.StopKind != "CANCEL" || out.Instance.StopRequestedAt == nil || out.Instance.State != "CANCELLING" {
t.Fatal("cancel intent incomplete")
}
drainStop(t, f, v.ID, "CANCELLED")
if f.count("pulse_tasks", "state='CANCELLED'") != 1 || f.count("weave_steps", "state='SKIPPED'") != 2 || f.count("weave_inbox", "state='PROCESSED'") != 1 || f.used() != 0 {
t.Fatal("cancel did not settle all work")
}
replay, err := api.Apply(ctx, f.token, v.ID, key, in)
if err != nil || !replay.Deduplicated || replay.Instance.State != "CANCELLING" || replay.Instance.ControlRevision != 2 {
t.Fatal("control receipt was mutated", err)
}
_, err = api.Apply(ctx, f.token, v.ID, string(executionID(t)), control.Input{Operation: "PAUSE", ExpectedControlRevision: 2, Reason: "cannot control terminal"})
if fault.Code(err) != "STATE_CONFLICT" {
t.Fatal("terminal accepted control", err)
}
})
t.Run("concurrent_same_identity_and_stale_control", func(t *testing.T) {
f, v := controlFixture(t, ctx, db, 1)
api := control.New(NewWorkflowControlStore(db))
key := string(executionID(t))
in := control.Input{Operation: "PAUSE", ExpectedControlRevision: 1, Reason: "one decision"}
var wg sync.WaitGroup
errs := make(chan error, 12)
fresh := make(chan bool, 12)
for i := 0; i < 12; i++ {
wg.Add(1)
go func() {
defer wg.Done()
out, err := api.Apply(ctx, f.token, v.ID, key, in)
errs <- err
fresh <- !out.Deduplicated
}()
}
wg.Wait()
close(errs)
close(fresh)
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
n := 0
for v := range fresh {
if v {
n++
}
}
if n != 1 || f.count("orbit_audit_events", "action='workflow.control'") != 1 || f.count("weave_instances", "control_revision=2 AND paused=TRUE") != 1 {
t.Fatal("duplicated control", n)
}
in.Operation = "CANCEL"
if _, err := api.Apply(ctx, f.token, v.ID, key, in); fault.Code(err) != "IDEMPOTENCY_CONFLICT" {
t.Fatal(err)
}
if _, err := api.Apply(ctx, f.token, v.ID, string(executionID(t)), in); fault.Code(err) != "STATE_CONFLICT" {
t.Fatal(err)
}
})
t.Run("permission_and_scope_still_checked_on_replay", func(t *testing.T) {
f, v := controlFixture(t, ctx, db, 1)
api := control.New(NewWorkflowControlStore(db))
key := string(executionID(t))
in := control.Input{Operation: "PAUSE", ExpectedControlRevision: 1, Reason: "authorized pause"}
if _, err := api.Apply(ctx, f.token, v.ID, key, in); err != nil {
t.Fatal(err)
}
f.exec(`UPDATE orbit_credentials SET permissions=JSON_SET(permissions,'$.queues',JSON_ARRAY(?)) WHERE namespace_id=? AND id=?`, string(executionID(t)), f.resources.NamespaceID, f.resources.CredentialID)
if _, err := api.Apply(ctx, f.token, v.ID, key, in); fault.Code(err) != "NOT_FOUND" {
t.Fatal("scope replay bypass", err)
}
grantWorkflowInstance(t, f)
if _, err := api.Apply(ctx, f.token, v.ID, key, in); fault.Code(err) != "PERMISSION_DENIED" {
t.Fatal("action replay bypass", err)
}
})
t.Run("control_audit_failure_rolls_back_everything", func(t *testing.T) {
f, v := controlFixture(t, ctx, db, 1)
api := control.New(NewWorkflowControlStore(db))
key := string(executionID(t))
in := control.Input{Operation: "CANCEL", ExpectedControlRevision: 1, Reason: "atomic decision"}
f.exec(`CREATE TRIGGER test_control_failure BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='control audit fault'`)
defer f.exec(`DROP TRIGGER IF EXISTS test_control_failure`)
if _, err := api.Apply(ctx, f.token, v.ID, key, in); err == nil {
t.Fatal("audit failure accepted")
}
if f.count("weave_instances", "control_revision=1 AND stop_kind IS NULL AND cancel_requested_at IS NULL") != 1 || f.count("pulse_execution_gates", "paused=FALSE") != 1 || f.count("weave_pending_advances", "1=1") != 0 {
t.Fatal("partial control mutation")
}
f.exec(`DROP TRIGGER test_control_failure`)
if _, err := api.Apply(ctx, f.token, v.ID, key, in); err != nil {
t.Fatal(err)
}
drainStop(t, f, v.ID, "CANCELLED")
})
t.Run("settlement_audit_failure_rolls_back_task_and_inbox", func(t *testing.T) {
f, v := controlFixture(t, ctx, db, 1)
applyControl(t, f, v, "CANCEL")
c := clockCandidate(t, f, v.ID, workflowRootKey)
f.exec(`CREATE TRIGGER test_settlement_failure BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='settlement audit fault'`)
defer f.exec(`DROP TRIGGER IF EXISTS test_settlement_failure`)
if done, err := newRuntime(t, db).process(ctx, c, false); err == nil || done {
t.Fatal("partial settlement committed")
}
if f.count("pulse_tasks", "state='WAITING' AND cancel_requested_at IS NULL") != 1 || f.count("weave_inbox", "1=1") != 0 {
t.Fatal("cancel or result escaped transaction")
}
f.exec(`DROP TRIGGER test_settlement_failure`)
drainStop(t, f, v.ID, "CANCELLED")
})
for _, outcome := range []string{"SUCCEEDED", "FAILED", "CANCELLED", "UNKNOWN"} {
t.Run("running_cancel_then_"+outcome, func(t *testing.T) {
f, v := controlFixture(t, ctx, db, 2)
ref := retryStart(t, f)
applyControl(t, f, v, "CANCEL")
stopStep(t, f, v.ID)
if f.get(ref.TaskID).State != "RUNNING" || f.get(ref.TaskID).CancelRequestedAt == nil || f.used() != 1 {
t.Fatal("cancel pretended running work ended")
}
retryFinish(t, f, ref, outcome, false)
want := "CANCELLED"
if outcome == "UNKNOWN" {
want = "NEEDS_REVIEW"
}
drainStop(t, f, v.ID, want)
after := runtimeView(t, f, v.ID, want)
if after.StopKind == nil || *after.StopKind != "CANCEL" || f.used() != 0 || f.count("pulse_tasks", "1=1") != 1 {
t.Fatal("stop intent lost or extra task")
}
})
}
t.Run("failure_policy_settles_parallel_and_freezes_first_reason", func(t *testing.T) {
f := newExecutionFixture(t, ctx, db)
spec := runtimeSpec(f.resources.QueueID, 1)
tail := spec.Body.Steps[0]
spec.Body.Steps = []definition.Step{runtimeParallel(spec), tail}
spec.FailurePolicy = "FAIL_AFTER_SETTLED"
v := runtimeStart(t, f, spec)
grantWorkflowControl(f)
left, right := retryStart(t, f), retryStart(t, f)
retryFinish(t, f, left, "FAILED", false)
runtimeConsume(t, f, newRuntime(t, db), left.TaskID)
stopping := runtimeView(t, f, v.ID, "CANCELLING")
if stopping.StopKind == nil || *stopping.StopKind != "FAILURE" {
t.Fatal("failure intent not persisted")
}
later := applyControl(t, f, stopping, "CANCEL").Instance
if *later.StopKind != "FAILURE" || !later.StopRequestedAt.Equal(*stopping.StopRequestedAt) {
t.Fatal("user cancel rewrote earlier failure")
}
stopStep(t, f, v.ID)
if f.get(right.TaskID).CancelRequestedAt == nil || f.used() != 1 {
t.Fatal("sibling cancellation missing")
}
retryFinish(t, f, right, "SUCCEEDED", false)
drainStop(t, f, v.ID, "FAILED")
if f.count("pulse_tasks", "1=1") != 2 || f.count("weave_steps", "`key`='join' AND state='CANCELLED'") != 1 || f.count("weave_steps", "`key`='task0' AND state='SKIPPED'") != 1 {
t.Fatal("failed join advanced tail")
}
})
t.Run("unknown_failure_cancels_sibling_but_never_terminal", func(t *testing.T) {
f := newExecutionFixture(t, ctx, db)
spec := runtimeSpec(f.resources.QueueID, 1)
tail := spec.Body.Steps[0]
spec.Body.Steps = []definition.Step{runtimeParallel(spec), tail}
spec.FailurePolicy = "FAIL_AFTER_SETTLED"
v := runtimeStart(t, f, spec)
left, right := retryStart(t, f), retryStart(t, f)
retryFinish(t, f, left, "UNKNOWN", false)
runtimeConsume(t, f, newRuntime(t, db), left.TaskID)
stopStep(t, f, v.ID)
stopStep(t, f, v.ID)
if f.get(right.TaskID).CancelRequestedAt == nil {
t.Fatal("review lost failure cancellation")
}
retryFinish(t, f, right, "CANCELLED", false)
drainStop(t, f, v.ID, "NEEDS_REVIEW")
if f.used() != 0 || f.count("weave_instances", "completed_at IS NULL AND stop_kind='FAILURE'") != 1 {
t.Fatal("unknown result terminalized")
}
})
for _, kind := range []string{"TIMER", "SIGNAL", "EARLY_SIGNAL"} {
t.Run("cancel_wait_"+kind, func(t *testing.T) {
f := newExecutionFixture(t, ctx, db)
spec := signalSpec(f.resources.QueueID, false)
if kind == "TIMER" {
spec = timerSpec(f.resources.QueueID, 30000, true)
}
v := runtimeStart(t, f, spec)
grantWorkflowControl(f)
grantWorkflowSignal(f)
if kind == "EARLY_SIGNAL" {
sendSignal(t, f, v, signalInput(v))
}
applyControl(t, f, v, "CANCEL")
if kind != "TIMER" {
late := sendSignal(t, f, v, signalInput(v))
if late.Signal.Disposition != "WORKFLOW_STOPPING" {
t.Fatal("late signal was admitted")
}
}
drainStop(t, f, v.ID, "CANCELLED")
if f.count("pulse_tasks", "1=1") != 0 {
t.Fatal("cancelled wait activated successor")
}
if kind == "EARLY_SIGNAL" && f.count("weave_signals", "state='CONSUMED'") != 1 {
t.Fatal("prior accepted signal lost")
}
})
}
for _, running := range []bool{false, true} {
name := "poll_between_rounds"
if running {
name = "poll_running_pending"
}
t.Run(name, func(t *testing.T) {
f := newClockFixture(t, ctx, db)
v := runtimeStart(t, f, pollingSpec(f.resources.QueueID))
grantWorkflowControl(f)
ref := retryStart(t, f)
if !running {
finishPoll(t, f, ref, "PENDING")
}
applyControl(t, f, v, "CANCEL")
stopStep(t, f, v.ID)
if running {
if f.get(ref.TaskID).CancelRequestedAt == nil {
t.Fatal("query did not receive cancel")
}
finishQueryOnly(t, f, ref, "PENDING")
}
drainStop(t, f, v.ID, "CANCELLED")
if f.count("pulse_tasks", "1=1") != 1 || f.count("weave_poll_states", "used_rounds=1 AND current_task_id IS NULL AND next_poll_at IS NULL AND resolved_at IS NOT NULL") != 1 {
t.Fatal("cancel query added a round or left budget open")
}
})
}
t.Run("more_than_32_blocked_steps_resume_root", func(t *testing.T) {
f, v := controlFixture(t, ctx, db, 60)
applyControl(t, f, v, "CANCEL")
drainStop(t, f, v.ID, "CANCELLED")
if f.count("weave_steps", "state='SKIPPED'") != 59 || f.count("orbit_audit_events", "action='workflow.advance' AND reason='WORKFLOW_STOP_CONTINUATION'") < 1 {
t.Fatal("bounded stop lost continuation")
}
})
t.Run("pause_blocks_leased_start_and_resume_preserves_attempt", func(t *testing.T) {
f, v := controlFixture(t, ctx, db, 1)
_, ref := f.claim()
paused := applyControl(t, f, v, "PAUSE").Instance
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); fault.Code(err) != "EXECUTION_PAUSED" {
t.Fatal("paused lease started", err)
}
resumed := applyControl(t, f, paused, "RESUME").Instance
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
t.Fatal(err)
}
if f.get(ref.TaskID).AttemptCount != 1 || resumed.ControlRevision != 3 {
t.Fatal("resume replaced original attempt")
}
retryFinish(t, f, ref, "SUCCEEDED", false)
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
runtimeView(t, f, v.ID, "SUCCEEDED")
})
t.Run("start_cancel_race_has_one_authority", func(t *testing.T) {
for i := 0; i < 8; i++ {
f, v := controlFixture(t, ctx, db, 1)
_, ref := f.claim()
api := control.New(NewWorkflowControlStore(db))
var startErr, controlErr error
var wg sync.WaitGroup
ready := make(chan struct{})
startID, key := executionID(t), string(executionID(t))
wg.Add(2)
go func() { defer wg.Done(); <-ready; _, startErr = f.a.Start(ctx, f.token, f.binding, startID, ref) }()
go func() {
defer wg.Done()
<-ready
_, controlErr = api.Apply(ctx, f.token, v.ID, key, control.Input{Operation: "CANCEL", ExpectedControlRevision: 1, Reason: "start cancellation race"})
}()
close(ready)
wg.Wait()
if controlErr != nil {
t.Fatal(controlErr)
}
if startErr != nil && fault.Code(startErr) != "EXECUTION_PAUSED" {
t.Fatal("unexpected start outcome", startErr)
}
stopStep(t, f, v.ID)
if startErr == nil {
if f.get(ref.TaskID).State != "RUNNING" || f.used() != 1 {
t.Fatal("authorized work released by cancellation")
}
retryFinish(t, f, ref, "CANCELLED", false)
} else if f.get(ref.TaskID).State != "CANCELLED" || f.used() != 0 {
t.Fatal("unstarted work not settled")
}
drainStop(t, f, v.ID, "CANCELLED")
}
})
t.Run("extra_gate_reference_cannot_cancel_unrelated_task", func(t *testing.T) {
f, v := controlFixture(t, ctx, db, 1)
extra := f.submit(1)
var gate, activation core.ID
if err := db.QueryRowContext(ctx, "SELECT i.gate_id,s.current_activation_id FROM weave_instances i JOIN weave_steps s ON s.namespace_id=i.namespace_id AND s.workflow_id=i.id AND s.`key`='task0' WHERE i.namespace_id=? AND i.id=?", f.resources.NamespaceID, v.ID).Scan(&gate, &activation); err != nil {
t.Fatal(err)
}
f.exec(`UPDATE pulse_tasks SET gate_id=?,owner_kind='workflow_activation',owner_id=? WHERE namespace_id=? AND id=?`, gate, activation, f.resources.NamespaceID, extra.ID)
defer f.exec(`UPDATE pulse_tasks SET gate_id=NULL,owner_kind=NULL,owner_id=NULL WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, extra.ID)
applyControl(t, f, v, "CANCEL")
rejected := false
for i := 0; i < 3; i++ {
c := clockCandidate(t, f, v.ID, workflowRootKey)
if _, err := newRuntime(t, db).process(ctx, c, false); err != nil {
rejected = true
break
}
}
if !rejected || f.get(extra.ID).CancelRequestedAt != nil || f.count("weave_instances", "state='CANCELLED'") != 0 {
t.Fatal("gate alone granted cancellation or termination")
}
f.exec(`UPDATE pulse_tasks SET gate_id=NULL,owner_kind=NULL,owner_id=NULL WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, extra.ID)
drainStop(t, f, v.ID, "CANCELLED")
if f.get(extra.ID).State != "WAITING" {
t.Fatal("unrelated task changed")
}
})
for _, withSignal := range []bool{false, true} {
name := "cancel_beats_unconsumed_timeout"
if withSignal {
name = "cancel_collects_signal_after_total_deadline"
}
t.Run(name, func(t *testing.T) {
f := newExecutionFixture(t, ctx, db)
spec := signalSpec(f.resources.QueueID, false)
spec.Body.Steps = spec.Body.Steps[:1]
spec.Body.Output = definition.Value{Kind: "REFERENCE", Reference: &definition.Reference{Source: "signal"}}
spec.TimeoutMS = 1000
spec.Body.Steps[0].Signal.TimeoutMS = 1000
v := runtimeStart(t, f, spec)
grantWorkflowControl(f)
grantWorkflowSignal(f)
var source workflowCandidate
if withSignal {
accepted := sendSignal(t, f, v, signalInput(v))
source = workflowCandidate{kind: "SIGNAL", ns: f.resources.NamespaceID, workflow: v.ID, id: accepted.Signal.ID, revision: accepted.Signal.Revision}
} else {
source = clockCandidate(t, f, v.ID, "signal")
}
applyControl(t, f, v, "CANCEL")
for time.Now().Before(v.DeadlineAt.Add(20 * time.Millisecond)) {
time.Sleep(20 * time.Millisecond)
}
if _, err := newRuntime(t, db).process(ctx, source, false); err != nil {
t.Fatal(err)
}
drainStop(t, f, v.ID, "CANCELLED")
if withSignal && f.count("weave_signals", "state='CONSUMED'") != 1 {
t.Fatal("accepted signal expired after cancellation")
}
if f.count("weave_steps", "state='NEEDS_REVIEW'") != 0 {
t.Fatal("later timeout rewrote earlier cancellation")
}
})
}
t.Log("versioned control, permission replay, audit rollback, safe cancellation, failure settlement, signal/POLL collection and bounded root continuation verified")
}

View File

@@ -52,7 +52,7 @@ type workflowPolicy struct {
func readWorkflowInstance(ctx context.Context, tx *sql.Tx, ns, id core.ID, lock bool) (instance.View, core.ID, error) {
var v instance.View
var gate core.ID
q := `SELECT i.id,i.namespace_id,i.definition_id,i.definition_version_id,v.version,i.gate_id,i.state,i.revision,i.paused,i.control_revision,i.input,i.output,COALESCE(i.business_key,''),i.deadline_at,i.cancel_requested_at,i.completed_at,i.created_at,i.updated_at
q := `SELECT i.id,i.namespace_id,i.definition_id,i.definition_version_id,v.version,i.gate_id,i.state,i.revision,i.paused,i.control_revision,i.input,i.output,COALESCE(i.business_key,''),i.deadline_at,i.cancel_requested_at,i.stop_kind,i.stop_requested_at,i.completed_at,i.created_at,i.updated_at
FROM weave_instances i JOIN weave_definition_versions v ON v.namespace_id=i.namespace_id AND v.definition_id=i.definition_id AND v.id=i.definition_version_id
JOIN weave_definitions d ON d.namespace_id=i.namespace_id AND d.id=i.definition_id
JOIN pulse_execution_gates g ON g.namespace_id=i.namespace_id AND g.id=i.gate_id
@@ -61,7 +61,7 @@ func readWorkflowInstance(ctx context.Context, tx *sql.Tx, ns, id core.ID, lock
q += ` FOR SHARE`
}
var input, output []byte
err := tx.QueryRowContext(ctx, q, ns, id).Scan(&v.ID, &v.NamespaceID, &v.DefinitionID, &v.DefinitionVersionID, &v.Version, &gate, &v.State, &v.Revision, &v.Paused, &v.ControlRevision, &input, &output, &v.BusinessKey, &v.DeadlineAt, &v.CancelRequestedAt, &v.CompletedAt, &v.CreatedAt, &v.UpdatedAt)
err := tx.QueryRowContext(ctx, q, ns, id).Scan(&v.ID, &v.NamespaceID, &v.DefinitionID, &v.DefinitionVersionID, &v.Version, &gate, &v.State, &v.Revision, &v.Paused, &v.ControlRevision, &input, &output, &v.BusinessKey, &v.DeadlineAt, &v.CancelRequestedAt, &v.StopKind, &v.StopRequestedAt, &v.CompletedAt, &v.CreatedAt, &v.UpdatedAt)
v.Input, v.Output = input, output
if errors.Is(err, sql.ErrNoRows) {
err = fault.NotFound

View File

@@ -152,6 +152,10 @@ func (r *WorkflowRuntime) process(ctx context.Context, c workflowCandidate, skip
_, err = tx.ExecContext(ctx, `DELETE FROM weave_pending_advances WHERE namespace_id=? AND id=?`, c.ns, c.id)
return err
}
if g.view.StopKind != nil {
done, err = r.settleWorkflow(ctx, tx, g, c, now)
return err
}
if !allow || g.view.State == weave.InstanceNeedsReview || g.view.State == weave.InstanceCancelling {
// 暂停/停用仍按原截止计时;核查和失败结清由控制运行器接续,不清除其状态。
if (g.view.State == weave.InstanceRunning || g.view.State == weave.InstanceWaiting) && !now.Before(g.view.DeadlineAt) {
@@ -192,7 +196,7 @@ func cloneWorkflowGraph(g *workflowGraph) *workflowGraph {
// progress 在结果已保存后执行有限决策。额度不足只回滚后继决策,不撤销已确认的结果。
func (r *WorkflowRuntime) progress(ctx context.Context, tx *sql.Tx, g *workflowGraph, allow bool, now time.Time) error {
if g.view.State.Terminal() || g.view.State == weave.InstanceNeedsReview || g.view.State == weave.InstanceCancelling {
if g.view.State.Terminal() || g.view.StopKind != nil || g.view.State == weave.InstanceNeedsReview || g.view.State == weave.InstanceCancelling {
return nil
}
allow = allow && now.Before(g.view.DeadlineAt)
@@ -232,6 +236,18 @@ func holdWorkflow(ctx context.Context, tx *sql.Tx, g *workflowGraph, state weave
if g.view.State.Terminal() {
return errors.New("cannot hold completed workflow")
}
if state == weave.InstanceCancelling || g.view.StopKind != nil || g.version.Specification.FailurePolicy == "FAIL_AFTER_SETTLED" {
kind := "FAILURE"
if g.view.CancelRequestedAt != nil {
kind = "CANCEL"
}
if err := recordWorkflowStop(ctx, tx, g, kind, now); err != nil {
return err
}
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
return err
}
}
if g.view.State == weave.InstanceNeedsReview {
state = weave.InstanceNeedsReview
}

View File

@@ -96,7 +96,7 @@ func lockWorkflowGraph(ctx context.Context, tx *sql.Tx, ns, id core.ID, skip boo
if len(g.steps) != int(version.Analysis.Steps)+1 || root == nil || root.view.ActivationID == nil {
return nil, false, errors.New("workflow step set incomplete")
}
allow := enabled && !gatePaused && !v.Paused && v.CancelRequestedAt == nil && (v.State == weave.InstanceRunning || v.State == weave.InstanceWaiting)
allow := enabled && v.StopKind == nil && !gatePaused && !v.Paused && v.CancelRequestedAt == nil && (v.State == weave.InstanceRunning || v.State == weave.InstanceWaiting)
return g, allow, nil
}

View File

@@ -262,4 +262,53 @@ func TestIntegrationWorkflowRuntimeProcess(t *testing.T) {
t.Fatal("final signal restart lost winner or issued an extra round")
}
t.Log("early signal persisted before wait activation across all-stop; final signal selected with running query kept permits, then survived another all-stop and accepted PENDING settlement without issuing a second query")
for _, p := range processes {
stop(p)
}
failedSpec := runtimeSpec(f.resources.QueueID, 1)
tail := failedSpec.Body.Steps[0]
failedSpec.Body.Steps = append(failedSpec.Body.Steps[:0], runtimeParallel(failedSpec), tail)
failedSpec.FailurePolicy = "FAIL_AFTER_SETTLED"
failed := runtimeStart(t, f, failedSpec)
left, right := retryStart(t, f), retryStart(t, f)
retryFinish(t, f, left, "FAILED", false)
start("failure-collection-a")
start("failure-collection-b")
wait(func() bool { return f.get(right.TaskID).CancelRequestedAt != nil }, "failure cancellation of parallel sibling")
if f.used() != 1 || f.get(right.TaskID).State != "RUNNING" {
t.Fatal("failure claimed sibling ended")
}
for _, p := range processes {
stop(p)
}
retryFinish(t, f, right, "SUCCEEDED", false)
if f.count("weave_instances", "id=? AND state='CANCELLING' AND stop_kind='FAILURE'", failed.ID) != 1 {
t.Fatal("all-stop lost failure intent")
}
start("failure-settlement-a")
start("failure-settlement-b")
wait(func() bool {
current, err := api.Get(ctx, f.token, failed.ID)
return err == nil && current.State == "FAILED"
}, "parallel failure settlement across restart")
if f.count("pulse_tasks", "1=1") != 10 || f.used() != 0 {
t.Fatal("failure restarted a branch or tail")
}
for _, p := range processes {
stop(p)
}
cancelled := runtimeStart(t, f, runtimeSpec(f.resources.QueueID, 2))
grantWorkflowControl(f)
ref = retryStart(t, f)
applyControl(t, f, cancelled, "CANCEL")
retryFinish(t, f, ref, "UNKNOWN", false)
start("cancel-review-a")
start("cancel-review-b")
wait(func() bool {
return f.count("weave_instances", "id=? AND state='NEEDS_REVIEW' AND stop_kind='CANCEL' AND completed_at IS NULL", cancelled.ID) == 1 && f.count("weave_pending_advances", "workflow_id=?", cancelled.ID) == 0
}, "cancel unknown review without false terminal state")
if f.used() != 0 || f.count("pulse_tasks", "1=1") != 11 {
t.Fatal("unknown cancel leaked permit or started tail")
}
t.Log("parallel failure intent and running sibling survived another all-stop, settled FAILED only after result collection; user cancellation plus UNKNOWN recovered NEEDS_REVIEW with no successor or false terminal state")
}

View File

@@ -0,0 +1,384 @@
package mysql
import (
"context"
"database/sql"
"errors"
"sort"
"time"
core "git.yuntuo.hk/k2safe/orbit/model"
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
"git.yuntuo.hk/k2safe/orbit/weave/engine"
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
)
// settleWorkflow 在根检查点下结清停止意图。每事务最多发送一个任务取消,避免取得结果流锁后再锁另一任务。
// 步骤收尾最多32项,超过后保留根接续;停止/核查不依赖节点内存恢复。
func (r *WorkflowRuntime) settleWorkflow(ctx context.Context, tx *sql.Tx, g *workflowGraph, c workflowCandidate, now time.Time) (bool, error) {
if g.view.StopKind == nil || g.view.StopRequestedAt == nil || (*g.view.StopKind != "CANCEL" && *g.view.StopKind != "FAILURE") {
return false, errors.New("workflow stop intent invalid")
}
if *g.view.StopKind == "FAILURE" && g.version.Specification.FailurePolicy != "FAIL_AFTER_SETTLED" {
return false, errors.New("workflow failure stop conflicts with fixed policy")
}
if *g.view.StopKind == "CANCEL" && g.view.CancelRequestedAt == nil {
return false, errors.New("workflow cancellation intent missing")
}
// 已持有应用/全部队列/实例/门控锁;门控漂移不能为停止中的工作重新授权。
if _, err := tx.ExecContext(ctx, `UPDATE pulse_execution_gates SET paused=TRUE,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND paused=FALSE`, now, g.view.NamespaceID, g.gate); err != nil {
return false, err
}
var taskID core.ID
err := tx.QueryRowContext(ctx, `SELECT id FROM pulse_tasks WHERE namespace_id=? AND gate_id=? AND state NOT IN ('SUCCEEDED','FAILED','CANCELLED') AND cancel_requested_at IS NULL ORDER BY id LIMIT 1`, g.view.NamespaceID, g.gate).Scan(&taskID)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return false, err
}
if err == nil {
if err := validateWorkflowStopTask(ctx, tx, g, taskID); err != nil {
return false, err
}
task, err := readExecutionTask(ctx, tx, g.view.NamespaceID, taskID)
if err != nil {
return false, err
}
if _, err := cancelExecutionTask(ctx, tx, g.view.NamespaceID, task, now, "WORKFLOW_STOP_REQUESTED"); err != nil {
return false, err
}
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
return false, err
}
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_TASK_STOP_REQUESTED", now)
}
var pending int
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_inbox WHERE namespace_id=? AND workflow_id=? AND state='PENDING'`, g.view.NamespaceID, g.view.ID).Scan(&pending); err != nil {
return false, err
}
if pending > 0 {
return false, saveWorkflowAdvance(ctx, tx, g, now.Add(time.Second))
}
keys := make([]string, 0, len(g.steps))
for key := range g.steps {
if key != workflowRootKey {
keys = append(keys, key)
}
}
sort.Strings(keys)
changed, active, review := 0, false, false
for _, key := range keys {
row := g.steps[key]
if row.view.State == weave.StepNeedsReview {
review = true
continue
}
if row.view.State == weave.StepSucceeded || row.view.State == weave.StepFailed || row.view.State == weave.StepCancelled || row.view.State == weave.StepSkipped {
continue
}
if changed == 32 {
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
return false, err
}
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_CONTINUATION", now)
}
if row.view.State == weave.StepBlocked {
res, err := tx.ExecContext(ctx, `UPDATE weave_steps SET state='SKIPPED',revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=? AND current_activation_id IS NULL AND state='BLOCKED'`, now, g.view.NamespaceID, row.view.ID, row.view.Revision)
if err := exactlyOne(res, err); err != nil {
return false, err
}
row.view.State = weave.StepSkipped
row.view.Revision++
changed++
if err := emitEvent(ctx, tx, g.view.NamespaceID, row.view.ID, row.view.Revision, "workflow_step", "workflow.step.changed", map[string]any{"workflow_id": g.view.ID, "key": key, "state": row.view.State, "reason": "WORKFLOW_STOP_REQUESTED"}); err != nil {
return false, err
}
continue
}
switch row.view.Kind {
case weave.NodeTask:
if row.view.TaskID == nil {
return false, errors.New("stopping task step missing task")
}
task, err := readTask(ctx, tx, g.view.NamespaceID, *row.view.TaskID, false)
if err != nil {
return false, err
}
if task.State.Terminal() || task.State == pulse.TaskNeedsReview {
return false, errors.New("stopping task missing result confirmation")
}
active = true
case weave.NodeSignal, weave.NodeTimer, weave.NodePoll:
node, _ := g.program.Step(key)
if row.view.Kind == weave.NodePoll {
var current *core.ID
if err := tx.QueryRowContext(ctx, `SELECT current_task_id FROM weave_poll_states WHERE namespace_id=? AND workflow_id=? AND activation_id=?`, g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(&current); err != nil {
return false, err
}
if current != nil {
active = true
continue
}
}
// 已在停止前可靠受理的信号保留为事实;停止后的新信号由受理事务拒绝。
if signalContract(node) != nil {
w, err := readWorkflowSignalWait(ctx, tx, g, row)
if err != nil {
return false, err
}
v, err := selectedWorkflowSignal(ctx, tx, g, row, w, now)
if err != nil || v != nil {
handled, done, reason, err := r.driveWorkflowSignal(ctx, tx, g, row, c, nil, false, now)
if err != nil {
return false, err
}
if !handled {
return false, errors.New("stopping signal did not resolve")
}
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
return false, err
}
if done && reason != "" {
return true, r.audit(ctx, tx, g, c.id, reason, now)
}
return done, nil
}
}
if err := cancelWorkflowWait(ctx, tx, g, row, now); err != nil {
return false, err
}
changed++
case weave.NodeBranch, weave.NodeParallel:
// 组合步骤只有在全部活动叶子结清后才能结清,避免掩盖在途工作。
default:
return false, errors.New("unsupported stopping step")
}
}
if active {
if err := saveWorkflowAdvance(ctx, tx, g, now.Add(time.Second)); err != nil {
return false, err
}
if changed > 0 {
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_COLLECTING", now)
}
return false, nil
}
settled, uncertain, err := verifyWorkflowStopWork(ctx, tx, g)
if err != nil {
return false, err
}
if !settled {
if err := saveWorkflowAdvance(ctx, tx, g, now.Add(time.Second)); err != nil {
return false, err
}
if changed > 0 {
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_COLLECTING", now)
}
return false, nil
}
if review || uncertain || g.view.State == weave.InstanceNeedsReview {
stateChanged := g.view.State != weave.InstanceNeedsReview
if stateChanged {
if err := holdWorkflow(ctx, tx, g, weave.InstanceNeedsReview, "WORKFLOW_STOP_REVIEW", now); err != nil {
return false, err
}
}
// 已无平台内有效授权时不逐秒空转;UNKNOWN不证明外部执行已结束,后续核查负责接续。
if _, err := tx.ExecContext(ctx, `UPDATE weave_signals SET state='REJECTED',disposition='WORKFLOW_STOPPED_REVIEW',revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND state='PENDING'`, now, g.view.NamespaceID, g.view.ID); err != nil {
return false, err
}
if err := deleteWorkflowClock(ctx, tx, g, g.steps[workflowRootKey]); err != nil {
return false, err
}
if changed > 0 || stateChanged {
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_REVIEW", now)
}
return false, nil
}
for _, key := range keys {
row := g.steps[key]
if row.view.Kind != weave.NodeBranch && row.view.Kind != weave.NodeParallel {
continue
}
if row.view.State != weave.StepActive && row.view.State != weave.StepWaiting {
continue
}
if changed == 32 {
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
return false, err
}
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_CONTINUATION", now)
}
if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepCancelled, nil, now); err != nil {
return false, err
}
changed++
}
if _, err := tx.ExecContext(ctx, `UPDATE weave_signals SET state='REJECTED',disposition='WORKFLOW_STOPPED',revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND state='PENDING'`, now, g.view.NamespaceID, g.view.ID); err != nil {
return false, err
}
next, rootState := weave.InstanceCancelled, weave.StepCancelled
if *g.view.StopKind == "FAILURE" {
next, rootState = weave.InstanceFailed, weave.StepFailed
}
if err := saveWorkflowResultStep(ctx, tx, g, g.steps[workflowRootKey], rootState, nil, now); err != nil {
return false, err
}
res, err := tx.ExecContext(ctx, `UPDATE weave_instances SET state=?,output=NULL,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, next, now, now, g.view.NamespaceID, g.view.ID, g.view.Revision)
if err := exactlyOne(res, err); err != nil {
return false, err
}
g.view.State = next
g.view.Output = nil
g.view.CompletedAt = &now
g.view.Revision++
if _, err := tx.ExecContext(ctx, `DELETE FROM weave_pending_advances WHERE namespace_id=? AND workflow_id=?`, g.view.NamespaceID, g.view.ID); err != nil {
return false, err
}
if err := emitEvent(ctx, tx, g.view.NamespaceID, g.view.ID, g.view.Revision, "workflow", "workflow.stopped", map[string]any{"state": next, "stop_kind": g.view.StopKind, "stop_requested_at": g.view.StopRequestedAt}); err != nil {
return false, err
}
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOPPED", now)
}
func cancelWorkflowWait(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, now time.Time) error {
if row.view.Kind == weave.NodeTimer {
var count int
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_waits WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND kind='TIMER' AND state='WAITING' AND signal_name IS NULL AND payload_version IS NULL AND winner_signal_id IS NULL`, g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(&count); err != nil {
return err
}
if count != 1 {
return errors.New("stopping timer wait invalid")
}
}
if row.view.Kind == weave.NodePoll {
var activated time.Time
if err := tx.QueryRowContext(ctx, `SELECT created_at FROM weave_activations WHERE namespace_id=? AND workflow_id=? AND step_id=? AND id=?`, g.view.NamespaceID, g.view.ID, row.view.ID, row.view.ActivationID).Scan(&activated); err != nil {
return err
}
p, _, err := readWorkflowPollState(ctx, tx, g, row, activated)
if err != nil {
return err
}
if p.CurrentTaskID != nil {
return errors.New("cannot cancel in-flight poll wait")
}
res, err := tx.ExecContext(ctx, `UPDATE weave_poll_states SET next_poll_at=NULL,resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND resolved_at IS NULL AND current_task_id IS NULL`, now, now, g.view.NamespaceID, p.ID)
if err := exactlyOne(res, err); err != nil {
return err
}
}
if _, err := tx.ExecContext(ctx, `UPDATE weave_waits SET state='CANCELLED',resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND state='WAITING'`, now, now, g.view.NamespaceID, g.view.ID, row.view.ActivationID); err != nil {
return err
}
if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepCancelled, nil, now); err != nil {
return err
}
return deleteWorkflowClock(ctx, tx, g, row)
}
// validateWorkflowStopTask 取消前核对实际任务到流程激活/步骤/队列的归属,不能仅凭gate_id关联。
func validateWorkflowStopTask(ctx context.Context, tx *sql.Tx, g *workflowGraph, id core.ID) error {
var kind string
var owner, queue, gate, workflow, step core.ID
var key string
var direct *core.ID
err := tx.QueryRowContext(ctx, "SELECT t.owner_kind,t.owner_id,t.queue_id,t.gate_id,a.workflow_id,a.step_id,s.`key`,a.task_id FROM pulse_tasks t JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id JOIN weave_steps s ON s.namespace_id=a.namespace_id AND s.workflow_id=a.workflow_id AND s.id=a.step_id WHERE t.namespace_id=? AND t.id=?", g.view.NamespaceID, id).Scan(&kind, &owner, &queue, &gate, &workflow, &step, &key, &direct)
if err != nil {
return err
}
row := g.steps[key]
node, ok := g.program.Step(key)
if kind != engine.TaskOwnerKind || gate != g.gate || workflow != g.view.ID || row == nil || row.view.ID != step || !ok {
return errors.New("stopping task ownership invalid")
}
if node.Task != nil {
if direct == nil || *direct != id || queue != node.Task.QueueID {
return errors.New("stopping task direct association invalid")
}
return nil
}
if node.Poll == nil || direct != nil || queue != node.Poll.Task.QueueID {
return errors.New("stopping query association invalid")
}
var count int
if err := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=? AND r.workflow_id=? AND r.activation_id=? AND r.task_id=? AND r.`round` BETWEEN 1 AND p.used_rounds", g.view.NamespaceID, g.view.ID, owner, id).Scan(&count); err != nil {
return err
}
if count != 1 {
return errors.New("stopping query round invalid")
}
return nil
}
// verifyWorkflowStopWork 终态前复核全部历史任务、双向归属、未结束尝试和许可,不能仅检查当前步骤。
func verifyWorkflowStopWork(ctx context.Context, tx *sql.Tx, g *workflowGraph) (settled, uncertain bool, err error) {
var invalid int
checks := []string{
`SELECT COUNT(*) FROM pulse_tasks t JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id WHERE a.namespace_id=? AND a.workflow_id=? AND t.owner_kind='workflow_activation' AND (t.gate_id IS NULL OR t.gate_id<>?)`,
`SELECT COUNT(*) FROM weave_activations a LEFT JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id WHERE a.namespace_id=? AND a.workflow_id=? AND a.task_id IS NOT NULL AND (t.id IS NULL OR t.owner_kind IS NULL OR t.owner_kind<>'workflow_activation' OR t.owner_id<>a.id OR t.gate_id IS NULL OR t.gate_id<>?)`,
`SELECT COUNT(*) FROM weave_poll_rounds r LEFT JOIN weave_activations a ON a.namespace_id=r.namespace_id AND a.workflow_id=r.workflow_id AND a.id=r.activation_id LEFT JOIN pulse_tasks t ON t.namespace_id=r.namespace_id AND t.id=r.task_id WHERE r.namespace_id=? AND r.workflow_id=? AND (a.id IS NULL OR t.id IS NULL OR t.owner_kind IS NULL OR t.owner_kind<>'workflow_activation' OR t.owner_id<>r.activation_id OR t.gate_id IS NULL OR t.gate_id<>?)`,
}
for _, q := range checks {
if err := tx.QueryRowContext(ctx, q, g.view.NamespaceID, g.view.ID, g.gate).Scan(&invalid); err != nil {
return false, false, err
}
if invalid != 0 {
return false, false, errors.New("workflow stop history association invalid")
}
}
// 一次读取关联元数据并通过索引EXISTS核对查询轮次,避免持有应用锁期间逐任务发出SQL。
rows, err := tx.QueryContext(ctx, "SELECT t.id,t.state,t.effect_uncertain,t.owner_kind,t.owner_id,t.queue_id,a.workflow_id,a.step_id,a.task_id,s.`key`,EXISTS(SELECT 1 FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=t.namespace_id AND r.workflow_id=a.workflow_id AND r.activation_id=a.id AND r.task_id=t.id AND r.`round` BETWEEN 1 AND p.used_rounds) FROM pulse_tasks t LEFT JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id LEFT JOIN weave_steps s ON s.namespace_id=a.namespace_id AND s.workflow_id=a.workflow_id AND s.id=a.step_id WHERE t.namespace_id=? AND t.gate_id=? ORDER BY t.id LIMIT 10001", g.view.NamespaceID, g.gate)
if err != nil {
return false, false, err
}
defer rows.Close()
count := 0
settled = true
for rows.Next() {
var id, queue core.ID
var owner, workflow, step, direct *core.ID
var kind, key sql.NullString
var state pulse.TaskState
var unknown, round bool
if err := rows.Scan(&id, &state, &unknown, &kind, &owner, &queue, &workflow, &step, &direct, &key, &round); err != nil {
return false, false, err
}
count++
if count > 10000 {
return false, false, errors.New("workflow historical task bound exceeded")
}
row := g.steps[key.String]
node, ok := g.program.Step(key.String)
if kind.String != engine.TaskOwnerKind || owner == nil || workflow == nil || *workflow != g.view.ID || step == nil || row == nil || row.view.ID != *step || !ok {
return false, false, errors.New("workflow stop task association invalid")
}
if node.Task != nil {
if direct == nil || *direct != id || queue != node.Task.QueueID {
return false, false, errors.New("workflow stop task parameters invalid")
}
} else if node.Poll == nil || direct != nil || queue != node.Poll.Task.QueueID || !round {
return false, false, errors.New("workflow stop query association invalid")
}
uncertain = uncertain || unknown || state == pulse.TaskNeedsReview
if !state.Terminal() && state != pulse.TaskNeedsReview {
settled = false
}
}
err = rows.Err()
rows.Close()
if err != nil {
return false, false, err
}
if !settled {
return false, uncertain, nil
}
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_attempts a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id WHERE t.namespace_id=? AND t.gate_id=? AND a.state IN ('LEASED','RUNNING')`, g.view.NamespaceID, g.gate).Scan(&invalid); err != nil {
return false, false, err
}
if invalid > 0 {
return false, uncertain, nil
}
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_permits p JOIN pulse_attempts a ON a.namespace_id=p.namespace_id AND a.id=p.attempt_id JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id WHERE t.namespace_id=? AND t.gate_id=? AND p.released_at IS NULL`, g.view.NamespaceID, g.gate).Scan(&invalid); err != nil {
return false, false, err
}
return invalid == 0, uncertain, nil
}

View File

@@ -242,7 +242,7 @@ func (r *WorkflowRuntime) applyWorkflowSignal(ctx context.Context, tx *sql.Tx, g
}
if state == weave.StepNeedsReview || state == weave.StepFailed {
next := weave.InstanceNeedsReview
if state == weave.StepFailed && (g.version.Specification.FailurePolicy == "FAIL_AFTER_SETTLED" || g.view.CancelRequestedAt != nil) {
if state == weave.StepFailed && (g.view.StopKind != nil || g.version.Specification.FailurePolicy == "FAIL_AFTER_SETTLED" || g.view.CancelRequestedAt != nil) {
next = weave.InstanceCancelling
}
if err := holdWorkflow(ctx, tx, g, next, reason, now); err != nil {
@@ -284,6 +284,14 @@ func (r *WorkflowRuntime) expireWorkflowSignal(ctx context.Context, tx *sql.Tx,
if v.State != "PENDING" || now.Before(v.ExpiresAt) {
return false, nil
}
if g.view.StopKind != nil && !g.view.State.Terminal() {
// 停止事务已关闭新等待;由根收集停止前信号,不能让稍后过期扫描把取消改成超时核查。
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
return false, err
}
_, err := tx.ExecContext(ctx, `UPDATE weave_signals SET available_at=DATE_ADD(?,INTERVAL 30 SECOND),revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND state='PENDING'`, now, now, c.ns, c.id)
return false, err
}
row := g.steps[v.TargetStepKey]
if row != nil && row.view.ActivationID != nil && !g.view.State.Terminal() && (row.view.State == weave.StepWaiting || row.view.State == weave.StepActive) {
w, err := readWorkflowSignalWait(ctx, tx, g, row)

View File

@@ -119,7 +119,7 @@ func selectedWorkflowSignal(ctx context.Context, tx *sql.Tx, g *workflowGraph, r
if w.state != "WAITING" {
return nil, nil
}
err := tx.QueryRowContext(ctx, `SELECT id FROM weave_signals WHERE namespace_id=? AND workflow_id=? AND target_step_key=? AND target_activation_sequence=? AND state='PENDING' AND type=? AND payload_version=? AND correlation_key=? AND accepted_at<? ORDER BY accepted_at,id LIMIT 1 FOR UPDATE`, g.view.NamespaceID, g.view.ID, row.view.Key, w.sequence, w.contract.Name, w.contract.PayloadVersion, []byte(w.correlation), w.deadline).Scan(&id)
err := tx.QueryRowContext(ctx, `SELECT id FROM weave_signals WHERE namespace_id=? AND workflow_id=? AND target_step_key=? AND target_activation_sequence=? AND state='PENDING' AND type=? AND payload_version=? AND correlation_key=? AND accepted_at<? AND (? IS NULL OR accepted_at<=?) ORDER BY accepted_at,id LIMIT 1 FOR UPDATE`, g.view.NamespaceID, g.view.ID, row.view.Key, w.sequence, w.contract.Name, w.contract.PayloadVersion, []byte(w.correlation), w.deadline, g.view.StopRequestedAt, g.view.StopRequestedAt).Scan(&id)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}

View File

@@ -124,6 +124,8 @@ func (s *WorkflowSignalStore) Send(ctx context.Context, t auth.Token, workflow c
state, disposition = "REJECTED", "STALE_ACTIVATION"
case g.view.State.Terminal():
state, disposition = "REJECTED", "WORKFLOW_SETTLED"
case g.view.StopKind != nil:
state, disposition = "REJECTED", "WORKFLOW_STOPPING"
case !now.Before(g.view.DeadlineAt):
state, disposition = "EXPIRED", "WORKFLOW_DEADLINE"
case row.view.State == weave.StepSkipped || row.view.State == weave.StepSucceeded || row.view.State == weave.StepFailed || row.view.State == weave.StepCancelled || row.view.State == weave.StepNeedsReview:

View File

@@ -0,0 +1,160 @@
package mysql
import (
"context"
"encoding/json"
"os"
"testing"
"time"
core "git.yuntuo.hk/k2safe/orbit/model"
driver "github.com/go-sql-driver/mysql"
)
// 用真实v20结构验证停止意图与根接续升级,保留原取消时间和业务预算。
func TestIntegrationWorkflowStopUpgrade(t *testing.T) {
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("requires disposable MySQL")
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
admin, err := Open(ctx, dsn, 2, 1)
if err != nil {
t.Fatal(err)
}
defer admin.Close()
name := "orbit_stop_upgrade_" + string(executionID(t))
if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+name+"`"); err != nil {
t.Fatal(err)
}
defer func() {
cleanup, stop := context.WithTimeout(context.Background(), 10*time.Second)
defer stop()
if _, err := admin.ExecContext(cleanup, "DROP DATABASE `"+name+"`"); err != nil {
t.Error(err)
}
}()
cfg, err := driver.ParseDSN(dsn)
if err != nil {
t.Fatal("invalid test DSN")
}
cfg.DBName = name
db, err := Open(ctx, cfg.FormatDSN(), 4, 2)
if err != nil {
t.Fatal(err)
}
defer db.Close()
exec := func(q string, args ...any) {
t.Helper()
if _, err := db.ExecContext(ctx, q, args...); err != nil {
t.Fatal(err)
}
}
exec(`CREATE TABLE orbit_schema_migrations (version BIGINT UNSIGNED PRIMARY KEY,name VARCHAR(191) NOT NULL,checksum CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,dirty BOOLEAN NOT NULL,applied_at DATETIME(6) NULL) ENGINE=InnoDB`)
migrations, err := Migrations()
if err != nil {
t.Fatal(err)
}
for _, m := range migrations {
if m.Version > 20 {
break
}
statements, err := splitSQL(m.SQL)
if err != nil {
t.Fatal(err)
}
for _, q := range statements {
exec(q)
}
exec(`INSERT INTO orbit_schema_migrations(version,name,checksum,dirty,applied_at) VALUES (?,?,?,FALSE,UTC_TIMESTAMP(6))`, m.Version, m.Name, m.Checksum)
}
var now time.Time
if err := db.QueryRowContext(ctx, `SELECT UTC_TIMESTAMP(6)`).Scan(&now); err != nil {
t.Fatal(err)
}
ns := executionID(t)
deadline := now.Add(time.Hour)
requested := now.Add(-time.Hour)
type fixture struct {
id, gate, root, activation core.ID
kind string
}
seed := func(state, policy string, cancelled, failed bool) fixture {
t.Helper()
f := fixture{executionID(t), executionID(t), executionID(t), executionID(t), "FAILURE"}
var cancelAt, completed *time.Time
if cancelled {
cancelAt = &requested
f.kind = "CANCEL"
}
if state == "CANCELLED" {
completed = &now
}
exec(`INSERT INTO pulse_execution_gates(namespace_id,id,paused) VALUES (?,?,FALSE)`, ns, f.gate)
exec(`INSERT INTO weave_instances(namespace_id,id,definition_id,definition_version_id,gate_id,state,control_revision,input,policy_snapshot,deadline_at,cancel_requested_at,completed_at,updated_at) VALUES (?,?,?,?,?,?,4,'{"n":9007199254740993}',JSON_OBJECT('failure_policy',?),?,?,?,?)`, ns, f.id, executionID(t), executionID(t), f.gate, state, policy, deadline, cancelAt, completed, now)
exec("INSERT INTO weave_steps(namespace_id,id,workflow_id,`key`,kind,state,current_activation_id,revision) VALUES (?,?,?,'$root','JOIN','ACTIVE',?,7)", ns, f.root, f.id, f.activation)
exec(`INSERT INTO weave_activations(namespace_id,id,workflow_id,step_id,sequence,state,input) VALUES (?,?,?,?,1,'ACTIVE','null')`, ns, f.activation, f.id, f.root)
if failed {
exec("INSERT INTO weave_steps(namespace_id,id,workflow_id,`key`,kind,state) VALUES (?,?,?,'failed','TASK','NEEDS_REVIEW')", ns, executionID(t), f.id)
}
return f
}
cancellation := seed("NEEDS_REVIEW", "REVIEW", true, false)
fail := seed("CANCELLING", "FAIL_AFTER_SETTLED", false, false)
uncertain := seed("NEEDS_REVIEW", "FAIL_AFTER_SETTLED", false, true)
untouched := seed("NEEDS_REVIEW", "REVIEW", false, true)
terminal := seed("CANCELLED", "REVIEW", true, false)
originalID := executionID(t)
original := `{"schema_version":1,"kind":"ROOT"}`
exec(`INSERT INTO weave_pending_advances(namespace_id,id,workflow_id,step_id,expected_step_revision,checkpoint,available_at,revision) VALUES (?,?,?,?,7,?,?,11)`, ns, originalID, fail.id, fail.root, original, deadline)
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
if err := CheckSchema(ctx, db); err != nil {
t.Fatal(err)
}
for _, f := range []fixture{cancellation, fail, uncertain} {
var kind string
var at, due, keptDeadline time.Time
var controlRevision, expected uint64
var paused bool
var raw []byte
if err := db.QueryRowContext(ctx, `SELECT i.stop_kind,i.stop_requested_at,i.deadline_at,i.control_revision,g.paused FROM weave_instances i JOIN pulse_execution_gates g ON g.namespace_id=i.namespace_id AND g.id=i.gate_id WHERE i.namespace_id=? AND i.id=?`, ns, f.id).Scan(&kind, &at, &keptDeadline, &controlRevision, &paused); err != nil {
t.Fatal(err)
}
want := now
if f.kind == "CANCEL" {
want = requested
}
if kind != f.kind || !at.Equal(want) || !keptDeadline.Equal(deadline) || controlRevision != 4 || !paused {
t.Fatal("migration reset stop identity, deadline or control version")
}
if err := db.QueryRowContext(ctx, `SELECT checkpoint,available_at,expected_step_revision FROM weave_pending_advances WHERE namespace_id=? AND workflow_id=?`, ns, f.id).Scan(&raw, &due, &expected); err != nil {
t.Fatal(err)
}
var cp workflowCheckpoint
if err := json.Unmarshal(raw, &cp); err != nil || cp.Kind != "ROOT" || cp.SchemaVersion != 1 || cp.ActivationID != nil || expected != 7 || !due.Before(deadline) {
t.Fatal("stop continuation missing or invalid", err)
}
}
var count int
for _, f := range []fixture{untouched, terminal} {
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_instances WHERE namespace_id=? AND id=? AND stop_kind IS NULL AND stop_requested_at IS NULL`, ns, f.id).Scan(&count); err != nil || count != 1 {
t.Fatal("unrelated or terminal workflow changed", err)
}
}
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_pending_advances WHERE namespace_id=?`, ns).Scan(&count); err != nil || count != 3 {
t.Fatal("backfill included unrelated workflow", count, err)
}
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_pending_advances WHERE namespace_id=? AND id=? AND revision=11 AND checkpoint=?`, ns, originalID, original).Scan(&count); err != nil || count != 1 {
t.Fatal("existing root identity changed", err)
}
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&count); err != nil || count != 0 {
t.Fatal("foreign keys introduced", err)
}
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
t.Log("real v20 stop intent backfill preserves cancellation time and fixed deadline/control revision, closes gate, preserves existing root identity; unrelated/terminal workflows unchanged; no foreign keys")
}