feat: persist workflow controls and settle stopped branches
This commit is contained in:
@@ -64,6 +64,8 @@ func TestIntegrationApplicationResourcesUpgrade(t *testing.T) {
|
||||
retryFinish(t, g, finished, "SUCCEEDED", false)
|
||||
assertApplicationCounts(t, f, 4, 2, 1)
|
||||
// 还原后续新增表和字段,得到真实 v9 结构;现有任务内容保持不变。
|
||||
f.exec(`ALTER TABLE pulse_tasks DROP INDEX ix_task_workflow_owner`)
|
||||
f.exec(`ALTER TABLE weave_instances DROP CHECK ck_workflow_stop_intent, DROP COLUMN stop_kind, DROP COLUMN stop_requested_at`)
|
||||
f.exec(`ALTER TABLE weave_signals DROP CHECK ck_signal_version_sequence, DROP CHECK ck_signal_pending_identity, DROP INDEX ix_signal_expiry, DROP INDEX ix_signal_activation, DROP COLUMN payload_version, DROP COLUMN target_activation_sequence, DROP COLUMN evidence_ref, DROP COLUMN disposition, DROP COLUMN available_at, MODIFY COLUMN type VARCHAR(64) NOT NULL COMMENT '事件或信号的稳定类型标识', MODIFY COLUMN correlation_key VARCHAR(191) NOT NULL COMMENT '外部业务操作关联键,防止旧信号误用于新工作'`)
|
||||
f.exec(`ALTER TABLE weave_waits DROP INDEX uq_wait_winner_signal, DROP COLUMN winner_signal_id`)
|
||||
f.exec(`ALTER TABLE weave_waits DROP COLUMN signal_name, DROP COLUMN payload_version, MODIFY COLUMN correlation_key VARCHAR(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_as_cs NOT NULL COMMENT '外部业务操作关联键,防止旧信号误用于新工作'`)
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
-- 独立保存首次停止意图,避免进入核查后丢失用户取消/失败结清的目标;不增加外键。
|
||||
ALTER TABLE weave_instances
|
||||
ADD COLUMN stop_kind VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NULL COMMENT '首次停止原因:CANCEL为用户取消,FAILURE为固定失败策略;为空表示尚未要求结清',
|
||||
ADD COLUMN stop_requested_at DATETIME(6) NULL COMMENT '首次停止意图的数据库时间,重报、进入核查或后来取消均不覆盖';
|
||||
|
||||
-- 历史取消保留原意图时间;旧失败结清及失败策略触发的核查保留原更新时间作为兼容边界。
|
||||
UPDATE weave_instances i SET stop_kind=CASE WHEN cancel_requested_at IS NOT NULL THEN 'CANCEL' ELSE 'FAILURE' END,
|
||||
stop_requested_at=COALESCE(cancel_requested_at,updated_at)
|
||||
WHERE state NOT IN ('SUCCEEDED','FAILED','CANCELLED') AND
|
||||
(cancel_requested_at IS NOT NULL OR state='CANCELLING' OR
|
||||
(JSON_UNQUOTE(JSON_EXTRACT(policy_snapshot,'$.failure_policy'))='FAIL_AFTER_SETTLED' AND EXISTS
|
||||
(SELECT 1 FROM weave_steps s WHERE s.namespace_id=i.namespace_id AND s.workflow_id=i.id AND s.`key`<>'$root' AND s.state IN ('FAILED','CANCELLED','NEEDS_REVIEW'))));
|
||||
|
||||
ALTER TABLE weave_instances
|
||||
ADD CONSTRAINT ck_workflow_stop_intent CHECK ((stop_kind IS NULL AND stop_requested_at IS NULL) OR (stop_kind IS NOT NULL AND stop_kind IN ('CANCEL','FAILURE') AND stop_requested_at IS NOT NULL));
|
||||
|
||||
-- 结清复用根检查点,已有根只提前检查,不更改固定版本、原预算和截止。
|
||||
INSERT INTO weave_pending_advances(namespace_id,id,workflow_id,step_id,expected_step_revision,checkpoint,available_at)
|
||||
SELECT i.namespace_id,RANDOM_BYTES(16),i.id,s.id,s.revision,JSON_OBJECT('schema_version',1,'kind','ROOT'),UTC_TIMESTAMP(6)
|
||||
FROM weave_instances i JOIN weave_steps s ON s.namespace_id=i.namespace_id AND s.workflow_id=i.id AND s.`key`='$root'
|
||||
JOIN weave_activations a ON a.namespace_id=s.namespace_id AND a.workflow_id=s.workflow_id AND a.step_id=s.id AND a.id=s.current_activation_id
|
||||
WHERE i.stop_kind IS NOT NULL AND i.state NOT IN ('SUCCEEDED','FAILED','CANCELLED') AND s.state='ACTIVE' AND a.state='ACTIVE'
|
||||
ON DUPLICATE KEY UPDATE available_at=LEAST(weave_pending_advances.available_at,VALUES(available_at));
|
||||
|
||||
UPDATE pulse_execution_gates g JOIN weave_instances i ON i.namespace_id=g.namespace_id AND i.gate_id=g.id
|
||||
SET g.paused=TRUE,g.revision=g.revision+1,g.updated_at=UTC_TIMESTAMP(6)
|
||||
WHERE i.stop_kind IS NOT NULL AND i.state NOT IN ('SUCCEEDED','FAILED','CANCELLED') AND g.paused=FALSE;
|
||||
|
||||
-- 反向核对流程所有者,防止无外键时遗漏门控被误改的历史任务。
|
||||
ALTER TABLE pulse_tasks ADD KEY ix_task_workflow_owner (namespace_id,owner_kind,owner_id,gate_id,id);
|
||||
@@ -89,7 +89,7 @@ func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
|
||||
indexes := func() []string {
|
||||
t.Helper()
|
||||
// v15替换retry_of索引、v17增加内部结果唯一索引、v20增加信号索引,下方单独验证定义。
|
||||
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result','ix_signal_expiry','ix_signal_activation','uq_wait_winner_signal') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
|
||||
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result','ix_signal_expiry','ix_signal_activation','uq_wait_winner_signal','ix_task_workflow_owner') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -179,6 +179,7 @@ func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
|
||||
}
|
||||
var collation string
|
||||
for _, index := range []struct{ table, name, want string }{
|
||||
{"pulse_tasks", "ix_task_workflow_owner", "namespace_id:1,owner_kind:1,owner_id:1,gate_id:1,id:1"},
|
||||
{"weave_signals", "ix_signal_expiry", "state:1,available_at:1,namespace_id:1,id:1"},
|
||||
{"weave_signals", "ix_signal_activation", "namespace_id:1,workflow_id:1,target_step_key:1,target_activation_sequence:1,state:1,accepted_at:1,id:1"},
|
||||
{"weave_waits", "uq_wait_winner_signal", "namespace_id:0,winner_signal_id:0"},
|
||||
|
||||
@@ -55,6 +55,12 @@ func (r *WorkflowRuntime) advanceClock(ctx context.Context, tx *sql.Tx, g *workf
|
||||
if row.view.State != weave.StepActive && row.view.State != weave.StepWaiting {
|
||||
return false, deleteWorkflowClock(ctx, tx, g, row)
|
||||
}
|
||||
if g.view.StopKind != nil {
|
||||
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return false, deleteWorkflowClock(ctx, tx, g, row)
|
||||
}
|
||||
var activated time.Time
|
||||
if err := tx.QueryRowContext(ctx, `SELECT created_at FROM weave_activations WHERE namespace_id=? AND id=? AND workflow_id=? AND step_id=?`, c.ns, checkpoint.ActivationID, c.workflow, step).Scan(&activated); err != nil {
|
||||
return false, err
|
||||
|
||||
@@ -187,7 +187,9 @@ func (r *WorkflowRuntime) consume(ctx context.Context, tx *sql.Tx, g *workflowGr
|
||||
}
|
||||
}
|
||||
// 单轮请求失败不能通过新建业务轮次绕过Pulse预算。
|
||||
if state != weave.StepSucceeded {
|
||||
if g.view.StopKind != nil && (state == weave.StepCancelled || state == weave.StepFailed) {
|
||||
reason = "POLL_STOP_REQUEST_SETTLED"
|
||||
} else if state != weave.StepSucceeded {
|
||||
state, reason = weave.StepNeedsReview, "POLL_REQUEST_REVIEW"
|
||||
} else if !source.occurred.Before(deadline) {
|
||||
state, output, reason = weave.StepNeedsReview, nil, "POLL_DEADLINE"
|
||||
@@ -198,7 +200,9 @@ func (r *WorkflowRuntime) consume(ctx context.Context, tx *sql.Tx, g *workflowGr
|
||||
state, output, reason = weave.StepNeedsReview, nil, "POLL_RESULT_INVALID"
|
||||
case outcome == engine.PollPending:
|
||||
state, output, reason = weave.StepWaiting, nil, "POLL_PENDING"
|
||||
if used >= max {
|
||||
if g.view.StopKind != nil {
|
||||
state, reason = weave.StepCancelled, "POLL_STOP_PENDING_SETTLED"
|
||||
} else if used >= max {
|
||||
state, reason = weave.StepNeedsReview, "POLL_ROUNDS_EXHAUSTED"
|
||||
}
|
||||
case outcome == engine.PollFailed:
|
||||
@@ -250,7 +254,7 @@ func (r *WorkflowRuntime) consume(ctx context.Context, tx *sql.Tx, g *workflowGr
|
||||
}
|
||||
if state == weave.StepNeedsReview || state == weave.StepFailed || state == weave.StepCancelled {
|
||||
next := weave.InstanceNeedsReview
|
||||
if (state == weave.StepFailed || state == weave.StepCancelled) && (g.version.Specification.FailurePolicy == "FAIL_AFTER_SETTLED" || g.view.CancelRequestedAt != nil) {
|
||||
if (state == weave.StepFailed || state == weave.StepCancelled) && (g.view.StopKind != nil || g.version.Specification.FailurePolicy == "FAIL_AFTER_SETTLED" || g.view.CancelRequestedAt != nil) {
|
||||
next = weave.InstanceCancelling
|
||||
}
|
||||
if err := holdWorkflow(ctx, tx, g, next, reason, now); err != nil {
|
||||
|
||||
136
internal/storage/mysql/workflow_controls.go
Normal file
136
internal/storage/mysql/workflow_controls.go
Normal file
@@ -0,0 +1,136 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/control"
|
||||
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
|
||||
)
|
||||
|
||||
// WorkflowControlStore 控制决定和门控同事务,不在请求中等待外部执行停止。
|
||||
type WorkflowControlStore struct{ *TaskStore }
|
||||
|
||||
func NewWorkflowControlStore(db *sql.DB) *WorkflowControlStore {
|
||||
return &WorkflowControlStore{NewTaskStore(db)}
|
||||
}
|
||||
|
||||
var _ control.Store = (*WorkflowControlStore)(nil)
|
||||
|
||||
func (s *WorkflowControlStore) Apply(ctx context.Context, t auth.Token, id core.ID, key string, in control.Input) (control.Result, error) {
|
||||
var out control.Result
|
||||
err := s.transaction(ctx, t, control.Action, func(tx *sql.Tx, p auth.Permissions) error {
|
||||
if _, err := lockApplicationResources(ctx, tx, t.NamespaceID, false); err != nil {
|
||||
return err
|
||||
}
|
||||
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "workflow.control", string(id), key, "workflow", id, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g, _, err := lockWorkflowGraph(ctx, tx, t.NamespaceID, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, q := range g.version.Analysis.Queues {
|
||||
if !p.AllowsQueue(q) {
|
||||
return fault.NotFound
|
||||
}
|
||||
}
|
||||
if cached != nil {
|
||||
if err := json.Unmarshal(cached, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
if out.Instance.NamespaceID != t.NamespaceID || out.Instance.ID != id || out.Instance.DefinitionVersionID != g.view.DefinitionVersionID {
|
||||
return errors.New("workflow control receipt mismatch")
|
||||
}
|
||||
out.Deduplicated = true
|
||||
return nil
|
||||
}
|
||||
if g.view.State.Terminal() || g.view.ControlRevision != in.ExpectedControlRevision {
|
||||
return fault.Conflict
|
||||
}
|
||||
before := g.view
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch in.Operation {
|
||||
case "PAUSE":
|
||||
if g.view.Paused {
|
||||
return fault.Conflict
|
||||
}
|
||||
g.view.Paused = true
|
||||
case "RESUME":
|
||||
if !g.view.Paused {
|
||||
return fault.Conflict
|
||||
}
|
||||
g.view.Paused = false
|
||||
case "CANCEL":
|
||||
if g.view.CancelRequestedAt != nil {
|
||||
return fault.Conflict
|
||||
}
|
||||
g.view.CancelRequestedAt = &now
|
||||
if err := recordWorkflowStop(ctx, tx, g, "CANCEL", now); err != nil {
|
||||
return err
|
||||
}
|
||||
if g.view.State != weave.InstanceNeedsReview {
|
||||
g.view.State = weave.InstanceCancelling
|
||||
}
|
||||
default:
|
||||
return fault.Invalid("unsupported workflow control operation")
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_instances SET paused=?,state=?,cancel_requested_at=?,control_revision=control_revision+1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND control_revision=? AND revision=?`, g.view.Paused, g.view.State, g.view.CancelRequestedAt, now, t.NamespaceID, id, in.ExpectedControlRevision, g.view.Revision)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return err
|
||||
}
|
||||
g.view.ControlRevision++
|
||||
g.view.Revision++
|
||||
gatePaused := g.view.Paused || g.view.StopKind != nil || g.view.State == weave.InstanceNeedsReview || g.view.State == weave.InstanceCancelling
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE pulse_execution_gates SET paused=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, gatePaused, now, t.NamespaceID, g.gate); err != nil {
|
||||
return err
|
||||
}
|
||||
if in.Operation == "CANCEL" || in.Operation == "RESUME" {
|
||||
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
meta := map[string]any{"operation": in.Operation, "paused": g.view.Paused, "state": g.view.State, "control_revision": g.view.ControlRevision, "stop_kind": g.view.StopKind, "stop_requested_at": g.view.StopRequestedAt, "cancel_requested_at": g.view.CancelRequestedAt}
|
||||
if err := emitEvent(ctx, tx, t.NamespaceID, id, g.view.Revision, "workflow", "workflow.controlled", meta); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := appendAudit(ctx, tx, t, identity, id, "workflow", "workflow.control", in.Reason, map[string]any{"paused": before.Paused, "state": before.State, "control_revision": before.ControlRevision, "stop_kind": before.StopKind}, meta); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Instance, _, err = readWorkflowInstance(ctx, tx, t.NamespaceID, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
// recordWorkflowStop 冻结首次停止原因;后来的用户取消不能抹去已启动的失败结清。
|
||||
// 调用方同时关闭门控、保存根检查点和审计,不能将此函数单独提交。
|
||||
func recordWorkflowStop(ctx context.Context, tx *sql.Tx, g *workflowGraph, kind string, now time.Time) error {
|
||||
if g.view.StopKind != nil {
|
||||
return nil
|
||||
}
|
||||
if kind != "CANCEL" && kind != "FAILURE" {
|
||||
return errors.New("invalid workflow stop kind")
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_instances SET stop_kind=?,stop_requested_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=? AND stop_kind IS NULL`, kind, now, now, g.view.NamespaceID, g.view.ID, g.view.Revision)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return err
|
||||
}
|
||||
g.view.StopKind = &kind
|
||||
g.view.StopRequestedAt = &now
|
||||
g.view.Revision++
|
||||
return nil
|
||||
}
|
||||
484
internal/storage/mysql/workflow_controls_integration_test.go
Normal file
484
internal/storage/mysql/workflow_controls_integration_test.go
Normal file
@@ -0,0 +1,484 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/control"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/definition"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/instance"
|
||||
)
|
||||
|
||||
func grantWorkflowControl(f *executionFixture) {
|
||||
f.exec(`UPDATE orbit_credentials SET permissions=JSON_ARRAY_APPEND(permissions,'$.actions','workflows:control') WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, f.resources.CredentialID)
|
||||
}
|
||||
func applyControl(t *testing.T, f *executionFixture, v instance.View, operation string) control.Result {
|
||||
t.Helper()
|
||||
out, err := control.New(NewWorkflowControlStore(f.db)).Apply(f.ctx, f.token, v.ID, string(executionID(t)), control.Input{Operation: operation, ExpectedControlRevision: v.ControlRevision, Reason: "controlled integration action"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func stopStep(t *testing.T, f *executionFixture, id core.ID) bool {
|
||||
t.Helper()
|
||||
f.exec(`UPDATE weave_pending_advances SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, id)
|
||||
rows, err := f.db.QueryContext(f.ctx, `SELECT 'INBOX',id,revision FROM weave_inbox WHERE namespace_id=? AND workflow_id=? AND state='PENDING' UNION ALL SELECT 'ROOT',id,revision FROM weave_pending_advances WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, id, f.resources.NamespaceID, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var candidates []workflowCandidate
|
||||
for rows.Next() {
|
||||
c := workflowCandidate{ns: f.resources.NamespaceID, workflow: id}
|
||||
if err := rows.Scan(&c.kind, &c.id, &c.revision); err != nil {
|
||||
rows.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
candidates = append(candidates, c)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range candidates {
|
||||
if _, err := newRuntime(t, f.db).process(f.ctx, c, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return len(candidates) > 0
|
||||
}
|
||||
func drainStop(t *testing.T, f *executionFixture, id core.ID, state string) {
|
||||
t.Helper()
|
||||
for i := 0; i < 40; i++ {
|
||||
if !stopStep(t, f, id) {
|
||||
runtimeView(t, f, id, state)
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("stop did not become quiescent")
|
||||
}
|
||||
func controlFixture(t *testing.T, ctx context.Context, db *sql.DB, count int) (*executionFixture, instance.View) {
|
||||
t.Helper()
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
v := runtimeStart(t, f, runtimeSpec(f.resources.QueueID, count))
|
||||
grantWorkflowControl(f)
|
||||
return f, v
|
||||
}
|
||||
func TestIntegrationWorkflowControls(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, stop := context.WithTimeout(context.Background(), 150*time.Second)
|
||||
defer stop()
|
||||
db, err := Open(ctx, dsn, 32, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Run("pause_collect_resume_uses_control_revision", func(t *testing.T) {
|
||||
f, v := controlFixture(t, ctx, db, 2)
|
||||
ref := retryStart(t, f)
|
||||
paused := applyControl(t, f, v, "PAUSE").Instance
|
||||
if !paused.Paused || paused.ControlRevision != 2 || f.used() != 1 {
|
||||
t.Fatal("pause stopped running work or failed version")
|
||||
}
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
after := runtimeView(t, f, v.ID, "RUNNING")
|
||||
if after.Revision <= paused.Revision || after.ControlRevision != paused.ControlRevision || f.count("pulse_tasks", "1=1") != 1 {
|
||||
t.Fatal("background result corrupted control version")
|
||||
}
|
||||
resumed := applyControl(t, f, paused, "RESUME").Instance
|
||||
if resumed.Paused || resumed.ControlRevision != 3 || !resumed.DeadlineAt.Equal(v.DeadlineAt) {
|
||||
t.Fatal("resume reset state or deadline")
|
||||
}
|
||||
runtimeAdvance(t, f, newRuntime(t, db), v.ID)
|
||||
if f.count("pulse_tasks", "1=1") != 2 {
|
||||
t.Fatal("resume lost successor")
|
||||
}
|
||||
})
|
||||
t.Run("cancel_waiting_cascades_in_application_and_preserves_receipt", func(t *testing.T) {
|
||||
f, v := controlFixture(t, ctx, db, 3)
|
||||
api := control.New(NewWorkflowControlStore(db))
|
||||
key := string(executionID(t))
|
||||
in := control.Input{Operation: "CANCEL", ExpectedControlRevision: 1, Reason: "stop all work"}
|
||||
out, err := api.Apply(ctx, f.token, v.ID, key, in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Instance.StopKind == nil || *out.Instance.StopKind != "CANCEL" || out.Instance.StopRequestedAt == nil || out.Instance.State != "CANCELLING" {
|
||||
t.Fatal("cancel intent incomplete")
|
||||
}
|
||||
drainStop(t, f, v.ID, "CANCELLED")
|
||||
if f.count("pulse_tasks", "state='CANCELLED'") != 1 || f.count("weave_steps", "state='SKIPPED'") != 2 || f.count("weave_inbox", "state='PROCESSED'") != 1 || f.used() != 0 {
|
||||
t.Fatal("cancel did not settle all work")
|
||||
}
|
||||
replay, err := api.Apply(ctx, f.token, v.ID, key, in)
|
||||
if err != nil || !replay.Deduplicated || replay.Instance.State != "CANCELLING" || replay.Instance.ControlRevision != 2 {
|
||||
t.Fatal("control receipt was mutated", err)
|
||||
}
|
||||
_, err = api.Apply(ctx, f.token, v.ID, string(executionID(t)), control.Input{Operation: "PAUSE", ExpectedControlRevision: 2, Reason: "cannot control terminal"})
|
||||
if fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("terminal accepted control", err)
|
||||
}
|
||||
})
|
||||
t.Run("concurrent_same_identity_and_stale_control", func(t *testing.T) {
|
||||
f, v := controlFixture(t, ctx, db, 1)
|
||||
api := control.New(NewWorkflowControlStore(db))
|
||||
key := string(executionID(t))
|
||||
in := control.Input{Operation: "PAUSE", ExpectedControlRevision: 1, Reason: "one decision"}
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 12)
|
||||
fresh := make(chan bool, 12)
|
||||
for i := 0; i < 12; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
out, err := api.Apply(ctx, f.token, v.ID, key, in)
|
||||
errs <- err
|
||||
fresh <- !out.Deduplicated
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
close(fresh)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
n := 0
|
||||
for v := range fresh {
|
||||
if v {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 || f.count("orbit_audit_events", "action='workflow.control'") != 1 || f.count("weave_instances", "control_revision=2 AND paused=TRUE") != 1 {
|
||||
t.Fatal("duplicated control", n)
|
||||
}
|
||||
in.Operation = "CANCEL"
|
||||
if _, err := api.Apply(ctx, f.token, v.ID, key, in); fault.Code(err) != "IDEMPOTENCY_CONFLICT" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := api.Apply(ctx, f.token, v.ID, string(executionID(t)), in); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
t.Run("permission_and_scope_still_checked_on_replay", func(t *testing.T) {
|
||||
f, v := controlFixture(t, ctx, db, 1)
|
||||
api := control.New(NewWorkflowControlStore(db))
|
||||
key := string(executionID(t))
|
||||
in := control.Input{Operation: "PAUSE", ExpectedControlRevision: 1, Reason: "authorized pause"}
|
||||
if _, err := api.Apply(ctx, f.token, v.ID, key, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.exec(`UPDATE orbit_credentials SET permissions=JSON_SET(permissions,'$.queues',JSON_ARRAY(?)) WHERE namespace_id=? AND id=?`, string(executionID(t)), f.resources.NamespaceID, f.resources.CredentialID)
|
||||
if _, err := api.Apply(ctx, f.token, v.ID, key, in); fault.Code(err) != "NOT_FOUND" {
|
||||
t.Fatal("scope replay bypass", err)
|
||||
}
|
||||
grantWorkflowInstance(t, f)
|
||||
if _, err := api.Apply(ctx, f.token, v.ID, key, in); fault.Code(err) != "PERMISSION_DENIED" {
|
||||
t.Fatal("action replay bypass", err)
|
||||
}
|
||||
})
|
||||
t.Run("control_audit_failure_rolls_back_everything", func(t *testing.T) {
|
||||
f, v := controlFixture(t, ctx, db, 1)
|
||||
api := control.New(NewWorkflowControlStore(db))
|
||||
key := string(executionID(t))
|
||||
in := control.Input{Operation: "CANCEL", ExpectedControlRevision: 1, Reason: "atomic decision"}
|
||||
f.exec(`CREATE TRIGGER test_control_failure BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='control audit fault'`)
|
||||
defer f.exec(`DROP TRIGGER IF EXISTS test_control_failure`)
|
||||
if _, err := api.Apply(ctx, f.token, v.ID, key, in); err == nil {
|
||||
t.Fatal("audit failure accepted")
|
||||
}
|
||||
if f.count("weave_instances", "control_revision=1 AND stop_kind IS NULL AND cancel_requested_at IS NULL") != 1 || f.count("pulse_execution_gates", "paused=FALSE") != 1 || f.count("weave_pending_advances", "1=1") != 0 {
|
||||
t.Fatal("partial control mutation")
|
||||
}
|
||||
f.exec(`DROP TRIGGER test_control_failure`)
|
||||
if _, err := api.Apply(ctx, f.token, v.ID, key, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
drainStop(t, f, v.ID, "CANCELLED")
|
||||
})
|
||||
t.Run("settlement_audit_failure_rolls_back_task_and_inbox", func(t *testing.T) {
|
||||
f, v := controlFixture(t, ctx, db, 1)
|
||||
applyControl(t, f, v, "CANCEL")
|
||||
c := clockCandidate(t, f, v.ID, workflowRootKey)
|
||||
f.exec(`CREATE TRIGGER test_settlement_failure BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='settlement audit fault'`)
|
||||
defer f.exec(`DROP TRIGGER IF EXISTS test_settlement_failure`)
|
||||
if done, err := newRuntime(t, db).process(ctx, c, false); err == nil || done {
|
||||
t.Fatal("partial settlement committed")
|
||||
}
|
||||
if f.count("pulse_tasks", "state='WAITING' AND cancel_requested_at IS NULL") != 1 || f.count("weave_inbox", "1=1") != 0 {
|
||||
t.Fatal("cancel or result escaped transaction")
|
||||
}
|
||||
f.exec(`DROP TRIGGER test_settlement_failure`)
|
||||
drainStop(t, f, v.ID, "CANCELLED")
|
||||
})
|
||||
for _, outcome := range []string{"SUCCEEDED", "FAILED", "CANCELLED", "UNKNOWN"} {
|
||||
t.Run("running_cancel_then_"+outcome, func(t *testing.T) {
|
||||
f, v := controlFixture(t, ctx, db, 2)
|
||||
ref := retryStart(t, f)
|
||||
applyControl(t, f, v, "CANCEL")
|
||||
stopStep(t, f, v.ID)
|
||||
if f.get(ref.TaskID).State != "RUNNING" || f.get(ref.TaskID).CancelRequestedAt == nil || f.used() != 1 {
|
||||
t.Fatal("cancel pretended running work ended")
|
||||
}
|
||||
retryFinish(t, f, ref, outcome, false)
|
||||
want := "CANCELLED"
|
||||
if outcome == "UNKNOWN" {
|
||||
want = "NEEDS_REVIEW"
|
||||
}
|
||||
drainStop(t, f, v.ID, want)
|
||||
after := runtimeView(t, f, v.ID, want)
|
||||
if after.StopKind == nil || *after.StopKind != "CANCEL" || f.used() != 0 || f.count("pulse_tasks", "1=1") != 1 {
|
||||
t.Fatal("stop intent lost or extra task")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("failure_policy_settles_parallel_and_freezes_first_reason", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
spec := runtimeSpec(f.resources.QueueID, 1)
|
||||
tail := spec.Body.Steps[0]
|
||||
spec.Body.Steps = []definition.Step{runtimeParallel(spec), tail}
|
||||
spec.FailurePolicy = "FAIL_AFTER_SETTLED"
|
||||
v := runtimeStart(t, f, spec)
|
||||
grantWorkflowControl(f)
|
||||
left, right := retryStart(t, f), retryStart(t, f)
|
||||
retryFinish(t, f, left, "FAILED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), left.TaskID)
|
||||
stopping := runtimeView(t, f, v.ID, "CANCELLING")
|
||||
if stopping.StopKind == nil || *stopping.StopKind != "FAILURE" {
|
||||
t.Fatal("failure intent not persisted")
|
||||
}
|
||||
later := applyControl(t, f, stopping, "CANCEL").Instance
|
||||
if *later.StopKind != "FAILURE" || !later.StopRequestedAt.Equal(*stopping.StopRequestedAt) {
|
||||
t.Fatal("user cancel rewrote earlier failure")
|
||||
}
|
||||
stopStep(t, f, v.ID)
|
||||
if f.get(right.TaskID).CancelRequestedAt == nil || f.used() != 1 {
|
||||
t.Fatal("sibling cancellation missing")
|
||||
}
|
||||
retryFinish(t, f, right, "SUCCEEDED", false)
|
||||
drainStop(t, f, v.ID, "FAILED")
|
||||
if f.count("pulse_tasks", "1=1") != 2 || f.count("weave_steps", "`key`='join' AND state='CANCELLED'") != 1 || f.count("weave_steps", "`key`='task0' AND state='SKIPPED'") != 1 {
|
||||
t.Fatal("failed join advanced tail")
|
||||
}
|
||||
})
|
||||
t.Run("unknown_failure_cancels_sibling_but_never_terminal", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
spec := runtimeSpec(f.resources.QueueID, 1)
|
||||
tail := spec.Body.Steps[0]
|
||||
spec.Body.Steps = []definition.Step{runtimeParallel(spec), tail}
|
||||
spec.FailurePolicy = "FAIL_AFTER_SETTLED"
|
||||
v := runtimeStart(t, f, spec)
|
||||
left, right := retryStart(t, f), retryStart(t, f)
|
||||
retryFinish(t, f, left, "UNKNOWN", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), left.TaskID)
|
||||
stopStep(t, f, v.ID)
|
||||
stopStep(t, f, v.ID)
|
||||
if f.get(right.TaskID).CancelRequestedAt == nil {
|
||||
t.Fatal("review lost failure cancellation")
|
||||
}
|
||||
retryFinish(t, f, right, "CANCELLED", false)
|
||||
drainStop(t, f, v.ID, "NEEDS_REVIEW")
|
||||
if f.used() != 0 || f.count("weave_instances", "completed_at IS NULL AND stop_kind='FAILURE'") != 1 {
|
||||
t.Fatal("unknown result terminalized")
|
||||
}
|
||||
})
|
||||
for _, kind := range []string{"TIMER", "SIGNAL", "EARLY_SIGNAL"} {
|
||||
t.Run("cancel_wait_"+kind, func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
spec := signalSpec(f.resources.QueueID, false)
|
||||
if kind == "TIMER" {
|
||||
spec = timerSpec(f.resources.QueueID, 30000, true)
|
||||
}
|
||||
v := runtimeStart(t, f, spec)
|
||||
grantWorkflowControl(f)
|
||||
grantWorkflowSignal(f)
|
||||
if kind == "EARLY_SIGNAL" {
|
||||
sendSignal(t, f, v, signalInput(v))
|
||||
}
|
||||
applyControl(t, f, v, "CANCEL")
|
||||
if kind != "TIMER" {
|
||||
late := sendSignal(t, f, v, signalInput(v))
|
||||
if late.Signal.Disposition != "WORKFLOW_STOPPING" {
|
||||
t.Fatal("late signal was admitted")
|
||||
}
|
||||
}
|
||||
drainStop(t, f, v.ID, "CANCELLED")
|
||||
if f.count("pulse_tasks", "1=1") != 0 {
|
||||
t.Fatal("cancelled wait activated successor")
|
||||
}
|
||||
if kind == "EARLY_SIGNAL" && f.count("weave_signals", "state='CONSUMED'") != 1 {
|
||||
t.Fatal("prior accepted signal lost")
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, running := range []bool{false, true} {
|
||||
name := "poll_between_rounds"
|
||||
if running {
|
||||
name = "poll_running_pending"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f := newClockFixture(t, ctx, db)
|
||||
v := runtimeStart(t, f, pollingSpec(f.resources.QueueID))
|
||||
grantWorkflowControl(f)
|
||||
ref := retryStart(t, f)
|
||||
if !running {
|
||||
finishPoll(t, f, ref, "PENDING")
|
||||
}
|
||||
applyControl(t, f, v, "CANCEL")
|
||||
stopStep(t, f, v.ID)
|
||||
if running {
|
||||
if f.get(ref.TaskID).CancelRequestedAt == nil {
|
||||
t.Fatal("query did not receive cancel")
|
||||
}
|
||||
finishQueryOnly(t, f, ref, "PENDING")
|
||||
}
|
||||
drainStop(t, f, v.ID, "CANCELLED")
|
||||
if f.count("pulse_tasks", "1=1") != 1 || f.count("weave_poll_states", "used_rounds=1 AND current_task_id IS NULL AND next_poll_at IS NULL AND resolved_at IS NOT NULL") != 1 {
|
||||
t.Fatal("cancel query added a round or left budget open")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("more_than_32_blocked_steps_resume_root", func(t *testing.T) {
|
||||
f, v := controlFixture(t, ctx, db, 60)
|
||||
applyControl(t, f, v, "CANCEL")
|
||||
drainStop(t, f, v.ID, "CANCELLED")
|
||||
if f.count("weave_steps", "state='SKIPPED'") != 59 || f.count("orbit_audit_events", "action='workflow.advance' AND reason='WORKFLOW_STOP_CONTINUATION'") < 1 {
|
||||
t.Fatal("bounded stop lost continuation")
|
||||
}
|
||||
})
|
||||
t.Run("pause_blocks_leased_start_and_resume_preserves_attempt", func(t *testing.T) {
|
||||
f, v := controlFixture(t, ctx, db, 1)
|
||||
_, ref := f.claim()
|
||||
paused := applyControl(t, f, v, "PAUSE").Instance
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); fault.Code(err) != "EXECUTION_PAUSED" {
|
||||
t.Fatal("paused lease started", err)
|
||||
}
|
||||
resumed := applyControl(t, f, paused, "RESUME").Instance
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.get(ref.TaskID).AttemptCount != 1 || resumed.ControlRevision != 3 {
|
||||
t.Fatal("resume replaced original attempt")
|
||||
}
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
runtimeView(t, f, v.ID, "SUCCEEDED")
|
||||
})
|
||||
t.Run("start_cancel_race_has_one_authority", func(t *testing.T) {
|
||||
for i := 0; i < 8; i++ {
|
||||
f, v := controlFixture(t, ctx, db, 1)
|
||||
_, ref := f.claim()
|
||||
api := control.New(NewWorkflowControlStore(db))
|
||||
var startErr, controlErr error
|
||||
var wg sync.WaitGroup
|
||||
ready := make(chan struct{})
|
||||
startID, key := executionID(t), string(executionID(t))
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); <-ready; _, startErr = f.a.Start(ctx, f.token, f.binding, startID, ref) }()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-ready
|
||||
_, controlErr = api.Apply(ctx, f.token, v.ID, key, control.Input{Operation: "CANCEL", ExpectedControlRevision: 1, Reason: "start cancellation race"})
|
||||
}()
|
||||
close(ready)
|
||||
wg.Wait()
|
||||
if controlErr != nil {
|
||||
t.Fatal(controlErr)
|
||||
}
|
||||
if startErr != nil && fault.Code(startErr) != "EXECUTION_PAUSED" {
|
||||
t.Fatal("unexpected start outcome", startErr)
|
||||
}
|
||||
stopStep(t, f, v.ID)
|
||||
if startErr == nil {
|
||||
if f.get(ref.TaskID).State != "RUNNING" || f.used() != 1 {
|
||||
t.Fatal("authorized work released by cancellation")
|
||||
}
|
||||
retryFinish(t, f, ref, "CANCELLED", false)
|
||||
} else if f.get(ref.TaskID).State != "CANCELLED" || f.used() != 0 {
|
||||
t.Fatal("unstarted work not settled")
|
||||
}
|
||||
drainStop(t, f, v.ID, "CANCELLED")
|
||||
}
|
||||
})
|
||||
t.Run("extra_gate_reference_cannot_cancel_unrelated_task", func(t *testing.T) {
|
||||
f, v := controlFixture(t, ctx, db, 1)
|
||||
extra := f.submit(1)
|
||||
var gate, activation core.ID
|
||||
if err := db.QueryRowContext(ctx, "SELECT i.gate_id,s.current_activation_id FROM weave_instances i JOIN weave_steps s ON s.namespace_id=i.namespace_id AND s.workflow_id=i.id AND s.`key`='task0' WHERE i.namespace_id=? AND i.id=?", f.resources.NamespaceID, v.ID).Scan(&gate, &activation); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.exec(`UPDATE pulse_tasks SET gate_id=?,owner_kind='workflow_activation',owner_id=? WHERE namespace_id=? AND id=?`, gate, activation, f.resources.NamespaceID, extra.ID)
|
||||
defer f.exec(`UPDATE pulse_tasks SET gate_id=NULL,owner_kind=NULL,owner_id=NULL WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, extra.ID)
|
||||
applyControl(t, f, v, "CANCEL")
|
||||
rejected := false
|
||||
for i := 0; i < 3; i++ {
|
||||
c := clockCandidate(t, f, v.ID, workflowRootKey)
|
||||
if _, err := newRuntime(t, db).process(ctx, c, false); err != nil {
|
||||
rejected = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !rejected || f.get(extra.ID).CancelRequestedAt != nil || f.count("weave_instances", "state='CANCELLED'") != 0 {
|
||||
t.Fatal("gate alone granted cancellation or termination")
|
||||
}
|
||||
f.exec(`UPDATE pulse_tasks SET gate_id=NULL,owner_kind=NULL,owner_id=NULL WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, extra.ID)
|
||||
drainStop(t, f, v.ID, "CANCELLED")
|
||||
if f.get(extra.ID).State != "WAITING" {
|
||||
t.Fatal("unrelated task changed")
|
||||
}
|
||||
})
|
||||
for _, withSignal := range []bool{false, true} {
|
||||
name := "cancel_beats_unconsumed_timeout"
|
||||
if withSignal {
|
||||
name = "cancel_collects_signal_after_total_deadline"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
spec := signalSpec(f.resources.QueueID, false)
|
||||
spec.Body.Steps = spec.Body.Steps[:1]
|
||||
spec.Body.Output = definition.Value{Kind: "REFERENCE", Reference: &definition.Reference{Source: "signal"}}
|
||||
spec.TimeoutMS = 1000
|
||||
spec.Body.Steps[0].Signal.TimeoutMS = 1000
|
||||
v := runtimeStart(t, f, spec)
|
||||
grantWorkflowControl(f)
|
||||
grantWorkflowSignal(f)
|
||||
var source workflowCandidate
|
||||
if withSignal {
|
||||
accepted := sendSignal(t, f, v, signalInput(v))
|
||||
source = workflowCandidate{kind: "SIGNAL", ns: f.resources.NamespaceID, workflow: v.ID, id: accepted.Signal.ID, revision: accepted.Signal.Revision}
|
||||
} else {
|
||||
source = clockCandidate(t, f, v.ID, "signal")
|
||||
}
|
||||
applyControl(t, f, v, "CANCEL")
|
||||
for time.Now().Before(v.DeadlineAt.Add(20 * time.Millisecond)) {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
if _, err := newRuntime(t, db).process(ctx, source, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
drainStop(t, f, v.ID, "CANCELLED")
|
||||
if withSignal && f.count("weave_signals", "state='CONSUMED'") != 1 {
|
||||
t.Fatal("accepted signal expired after cancellation")
|
||||
}
|
||||
if f.count("weave_steps", "state='NEEDS_REVIEW'") != 0 {
|
||||
t.Fatal("later timeout rewrote earlier cancellation")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Log("versioned control, permission replay, audit rollback, safe cancellation, failure settlement, signal/POLL collection and bounded root continuation verified")
|
||||
}
|
||||
@@ -52,7 +52,7 @@ type workflowPolicy struct {
|
||||
func readWorkflowInstance(ctx context.Context, tx *sql.Tx, ns, id core.ID, lock bool) (instance.View, core.ID, error) {
|
||||
var v instance.View
|
||||
var gate core.ID
|
||||
q := `SELECT i.id,i.namespace_id,i.definition_id,i.definition_version_id,v.version,i.gate_id,i.state,i.revision,i.paused,i.control_revision,i.input,i.output,COALESCE(i.business_key,''),i.deadline_at,i.cancel_requested_at,i.completed_at,i.created_at,i.updated_at
|
||||
q := `SELECT i.id,i.namespace_id,i.definition_id,i.definition_version_id,v.version,i.gate_id,i.state,i.revision,i.paused,i.control_revision,i.input,i.output,COALESCE(i.business_key,''),i.deadline_at,i.cancel_requested_at,i.stop_kind,i.stop_requested_at,i.completed_at,i.created_at,i.updated_at
|
||||
FROM weave_instances i JOIN weave_definition_versions v ON v.namespace_id=i.namespace_id AND v.definition_id=i.definition_id AND v.id=i.definition_version_id
|
||||
JOIN weave_definitions d ON d.namespace_id=i.namespace_id AND d.id=i.definition_id
|
||||
JOIN pulse_execution_gates g ON g.namespace_id=i.namespace_id AND g.id=i.gate_id
|
||||
@@ -61,7 +61,7 @@ func readWorkflowInstance(ctx context.Context, tx *sql.Tx, ns, id core.ID, lock
|
||||
q += ` FOR SHARE`
|
||||
}
|
||||
var input, output []byte
|
||||
err := tx.QueryRowContext(ctx, q, ns, id).Scan(&v.ID, &v.NamespaceID, &v.DefinitionID, &v.DefinitionVersionID, &v.Version, &gate, &v.State, &v.Revision, &v.Paused, &v.ControlRevision, &input, &output, &v.BusinessKey, &v.DeadlineAt, &v.CancelRequestedAt, &v.CompletedAt, &v.CreatedAt, &v.UpdatedAt)
|
||||
err := tx.QueryRowContext(ctx, q, ns, id).Scan(&v.ID, &v.NamespaceID, &v.DefinitionID, &v.DefinitionVersionID, &v.Version, &gate, &v.State, &v.Revision, &v.Paused, &v.ControlRevision, &input, &output, &v.BusinessKey, &v.DeadlineAt, &v.CancelRequestedAt, &v.StopKind, &v.StopRequestedAt, &v.CompletedAt, &v.CreatedAt, &v.UpdatedAt)
|
||||
v.Input, v.Output = input, output
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
err = fault.NotFound
|
||||
|
||||
@@ -152,6 +152,10 @@ func (r *WorkflowRuntime) process(ctx context.Context, c workflowCandidate, skip
|
||||
_, err = tx.ExecContext(ctx, `DELETE FROM weave_pending_advances WHERE namespace_id=? AND id=?`, c.ns, c.id)
|
||||
return err
|
||||
}
|
||||
if g.view.StopKind != nil {
|
||||
done, err = r.settleWorkflow(ctx, tx, g, c, now)
|
||||
return err
|
||||
}
|
||||
if !allow || g.view.State == weave.InstanceNeedsReview || g.view.State == weave.InstanceCancelling {
|
||||
// 暂停/停用仍按原截止计时;核查和失败结清由控制运行器接续,不清除其状态。
|
||||
if (g.view.State == weave.InstanceRunning || g.view.State == weave.InstanceWaiting) && !now.Before(g.view.DeadlineAt) {
|
||||
@@ -192,7 +196,7 @@ func cloneWorkflowGraph(g *workflowGraph) *workflowGraph {
|
||||
|
||||
// progress 在结果已保存后执行有限决策。额度不足只回滚后继决策,不撤销已确认的结果。
|
||||
func (r *WorkflowRuntime) progress(ctx context.Context, tx *sql.Tx, g *workflowGraph, allow bool, now time.Time) error {
|
||||
if g.view.State.Terminal() || g.view.State == weave.InstanceNeedsReview || g.view.State == weave.InstanceCancelling {
|
||||
if g.view.State.Terminal() || g.view.StopKind != nil || g.view.State == weave.InstanceNeedsReview || g.view.State == weave.InstanceCancelling {
|
||||
return nil
|
||||
}
|
||||
allow = allow && now.Before(g.view.DeadlineAt)
|
||||
@@ -232,6 +236,18 @@ func holdWorkflow(ctx context.Context, tx *sql.Tx, g *workflowGraph, state weave
|
||||
if g.view.State.Terminal() {
|
||||
return errors.New("cannot hold completed workflow")
|
||||
}
|
||||
if state == weave.InstanceCancelling || g.view.StopKind != nil || g.version.Specification.FailurePolicy == "FAIL_AFTER_SETTLED" {
|
||||
kind := "FAILURE"
|
||||
if g.view.CancelRequestedAt != nil {
|
||||
kind = "CANCEL"
|
||||
}
|
||||
if err := recordWorkflowStop(ctx, tx, g, kind, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if g.view.State == weave.InstanceNeedsReview {
|
||||
state = weave.InstanceNeedsReview
|
||||
}
|
||||
|
||||
@@ -96,7 +96,7 @@ func lockWorkflowGraph(ctx context.Context, tx *sql.Tx, ns, id core.ID, skip boo
|
||||
if len(g.steps) != int(version.Analysis.Steps)+1 || root == nil || root.view.ActivationID == nil {
|
||||
return nil, false, errors.New("workflow step set incomplete")
|
||||
}
|
||||
allow := enabled && !gatePaused && !v.Paused && v.CancelRequestedAt == nil && (v.State == weave.InstanceRunning || v.State == weave.InstanceWaiting)
|
||||
allow := enabled && v.StopKind == nil && !gatePaused && !v.Paused && v.CancelRequestedAt == nil && (v.State == weave.InstanceRunning || v.State == weave.InstanceWaiting)
|
||||
return g, allow, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -262,4 +262,53 @@ func TestIntegrationWorkflowRuntimeProcess(t *testing.T) {
|
||||
t.Fatal("final signal restart lost winner or issued an extra round")
|
||||
}
|
||||
t.Log("early signal persisted before wait activation across all-stop; final signal selected with running query kept permits, then survived another all-stop and accepted PENDING settlement without issuing a second query")
|
||||
for _, p := range processes {
|
||||
stop(p)
|
||||
}
|
||||
failedSpec := runtimeSpec(f.resources.QueueID, 1)
|
||||
tail := failedSpec.Body.Steps[0]
|
||||
failedSpec.Body.Steps = append(failedSpec.Body.Steps[:0], runtimeParallel(failedSpec), tail)
|
||||
failedSpec.FailurePolicy = "FAIL_AFTER_SETTLED"
|
||||
failed := runtimeStart(t, f, failedSpec)
|
||||
left, right := retryStart(t, f), retryStart(t, f)
|
||||
retryFinish(t, f, left, "FAILED", false)
|
||||
start("failure-collection-a")
|
||||
start("failure-collection-b")
|
||||
wait(func() bool { return f.get(right.TaskID).CancelRequestedAt != nil }, "failure cancellation of parallel sibling")
|
||||
if f.used() != 1 || f.get(right.TaskID).State != "RUNNING" {
|
||||
t.Fatal("failure claimed sibling ended")
|
||||
}
|
||||
for _, p := range processes {
|
||||
stop(p)
|
||||
}
|
||||
retryFinish(t, f, right, "SUCCEEDED", false)
|
||||
if f.count("weave_instances", "id=? AND state='CANCELLING' AND stop_kind='FAILURE'", failed.ID) != 1 {
|
||||
t.Fatal("all-stop lost failure intent")
|
||||
}
|
||||
start("failure-settlement-a")
|
||||
start("failure-settlement-b")
|
||||
wait(func() bool {
|
||||
current, err := api.Get(ctx, f.token, failed.ID)
|
||||
return err == nil && current.State == "FAILED"
|
||||
}, "parallel failure settlement across restart")
|
||||
if f.count("pulse_tasks", "1=1") != 10 || f.used() != 0 {
|
||||
t.Fatal("failure restarted a branch or tail")
|
||||
}
|
||||
for _, p := range processes {
|
||||
stop(p)
|
||||
}
|
||||
cancelled := runtimeStart(t, f, runtimeSpec(f.resources.QueueID, 2))
|
||||
grantWorkflowControl(f)
|
||||
ref = retryStart(t, f)
|
||||
applyControl(t, f, cancelled, "CANCEL")
|
||||
retryFinish(t, f, ref, "UNKNOWN", false)
|
||||
start("cancel-review-a")
|
||||
start("cancel-review-b")
|
||||
wait(func() bool {
|
||||
return f.count("weave_instances", "id=? AND state='NEEDS_REVIEW' AND stop_kind='CANCEL' AND completed_at IS NULL", cancelled.ID) == 1 && f.count("weave_pending_advances", "workflow_id=?", cancelled.ID) == 0
|
||||
}, "cancel unknown review without false terminal state")
|
||||
if f.used() != 0 || f.count("pulse_tasks", "1=1") != 11 {
|
||||
t.Fatal("unknown cancel leaked permit or started tail")
|
||||
}
|
||||
t.Log("parallel failure intent and running sibling survived another all-stop, settled FAILED only after result collection; user cancellation plus UNKNOWN recovered NEEDS_REVIEW with no successor or false terminal state")
|
||||
}
|
||||
|
||||
384
internal/storage/mysql/workflow_settlement.go
Normal file
384
internal/storage/mysql/workflow_settlement.go
Normal file
@@ -0,0 +1,384 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/engine"
|
||||
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
|
||||
)
|
||||
|
||||
// settleWorkflow 在根检查点下结清停止意图。每事务最多发送一个任务取消,避免取得结果流锁后再锁另一任务。
|
||||
// 步骤收尾最多32项,超过后保留根接续;停止/核查不依赖节点内存恢复。
|
||||
func (r *WorkflowRuntime) settleWorkflow(ctx context.Context, tx *sql.Tx, g *workflowGraph, c workflowCandidate, now time.Time) (bool, error) {
|
||||
if g.view.StopKind == nil || g.view.StopRequestedAt == nil || (*g.view.StopKind != "CANCEL" && *g.view.StopKind != "FAILURE") {
|
||||
return false, errors.New("workflow stop intent invalid")
|
||||
}
|
||||
if *g.view.StopKind == "FAILURE" && g.version.Specification.FailurePolicy != "FAIL_AFTER_SETTLED" {
|
||||
return false, errors.New("workflow failure stop conflicts with fixed policy")
|
||||
}
|
||||
if *g.view.StopKind == "CANCEL" && g.view.CancelRequestedAt == nil {
|
||||
return false, errors.New("workflow cancellation intent missing")
|
||||
}
|
||||
// 已持有应用/全部队列/实例/门控锁;门控漂移不能为停止中的工作重新授权。
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE pulse_execution_gates SET paused=TRUE,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND paused=FALSE`, now, g.view.NamespaceID, g.gate); err != nil {
|
||||
return false, err
|
||||
}
|
||||
var taskID core.ID
|
||||
err := tx.QueryRowContext(ctx, `SELECT id FROM pulse_tasks WHERE namespace_id=? AND gate_id=? AND state NOT IN ('SUCCEEDED','FAILED','CANCELLED') AND cancel_requested_at IS NULL ORDER BY id LIMIT 1`, g.view.NamespaceID, g.gate).Scan(&taskID)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return false, err
|
||||
}
|
||||
if err == nil {
|
||||
if err := validateWorkflowStopTask(ctx, tx, g, taskID); err != nil {
|
||||
return false, err
|
||||
}
|
||||
task, err := readExecutionTask(ctx, tx, g.view.NamespaceID, taskID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err := cancelExecutionTask(ctx, tx, g.view.NamespaceID, task, now, "WORKFLOW_STOP_REQUESTED"); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_TASK_STOP_REQUESTED", now)
|
||||
}
|
||||
var pending int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_inbox WHERE namespace_id=? AND workflow_id=? AND state='PENDING'`, g.view.NamespaceID, g.view.ID).Scan(&pending); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if pending > 0 {
|
||||
return false, saveWorkflowAdvance(ctx, tx, g, now.Add(time.Second))
|
||||
}
|
||||
keys := make([]string, 0, len(g.steps))
|
||||
for key := range g.steps {
|
||||
if key != workflowRootKey {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
changed, active, review := 0, false, false
|
||||
for _, key := range keys {
|
||||
row := g.steps[key]
|
||||
if row.view.State == weave.StepNeedsReview {
|
||||
review = true
|
||||
continue
|
||||
}
|
||||
if row.view.State == weave.StepSucceeded || row.view.State == weave.StepFailed || row.view.State == weave.StepCancelled || row.view.State == weave.StepSkipped {
|
||||
continue
|
||||
}
|
||||
if changed == 32 {
|
||||
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_CONTINUATION", now)
|
||||
}
|
||||
if row.view.State == weave.StepBlocked {
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_steps SET state='SKIPPED',revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=? AND current_activation_id IS NULL AND state='BLOCKED'`, now, g.view.NamespaceID, row.view.ID, row.view.Revision)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return false, err
|
||||
}
|
||||
row.view.State = weave.StepSkipped
|
||||
row.view.Revision++
|
||||
changed++
|
||||
if err := emitEvent(ctx, tx, g.view.NamespaceID, row.view.ID, row.view.Revision, "workflow_step", "workflow.step.changed", map[string]any{"workflow_id": g.view.ID, "key": key, "state": row.view.State, "reason": "WORKFLOW_STOP_REQUESTED"}); err != nil {
|
||||
return false, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
switch row.view.Kind {
|
||||
case weave.NodeTask:
|
||||
if row.view.TaskID == nil {
|
||||
return false, errors.New("stopping task step missing task")
|
||||
}
|
||||
task, err := readTask(ctx, tx, g.view.NamespaceID, *row.view.TaskID, false)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if task.State.Terminal() || task.State == pulse.TaskNeedsReview {
|
||||
return false, errors.New("stopping task missing result confirmation")
|
||||
}
|
||||
active = true
|
||||
case weave.NodeSignal, weave.NodeTimer, weave.NodePoll:
|
||||
node, _ := g.program.Step(key)
|
||||
if row.view.Kind == weave.NodePoll {
|
||||
var current *core.ID
|
||||
if err := tx.QueryRowContext(ctx, `SELECT current_task_id FROM weave_poll_states WHERE namespace_id=? AND workflow_id=? AND activation_id=?`, g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(¤t); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if current != nil {
|
||||
active = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
// 已在停止前可靠受理的信号保留为事实;停止后的新信号由受理事务拒绝。
|
||||
if signalContract(node) != nil {
|
||||
w, err := readWorkflowSignalWait(ctx, tx, g, row)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
v, err := selectedWorkflowSignal(ctx, tx, g, row, w, now)
|
||||
if err != nil || v != nil {
|
||||
handled, done, reason, err := r.driveWorkflowSignal(ctx, tx, g, row, c, nil, false, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !handled {
|
||||
return false, errors.New("stopping signal did not resolve")
|
||||
}
|
||||
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if done && reason != "" {
|
||||
return true, r.audit(ctx, tx, g, c.id, reason, now)
|
||||
}
|
||||
return done, nil
|
||||
}
|
||||
}
|
||||
if err := cancelWorkflowWait(ctx, tx, g, row, now); err != nil {
|
||||
return false, err
|
||||
}
|
||||
changed++
|
||||
case weave.NodeBranch, weave.NodeParallel:
|
||||
// 组合步骤只有在全部活动叶子结清后才能结清,避免掩盖在途工作。
|
||||
default:
|
||||
return false, errors.New("unsupported stopping step")
|
||||
}
|
||||
}
|
||||
if active {
|
||||
if err := saveWorkflowAdvance(ctx, tx, g, now.Add(time.Second)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if changed > 0 {
|
||||
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_COLLECTING", now)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
settled, uncertain, err := verifyWorkflowStopWork(ctx, tx, g)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !settled {
|
||||
if err := saveWorkflowAdvance(ctx, tx, g, now.Add(time.Second)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if changed > 0 {
|
||||
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_COLLECTING", now)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
if review || uncertain || g.view.State == weave.InstanceNeedsReview {
|
||||
stateChanged := g.view.State != weave.InstanceNeedsReview
|
||||
if stateChanged {
|
||||
if err := holdWorkflow(ctx, tx, g, weave.InstanceNeedsReview, "WORKFLOW_STOP_REVIEW", now); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
// 已无平台内有效授权时不逐秒空转;UNKNOWN不证明外部执行已结束,后续核查负责接续。
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE weave_signals SET state='REJECTED',disposition='WORKFLOW_STOPPED_REVIEW',revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND state='PENDING'`, now, g.view.NamespaceID, g.view.ID); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := deleteWorkflowClock(ctx, tx, g, g.steps[workflowRootKey]); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if changed > 0 || stateChanged {
|
||||
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_REVIEW", now)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
for _, key := range keys {
|
||||
row := g.steps[key]
|
||||
if row.view.Kind != weave.NodeBranch && row.view.Kind != weave.NodeParallel {
|
||||
continue
|
||||
}
|
||||
if row.view.State != weave.StepActive && row.view.State != weave.StepWaiting {
|
||||
continue
|
||||
}
|
||||
if changed == 32 {
|
||||
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_CONTINUATION", now)
|
||||
}
|
||||
if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepCancelled, nil, now); err != nil {
|
||||
return false, err
|
||||
}
|
||||
changed++
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE weave_signals SET state='REJECTED',disposition='WORKFLOW_STOPPED',revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND state='PENDING'`, now, g.view.NamespaceID, g.view.ID); err != nil {
|
||||
return false, err
|
||||
}
|
||||
next, rootState := weave.InstanceCancelled, weave.StepCancelled
|
||||
if *g.view.StopKind == "FAILURE" {
|
||||
next, rootState = weave.InstanceFailed, weave.StepFailed
|
||||
}
|
||||
if err := saveWorkflowResultStep(ctx, tx, g, g.steps[workflowRootKey], rootState, nil, now); err != nil {
|
||||
return false, err
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_instances SET state=?,output=NULL,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, next, now, now, g.view.NamespaceID, g.view.ID, g.view.Revision)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return false, err
|
||||
}
|
||||
g.view.State = next
|
||||
g.view.Output = nil
|
||||
g.view.CompletedAt = &now
|
||||
g.view.Revision++
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM weave_pending_advances WHERE namespace_id=? AND workflow_id=?`, g.view.NamespaceID, g.view.ID); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, g.view.NamespaceID, g.view.ID, g.view.Revision, "workflow", "workflow.stopped", map[string]any{"state": next, "stop_kind": g.view.StopKind, "stop_requested_at": g.view.StopRequestedAt}); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOPPED", now)
|
||||
}
|
||||
|
||||
func cancelWorkflowWait(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, now time.Time) error {
|
||||
if row.view.Kind == weave.NodeTimer {
|
||||
var count int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_waits WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND kind='TIMER' AND state='WAITING' AND signal_name IS NULL AND payload_version IS NULL AND winner_signal_id IS NULL`, g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(&count); err != nil {
|
||||
return err
|
||||
}
|
||||
if count != 1 {
|
||||
return errors.New("stopping timer wait invalid")
|
||||
}
|
||||
}
|
||||
if row.view.Kind == weave.NodePoll {
|
||||
var activated time.Time
|
||||
if err := tx.QueryRowContext(ctx, `SELECT created_at FROM weave_activations WHERE namespace_id=? AND workflow_id=? AND step_id=? AND id=?`, g.view.NamespaceID, g.view.ID, row.view.ID, row.view.ActivationID).Scan(&activated); err != nil {
|
||||
return err
|
||||
}
|
||||
p, _, err := readWorkflowPollState(ctx, tx, g, row, activated)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if p.CurrentTaskID != nil {
|
||||
return errors.New("cannot cancel in-flight poll wait")
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_poll_states SET next_poll_at=NULL,resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND resolved_at IS NULL AND current_task_id IS NULL`, now, now, g.view.NamespaceID, p.ID)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE weave_waits SET state='CANCELLED',resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND state='WAITING'`, now, now, g.view.NamespaceID, g.view.ID, row.view.ActivationID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepCancelled, nil, now); err != nil {
|
||||
return err
|
||||
}
|
||||
return deleteWorkflowClock(ctx, tx, g, row)
|
||||
}
|
||||
|
||||
// validateWorkflowStopTask 取消前核对实际任务到流程激活/步骤/队列的归属,不能仅凭gate_id关联。
|
||||
func validateWorkflowStopTask(ctx context.Context, tx *sql.Tx, g *workflowGraph, id core.ID) error {
|
||||
var kind string
|
||||
var owner, queue, gate, workflow, step core.ID
|
||||
var key string
|
||||
var direct *core.ID
|
||||
err := tx.QueryRowContext(ctx, "SELECT t.owner_kind,t.owner_id,t.queue_id,t.gate_id,a.workflow_id,a.step_id,s.`key`,a.task_id FROM pulse_tasks t JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id JOIN weave_steps s ON s.namespace_id=a.namespace_id AND s.workflow_id=a.workflow_id AND s.id=a.step_id WHERE t.namespace_id=? AND t.id=?", g.view.NamespaceID, id).Scan(&kind, &owner, &queue, &gate, &workflow, &step, &key, &direct)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
row := g.steps[key]
|
||||
node, ok := g.program.Step(key)
|
||||
if kind != engine.TaskOwnerKind || gate != g.gate || workflow != g.view.ID || row == nil || row.view.ID != step || !ok {
|
||||
return errors.New("stopping task ownership invalid")
|
||||
}
|
||||
if node.Task != nil {
|
||||
if direct == nil || *direct != id || queue != node.Task.QueueID {
|
||||
return errors.New("stopping task direct association invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if node.Poll == nil || direct != nil || queue != node.Poll.Task.QueueID {
|
||||
return errors.New("stopping query association invalid")
|
||||
}
|
||||
var count int
|
||||
if err := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=? AND r.workflow_id=? AND r.activation_id=? AND r.task_id=? AND r.`round` BETWEEN 1 AND p.used_rounds", g.view.NamespaceID, g.view.ID, owner, id).Scan(&count); err != nil {
|
||||
return err
|
||||
}
|
||||
if count != 1 {
|
||||
return errors.New("stopping query round invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifyWorkflowStopWork 终态前复核全部历史任务、双向归属、未结束尝试和许可,不能仅检查当前步骤。
|
||||
func verifyWorkflowStopWork(ctx context.Context, tx *sql.Tx, g *workflowGraph) (settled, uncertain bool, err error) {
|
||||
var invalid int
|
||||
checks := []string{
|
||||
`SELECT COUNT(*) FROM pulse_tasks t JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id WHERE a.namespace_id=? AND a.workflow_id=? AND t.owner_kind='workflow_activation' AND (t.gate_id IS NULL OR t.gate_id<>?)`,
|
||||
`SELECT COUNT(*) FROM weave_activations a LEFT JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id WHERE a.namespace_id=? AND a.workflow_id=? AND a.task_id IS NOT NULL AND (t.id IS NULL OR t.owner_kind IS NULL OR t.owner_kind<>'workflow_activation' OR t.owner_id<>a.id OR t.gate_id IS NULL OR t.gate_id<>?)`,
|
||||
`SELECT COUNT(*) FROM weave_poll_rounds r LEFT JOIN weave_activations a ON a.namespace_id=r.namespace_id AND a.workflow_id=r.workflow_id AND a.id=r.activation_id LEFT JOIN pulse_tasks t ON t.namespace_id=r.namespace_id AND t.id=r.task_id WHERE r.namespace_id=? AND r.workflow_id=? AND (a.id IS NULL OR t.id IS NULL OR t.owner_kind IS NULL OR t.owner_kind<>'workflow_activation' OR t.owner_id<>r.activation_id OR t.gate_id IS NULL OR t.gate_id<>?)`,
|
||||
}
|
||||
for _, q := range checks {
|
||||
if err := tx.QueryRowContext(ctx, q, g.view.NamespaceID, g.view.ID, g.gate).Scan(&invalid); err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
if invalid != 0 {
|
||||
return false, false, errors.New("workflow stop history association invalid")
|
||||
}
|
||||
}
|
||||
// 一次读取关联元数据并通过索引EXISTS核对查询轮次,避免持有应用锁期间逐任务发出SQL。
|
||||
rows, err := tx.QueryContext(ctx, "SELECT t.id,t.state,t.effect_uncertain,t.owner_kind,t.owner_id,t.queue_id,a.workflow_id,a.step_id,a.task_id,s.`key`,EXISTS(SELECT 1 FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=t.namespace_id AND r.workflow_id=a.workflow_id AND r.activation_id=a.id AND r.task_id=t.id AND r.`round` BETWEEN 1 AND p.used_rounds) FROM pulse_tasks t LEFT JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id LEFT JOIN weave_steps s ON s.namespace_id=a.namespace_id AND s.workflow_id=a.workflow_id AND s.id=a.step_id WHERE t.namespace_id=? AND t.gate_id=? ORDER BY t.id LIMIT 10001", g.view.NamespaceID, g.gate)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
count := 0
|
||||
settled = true
|
||||
for rows.Next() {
|
||||
var id, queue core.ID
|
||||
var owner, workflow, step, direct *core.ID
|
||||
var kind, key sql.NullString
|
||||
var state pulse.TaskState
|
||||
var unknown, round bool
|
||||
if err := rows.Scan(&id, &state, &unknown, &kind, &owner, &queue, &workflow, &step, &direct, &key, &round); err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
count++
|
||||
if count > 10000 {
|
||||
return false, false, errors.New("workflow historical task bound exceeded")
|
||||
}
|
||||
row := g.steps[key.String]
|
||||
node, ok := g.program.Step(key.String)
|
||||
if kind.String != engine.TaskOwnerKind || owner == nil || workflow == nil || *workflow != g.view.ID || step == nil || row == nil || row.view.ID != *step || !ok {
|
||||
return false, false, errors.New("workflow stop task association invalid")
|
||||
}
|
||||
if node.Task != nil {
|
||||
if direct == nil || *direct != id || queue != node.Task.QueueID {
|
||||
return false, false, errors.New("workflow stop task parameters invalid")
|
||||
}
|
||||
} else if node.Poll == nil || direct != nil || queue != node.Poll.Task.QueueID || !round {
|
||||
return false, false, errors.New("workflow stop query association invalid")
|
||||
}
|
||||
uncertain = uncertain || unknown || state == pulse.TaskNeedsReview
|
||||
if !state.Terminal() && state != pulse.TaskNeedsReview {
|
||||
settled = false
|
||||
}
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
if !settled {
|
||||
return false, uncertain, nil
|
||||
}
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_attempts a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id WHERE t.namespace_id=? AND t.gate_id=? AND a.state IN ('LEASED','RUNNING')`, g.view.NamespaceID, g.gate).Scan(&invalid); err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
if invalid > 0 {
|
||||
return false, uncertain, nil
|
||||
}
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_permits p JOIN pulse_attempts a ON a.namespace_id=p.namespace_id AND a.id=p.attempt_id JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id WHERE t.namespace_id=? AND t.gate_id=? AND p.released_at IS NULL`, g.view.NamespaceID, g.gate).Scan(&invalid); err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
return invalid == 0, uncertain, nil
|
||||
}
|
||||
@@ -242,7 +242,7 @@ func (r *WorkflowRuntime) applyWorkflowSignal(ctx context.Context, tx *sql.Tx, g
|
||||
}
|
||||
if state == weave.StepNeedsReview || state == weave.StepFailed {
|
||||
next := weave.InstanceNeedsReview
|
||||
if state == weave.StepFailed && (g.version.Specification.FailurePolicy == "FAIL_AFTER_SETTLED" || g.view.CancelRequestedAt != nil) {
|
||||
if state == weave.StepFailed && (g.view.StopKind != nil || g.version.Specification.FailurePolicy == "FAIL_AFTER_SETTLED" || g.view.CancelRequestedAt != nil) {
|
||||
next = weave.InstanceCancelling
|
||||
}
|
||||
if err := holdWorkflow(ctx, tx, g, next, reason, now); err != nil {
|
||||
@@ -284,6 +284,14 @@ func (r *WorkflowRuntime) expireWorkflowSignal(ctx context.Context, tx *sql.Tx,
|
||||
if v.State != "PENDING" || now.Before(v.ExpiresAt) {
|
||||
return false, nil
|
||||
}
|
||||
if g.view.StopKind != nil && !g.view.State.Terminal() {
|
||||
// 停止事务已关闭新等待;由根收集停止前信号,不能让稍后过期扫描把取消改成超时核查。
|
||||
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
|
||||
return false, err
|
||||
}
|
||||
_, err := tx.ExecContext(ctx, `UPDATE weave_signals SET available_at=DATE_ADD(?,INTERVAL 30 SECOND),revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND state='PENDING'`, now, now, c.ns, c.id)
|
||||
return false, err
|
||||
}
|
||||
row := g.steps[v.TargetStepKey]
|
||||
if row != nil && row.view.ActivationID != nil && !g.view.State.Terminal() && (row.view.State == weave.StepWaiting || row.view.State == weave.StepActive) {
|
||||
w, err := readWorkflowSignalWait(ctx, tx, g, row)
|
||||
|
||||
@@ -119,7 +119,7 @@ func selectedWorkflowSignal(ctx context.Context, tx *sql.Tx, g *workflowGraph, r
|
||||
if w.state != "WAITING" {
|
||||
return nil, nil
|
||||
}
|
||||
err := tx.QueryRowContext(ctx, `SELECT id FROM weave_signals WHERE namespace_id=? AND workflow_id=? AND target_step_key=? AND target_activation_sequence=? AND state='PENDING' AND type=? AND payload_version=? AND correlation_key=? AND accepted_at<? ORDER BY accepted_at,id LIMIT 1 FOR UPDATE`, g.view.NamespaceID, g.view.ID, row.view.Key, w.sequence, w.contract.Name, w.contract.PayloadVersion, []byte(w.correlation), w.deadline).Scan(&id)
|
||||
err := tx.QueryRowContext(ctx, `SELECT id FROM weave_signals WHERE namespace_id=? AND workflow_id=? AND target_step_key=? AND target_activation_sequence=? AND state='PENDING' AND type=? AND payload_version=? AND correlation_key=? AND accepted_at<? AND (? IS NULL OR accepted_at<=?) ORDER BY accepted_at,id LIMIT 1 FOR UPDATE`, g.view.NamespaceID, g.view.ID, row.view.Key, w.sequence, w.contract.Name, w.contract.PayloadVersion, []byte(w.correlation), w.deadline, g.view.StopRequestedAt, g.view.StopRequestedAt).Scan(&id)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -124,6 +124,8 @@ func (s *WorkflowSignalStore) Send(ctx context.Context, t auth.Token, workflow c
|
||||
state, disposition = "REJECTED", "STALE_ACTIVATION"
|
||||
case g.view.State.Terminal():
|
||||
state, disposition = "REJECTED", "WORKFLOW_SETTLED"
|
||||
case g.view.StopKind != nil:
|
||||
state, disposition = "REJECTED", "WORKFLOW_STOPPING"
|
||||
case !now.Before(g.view.DeadlineAt):
|
||||
state, disposition = "EXPIRED", "WORKFLOW_DEADLINE"
|
||||
case row.view.State == weave.StepSkipped || row.view.State == weave.StepSucceeded || row.view.State == weave.StepFailed || row.view.State == weave.StepCancelled || row.view.State == weave.StepNeedsReview:
|
||||
|
||||
160
internal/storage/mysql/workflow_stop_upgrade_integration_test.go
Normal file
160
internal/storage/mysql/workflow_stop_upgrade_integration_test.go
Normal file
@@ -0,0 +1,160 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
driver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// 用真实v20结构验证停止意图与根接续升级,保留原取消时间和业务预算。
|
||||
func TestIntegrationWorkflowStopUpgrade(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
admin, err := Open(ctx, dsn, 2, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer admin.Close()
|
||||
name := "orbit_stop_upgrade_" + string(executionID(t))
|
||||
if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+name+"`"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
cleanup, stop := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer stop()
|
||||
if _, err := admin.ExecContext(cleanup, "DROP DATABASE `"+name+"`"); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
cfg, err := driver.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
t.Fatal("invalid test DSN")
|
||||
}
|
||||
cfg.DBName = name
|
||||
db, err := Open(ctx, cfg.FormatDSN(), 4, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
exec := func(q string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := db.ExecContext(ctx, q, args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
exec(`CREATE TABLE orbit_schema_migrations (version BIGINT UNSIGNED PRIMARY KEY,name VARCHAR(191) NOT NULL,checksum CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,dirty BOOLEAN NOT NULL,applied_at DATETIME(6) NULL) ENGINE=InnoDB`)
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range migrations {
|
||||
if m.Version > 20 {
|
||||
break
|
||||
}
|
||||
statements, err := splitSQL(m.SQL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, q := range statements {
|
||||
exec(q)
|
||||
}
|
||||
exec(`INSERT INTO orbit_schema_migrations(version,name,checksum,dirty,applied_at) VALUES (?,?,?,FALSE,UTC_TIMESTAMP(6))`, m.Version, m.Name, m.Checksum)
|
||||
}
|
||||
var now time.Time
|
||||
if err := db.QueryRowContext(ctx, `SELECT UTC_TIMESTAMP(6)`).Scan(&now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ns := executionID(t)
|
||||
deadline := now.Add(time.Hour)
|
||||
requested := now.Add(-time.Hour)
|
||||
type fixture struct {
|
||||
id, gate, root, activation core.ID
|
||||
kind string
|
||||
}
|
||||
seed := func(state, policy string, cancelled, failed bool) fixture {
|
||||
t.Helper()
|
||||
f := fixture{executionID(t), executionID(t), executionID(t), executionID(t), "FAILURE"}
|
||||
var cancelAt, completed *time.Time
|
||||
if cancelled {
|
||||
cancelAt = &requested
|
||||
f.kind = "CANCEL"
|
||||
}
|
||||
if state == "CANCELLED" {
|
||||
completed = &now
|
||||
}
|
||||
exec(`INSERT INTO pulse_execution_gates(namespace_id,id,paused) VALUES (?,?,FALSE)`, ns, f.gate)
|
||||
exec(`INSERT INTO weave_instances(namespace_id,id,definition_id,definition_version_id,gate_id,state,control_revision,input,policy_snapshot,deadline_at,cancel_requested_at,completed_at,updated_at) VALUES (?,?,?,?,?,?,4,'{"n":9007199254740993}',JSON_OBJECT('failure_policy',?),?,?,?,?)`, ns, f.id, executionID(t), executionID(t), f.gate, state, policy, deadline, cancelAt, completed, now)
|
||||
exec("INSERT INTO weave_steps(namespace_id,id,workflow_id,`key`,kind,state,current_activation_id,revision) VALUES (?,?,?,'$root','JOIN','ACTIVE',?,7)", ns, f.root, f.id, f.activation)
|
||||
exec(`INSERT INTO weave_activations(namespace_id,id,workflow_id,step_id,sequence,state,input) VALUES (?,?,?,?,1,'ACTIVE','null')`, ns, f.activation, f.id, f.root)
|
||||
if failed {
|
||||
exec("INSERT INTO weave_steps(namespace_id,id,workflow_id,`key`,kind,state) VALUES (?,?,?,'failed','TASK','NEEDS_REVIEW')", ns, executionID(t), f.id)
|
||||
}
|
||||
return f
|
||||
}
|
||||
cancellation := seed("NEEDS_REVIEW", "REVIEW", true, false)
|
||||
fail := seed("CANCELLING", "FAIL_AFTER_SETTLED", false, false)
|
||||
uncertain := seed("NEEDS_REVIEW", "FAIL_AFTER_SETTLED", false, true)
|
||||
untouched := seed("NEEDS_REVIEW", "REVIEW", false, true)
|
||||
terminal := seed("CANCELLED", "REVIEW", true, false)
|
||||
originalID := executionID(t)
|
||||
original := `{"schema_version":1,"kind":"ROOT"}`
|
||||
exec(`INSERT INTO weave_pending_advances(namespace_id,id,workflow_id,step_id,expected_step_revision,checkpoint,available_at,revision) VALUES (?,?,?,?,7,?,?,11)`, ns, originalID, fail.id, fail.root, original, deadline)
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := CheckSchema(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range []fixture{cancellation, fail, uncertain} {
|
||||
var kind string
|
||||
var at, due, keptDeadline time.Time
|
||||
var controlRevision, expected uint64
|
||||
var paused bool
|
||||
var raw []byte
|
||||
if err := db.QueryRowContext(ctx, `SELECT i.stop_kind,i.stop_requested_at,i.deadline_at,i.control_revision,g.paused FROM weave_instances i JOIN pulse_execution_gates g ON g.namespace_id=i.namespace_id AND g.id=i.gate_id WHERE i.namespace_id=? AND i.id=?`, ns, f.id).Scan(&kind, &at, &keptDeadline, &controlRevision, &paused); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := now
|
||||
if f.kind == "CANCEL" {
|
||||
want = requested
|
||||
}
|
||||
if kind != f.kind || !at.Equal(want) || !keptDeadline.Equal(deadline) || controlRevision != 4 || !paused {
|
||||
t.Fatal("migration reset stop identity, deadline or control version")
|
||||
}
|
||||
if err := db.QueryRowContext(ctx, `SELECT checkpoint,available_at,expected_step_revision FROM weave_pending_advances WHERE namespace_id=? AND workflow_id=?`, ns, f.id).Scan(&raw, &due, &expected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var cp workflowCheckpoint
|
||||
if err := json.Unmarshal(raw, &cp); err != nil || cp.Kind != "ROOT" || cp.SchemaVersion != 1 || cp.ActivationID != nil || expected != 7 || !due.Before(deadline) {
|
||||
t.Fatal("stop continuation missing or invalid", err)
|
||||
}
|
||||
}
|
||||
var count int
|
||||
for _, f := range []fixture{untouched, terminal} {
|
||||
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_instances WHERE namespace_id=? AND id=? AND stop_kind IS NULL AND stop_requested_at IS NULL`, ns, f.id).Scan(&count); err != nil || count != 1 {
|
||||
t.Fatal("unrelated or terminal workflow changed", err)
|
||||
}
|
||||
}
|
||||
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_pending_advances WHERE namespace_id=?`, ns).Scan(&count); err != nil || count != 3 {
|
||||
t.Fatal("backfill included unrelated workflow", count, err)
|
||||
}
|
||||
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_pending_advances WHERE namespace_id=? AND id=? AND revision=11 AND checkpoint=?`, ns, originalID, original).Scan(&count); err != nil || count != 1 {
|
||||
t.Fatal("existing root identity changed", err)
|
||||
}
|
||||
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatal("foreign keys introduced", err)
|
||||
}
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("real v20 stop intent backfill preserves cancellation time and fixed deadline/control revision, closes gate, preserves existing root identity; unrelated/terminal workflows unchanged; no foreign keys")
|
||||
}
|
||||
Reference in New Issue
Block a user