385 lines
18 KiB
Go
385 lines
18 KiB
Go
package mysql
|
||
|
||
import (
|
||
"context"
|
||
"database/sql"
|
||
"errors"
|
||
"sort"
|
||
"time"
|
||
|
||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||
"git.yuntuo.hk/k2safe/orbit/weave/engine"
|
||
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
|
||
)
|
||
|
||
// settleWorkflow 在根检查点下结清停止意图。每事务最多发送一个任务取消，避免取得结果流锁后再锁另一任务。
|
||
// 步骤收尾最多32项，超过后保留根接续；停止/核查不依赖节点内存恢复。
|
||
func (r *WorkflowRuntime) settleWorkflow(ctx context.Context, tx *sql.Tx, g *workflowGraph, c workflowCandidate, now time.Time) (bool, error) {
|
||
if g.view.StopKind == nil || g.view.StopRequestedAt == nil || (*g.view.StopKind != "CANCEL" && *g.view.StopKind != "FAILURE") {
|
||
return false, errors.New("workflow stop intent invalid")
|
||
}
|
||
if *g.view.StopKind == "FAILURE" && g.version.Specification.FailurePolicy != "FAIL_AFTER_SETTLED" {
|
||
return false, errors.New("workflow failure stop conflicts with fixed policy")
|
||
}
|
||
if *g.view.StopKind == "CANCEL" && g.view.CancelRequestedAt == nil {
|
||
return false, errors.New("workflow cancellation intent missing")
|
||
}
|
||
// 已持有应用/全部队列/实例/门控锁；门控漂移不能为停止中的工作重新授权。
|
||
if _, err := tx.ExecContext(ctx, `UPDATE pulse_execution_gates SET paused=TRUE,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND paused=FALSE`, now, g.view.NamespaceID, g.gate); err != nil {
|
||
return false, err
|
||
}
|
||
var taskID core.ID
|
||
err := tx.QueryRowContext(ctx, `SELECT id FROM pulse_tasks WHERE namespace_id=? AND gate_id=? AND state NOT IN ('SUCCEEDED','FAILED','CANCELLED') AND cancel_requested_at IS NULL ORDER BY id LIMIT 1`, g.view.NamespaceID, g.gate).Scan(&taskID)
|
||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||
return false, err
|
||
}
|
||
if err == nil {
|
||
if err := validateWorkflowStopTask(ctx, tx, g, taskID); err != nil {
|
||
return false, err
|
||
}
|
||
task, err := readExecutionTask(ctx, tx, g.view.NamespaceID, taskID)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if _, err := cancelExecutionTask(ctx, tx, g.view.NamespaceID, task, now, "WORKFLOW_STOP_REQUESTED"); err != nil {
|
||
return false, err
|
||
}
|
||
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
|
||
return false, err
|
||
}
|
||
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_TASK_STOP_REQUESTED", now)
|
||
}
|
||
var pending int
|
||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_inbox WHERE namespace_id=? AND workflow_id=? AND state='PENDING'`, g.view.NamespaceID, g.view.ID).Scan(&pending); err != nil {
|
||
return false, err
|
||
}
|
||
if pending > 0 {
|
||
return false, saveWorkflowAdvance(ctx, tx, g, now.Add(time.Second))
|
||
}
|
||
keys := make([]string, 0, len(g.steps))
|
||
for key := range g.steps {
|
||
if key != workflowRootKey {
|
||
keys = append(keys, key)
|
||
}
|
||
}
|
||
sort.Strings(keys)
|
||
changed, active, review := 0, false, false
|
||
for _, key := range keys {
|
||
row := g.steps[key]
|
||
if row.view.State == weave.StepNeedsReview {
|
||
review = true
|
||
continue
|
||
}
|
||
if row.view.State == weave.StepSucceeded || row.view.State == weave.StepFailed || row.view.State == weave.StepCancelled || row.view.State == weave.StepSkipped {
|
||
continue
|
||
}
|
||
if changed == 32 {
|
||
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
|
||
return false, err
|
||
}
|
||
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_CONTINUATION", now)
|
||
}
|
||
if row.view.State == weave.StepBlocked {
|
||
res, err := tx.ExecContext(ctx, `UPDATE weave_steps SET state='SKIPPED',revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=? AND current_activation_id IS NULL AND state='BLOCKED'`, now, g.view.NamespaceID, row.view.ID, row.view.Revision)
|
||
if err := exactlyOne(res, err); err != nil {
|
||
return false, err
|
||
}
|
||
row.view.State = weave.StepSkipped
|
||
row.view.Revision++
|
||
changed++
|
||
if err := emitEvent(ctx, tx, g.view.NamespaceID, row.view.ID, row.view.Revision, "workflow_step", "workflow.step.changed", map[string]any{"workflow_id": g.view.ID, "key": key, "state": row.view.State, "reason": "WORKFLOW_STOP_REQUESTED"}); err != nil {
|
||
return false, err
|
||
}
|
||
continue
|
||
}
|
||
switch row.view.Kind {
|
||
case weave.NodeTask:
|
||
if row.view.TaskID == nil {
|
||
return false, errors.New("stopping task step missing task")
|
||
}
|
||
task, err := readTask(ctx, tx, g.view.NamespaceID, *row.view.TaskID, false)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if task.State.Terminal() || task.State == pulse.TaskNeedsReview {
|
||
return false, errors.New("stopping task missing result confirmation")
|
||
}
|
||
active = true
|
||
case weave.NodeSignal, weave.NodeTimer, weave.NodePoll:
|
||
node, _ := g.program.Step(key)
|
||
if row.view.Kind == weave.NodePoll {
|
||
var current *core.ID
|
||
if err := tx.QueryRowContext(ctx, `SELECT current_task_id FROM weave_poll_states WHERE namespace_id=? AND workflow_id=? AND activation_id=?`, g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(¤t); err != nil {
|
||
return false, err
|
||
}
|
||
if current != nil {
|
||
active = true
|
||
continue
|
||
}
|
||
}
|
||
// 已在停止前可靠受理的信号保留为事实；停止后的新信号由受理事务拒绝。
|
||
if signalContract(node) != nil {
|
||
w, err := readWorkflowSignalWait(ctx, tx, g, row)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
v, err := selectedWorkflowSignal(ctx, tx, g, row, w, now)
|
||
if err != nil || v != nil {
|
||
handled, done, reason, err := r.driveWorkflowSignal(ctx, tx, g, row, c, nil, false, now)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if !handled {
|
||
return false, errors.New("stopping signal did not resolve")
|
||
}
|
||
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
|
||
return false, err
|
||
}
|
||
if done && reason != "" {
|
||
return true, r.audit(ctx, tx, g, c.id, reason, now)
|
||
}
|
||
return done, nil
|
||
}
|
||
}
|
||
if err := cancelWorkflowWait(ctx, tx, g, row, now); err != nil {
|
||
return false, err
|
||
}
|
||
changed++
|
||
case weave.NodeBranch, weave.NodeParallel:
|
||
// 组合步骤只有在全部活动叶子结清后才能结清，避免掩盖在途工作。
|
||
default:
|
||
return false, errors.New("unsupported stopping step")
|
||
}
|
||
}
|
||
if active {
|
||
if err := saveWorkflowAdvance(ctx, tx, g, now.Add(time.Second)); err != nil {
|
||
return false, err
|
||
}
|
||
if changed > 0 {
|
||
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_COLLECTING", now)
|
||
}
|
||
return false, nil
|
||
}
|
||
settled, uncertain, err := verifyWorkflowStopWork(ctx, tx, g)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if !settled {
|
||
if err := saveWorkflowAdvance(ctx, tx, g, now.Add(time.Second)); err != nil {
|
||
return false, err
|
||
}
|
||
if changed > 0 {
|
||
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_COLLECTING", now)
|
||
}
|
||
return false, nil
|
||
}
|
||
if review || uncertain || g.view.State == weave.InstanceNeedsReview {
|
||
stateChanged := g.view.State != weave.InstanceNeedsReview
|
||
if stateChanged {
|
||
if err := holdWorkflow(ctx, tx, g, weave.InstanceNeedsReview, "WORKFLOW_STOP_REVIEW", now); err != nil {
|
||
return false, err
|
||
}
|
||
}
|
||
// 已无平台内有效授权时不逐秒空转；UNKNOWN不证明外部执行已结束，后续核查负责接续。
|
||
if _, err := tx.ExecContext(ctx, `UPDATE weave_signals SET state='REJECTED',disposition='WORKFLOW_STOPPED_REVIEW',revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND state='PENDING'`, now, g.view.NamespaceID, g.view.ID); err != nil {
|
||
return false, err
|
||
}
|
||
if err := deleteWorkflowClock(ctx, tx, g, g.steps[workflowRootKey]); err != nil {
|
||
return false, err
|
||
}
|
||
if changed > 0 || stateChanged {
|
||
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_REVIEW", now)
|
||
}
|
||
return false, nil
|
||
}
|
||
for _, key := range keys {
|
||
row := g.steps[key]
|
||
if row.view.Kind != weave.NodeBranch && row.view.Kind != weave.NodeParallel {
|
||
continue
|
||
}
|
||
if row.view.State != weave.StepActive && row.view.State != weave.StepWaiting {
|
||
continue
|
||
}
|
||
if changed == 32 {
|
||
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
|
||
return false, err
|
||
}
|
||
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_CONTINUATION", now)
|
||
}
|
||
if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepCancelled, nil, now); err != nil {
|
||
return false, err
|
||
}
|
||
changed++
|
||
}
|
||
if _, err := tx.ExecContext(ctx, `UPDATE weave_signals SET state='REJECTED',disposition='WORKFLOW_STOPPED',revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND state='PENDING'`, now, g.view.NamespaceID, g.view.ID); err != nil {
|
||
return false, err
|
||
}
|
||
next, rootState := weave.InstanceCancelled, weave.StepCancelled
|
||
if *g.view.StopKind == "FAILURE" {
|
||
next, rootState = weave.InstanceFailed, weave.StepFailed
|
||
}
|
||
if err := saveWorkflowResultStep(ctx, tx, g, g.steps[workflowRootKey], rootState, nil, now); err != nil {
|
||
return false, err
|
||
}
|
||
res, err := tx.ExecContext(ctx, `UPDATE weave_instances SET state=?,output=NULL,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, next, now, now, g.view.NamespaceID, g.view.ID, g.view.Revision)
|
||
if err := exactlyOne(res, err); err != nil {
|
||
return false, err
|
||
}
|
||
g.view.State = next
|
||
g.view.Output = nil
|
||
g.view.CompletedAt = &now
|
||
g.view.Revision++
|
||
if _, err := tx.ExecContext(ctx, `DELETE FROM weave_pending_advances WHERE namespace_id=? AND workflow_id=?`, g.view.NamespaceID, g.view.ID); err != nil {
|
||
return false, err
|
||
}
|
||
if err := emitEvent(ctx, tx, g.view.NamespaceID, g.view.ID, g.view.Revision, "workflow", "workflow.stopped", map[string]any{"state": next, "stop_kind": g.view.StopKind, "stop_requested_at": g.view.StopRequestedAt}); err != nil {
|
||
return false, err
|
||
}
|
||
return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOPPED", now)
|
||
}
|
||
|
||
func cancelWorkflowWait(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, now time.Time) error {
|
||
if row.view.Kind == weave.NodeTimer {
|
||
var count int
|
||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_waits WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND kind='TIMER' AND state='WAITING' AND signal_name IS NULL AND payload_version IS NULL AND winner_signal_id IS NULL`, g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(&count); err != nil {
|
||
return err
|
||
}
|
||
if count != 1 {
|
||
return errors.New("stopping timer wait invalid")
|
||
}
|
||
}
|
||
if row.view.Kind == weave.NodePoll {
|
||
var activated time.Time
|
||
if err := tx.QueryRowContext(ctx, `SELECT created_at FROM weave_activations WHERE namespace_id=? AND workflow_id=? AND step_id=? AND id=?`, g.view.NamespaceID, g.view.ID, row.view.ID, row.view.ActivationID).Scan(&activated); err != nil {
|
||
return err
|
||
}
|
||
p, _, err := readWorkflowPollState(ctx, tx, g, row, activated)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if p.CurrentTaskID != nil {
|
||
return errors.New("cannot cancel in-flight poll wait")
|
||
}
|
||
res, err := tx.ExecContext(ctx, `UPDATE weave_poll_states SET next_poll_at=NULL,resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND resolved_at IS NULL AND current_task_id IS NULL`, now, now, g.view.NamespaceID, p.ID)
|
||
if err := exactlyOne(res, err); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if _, err := tx.ExecContext(ctx, `UPDATE weave_waits SET state='CANCELLED',resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND state='WAITING'`, now, now, g.view.NamespaceID, g.view.ID, row.view.ActivationID); err != nil {
|
||
return err
|
||
}
|
||
if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepCancelled, nil, now); err != nil {
|
||
return err
|
||
}
|
||
return deleteWorkflowClock(ctx, tx, g, row)
|
||
}
|
||
|
||
// validateWorkflowStopTask 取消前核对实际任务到流程激活/步骤/队列的归属，不能仅凭gate_id关联。
|
||
func validateWorkflowStopTask(ctx context.Context, tx *sql.Tx, g *workflowGraph, id core.ID) error {
|
||
var kind string
|
||
var owner, queue, gate, workflow, step core.ID
|
||
var key string
|
||
var direct *core.ID
|
||
err := tx.QueryRowContext(ctx, "SELECT t.owner_kind,t.owner_id,t.queue_id,t.gate_id,a.workflow_id,a.step_id,s.`key`,a.task_id FROM pulse_tasks t JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id JOIN weave_steps s ON s.namespace_id=a.namespace_id AND s.workflow_id=a.workflow_id AND s.id=a.step_id WHERE t.namespace_id=? AND t.id=?", g.view.NamespaceID, id).Scan(&kind, &owner, &queue, &gate, &workflow, &step, &key, &direct)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
row := g.steps[key]
|
||
node, ok := g.program.Step(key)
|
||
if kind != engine.TaskOwnerKind || gate != g.gate || workflow != g.view.ID || row == nil || row.view.ID != step || !ok {
|
||
return errors.New("stopping task ownership invalid")
|
||
}
|
||
if node.Task != nil {
|
||
if direct == nil || *direct != id || queue != node.Task.QueueID {
|
||
return errors.New("stopping task direct association invalid")
|
||
}
|
||
return nil
|
||
}
|
||
if node.Poll == nil || direct != nil || queue != node.Poll.Task.QueueID {
|
||
return errors.New("stopping query association invalid")
|
||
}
|
||
var count int
|
||
if err := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=? AND r.workflow_id=? AND r.activation_id=? AND r.task_id=? AND r.`round` BETWEEN 1 AND p.used_rounds", g.view.NamespaceID, g.view.ID, owner, id).Scan(&count); err != nil {
|
||
return err
|
||
}
|
||
if count != 1 {
|
||
return errors.New("stopping query round invalid")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// verifyWorkflowStopWork 终态前复核全部历史任务、双向归属、未结束尝试和许可，不能仅检查当前步骤。
|
||
func verifyWorkflowStopWork(ctx context.Context, tx *sql.Tx, g *workflowGraph) (settled, uncertain bool, err error) {
|
||
var invalid int
|
||
checks := []string{
|
||
`SELECT COUNT(*) FROM pulse_tasks t JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id WHERE a.namespace_id=? AND a.workflow_id=? AND t.owner_kind='workflow_activation' AND (t.gate_id IS NULL OR t.gate_id<>?)`,
|
||
`SELECT COUNT(*) FROM weave_activations a LEFT JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id WHERE a.namespace_id=? AND a.workflow_id=? AND a.task_id IS NOT NULL AND (t.id IS NULL OR t.owner_kind IS NULL OR t.owner_kind<>'workflow_activation' OR t.owner_id<>a.id OR t.gate_id IS NULL OR t.gate_id<>?)`,
|
||
`SELECT COUNT(*) FROM weave_poll_rounds r LEFT JOIN weave_activations a ON a.namespace_id=r.namespace_id AND a.workflow_id=r.workflow_id AND a.id=r.activation_id LEFT JOIN pulse_tasks t ON t.namespace_id=r.namespace_id AND t.id=r.task_id WHERE r.namespace_id=? AND r.workflow_id=? AND (a.id IS NULL OR t.id IS NULL OR t.owner_kind IS NULL OR t.owner_kind<>'workflow_activation' OR t.owner_id<>r.activation_id OR t.gate_id IS NULL OR t.gate_id<>?)`,
|
||
}
|
||
for _, q := range checks {
|
||
if err := tx.QueryRowContext(ctx, q, g.view.NamespaceID, g.view.ID, g.gate).Scan(&invalid); err != nil {
|
||
return false, false, err
|
||
}
|
||
if invalid != 0 {
|
||
return false, false, errors.New("workflow stop history association invalid")
|
||
}
|
||
}
|
||
// 一次读取关联元数据并通过索引EXISTS核对查询轮次，避免持有应用锁期间逐任务发出SQL。
|
||
rows, err := tx.QueryContext(ctx, "SELECT t.id,t.state,t.effect_uncertain,t.owner_kind,t.owner_id,t.queue_id,a.workflow_id,a.step_id,a.task_id,s.`key`,EXISTS(SELECT 1 FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=t.namespace_id AND r.workflow_id=a.workflow_id AND r.activation_id=a.id AND r.task_id=t.id AND r.`round` BETWEEN 1 AND p.used_rounds) FROM pulse_tasks t LEFT JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id LEFT JOIN weave_steps s ON s.namespace_id=a.namespace_id AND s.workflow_id=a.workflow_id AND s.id=a.step_id WHERE t.namespace_id=? AND t.gate_id=? ORDER BY t.id LIMIT 10001", g.view.NamespaceID, g.gate)
|
||
if err != nil {
|
||
return false, false, err
|
||
}
|
||
defer rows.Close()
|
||
count := 0
|
||
settled = true
|
||
for rows.Next() {
|
||
var id, queue core.ID
|
||
var owner, workflow, step, direct *core.ID
|
||
var kind, key sql.NullString
|
||
var state pulse.TaskState
|
||
var unknown, round bool
|
||
if err := rows.Scan(&id, &state, &unknown, &kind, &owner, &queue, &workflow, &step, &direct, &key, &round); err != nil {
|
||
return false, false, err
|
||
}
|
||
count++
|
||
if count > 10000 {
|
||
return false, false, errors.New("workflow historical task bound exceeded")
|
||
}
|
||
row := g.steps[key.String]
|
||
node, ok := g.program.Step(key.String)
|
||
if kind.String != engine.TaskOwnerKind || owner == nil || workflow == nil || *workflow != g.view.ID || step == nil || row == nil || row.view.ID != *step || !ok {
|
||
return false, false, errors.New("workflow stop task association invalid")
|
||
}
|
||
if node.Task != nil {
|
||
if direct == nil || *direct != id || queue != node.Task.QueueID {
|
||
return false, false, errors.New("workflow stop task parameters invalid")
|
||
}
|
||
} else if node.Poll == nil || direct != nil || queue != node.Poll.Task.QueueID || !round {
|
||
return false, false, errors.New("workflow stop query association invalid")
|
||
}
|
||
uncertain = uncertain || unknown || state == pulse.TaskNeedsReview
|
||
if !state.Terminal() && state != pulse.TaskNeedsReview {
|
||
settled = false
|
||
}
|
||
}
|
||
err = rows.Err()
|
||
rows.Close()
|
||
if err != nil {
|
||
return false, false, err
|
||
}
|
||
if !settled {
|
||
return false, uncertain, nil
|
||
}
|
||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_attempts a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id WHERE t.namespace_id=? AND t.gate_id=? AND a.state IN ('LEASED','RUNNING')`, g.view.NamespaceID, g.gate).Scan(&invalid); err != nil {
|
||
return false, false, err
|
||
}
|
||
if invalid > 0 {
|
||
return false, uncertain, nil
|
||
}
|
||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_permits p JOIN pulse_attempts a ON a.namespace_id=p.namespace_id AND a.id=p.attempt_id JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id WHERE t.namespace_id=? AND t.gate_id=? AND p.released_at IS NULL`, g.view.NamespaceID, g.gate).Scan(&invalid); err != nil {
|
||
return false, false, err
|
||
}
|
||
return invalid == 0, uncertain, nil
|
||
}
|