package mysql import ( "context" "database/sql" "errors" "sort" "time" core "git.yuntuo.hk/k2safe/orbit/model" pulse "git.yuntuo.hk/k2safe/orbit/pulse/model" "git.yuntuo.hk/k2safe/orbit/weave/engine" weave "git.yuntuo.hk/k2safe/orbit/weave/model" ) // settleWorkflow 在根检查点下结清停止意图。每事务最多发送一个任务取消,避免取得结果流锁后再锁另一任务。 // 步骤收尾最多32项,超过后保留根接续;停止/核查不依赖节点内存恢复。 func (r *WorkflowRuntime) settleWorkflow(ctx context.Context, tx *sql.Tx, g *workflowGraph, c workflowCandidate, now time.Time) (bool, error) { if g.view.StopKind == nil || g.view.StopRequestedAt == nil || (*g.view.StopKind != "CANCEL" && *g.view.StopKind != "FAILURE") { return false, errors.New("workflow stop intent invalid") } if *g.view.StopKind == "FAILURE" && g.version.Specification.FailurePolicy != "FAIL_AFTER_SETTLED" { return false, errors.New("workflow failure stop conflicts with fixed policy") } if *g.view.StopKind == "CANCEL" && g.view.CancelRequestedAt == nil { return false, errors.New("workflow cancellation intent missing") } // 已持有应用/全部队列/实例/门控锁;门控漂移不能为停止中的工作重新授权。 if _, err := tx.ExecContext(ctx, `UPDATE pulse_execution_gates SET paused=TRUE,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND paused=FALSE`, now, g.view.NamespaceID, g.gate); err != nil { return false, err } var taskID core.ID err := tx.QueryRowContext(ctx, `SELECT id FROM pulse_tasks WHERE namespace_id=? AND gate_id=? AND state NOT IN ('SUCCEEDED','FAILED','CANCELLED') AND cancel_requested_at IS NULL ORDER BY id LIMIT 1`, g.view.NamespaceID, g.gate).Scan(&taskID) if err != nil && !errors.Is(err, sql.ErrNoRows) { return false, err } if err == nil { if err := validateWorkflowStopTask(ctx, tx, g, taskID); err != nil { return false, err } task, err := readExecutionTask(ctx, tx, g.view.NamespaceID, taskID) if err != nil { return false, err } if _, err := cancelExecutionTask(ctx, tx, g.view.NamespaceID, task, now, "WORKFLOW_STOP_REQUESTED"); err != nil { return false, err } if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil { return false, err } return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_TASK_STOP_REQUESTED", now) } var pending int if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_inbox WHERE namespace_id=? AND workflow_id=? AND state='PENDING'`, g.view.NamespaceID, g.view.ID).Scan(&pending); err != nil { return false, err } if pending > 0 { return false, saveWorkflowAdvance(ctx, tx, g, now.Add(time.Second)) } keys := make([]string, 0, len(g.steps)) for key := range g.steps { if key != workflowRootKey { keys = append(keys, key) } } sort.Strings(keys) changed, active, review := 0, false, false for _, key := range keys { row := g.steps[key] if row.view.State == weave.StepNeedsReview { review = true continue } if row.view.State == weave.StepSucceeded || row.view.State == weave.StepFailed || row.view.State == weave.StepCancelled || row.view.State == weave.StepSkipped { continue } if changed == 32 { if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil { return false, err } return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_CONTINUATION", now) } if row.view.State == weave.StepBlocked { res, err := tx.ExecContext(ctx, `UPDATE weave_steps SET state='SKIPPED',revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=? AND current_activation_id IS NULL AND state='BLOCKED'`, now, g.view.NamespaceID, row.view.ID, row.view.Revision) if err := exactlyOne(res, err); err != nil { return false, err } row.view.State = weave.StepSkipped row.view.Revision++ changed++ if err := emitEvent(ctx, tx, g.view.NamespaceID, row.view.ID, row.view.Revision, "workflow_step", "workflow.step.changed", map[string]any{"workflow_id": g.view.ID, "key": key, "state": row.view.State, "reason": "WORKFLOW_STOP_REQUESTED"}); err != nil { return false, err } continue } switch row.view.Kind { case weave.NodeTask: if row.view.TaskID == nil { return false, errors.New("stopping task step missing task") } task, err := readTask(ctx, tx, g.view.NamespaceID, *row.view.TaskID, false) if err != nil { return false, err } if task.State.Terminal() || task.State == pulse.TaskNeedsReview { return false, errors.New("stopping task missing result confirmation") } active = true case weave.NodeSignal, weave.NodeTimer, weave.NodePoll: node, _ := g.program.Step(key) if row.view.Kind == weave.NodePoll { var current *core.ID if err := tx.QueryRowContext(ctx, `SELECT current_task_id FROM weave_poll_states WHERE namespace_id=? AND workflow_id=? AND activation_id=?`, g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(¤t); err != nil { return false, err } if current != nil { active = true continue } } // 已在停止前可靠受理的信号保留为事实;停止后的新信号由受理事务拒绝。 if signalContract(node) != nil { w, err := readWorkflowSignalWait(ctx, tx, g, row) if err != nil { return false, err } v, err := selectedWorkflowSignal(ctx, tx, g, row, w, now) if err != nil || v != nil { handled, done, reason, err := r.driveWorkflowSignal(ctx, tx, g, row, c, nil, false, now) if err != nil { return false, err } if !handled { return false, errors.New("stopping signal did not resolve") } if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil { return false, err } if done && reason != "" { return true, r.audit(ctx, tx, g, c.id, reason, now) } return done, nil } } if err := cancelWorkflowWait(ctx, tx, g, row, now); err != nil { return false, err } changed++ case weave.NodeBranch, weave.NodeParallel: // 组合步骤只有在全部活动叶子结清后才能结清,避免掩盖在途工作。 default: return false, errors.New("unsupported stopping step") } } if active { if err := saveWorkflowAdvance(ctx, tx, g, now.Add(time.Second)); err != nil { return false, err } if changed > 0 { return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_COLLECTING", now) } return false, nil } settled, uncertain, err := verifyWorkflowStopWork(ctx, tx, g) if err != nil { return false, err } if !settled { if err := saveWorkflowAdvance(ctx, tx, g, now.Add(time.Second)); err != nil { return false, err } if changed > 0 { return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_COLLECTING", now) } return false, nil } if review || uncertain || g.view.State == weave.InstanceNeedsReview { stateChanged := g.view.State != weave.InstanceNeedsReview if stateChanged { if err := holdWorkflow(ctx, tx, g, weave.InstanceNeedsReview, "WORKFLOW_STOP_REVIEW", now); err != nil { return false, err } } // 已无平台内有效授权时不逐秒空转;UNKNOWN不证明外部执行已结束,后续核查负责接续。 if _, err := tx.ExecContext(ctx, `UPDATE weave_signals SET state='REJECTED',disposition='WORKFLOW_STOPPED_REVIEW',revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND state='PENDING'`, now, g.view.NamespaceID, g.view.ID); err != nil { return false, err } if err := deleteWorkflowClock(ctx, tx, g, g.steps[workflowRootKey]); err != nil { return false, err } if changed > 0 || stateChanged { return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_REVIEW", now) } return false, nil } for _, key := range keys { row := g.steps[key] if row.view.Kind != weave.NodeBranch && row.view.Kind != weave.NodeParallel { continue } if row.view.State != weave.StepActive && row.view.State != weave.StepWaiting { continue } if changed == 32 { if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil { return false, err } return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOP_CONTINUATION", now) } if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepCancelled, nil, now); err != nil { return false, err } changed++ } if _, err := tx.ExecContext(ctx, `UPDATE weave_signals SET state='REJECTED',disposition='WORKFLOW_STOPPED',revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND state='PENDING'`, now, g.view.NamespaceID, g.view.ID); err != nil { return false, err } next, rootState := weave.InstanceCancelled, weave.StepCancelled if *g.view.StopKind == "FAILURE" { next, rootState = weave.InstanceFailed, weave.StepFailed } if err := saveWorkflowResultStep(ctx, tx, g, g.steps[workflowRootKey], rootState, nil, now); err != nil { return false, err } res, err := tx.ExecContext(ctx, `UPDATE weave_instances SET state=?,output=NULL,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, next, now, now, g.view.NamespaceID, g.view.ID, g.view.Revision) if err := exactlyOne(res, err); err != nil { return false, err } g.view.State = next g.view.Output = nil g.view.CompletedAt = &now g.view.Revision++ if _, err := tx.ExecContext(ctx, `DELETE FROM weave_pending_advances WHERE namespace_id=? AND workflow_id=?`, g.view.NamespaceID, g.view.ID); err != nil { return false, err } if err := emitEvent(ctx, tx, g.view.NamespaceID, g.view.ID, g.view.Revision, "workflow", "workflow.stopped", map[string]any{"state": next, "stop_kind": g.view.StopKind, "stop_requested_at": g.view.StopRequestedAt}); err != nil { return false, err } return true, r.audit(ctx, tx, g, c.id, "WORKFLOW_STOPPED", now) } func cancelWorkflowWait(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, now time.Time) error { if row.view.Kind == weave.NodeTimer { var count int if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_waits WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND kind='TIMER' AND state='WAITING' AND signal_name IS NULL AND payload_version IS NULL AND winner_signal_id IS NULL`, g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(&count); err != nil { return err } if count != 1 { return errors.New("stopping timer wait invalid") } } if row.view.Kind == weave.NodePoll { var activated time.Time if err := tx.QueryRowContext(ctx, `SELECT created_at FROM weave_activations WHERE namespace_id=? AND workflow_id=? AND step_id=? AND id=?`, g.view.NamespaceID, g.view.ID, row.view.ID, row.view.ActivationID).Scan(&activated); err != nil { return err } p, _, err := readWorkflowPollState(ctx, tx, g, row, activated) if err != nil { return err } if p.CurrentTaskID != nil { return errors.New("cannot cancel in-flight poll wait") } res, err := tx.ExecContext(ctx, `UPDATE weave_poll_states SET next_poll_at=NULL,resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND resolved_at IS NULL AND current_task_id IS NULL`, now, now, g.view.NamespaceID, p.ID) if err := exactlyOne(res, err); err != nil { return err } } if _, err := tx.ExecContext(ctx, `UPDATE weave_waits SET state='CANCELLED',resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND state='WAITING'`, now, now, g.view.NamespaceID, g.view.ID, row.view.ActivationID); err != nil { return err } if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepCancelled, nil, now); err != nil { return err } return deleteWorkflowClock(ctx, tx, g, row) } // validateWorkflowStopTask 取消前核对实际任务到流程激活/步骤/队列的归属,不能仅凭gate_id关联。 func validateWorkflowStopTask(ctx context.Context, tx *sql.Tx, g *workflowGraph, id core.ID) error { var kind string var owner, queue, gate, workflow, step core.ID var key string var direct *core.ID err := tx.QueryRowContext(ctx, "SELECT t.owner_kind,t.owner_id,t.queue_id,t.gate_id,a.workflow_id,a.step_id,s.`key`,a.task_id FROM pulse_tasks t JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id JOIN weave_steps s ON s.namespace_id=a.namespace_id AND s.workflow_id=a.workflow_id AND s.id=a.step_id WHERE t.namespace_id=? AND t.id=?", g.view.NamespaceID, id).Scan(&kind, &owner, &queue, &gate, &workflow, &step, &key, &direct) if err != nil { return err } row := g.steps[key] node, ok := g.program.Step(key) if kind != engine.TaskOwnerKind || gate != g.gate || workflow != g.view.ID || row == nil || row.view.ID != step || !ok { return errors.New("stopping task ownership invalid") } if node.Task != nil { if direct == nil || *direct != id || queue != node.Task.QueueID { return errors.New("stopping task direct association invalid") } return nil } if node.Poll == nil || direct != nil || queue != node.Poll.Task.QueueID { return errors.New("stopping query association invalid") } var count int if err := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=? AND r.workflow_id=? AND r.activation_id=? AND r.task_id=? AND r.`round` BETWEEN 1 AND p.used_rounds", g.view.NamespaceID, g.view.ID, owner, id).Scan(&count); err != nil { return err } if count != 1 { return errors.New("stopping query round invalid") } return nil } // verifyWorkflowStopWork 终态前复核全部历史任务、双向归属、未结束尝试和许可,不能仅检查当前步骤。 func verifyWorkflowStopWork(ctx context.Context, tx *sql.Tx, g *workflowGraph) (settled, uncertain bool, err error) { var invalid int checks := []string{ `SELECT COUNT(*) FROM pulse_tasks t JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id WHERE a.namespace_id=? AND a.workflow_id=? AND t.owner_kind='workflow_activation' AND (t.gate_id IS NULL OR t.gate_id<>?)`, `SELECT COUNT(*) FROM weave_activations a LEFT JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id WHERE a.namespace_id=? AND a.workflow_id=? AND a.task_id IS NOT NULL AND (t.id IS NULL OR t.owner_kind IS NULL OR t.owner_kind<>'workflow_activation' OR t.owner_id<>a.id OR t.gate_id IS NULL OR t.gate_id<>?)`, `SELECT COUNT(*) FROM weave_poll_rounds r LEFT JOIN weave_activations a ON a.namespace_id=r.namespace_id AND a.workflow_id=r.workflow_id AND a.id=r.activation_id LEFT JOIN pulse_tasks t ON t.namespace_id=r.namespace_id AND t.id=r.task_id WHERE r.namespace_id=? AND r.workflow_id=? AND (a.id IS NULL OR t.id IS NULL OR t.owner_kind IS NULL OR t.owner_kind<>'workflow_activation' OR t.owner_id<>r.activation_id OR t.gate_id IS NULL OR t.gate_id<>?)`, } for _, q := range checks { if err := tx.QueryRowContext(ctx, q, g.view.NamespaceID, g.view.ID, g.gate).Scan(&invalid); err != nil { return false, false, err } if invalid != 0 { return false, false, errors.New("workflow stop history association invalid") } } // 一次读取关联元数据并通过索引EXISTS核对查询轮次,避免持有应用锁期间逐任务发出SQL。 rows, err := tx.QueryContext(ctx, "SELECT t.id,t.state,t.effect_uncertain,t.owner_kind,t.owner_id,t.queue_id,a.workflow_id,a.step_id,a.task_id,s.`key`,EXISTS(SELECT 1 FROM weave_poll_rounds r JOIN weave_poll_states p ON p.namespace_id=r.namespace_id AND p.workflow_id=r.workflow_id AND p.activation_id=r.activation_id WHERE r.namespace_id=t.namespace_id AND r.workflow_id=a.workflow_id AND r.activation_id=a.id AND r.task_id=t.id AND r.`round` BETWEEN 1 AND p.used_rounds) FROM pulse_tasks t LEFT JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id LEFT JOIN weave_steps s ON s.namespace_id=a.namespace_id AND s.workflow_id=a.workflow_id AND s.id=a.step_id WHERE t.namespace_id=? AND t.gate_id=? ORDER BY t.id LIMIT 10001", g.view.NamespaceID, g.gate) if err != nil { return false, false, err } defer rows.Close() count := 0 settled = true for rows.Next() { var id, queue core.ID var owner, workflow, step, direct *core.ID var kind, key sql.NullString var state pulse.TaskState var unknown, round bool if err := rows.Scan(&id, &state, &unknown, &kind, &owner, &queue, &workflow, &step, &direct, &key, &round); err != nil { return false, false, err } count++ if count > 10000 { return false, false, errors.New("workflow historical task bound exceeded") } row := g.steps[key.String] node, ok := g.program.Step(key.String) if kind.String != engine.TaskOwnerKind || owner == nil || workflow == nil || *workflow != g.view.ID || step == nil || row == nil || row.view.ID != *step || !ok { return false, false, errors.New("workflow stop task association invalid") } if node.Task != nil { if direct == nil || *direct != id || queue != node.Task.QueueID { return false, false, errors.New("workflow stop task parameters invalid") } } else if node.Poll == nil || direct != nil || queue != node.Poll.Task.QueueID || !round { return false, false, errors.New("workflow stop query association invalid") } uncertain = uncertain || unknown || state == pulse.TaskNeedsReview if !state.Terminal() && state != pulse.TaskNeedsReview { settled = false } } err = rows.Err() rows.Close() if err != nil { return false, false, err } if !settled { return false, uncertain, nil } if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_attempts a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id WHERE t.namespace_id=? AND t.gate_id=? AND a.state IN ('LEASED','RUNNING')`, g.view.NamespaceID, g.gate).Scan(&invalid); err != nil { return false, false, err } if invalid > 0 { return false, uncertain, nil } if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_permits p JOIN pulse_attempts a ON a.namespace_id=p.namespace_id AND a.id=p.attempt_id JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id WHERE t.namespace_id=? AND t.gate_id=? AND p.released_at IS NULL`, g.view.NamespaceID, g.gate).Scan(&invalid); err != nil { return false, false, err } return invalid == 0, uncertain, nil }