package mysql import ( "bytes" "context" "database/sql" "encoding/hex" "encoding/json" "errors" "time" "git.yuntuo.hk/k2safe/orbit/internal/fault" "git.yuntuo.hk/k2safe/orbit/internal/jsonvalue" core "git.yuntuo.hk/k2safe/orbit/model" "git.yuntuo.hk/k2safe/orbit/platform/auth" "git.yuntuo.hk/k2safe/orbit/pulse/service" "git.yuntuo.hk/k2safe/orbit/weave/definition" "git.yuntuo.hk/k2safe/orbit/weave/engine" weave "git.yuntuo.hk/k2safe/orbit/weave/model" "git.yuntuo.hk/k2safe/orbit/weave/recovery" ) // WorkflowRecoveryStore 管理核查事实与接续决定;普通Pulse人工入口仍拒绝流程所有者。 type WorkflowRecoveryStore struct{ *TaskStore } func NewWorkflowRecoveryStore(db *sql.DB) *WorkflowRecoveryStore { return &WorkflowRecoveryStore{NewTaskStore(db)} } var _ recovery.Store = (*WorkflowRecoveryStore)(nil) type workflowRecoveryApply func(*sql.Tx, *workflowGraph, core.ID, time.Time, *recovery.Result) (map[string]any, error) func (s *WorkflowRecoveryStore) mutate(ctx context.Context, t auth.Token, id core.ID, key, operation string, command any, decision recovery.Decision, apply workflowRecoveryApply) (recovery.Result, error) { var out recovery.Result err := s.transaction(ctx, t, recovery.Action, func(tx *sql.Tx, p auth.Permissions) error { if _, err := lockApplicationResources(ctx, tx, t.NamespaceID, false); err != nil { return err } identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, operation, string(id), key, "workflow", id, command) if err != nil { return err } g, _, err := lockWorkflowGraph(ctx, tx, t.NamespaceID, id, false) if err != nil { return err } for _, q := range g.version.Analysis.Queues { if !p.AllowsQueue(q) { return fault.NotFound } } if cached != nil { if err := json.Unmarshal(cached, &out); err != nil { return err } if out.Instance.NamespaceID != t.NamespaceID || out.Instance.ID != id || out.Instance.DefinitionVersionID != g.view.DefinitionVersionID { return errors.New("workflow recovery receipt mismatch") } out.Deduplicated = true return nil } if g.view.State != weave.InstanceNeedsReview || g.view.Revision != decision.ExpectedRevision { return fault.Conflict } // 新事实须先进入当前图;损坏来源不能用一句人工声明绕过隔离。 var pending int if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_inbox WHERE namespace_id=? AND workflow_id=? AND state IN ('PENDING','QUARANTINED')`, t.NamespaceID, id).Scan(&pending); err != nil { return err } if pending != 0 { return fault.Conflict } now, err := dbNow(ctx, tx) if err != nil { return err } before := map[string]any{"state": g.view.State, "revision": g.view.Revision, "control_revision": g.view.ControlRevision, "stop_kind": g.view.StopKind} // 核查期间始终关闭新执行授权;Continue仅在下面完成全部证据校验后按原paused决定是否打开。 if _, err := tx.ExecContext(ctx, `UPDATE pulse_execution_gates SET paused=TRUE,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND paused=FALSE`, now, t.NamespaceID, g.gate); err != nil { return err } meta, err := apply(tx, g, identity, now, &out) if err != nil { return err } res, err := tx.ExecContext(ctx, `UPDATE weave_instances SET revision=revision+1,control_revision=control_revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, now, t.NamespaceID, id, g.view.Revision) if err := exactlyOne(res, err); err != nil { return err } g.view.Revision++ g.view.ControlRevision++ meta["state"], meta["revision"], meta["control_revision"], meta["stop_kind"] = g.view.State, g.view.Revision, g.view.ControlRevision, g.view.StopKind out.AuditID, err = appendAuditWithEvidence(ctx, tx, t, identity, id, "workflow", operation, decision.Reason, decision.EvidenceRef, before, meta) if err != nil { return err } if err := emitEvent(ctx, tx, t.NamespaceID, id, g.view.Revision, "workflow", "workflow.recovered", map[string]any{"operation": operation, "audit_id": out.AuditID, "state": g.view.State}); err != nil { return err } if g.view.StopKind != nil || operation == "workflow.review.continue" { if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil { return err } } out.Instance, _, err = readWorkflowInstance(ctx, tx, t.NamespaceID, id, false) if err != nil { return err } if err := saveIdentity(ctx, tx, t.NamespaceID, identity, out); err != nil { return err } if operation == "workflow.task.review" { // 只在全部管理状态、回执和审计完成后获取结果流锁;之后不再锁任务。 return emitEvent(ctx, tx, t.NamespaceID, out.Task.ID, out.Task.Revision, "task", resultEvent(out.Task.State), map[string]any{"task_id": out.Task.ID, "state": out.Task.State, "effect_uncertain": false, "decision": "MANUAL_REVIEW", "audit_id": out.TaskAuditID, "workflow_audit_id": out.AuditID}) } return nil }) return out, err } func (s *WorkflowRecoveryStore) ReviewTask(ctx context.Context, t auth.Token, id core.ID, key string, in recovery.ReviewTaskInput) (recovery.Result, error) { return s.mutate(ctx, t, id, key, "workflow.task.review", in, in.Decision, func(tx *sql.Tx, g *workflowGraph, identity core.ID, now time.Time, out *recovery.Result) (map[string]any, error) { var owned int if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_tasks t JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id WHERE t.namespace_id=? AND t.id=? AND t.owner_kind='workflow_activation' AND a.workflow_id=?`, t.NamespaceID, in.TaskID, id).Scan(&owned); err != nil { return nil, err } if owned != 1 { return nil, fault.NotFound } if err := validateWorkflowStopTask(ctx, tx, g, in.TaskID); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, fault.NotFound } return nil, err } task, err := readExecutionTask(ctx, tx, t.NamespaceID, in.TaskID) if err != nil { return nil, err } if err := taskExecutionSettled(ctx, tx, t.NamespaceID, in.TaskID); err != nil { return nil, err } taskOperation, err := core.NewID() if err != nil { return nil, err } v, err := applyTaskReview(ctx, tx, t, task, taskOperation, service.ReviewInput{ExpectedRevision: in.ExpectedTaskRevision, Reason: in.Reason, EvidenceRef: in.EvidenceRef, ExecutionSettled: in.ExecutionSettled, Outcome: in.Outcome, Result: in.Result}, now) if err != nil { return nil, err } out.Task, out.TaskAuditID = &v.Task, &v.AuditID return map[string]any{"task_id": in.TaskID, "task_audit_id": v.AuditID, "task_operation_id": taskOperation, "workflow_operation_id": identity, "execution_settled": true, "outcome": in.Outcome}, nil }) } func workflowRecoveryStep(g *workflowGraph, in recovery.StepDecision) (*workflowStep, error) { row, err := workflowRecoveryTarget(g, in) if err != nil { return nil, err } if row.view.State != weave.StepNeedsReview && row.view.State != weave.StepFailed && row.view.State != weave.StepCancelled { return nil, fault.Conflict } return row, nil } func workflowRecoveryTarget(g *workflowGraph, in recovery.StepDecision) (*workflowStep, error) { row := g.steps[in.StepKey] if row == nil || in.StepKey == workflowRootKey { return nil, fault.NotFound } if row.view.ActivationID == nil || *row.view.ActivationID != in.ActivationID || row.view.Revision != in.ExpectedStepRevision { return nil, fault.Conflict } return row, nil } // workflowStepSettled 检查全部历史归属,再核验本步骤的执行与许可;兄弟分支可继续收集在途结果。 func workflowStepSettled(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep) error { if _, _, err := verifyWorkflowStopWork(ctx, tx, g); err != nil { return err } queries := []string{ `SELECT COUNT(*) FROM weave_activations a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.owner_id=a.id AND t.owner_kind='workflow_activation' WHERE a.namespace_id=? AND a.workflow_id=? AND a.step_id=? AND (t.state NOT IN ('SUCCEEDED','FAILED','CANCELLED') OR t.effect_uncertain=TRUE)`, `SELECT COUNT(*) FROM weave_activations a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.owner_id=a.id AND t.owner_kind='workflow_activation' JOIN pulse_attempts x ON x.namespace_id=t.namespace_id AND x.task_id=t.id WHERE a.namespace_id=? AND a.workflow_id=? AND a.step_id=? AND x.state IN ('LEASED','RUNNING')`, `SELECT COUNT(*) FROM weave_activations a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.owner_id=a.id AND t.owner_kind='workflow_activation' JOIN pulse_attempts x ON x.namespace_id=t.namespace_id AND x.task_id=t.id JOIN pulse_permits p ON p.namespace_id=x.namespace_id AND p.attempt_id=x.id WHERE a.namespace_id=? AND a.workflow_id=? AND a.step_id=? AND p.released_at IS NULL`, } for _, q := range queries { var count int if err := tx.QueryRowContext(ctx, q, g.view.NamespaceID, g.view.ID, row.view.ID).Scan(&count); err != nil { return err } if count != 0 { return fault.Conflict } } return nil } func (s *WorkflowRecoveryStore) ResolveStep(ctx context.Context, t auth.Token, id core.ID, key string, in recovery.ResolveStepInput) (recovery.Result, error) { return s.mutate(ctx, t, id, key, "workflow.step.resolve", in, in.Decision, func(tx *sql.Tx, g *workflowGraph, _ core.ID, now time.Time, out *recovery.Result) (map[string]any, error) { row, err := workflowRecoveryStep(g, in.StepDecision) if err != nil { return nil, err } node, ok := g.program.Step(row.view.Key) if !ok || node.Task == nil && node.Poll == nil && node.Signal == nil && node.Timer == nil { return nil, fault.Conflict } if err := workflowStepSettled(ctx, tx, g, row); err != nil { return nil, err } if in.Outcome == "SUCCEEDED" { var typ definition.Type switch { case node.Task != nil: typ = node.Task.ResultType case node.Signal != nil: typ = node.Signal.PayloadType case node.Poll != nil: kind, err := g.program.ClassifyPoll(row.view.Key, in.Result) if err != nil || kind != engine.PollSucceeded { return nil, fault.Invalid("poll recovery requires a confirmed business success result") } typ = node.Poll.Task.ResultType case node.Timer != nil: origin, err := workflowActivationOrigin(ctx, tx, g, row) if err != nil { return nil, err } if now.Before(origin.started.Add(time.Duration(node.Timer.DelayMS) * time.Millisecond)) { return nil, fault.Conflict } typ = definition.Type{Kind: "NULL"} } if err := definition.ValidateData(typ, in.Result); err != nil { return nil, fault.Invalid("review result violates fixed step contract") } } beforeState, beforeOutput := row.view.State, row.output if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepState(in.Outcome), in.Result, now); err != nil { return nil, err } if err := deleteWorkflowClock(ctx, tx, g, row); err != nil { return nil, err } // 等待、轮次和执行来源保持原样;本次逻辑结论由管理审计与激活结果表示。 out.Step = &row.view digest, err := requestDigest(in.Result) if err != nil { return nil, err } return map[string]any{"step_id": row.view.ID, "activation_id": row.view.ActivationID, "previous_state": beforeState, "previous_output": json.RawMessage(beforeOutput), "outcome": in.Outcome, "result_digest": hex.EncodeToString(digest[:]), "execution_settled": true}, nil }) } func (s *WorkflowRecoveryStore) Continue(ctx context.Context, t auth.Token, id core.ID, key string, in recovery.ContinueInput) (recovery.Result, error) { return s.mutate(ctx, t, id, key, "workflow.review.continue", in, in.Decision, func(tx *sql.Tx, g *workflowGraph, _ core.ID, now time.Time, _ *recovery.Result) (map[string]any, error) { for key, row := range g.steps { if key == workflowRootKey { continue } if row.view.State == weave.StepNeedsReview || g.view.StopKind == nil && (row.view.State == weave.StepFailed || row.view.State == weave.StepCancelled) { return nil, fault.Conflict } } _, uncertain, err := verifyWorkflowStopWork(ctx, tx, g) if err != nil { return nil, err } if uncertain { return nil, fault.Conflict } state := weave.InstanceRunning if g.view.StopKind != nil { state = weave.InstanceCancelling } else if !now.Before(g.view.DeadlineAt) { return nil, fault.Conflict } res, err := tx.ExecContext(ctx, `UPDATE weave_instances SET state=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=? AND state='NEEDS_REVIEW'`, state, now, t.NamespaceID, id, g.view.Revision) if err := exactlyOne(res, err); err != nil { return nil, err } g.view.State, g.view.Revision = state, g.view.Revision+1 if _, err := tx.ExecContext(ctx, `UPDATE pulse_execution_gates SET paused=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, g.view.Paused || g.view.StopKind != nil, now, t.NamespaceID, g.gate); err != nil { return nil, err } return map[string]any{"review_resolved": true, "paused": g.view.Paused, "deadline_at": g.view.DeadlineAt}, nil }) } func sameWorkflowJSON(a, b []byte) bool { left, err := jsonvalue.Canonical(a) if err != nil { return false } right, err := jsonvalue.Canonical(b) return err == nil && bytes.Equal(left, right) }