feat: add audited workflow review and bounded task recovery
This commit is contained in:
292
internal/storage/mysql/workflow_recovery.go
Normal file
292
internal/storage/mysql/workflow_recovery.go
Normal file
@@ -0,0 +1,292 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/jsonvalue"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/definition"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/engine"
|
||||
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/recovery"
|
||||
)
|
||||
|
||||
// WorkflowRecoveryStore 管理核查事实与接续决定；普通Pulse人工入口仍拒绝流程所有者。
|
||||
type WorkflowRecoveryStore struct{ *TaskStore }
|
||||
|
||||
func NewWorkflowRecoveryStore(db *sql.DB) *WorkflowRecoveryStore {
|
||||
return &WorkflowRecoveryStore{NewTaskStore(db)}
|
||||
}
|
||||
|
||||
var _ recovery.Store = (*WorkflowRecoveryStore)(nil)
|
||||
|
||||
type workflowRecoveryApply func(*sql.Tx, *workflowGraph, core.ID, time.Time, *recovery.Result) (map[string]any, error)
|
||||
|
||||
func (s *WorkflowRecoveryStore) mutate(ctx context.Context, t auth.Token, id core.ID, key, operation string, command any, decision recovery.Decision, apply workflowRecoveryApply) (recovery.Result, error) {
|
||||
var out recovery.Result
|
||||
err := s.transaction(ctx, t, recovery.Action, func(tx *sql.Tx, p auth.Permissions) error {
|
||||
if _, err := lockApplicationResources(ctx, tx, t.NamespaceID, false); err != nil {
|
||||
return err
|
||||
}
|
||||
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, operation, string(id), key, "workflow", id, command)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g, _, err := lockWorkflowGraph(ctx, tx, t.NamespaceID, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, q := range g.version.Analysis.Queues {
|
||||
if !p.AllowsQueue(q) {
|
||||
return fault.NotFound
|
||||
}
|
||||
}
|
||||
if cached != nil {
|
||||
if err := json.Unmarshal(cached, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
if out.Instance.NamespaceID != t.NamespaceID || out.Instance.ID != id || out.Instance.DefinitionVersionID != g.view.DefinitionVersionID {
|
||||
return errors.New("workflow recovery receipt mismatch")
|
||||
}
|
||||
out.Deduplicated = true
|
||||
return nil
|
||||
}
|
||||
if g.view.State != weave.InstanceNeedsReview || g.view.Revision != decision.ExpectedRevision {
|
||||
return fault.Conflict
|
||||
}
|
||||
// 新事实须先进入当前图；损坏来源不能用一句人工声明绕过隔离。
|
||||
var pending int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_inbox WHERE namespace_id=? AND workflow_id=? AND state IN ('PENDING','QUARANTINED')`, t.NamespaceID, id).Scan(&pending); err != nil {
|
||||
return err
|
||||
}
|
||||
if pending != 0 {
|
||||
return fault.Conflict
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
before := map[string]any{"state": g.view.State, "revision": g.view.Revision, "control_revision": g.view.ControlRevision, "stop_kind": g.view.StopKind}
|
||||
// 核查期间始终关闭新执行授权；Continue仅在下面完成全部证据校验后按原paused决定是否打开。
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE pulse_execution_gates SET paused=TRUE,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND paused=FALSE`, now, t.NamespaceID, g.gate); err != nil {
|
||||
return err
|
||||
}
|
||||
meta, err := apply(tx, g, identity, now, &out)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_instances SET revision=revision+1,control_revision=control_revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, now, t.NamespaceID, id, g.view.Revision)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return err
|
||||
}
|
||||
g.view.Revision++
|
||||
g.view.ControlRevision++
|
||||
meta["state"], meta["revision"], meta["control_revision"], meta["stop_kind"] = g.view.State, g.view.Revision, g.view.ControlRevision, g.view.StopKind
|
||||
out.AuditID, err = appendAuditWithEvidence(ctx, tx, t, identity, id, "workflow", operation, decision.Reason, decision.EvidenceRef, before, meta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, t.NamespaceID, id, g.view.Revision, "workflow", "workflow.recovered", map[string]any{"operation": operation, "audit_id": out.AuditID, "state": g.view.State}); err != nil {
|
||||
return err
|
||||
}
|
||||
if g.view.StopKind != nil || operation == "workflow.review.continue" {
|
||||
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
out.Instance, _, err = readWorkflowInstance(ctx, tx, t.NamespaceID, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := saveIdentity(ctx, tx, t.NamespaceID, identity, out); err != nil {
|
||||
return err
|
||||
}
|
||||
if operation == "workflow.task.review" {
|
||||
// 只在全部管理状态、回执和审计完成后获取结果流锁；之后不再锁任务。
|
||||
return emitEvent(ctx, tx, t.NamespaceID, out.Task.ID, out.Task.Revision, "task", resultEvent(out.Task.State), map[string]any{"task_id": out.Task.ID, "state": out.Task.State, "effect_uncertain": false, "decision": "MANUAL_REVIEW", "audit_id": out.TaskAuditID, "workflow_audit_id": out.AuditID})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (s *WorkflowRecoveryStore) ReviewTask(ctx context.Context, t auth.Token, id core.ID, key string, in recovery.ReviewTaskInput) (recovery.Result, error) {
|
||||
return s.mutate(ctx, t, id, key, "workflow.task.review", in, in.Decision, func(tx *sql.Tx, g *workflowGraph, identity core.ID, now time.Time, out *recovery.Result) (map[string]any, error) {
|
||||
var owned int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_tasks t JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id WHERE t.namespace_id=? AND t.id=? AND t.owner_kind='workflow_activation' AND a.workflow_id=?`, t.NamespaceID, in.TaskID, id).Scan(&owned); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if owned != 1 {
|
||||
return nil, fault.NotFound
|
||||
}
|
||||
if err := validateWorkflowStopTask(ctx, tx, g, in.TaskID); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, fault.NotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
task, err := readExecutionTask(ctx, tx, t.NamespaceID, in.TaskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := taskExecutionSettled(ctx, tx, t.NamespaceID, in.TaskID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
taskOperation, err := core.NewID()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
v, err := applyTaskReview(ctx, tx, t, task, taskOperation, service.ReviewInput{ExpectedRevision: in.ExpectedTaskRevision, Reason: in.Reason, EvidenceRef: in.EvidenceRef, ExecutionSettled: in.ExecutionSettled, Outcome: in.Outcome, Result: in.Result}, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out.Task, out.TaskAuditID = &v.Task, &v.AuditID
|
||||
return map[string]any{"task_id": in.TaskID, "task_audit_id": v.AuditID, "task_operation_id": taskOperation, "workflow_operation_id": identity, "execution_settled": true, "outcome": in.Outcome}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func workflowRecoveryStep(g *workflowGraph, in recovery.StepDecision) (*workflowStep, error) {
|
||||
row := g.steps[in.StepKey]
|
||||
if row == nil || in.StepKey == workflowRootKey {
|
||||
return nil, fault.NotFound
|
||||
}
|
||||
if row.view.ActivationID == nil || *row.view.ActivationID != in.ActivationID || row.view.Revision != in.ExpectedStepRevision {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
if row.view.State != weave.StepNeedsReview && row.view.State != weave.StepFailed && row.view.State != weave.StepCancelled {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
return row, nil
|
||||
}
|
||||
|
||||
// workflowStepSettled 检查全部历史归属，再核验本步骤的执行与许可；兄弟分支可继续收集在途结果。
|
||||
func workflowStepSettled(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep) error {
|
||||
if _, _, err := verifyWorkflowStopWork(ctx, tx, g); err != nil {
|
||||
return err
|
||||
}
|
||||
queries := []string{
|
||||
`SELECT COUNT(*) FROM weave_activations a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.owner_id=a.id AND t.owner_kind='workflow_activation' WHERE a.namespace_id=? AND a.workflow_id=? AND a.step_id=? AND (t.state NOT IN ('SUCCEEDED','FAILED','CANCELLED') OR t.effect_uncertain=TRUE)`,
|
||||
`SELECT COUNT(*) FROM weave_activations a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.owner_id=a.id AND t.owner_kind='workflow_activation' JOIN pulse_attempts x ON x.namespace_id=t.namespace_id AND x.task_id=t.id WHERE a.namespace_id=? AND a.workflow_id=? AND a.step_id=? AND x.state IN ('LEASED','RUNNING')`,
|
||||
`SELECT COUNT(*) FROM weave_activations a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.owner_id=a.id AND t.owner_kind='workflow_activation' JOIN pulse_attempts x ON x.namespace_id=t.namespace_id AND x.task_id=t.id JOIN pulse_permits p ON p.namespace_id=x.namespace_id AND p.attempt_id=x.id WHERE a.namespace_id=? AND a.workflow_id=? AND a.step_id=? AND p.released_at IS NULL`,
|
||||
}
|
||||
for _, q := range queries {
|
||||
var count int
|
||||
if err := tx.QueryRowContext(ctx, q, g.view.NamespaceID, g.view.ID, row.view.ID).Scan(&count); err != nil {
|
||||
return err
|
||||
}
|
||||
if count != 0 {
|
||||
return fault.Conflict
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *WorkflowRecoveryStore) ResolveStep(ctx context.Context, t auth.Token, id core.ID, key string, in recovery.ResolveStepInput) (recovery.Result, error) {
|
||||
return s.mutate(ctx, t, id, key, "workflow.step.resolve", in, in.Decision, func(tx *sql.Tx, g *workflowGraph, _ core.ID, now time.Time, out *recovery.Result) (map[string]any, error) {
|
||||
row, err := workflowRecoveryStep(g, in.StepDecision)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node, ok := g.program.Step(row.view.Key)
|
||||
if !ok || node.Task == nil && node.Poll == nil && node.Signal == nil && node.Timer == nil {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
if err := workflowStepSettled(ctx, tx, g, row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if in.Outcome == "SUCCEEDED" {
|
||||
var typ definition.Type
|
||||
switch {
|
||||
case node.Task != nil:
|
||||
typ = node.Task.ResultType
|
||||
case node.Signal != nil:
|
||||
typ = node.Signal.PayloadType
|
||||
case node.Poll != nil:
|
||||
kind, err := g.program.ClassifyPoll(row.view.Key, in.Result)
|
||||
if err != nil || kind != engine.PollSucceeded {
|
||||
return nil, fault.Invalid("poll recovery requires a confirmed business success result")
|
||||
}
|
||||
typ = node.Poll.Task.ResultType
|
||||
case node.Timer != nil:
|
||||
var activated time.Time
|
||||
if err := tx.QueryRowContext(ctx, `SELECT created_at FROM weave_activations WHERE namespace_id=? AND id=?`, t.NamespaceID, row.view.ActivationID).Scan(&activated); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if now.Before(activated.Add(time.Duration(node.Timer.DelayMS) * time.Millisecond)) {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
typ = definition.Type{Kind: "NULL"}
|
||||
}
|
||||
if err := definition.ValidateData(typ, in.Result); err != nil {
|
||||
return nil, fault.Invalid("review result violates fixed step contract")
|
||||
}
|
||||
}
|
||||
beforeState, beforeOutput := row.view.State, row.output
|
||||
if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepState(in.Outcome), in.Result, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := deleteWorkflowClock(ctx, tx, g, row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// 等待、轮次和执行来源保持原样；本次逻辑结论由管理审计与激活结果表示。
|
||||
out.Step = &row.view
|
||||
digest, err := requestDigest(in.Result)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"step_id": row.view.ID, "activation_id": row.view.ActivationID, "previous_state": beforeState, "previous_output": json.RawMessage(beforeOutput), "outcome": in.Outcome, "result_digest": hex.EncodeToString(digest[:]), "execution_settled": true}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *WorkflowRecoveryStore) Continue(ctx context.Context, t auth.Token, id core.ID, key string, in recovery.ContinueInput) (recovery.Result, error) {
|
||||
return s.mutate(ctx, t, id, key, "workflow.review.continue", in, in.Decision, func(tx *sql.Tx, g *workflowGraph, _ core.ID, now time.Time, _ *recovery.Result) (map[string]any, error) {
|
||||
for key, row := range g.steps {
|
||||
if key == workflowRootKey {
|
||||
continue
|
||||
}
|
||||
if row.view.State == weave.StepNeedsReview || g.view.StopKind == nil && (row.view.State == weave.StepFailed || row.view.State == weave.StepCancelled) {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
}
|
||||
_, uncertain, err := verifyWorkflowStopWork(ctx, tx, g)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if uncertain {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
state := weave.InstanceRunning
|
||||
if g.view.StopKind != nil {
|
||||
state = weave.InstanceCancelling
|
||||
} else if !now.Before(g.view.DeadlineAt) {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_instances SET state=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=? AND state='NEEDS_REVIEW'`, state, now, t.NamespaceID, id, g.view.Revision)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g.view.State, g.view.Revision = state, g.view.Revision+1
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE pulse_execution_gates SET paused=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, g.view.Paused || g.view.StopKind != nil, now, t.NamespaceID, g.gate); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"review_resolved": true, "paused": g.view.Paused, "deadline_at": g.view.DeadlineAt}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func sameWorkflowJSON(a, b []byte) bool {
|
||||
left, err := jsonvalue.Canonical(a)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
right, err := jsonvalue.Canonical(b)
|
||||
return err == nil && bytes.Equal(left, right)
|
||||
}
|
||||
Reference in New Issue
Block a user