feat: add audited workflow review and bounded task recovery
This commit is contained in:
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
@@ -61,6 +62,11 @@ func allowTaskRecovery(ctx context.Context, tx *sql.Tx, ns, id core.ID, allowSch
|
||||
} else if ownerID != nil {
|
||||
return fault.Conflict
|
||||
}
|
||||
return taskExecutionSettled(ctx, tx, ns, id)
|
||||
}
|
||||
|
||||
// taskExecutionSettled 仅检查平台内已授权尝试和许可；调用者仍须提供外部执行已结束的证据。
|
||||
func taskExecutionSettled(ctx context.Context, tx *sql.Tx, ns, id core.ID) error {
|
||||
var active int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_attempts WHERE namespace_id=? AND task_id=? AND state IN ('LEASED','RUNNING')`, ns, id).Scan(&active); err != nil {
|
||||
return err
|
||||
@@ -102,23 +108,7 @@ func (s *TaskStore) ReviewTask(ctx context.Context, t auth.Token, id core.ID, ke
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,result=?,effect_uncertain=FALSE,completed_at=?,updated_at=?,revision=revision+1 WHERE namespace_id=? AND id=? AND revision=? AND state='NEEDS_REVIEW'`, in.Outcome, string(in.Result), now, now, t.NamespaceID, id, in.ExpectedRevision)
|
||||
if err := exactlyOne(changed, err); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := releaseQueueTask(ctx, tx, t.NamespaceID, v.QueueID); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Task, err = readTask(ctx, tx, t.NamespaceID, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digest, err := requestDigest(in.Result)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
after := map[string]any{"state": in.Outcome, "revision": fmt.Sprint(out.Task.Revision), "effect_uncertain": false, "execution_settled": true, "result_digest": hex.EncodeToString(digest[:])}
|
||||
out.AuditID, err = appendAuditWithEvidence(ctx, tx, t, identity, id, "task", "task.review", in.Reason, in.EvidenceRef, map[string]any{"state": v.State, "revision": fmt.Sprint(v.Revision), "effect_uncertain": v.EffectUncertain}, after)
|
||||
out, err = applyTaskReview(ctx, tx, t, task, identity, in, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -183,3 +173,33 @@ func (s *TaskStore) RetryTask(ctx context.Context, t auth.Token, id core.ID, key
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
// applyTaskReview 核查状态、额度与任务审计同事务。调用方已校验归属并持锁，须在最后发布结果事件。
|
||||
func applyTaskReview(ctx context.Context, tx *sql.Tx, t auth.Token, task executionTask, identity core.ID, in service.ReviewInput, now time.Time) (service.ReviewResult, error) {
|
||||
var out service.ReviewResult
|
||||
v := task.view
|
||||
if v.State != pulse.TaskNeedsReview || v.Revision != in.ExpectedRevision || task.lease != nil {
|
||||
return out, fault.Conflict
|
||||
}
|
||||
changed, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,result=?,effect_uncertain=FALSE,completed_at=?,updated_at=?,revision=revision+1 WHERE namespace_id=? AND id=? AND revision=? AND state='NEEDS_REVIEW'`, in.Outcome, string(in.Result), now, now, t.NamespaceID, v.ID, in.ExpectedRevision)
|
||||
if err := exactlyOne(changed, err); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if err := releaseQueueTask(ctx, tx, t.NamespaceID, v.QueueID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Task, err = readTask(ctx, tx, t.NamespaceID, v.ID, false)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
digest, err := requestDigest(in.Result)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
after := map[string]any{"state": in.Outcome, "revision": fmt.Sprint(out.Task.Revision), "effect_uncertain": false, "execution_settled": true, "result_digest": hex.EncodeToString(digest[:])}
|
||||
out.AuditID, err = appendAuditWithEvidence(ctx, tx, t, identity, v.ID, "task", "task.review", in.Reason, in.EvidenceRef, map[string]any{"state": v.State, "revision": fmt.Sprint(v.Revision), "effect_uncertain": v.EffectUncertain}, after)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
292
internal/storage/mysql/workflow_recovery.go
Normal file
292
internal/storage/mysql/workflow_recovery.go
Normal file
@@ -0,0 +1,292 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/jsonvalue"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/definition"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/engine"
|
||||
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/recovery"
|
||||
)
|
||||
|
||||
// WorkflowRecoveryStore 管理核查事实与接续决定；普通Pulse人工入口仍拒绝流程所有者。
|
||||
type WorkflowRecoveryStore struct{ *TaskStore }
|
||||
|
||||
func NewWorkflowRecoveryStore(db *sql.DB) *WorkflowRecoveryStore {
|
||||
return &WorkflowRecoveryStore{NewTaskStore(db)}
|
||||
}
|
||||
|
||||
var _ recovery.Store = (*WorkflowRecoveryStore)(nil)
|
||||
|
||||
type workflowRecoveryApply func(*sql.Tx, *workflowGraph, core.ID, time.Time, *recovery.Result) (map[string]any, error)
|
||||
|
||||
func (s *WorkflowRecoveryStore) mutate(ctx context.Context, t auth.Token, id core.ID, key, operation string, command any, decision recovery.Decision, apply workflowRecoveryApply) (recovery.Result, error) {
|
||||
var out recovery.Result
|
||||
err := s.transaction(ctx, t, recovery.Action, func(tx *sql.Tx, p auth.Permissions) error {
|
||||
if _, err := lockApplicationResources(ctx, tx, t.NamespaceID, false); err != nil {
|
||||
return err
|
||||
}
|
||||
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, operation, string(id), key, "workflow", id, command)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g, _, err := lockWorkflowGraph(ctx, tx, t.NamespaceID, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, q := range g.version.Analysis.Queues {
|
||||
if !p.AllowsQueue(q) {
|
||||
return fault.NotFound
|
||||
}
|
||||
}
|
||||
if cached != nil {
|
||||
if err := json.Unmarshal(cached, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
if out.Instance.NamespaceID != t.NamespaceID || out.Instance.ID != id || out.Instance.DefinitionVersionID != g.view.DefinitionVersionID {
|
||||
return errors.New("workflow recovery receipt mismatch")
|
||||
}
|
||||
out.Deduplicated = true
|
||||
return nil
|
||||
}
|
||||
if g.view.State != weave.InstanceNeedsReview || g.view.Revision != decision.ExpectedRevision {
|
||||
return fault.Conflict
|
||||
}
|
||||
// 新事实须先进入当前图；损坏来源不能用一句人工声明绕过隔离。
|
||||
var pending int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_inbox WHERE namespace_id=? AND workflow_id=? AND state IN ('PENDING','QUARANTINED')`, t.NamespaceID, id).Scan(&pending); err != nil {
|
||||
return err
|
||||
}
|
||||
if pending != 0 {
|
||||
return fault.Conflict
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
before := map[string]any{"state": g.view.State, "revision": g.view.Revision, "control_revision": g.view.ControlRevision, "stop_kind": g.view.StopKind}
|
||||
// 核查期间始终关闭新执行授权；Continue仅在下面完成全部证据校验后按原paused决定是否打开。
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE pulse_execution_gates SET paused=TRUE,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND paused=FALSE`, now, t.NamespaceID, g.gate); err != nil {
|
||||
return err
|
||||
}
|
||||
meta, err := apply(tx, g, identity, now, &out)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_instances SET revision=revision+1,control_revision=control_revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, now, t.NamespaceID, id, g.view.Revision)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return err
|
||||
}
|
||||
g.view.Revision++
|
||||
g.view.ControlRevision++
|
||||
meta["state"], meta["revision"], meta["control_revision"], meta["stop_kind"] = g.view.State, g.view.Revision, g.view.ControlRevision, g.view.StopKind
|
||||
out.AuditID, err = appendAuditWithEvidence(ctx, tx, t, identity, id, "workflow", operation, decision.Reason, decision.EvidenceRef, before, meta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, t.NamespaceID, id, g.view.Revision, "workflow", "workflow.recovered", map[string]any{"operation": operation, "audit_id": out.AuditID, "state": g.view.State}); err != nil {
|
||||
return err
|
||||
}
|
||||
if g.view.StopKind != nil || operation == "workflow.review.continue" {
|
||||
if err := saveWorkflowAdvance(ctx, tx, g, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
out.Instance, _, err = readWorkflowInstance(ctx, tx, t.NamespaceID, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := saveIdentity(ctx, tx, t.NamespaceID, identity, out); err != nil {
|
||||
return err
|
||||
}
|
||||
if operation == "workflow.task.review" {
|
||||
// 只在全部管理状态、回执和审计完成后获取结果流锁；之后不再锁任务。
|
||||
return emitEvent(ctx, tx, t.NamespaceID, out.Task.ID, out.Task.Revision, "task", resultEvent(out.Task.State), map[string]any{"task_id": out.Task.ID, "state": out.Task.State, "effect_uncertain": false, "decision": "MANUAL_REVIEW", "audit_id": out.TaskAuditID, "workflow_audit_id": out.AuditID})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (s *WorkflowRecoveryStore) ReviewTask(ctx context.Context, t auth.Token, id core.ID, key string, in recovery.ReviewTaskInput) (recovery.Result, error) {
|
||||
return s.mutate(ctx, t, id, key, "workflow.task.review", in, in.Decision, func(tx *sql.Tx, g *workflowGraph, identity core.ID, now time.Time, out *recovery.Result) (map[string]any, error) {
|
||||
var owned int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_tasks t JOIN weave_activations a ON a.namespace_id=t.namespace_id AND a.id=t.owner_id WHERE t.namespace_id=? AND t.id=? AND t.owner_kind='workflow_activation' AND a.workflow_id=?`, t.NamespaceID, in.TaskID, id).Scan(&owned); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if owned != 1 {
|
||||
return nil, fault.NotFound
|
||||
}
|
||||
if err := validateWorkflowStopTask(ctx, tx, g, in.TaskID); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, fault.NotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
task, err := readExecutionTask(ctx, tx, t.NamespaceID, in.TaskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := taskExecutionSettled(ctx, tx, t.NamespaceID, in.TaskID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
taskOperation, err := core.NewID()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
v, err := applyTaskReview(ctx, tx, t, task, taskOperation, service.ReviewInput{ExpectedRevision: in.ExpectedTaskRevision, Reason: in.Reason, EvidenceRef: in.EvidenceRef, ExecutionSettled: in.ExecutionSettled, Outcome: in.Outcome, Result: in.Result}, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out.Task, out.TaskAuditID = &v.Task, &v.AuditID
|
||||
return map[string]any{"task_id": in.TaskID, "task_audit_id": v.AuditID, "task_operation_id": taskOperation, "workflow_operation_id": identity, "execution_settled": true, "outcome": in.Outcome}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func workflowRecoveryStep(g *workflowGraph, in recovery.StepDecision) (*workflowStep, error) {
|
||||
row := g.steps[in.StepKey]
|
||||
if row == nil || in.StepKey == workflowRootKey {
|
||||
return nil, fault.NotFound
|
||||
}
|
||||
if row.view.ActivationID == nil || *row.view.ActivationID != in.ActivationID || row.view.Revision != in.ExpectedStepRevision {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
if row.view.State != weave.StepNeedsReview && row.view.State != weave.StepFailed && row.view.State != weave.StepCancelled {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
return row, nil
|
||||
}
|
||||
|
||||
// workflowStepSettled 检查全部历史归属，再核验本步骤的执行与许可；兄弟分支可继续收集在途结果。
|
||||
func workflowStepSettled(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep) error {
|
||||
if _, _, err := verifyWorkflowStopWork(ctx, tx, g); err != nil {
|
||||
return err
|
||||
}
|
||||
queries := []string{
|
||||
`SELECT COUNT(*) FROM weave_activations a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.owner_id=a.id AND t.owner_kind='workflow_activation' WHERE a.namespace_id=? AND a.workflow_id=? AND a.step_id=? AND (t.state NOT IN ('SUCCEEDED','FAILED','CANCELLED') OR t.effect_uncertain=TRUE)`,
|
||||
`SELECT COUNT(*) FROM weave_activations a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.owner_id=a.id AND t.owner_kind='workflow_activation' JOIN pulse_attempts x ON x.namespace_id=t.namespace_id AND x.task_id=t.id WHERE a.namespace_id=? AND a.workflow_id=? AND a.step_id=? AND x.state IN ('LEASED','RUNNING')`,
|
||||
`SELECT COUNT(*) FROM weave_activations a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.owner_id=a.id AND t.owner_kind='workflow_activation' JOIN pulse_attempts x ON x.namespace_id=t.namespace_id AND x.task_id=t.id JOIN pulse_permits p ON p.namespace_id=x.namespace_id AND p.attempt_id=x.id WHERE a.namespace_id=? AND a.workflow_id=? AND a.step_id=? AND p.released_at IS NULL`,
|
||||
}
|
||||
for _, q := range queries {
|
||||
var count int
|
||||
if err := tx.QueryRowContext(ctx, q, g.view.NamespaceID, g.view.ID, row.view.ID).Scan(&count); err != nil {
|
||||
return err
|
||||
}
|
||||
if count != 0 {
|
||||
return fault.Conflict
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *WorkflowRecoveryStore) ResolveStep(ctx context.Context, t auth.Token, id core.ID, key string, in recovery.ResolveStepInput) (recovery.Result, error) {
|
||||
return s.mutate(ctx, t, id, key, "workflow.step.resolve", in, in.Decision, func(tx *sql.Tx, g *workflowGraph, _ core.ID, now time.Time, out *recovery.Result) (map[string]any, error) {
|
||||
row, err := workflowRecoveryStep(g, in.StepDecision)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node, ok := g.program.Step(row.view.Key)
|
||||
if !ok || node.Task == nil && node.Poll == nil && node.Signal == nil && node.Timer == nil {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
if err := workflowStepSettled(ctx, tx, g, row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if in.Outcome == "SUCCEEDED" {
|
||||
var typ definition.Type
|
||||
switch {
|
||||
case node.Task != nil:
|
||||
typ = node.Task.ResultType
|
||||
case node.Signal != nil:
|
||||
typ = node.Signal.PayloadType
|
||||
case node.Poll != nil:
|
||||
kind, err := g.program.ClassifyPoll(row.view.Key, in.Result)
|
||||
if err != nil || kind != engine.PollSucceeded {
|
||||
return nil, fault.Invalid("poll recovery requires a confirmed business success result")
|
||||
}
|
||||
typ = node.Poll.Task.ResultType
|
||||
case node.Timer != nil:
|
||||
var activated time.Time
|
||||
if err := tx.QueryRowContext(ctx, `SELECT created_at FROM weave_activations WHERE namespace_id=? AND id=?`, t.NamespaceID, row.view.ActivationID).Scan(&activated); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if now.Before(activated.Add(time.Duration(node.Timer.DelayMS) * time.Millisecond)) {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
typ = definition.Type{Kind: "NULL"}
|
||||
}
|
||||
if err := definition.ValidateData(typ, in.Result); err != nil {
|
||||
return nil, fault.Invalid("review result violates fixed step contract")
|
||||
}
|
||||
}
|
||||
beforeState, beforeOutput := row.view.State, row.output
|
||||
if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepState(in.Outcome), in.Result, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := deleteWorkflowClock(ctx, tx, g, row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// 等待、轮次和执行来源保持原样；本次逻辑结论由管理审计与激活结果表示。
|
||||
out.Step = &row.view
|
||||
digest, err := requestDigest(in.Result)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"step_id": row.view.ID, "activation_id": row.view.ActivationID, "previous_state": beforeState, "previous_output": json.RawMessage(beforeOutput), "outcome": in.Outcome, "result_digest": hex.EncodeToString(digest[:]), "execution_settled": true}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *WorkflowRecoveryStore) Continue(ctx context.Context, t auth.Token, id core.ID, key string, in recovery.ContinueInput) (recovery.Result, error) {
|
||||
return s.mutate(ctx, t, id, key, "workflow.review.continue", in, in.Decision, func(tx *sql.Tx, g *workflowGraph, _ core.ID, now time.Time, _ *recovery.Result) (map[string]any, error) {
|
||||
for key, row := range g.steps {
|
||||
if key == workflowRootKey {
|
||||
continue
|
||||
}
|
||||
if row.view.State == weave.StepNeedsReview || g.view.StopKind == nil && (row.view.State == weave.StepFailed || row.view.State == weave.StepCancelled) {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
}
|
||||
_, uncertain, err := verifyWorkflowStopWork(ctx, tx, g)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if uncertain {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
state := weave.InstanceRunning
|
||||
if g.view.StopKind != nil {
|
||||
state = weave.InstanceCancelling
|
||||
} else if !now.Before(g.view.DeadlineAt) {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_instances SET state=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=? AND state='NEEDS_REVIEW'`, state, now, t.NamespaceID, id, g.view.Revision)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g.view.State, g.view.Revision = state, g.view.Revision+1
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE pulse_execution_gates SET paused=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, g.view.Paused || g.view.StopKind != nil, now, t.NamespaceID, g.gate); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"review_resolved": true, "paused": g.view.Paused, "deadline_at": g.view.DeadlineAt}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func sameWorkflowJSON(a, b []byte) bool {
|
||||
left, err := jsonvalue.Canonical(a)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
right, err := jsonvalue.Canonical(b)
|
||||
return err == nil && bytes.Equal(left, right)
|
||||
}
|
||||
507
internal/storage/mysql/workflow_recovery_integration_test.go
Normal file
507
internal/storage/mysql/workflow_recovery_integration_test.go
Normal file
@@ -0,0 +1,507 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/definition"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/instance"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/recovery"
|
||||
)
|
||||
|
||||
func grantWorkflowRecovery(f *executionFixture) {
|
||||
f.exec(`UPDATE orbit_credentials SET permissions=JSON_ARRAY_APPEND(permissions,'$.actions','workflows:recover','$.actions','workflows:control','$.actions','tasks:review') WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, f.resources.CredentialID)
|
||||
}
|
||||
func recoveryDecision(t *testing.T, f *executionFixture, id core.ID) recovery.Decision {
|
||||
t.Helper()
|
||||
v := runtimeView(t, f, id, "NEEDS_REVIEW")
|
||||
return recovery.Decision{ExpectedRevision: v.Revision, Reason: "verified external result", EvidenceRef: "evidence://recovery/verified"}
|
||||
}
|
||||
func recoveryStep(t *testing.T, f *executionFixture, id core.ID, key string) recovery.StepDecision {
|
||||
t.Helper()
|
||||
out := recovery.StepDecision{Decision: recoveryDecision(t, f, id), StepKey: key}
|
||||
if err := f.db.QueryRowContext(f.ctx, "SELECT current_activation_id,revision FROM weave_steps WHERE namespace_id=? AND workflow_id=? AND `key`=?", f.resources.NamespaceID, id, key).Scan(&out.ActivationID, &out.ExpectedStepRevision); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func reviewWorkflowTask(t *testing.T, f *executionFixture, id, task core.ID, outcome string) recovery.Result {
|
||||
t.Helper()
|
||||
v, err := f.tasks.GetTask(f.ctx, f.token, task)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := recovery.New(NewWorkflowRecoveryStore(f.db)).ReviewTask(f.ctx, f.token, id, string(executionID(t)), recovery.ReviewTaskInput{Decision: recoveryDecision(t, f, id), TaskID: task, ExpectedTaskRevision: v.Revision, ExecutionSettled: true, Outcome: pulse.TaskState(outcome), Result: json.RawMessage(`{"n":9007199254740993}`)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func resolveWorkflowStep(t *testing.T, f *executionFixture, id core.ID, key, outcome, result string) recovery.Result {
|
||||
t.Helper()
|
||||
out, err := recovery.New(NewWorkflowRecoveryStore(f.db)).ResolveStep(f.ctx, f.token, id, string(executionID(t)), recovery.ResolveStepInput{StepDecision: recoveryStep(t, f, id, key), ExecutionSettled: true, Outcome: outcome, Result: json.RawMessage(result)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func continueWorkflow(t *testing.T, f *executionFixture, id core.ID) recovery.Result {
|
||||
t.Helper()
|
||||
out, err := recovery.New(NewWorkflowRecoveryStore(f.db)).Continue(f.ctx, f.token, id, string(executionID(t)), recovery.ContinueInput{Decision: recoveryDecision(t, f, id), ReviewResolved: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func TestIntegrationWorkflowRecovery(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, stop := context.WithTimeout(context.Background(), 150*time.Second)
|
||||
defer stop()
|
||||
db, err := Open(ctx, dsn, 32, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
api := recovery.New(NewWorkflowRecoveryStore(db))
|
||||
fixture := func(t *testing.T, n int) (*executionFixture, instance.View, core.ID) {
|
||||
f, v := controlFixture(t, ctx, db, n)
|
||||
grantWorkflowRecovery(f)
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "UNKNOWN", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
return f, v, ref.TaskID
|
||||
}
|
||||
t.Run("unknown_review_resolve_continue_preserves_history", func(t *testing.T) {
|
||||
f, v, task := fixture(t, 2)
|
||||
old := recoveryStep(t, f, v.ID, "task0")
|
||||
_, err := api.ResolveStep(ctx, f.token, v.ID, string(executionID(t)), recovery.ResolveStepInput{StepDecision: old, ExecutionSettled: true, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"n":9007199254740993}`)})
|
||||
if fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("unresolved execution bypassed", err)
|
||||
}
|
||||
tv, _ := f.tasks.GetTask(ctx, f.token, task)
|
||||
_, err = f.tasks.ReviewTask(ctx, f.token, task, string(executionID(t)), service.ReviewInput{ExpectedRevision: tv.Revision, ExecutionSettled: true, Outcome: "SUCCEEDED", Reason: "cannot bypass workflow", EvidenceRef: "evidence://1", Result: json.RawMessage(`{"n":9007199254740993}`)})
|
||||
if fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("ordinary review bypassed owner", err)
|
||||
}
|
||||
review := reviewWorkflowTask(t, f, v.ID, task, "SUCCEEDED")
|
||||
if review.Task.EffectUncertain || review.Instance.State != "NEEDS_REVIEW" || review.TaskAuditID == nil || f.count("pulse_attempts", "state='UNKNOWN'") != 1 || f.count("orbit_audit_events", "action IN ('workflow.task.review','task.review')") != 2 {
|
||||
t.Fatal("review lost history or released workflow")
|
||||
}
|
||||
// 新执行结果先可靠消费，再由显式步骤核查决定业务输出。
|
||||
runtimeConsume(t, f, newRuntime(t, db), task)
|
||||
resolved := resolveWorkflowStep(t, f, v.ID, "task0", "SUCCEEDED", `{"n":9007199254740993}`)
|
||||
if resolved.Instance.State != "NEEDS_REVIEW" || f.count("pulse_tasks", "1=1") != 1 {
|
||||
t.Fatal("resolution auto continued")
|
||||
}
|
||||
continued := continueWorkflow(t, f, v.ID)
|
||||
if continued.Instance.State != "RUNNING" || !continued.Instance.DeadlineAt.Equal(v.DeadlineAt) {
|
||||
t.Fatal("continue changed deadline")
|
||||
}
|
||||
runtimeAdvance(t, f, newRuntime(t, db), v.ID)
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
runtimeView(t, f, v.ID, "SUCCEEDED")
|
||||
if f.count("pulse_attempts", "state='UNKNOWN'") != 1 || f.used() != 0 {
|
||||
t.Fatal("old execution rewritten or permit leaked")
|
||||
}
|
||||
})
|
||||
t.Run("task_retry_new_activation_remaining_budget_and_stale_result", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
spec := runtimeSpec(f.resources.QueueID, 1)
|
||||
spec.Body.Steps[0].Task.Retry.MaxAttempts = 2
|
||||
spec.Body.Steps[0].Task.Retry.InitialDelayMS = 1000
|
||||
spec.Body.Steps[0].Task.Retry.MaxDelayMS = 1000
|
||||
v := runtimeStart(t, f, spec)
|
||||
grantWorkflowRecovery(f)
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "FAILED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
oldTask, _ := f.tasks.GetTask(ctx, f.token, ref.TaskID)
|
||||
in := recovery.RetryStepInput{StepDecision: recoveryStep(t, f, v.ID, "task0"), SafeToRetry: true}
|
||||
key := string(executionID(t))
|
||||
out, err := api.RetryStep(ctx, f.token, v.ID, key, in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Task.ID == ref.TaskID || out.Task.RetryOf == nil || *out.Task.RetryOf != ref.TaskID || out.Task.Retry.MaxAttempts != oldTask.Retry.MaxAttempts-1 || !out.Task.DeadlineAt.Equal(oldTask.DeadlineAt) || !sameWorkflowJSON(out.Task.Payload, oldTask.Payload) || *out.Step.ActivationID == in.ActivationID {
|
||||
t.Fatal("retry replaced identity or reset budget")
|
||||
}
|
||||
if f.count("weave_activations", "id=? AND state='FAILED'", in.ActivationID) != 1 {
|
||||
t.Fatal("old activation rewritten")
|
||||
}
|
||||
// 重放旧激活的真实事件，只留痕而不能完成新任务。
|
||||
f.exec(`UPDATE weave_inbox SET state='PENDING',processed_at=NULL WHERE namespace_id=? AND source_task_id=?`, f.resources.NamespaceID, ref.TaskID)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
if f.count("weave_inbox", "source_task_id=? AND error_code='STALE_ACTIVATION'", ref.TaskID) != 1 {
|
||||
t.Fatal("old activation not isolated")
|
||||
}
|
||||
replayed, err := api.RetryStep(ctx, f.token, v.ID, key, in)
|
||||
if err != nil || !replayed.Deduplicated || replayed.Task.ID != out.Task.ID {
|
||||
t.Fatal("retry replay changed", err)
|
||||
}
|
||||
oldStep := in
|
||||
oldStep.ExpectedRevision = recoveryDecision(t, f, v.ID).ExpectedRevision
|
||||
if _, err := api.RetryStep(ctx, f.token, v.ID, string(executionID(t)), oldStep); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("old activation controlled new work", err)
|
||||
}
|
||||
continueWorkflow(t, f, v.ID)
|
||||
ref = retryStart(t, f)
|
||||
retryFinish(t, f, ref, "FAILED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
_, err = api.RetryStep(ctx, f.token, v.ID, string(executionID(t)), recovery.RetryStepInput{StepDecision: recoveryStep(t, f, v.ID, "task0"), SafeToRetry: true})
|
||||
if oldTask.Retry.MaxAttempts == 2 && fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("exhausted original budget reset", err)
|
||||
}
|
||||
})
|
||||
t.Run("same_review_key_concurrency_and_revoked_replay", func(t *testing.T) {
|
||||
f, v, task := fixture(t, 1)
|
||||
tv, _ := f.tasks.GetTask(ctx, f.token, task)
|
||||
in := recovery.ReviewTaskInput{Decision: recoveryDecision(t, f, v.ID), TaskID: task, ExpectedTaskRevision: tv.Revision, ExecutionSettled: true, Outcome: "FAILED", Result: json.RawMessage(`{"known":true}`)}
|
||||
key := string(executionID(t))
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 12)
|
||||
fresh := make(chan bool, 12)
|
||||
for i := 0; i < 12; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
v, e := api.ReviewTask(ctx, f.token, v.ID, key, in)
|
||||
errs <- e
|
||||
fresh <- !v.Deduplicated
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
close(fresh)
|
||||
for e := range errs {
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
n := 0
|
||||
for ok := range fresh {
|
||||
if ok {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 || f.count("orbit_audit_events", "action='workflow.task.review'") != 1 || f.count("weave_inbox", "source_task_id=?", task) != 2 {
|
||||
t.Fatal("duplicate review effects")
|
||||
}
|
||||
f.exec(`UPDATE orbit_credentials SET permissions=JSON_SET(permissions,'$.queues',JSON_ARRAY(?)) WHERE namespace_id=? AND id=?`, string(executionID(t)), f.resources.NamespaceID, f.resources.CredentialID)
|
||||
if _, err := api.ReviewTask(ctx, f.token, v.ID, key, in); fault.Code(err) != "NOT_FOUND" {
|
||||
t.Fatal("cached receipt bypassed queue permission", err)
|
||||
}
|
||||
})
|
||||
t.Run("review_result_stream_failure_rolls_back_all", func(t *testing.T) {
|
||||
f, v, task := fixture(t, 1)
|
||||
tv, _ := f.tasks.GetTask(ctx, f.token, task)
|
||||
decision := recoveryDecision(t, f, v.ID)
|
||||
trigger := "reject_workflow_review_" + string(executionID(t))
|
||||
f.exec("CREATE TRIGGER " + trigger + " BEFORE INSERT ON orbit_result_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='controlled review stream failure'")
|
||||
in := recovery.ReviewTaskInput{Decision: decision, TaskID: task, ExpectedTaskRevision: tv.Revision, ExecutionSettled: true, Outcome: "FAILED", Result: json.RawMessage(`{"known":true}`)}
|
||||
key := string(executionID(t))
|
||||
_, err := api.ReviewTask(ctx, f.token, v.ID, key, in)
|
||||
f.exec("DROP TRIGGER " + trigger)
|
||||
if err == nil {
|
||||
t.Fatal("injected failure accepted")
|
||||
}
|
||||
if f.count("pulse_tasks", "id=? AND state='NEEDS_REVIEW' AND effect_uncertain=TRUE", task) != 1 || f.count("orbit_audit_events", "action IN ('workflow.task.review','task.review')") != 0 || f.count("weave_inbox", "source_task_id=?", task) != 1 || recoveryDecision(t, f, v.ID).ExpectedRevision != decision.ExpectedRevision {
|
||||
t.Fatal("partial review committed")
|
||||
}
|
||||
if _, err := api.ReviewTask(ctx, f.token, v.ID, key, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
for _, policy := range []string{"REVIEW", "FAIL_AFTER_SETTLED"} {
|
||||
t.Run("resolve_unknown_with_stop_"+policy, func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
spec := runtimeSpec(f.resources.QueueID, 2)
|
||||
spec.FailurePolicy = policy
|
||||
v := runtimeStart(t, f, spec)
|
||||
grantWorkflowRecovery(f)
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "UNKNOWN", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
if policy == "REVIEW" {
|
||||
applyControl(t, f, runtimeView(t, f, v.ID, "NEEDS_REVIEW"), "CANCEL")
|
||||
}
|
||||
drainStop(t, f, v.ID, "NEEDS_REVIEW")
|
||||
before := runtimeView(t, f, v.ID, "NEEDS_REVIEW")
|
||||
reviewWorkflowTask(t, f, v.ID, ref.TaskID, "FAILED")
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
resolveWorkflowStep(t, f, v.ID, "task0", "FAILED", `{"known":true}`)
|
||||
continueWorkflow(t, f, v.ID)
|
||||
want := "CANCELLED"
|
||||
if policy == "FAIL_AFTER_SETTLED" {
|
||||
want = "FAILED"
|
||||
}
|
||||
drainStop(t, f, v.ID, want)
|
||||
after := runtimeView(t, f, v.ID, want)
|
||||
if *after.StopKind != *before.StopKind || !after.StopRequestedAt.Equal(*before.StopRequestedAt) || f.count("pulse_tasks", "1=1") != 1 {
|
||||
t.Fatal("recovery changed original stop intent")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("poll_review_cannot_treat_pending_as_success", func(t *testing.T) {
|
||||
f := newClockFixture(t, ctx, db)
|
||||
spec := pollingSpec(f.resources.QueueID)
|
||||
spec.Body.Steps[0].Poll.MaxRounds = 1
|
||||
v := runtimeStart(t, f, spec)
|
||||
grantWorkflowRecovery(f)
|
||||
ref := retryStart(t, f)
|
||||
finishPoll(t, f, ref, "PENDING")
|
||||
in := recovery.ResolveStepInput{StepDecision: recoveryStep(t, f, v.ID, "query"), ExecutionSettled: true, Outcome: "SUCCEEDED", Result: json.RawMessage(`"PENDING"`)}
|
||||
if _, err := api.ResolveStep(ctx, f.token, v.ID, string(executionID(t)), in); fault.Code(err) != "INVALID_ARGUMENT" {
|
||||
t.Fatal("PENDING became business success", err)
|
||||
}
|
||||
resolveWorkflowStep(t, f, v.ID, "query", "SUCCEEDED", `"DONE"`)
|
||||
continueWorkflow(t, f, v.ID)
|
||||
runtimeAdvance(t, f, newRuntime(t, db), v.ID)
|
||||
runtimeView(t, f, v.ID, "SUCCEEDED")
|
||||
if f.count("weave_poll_states", "used_rounds=1 AND max_rounds=1") != 1 || f.count("pulse_tasks", "1=1") != 1 {
|
||||
t.Fatal("review issued extra query or reset rounds")
|
||||
}
|
||||
})
|
||||
t.Run("continue_preserves_pause_and_rejects_stale_evidence", func(t *testing.T) {
|
||||
f, v, task := fixture(t, 2)
|
||||
stale := recoveryDecision(t, f, v.ID)
|
||||
applyControl(t, f, runtimeView(t, f, v.ID, "NEEDS_REVIEW"), "PAUSE")
|
||||
tv := f.get(task)
|
||||
in := recovery.ReviewTaskInput{Decision: stale, TaskID: task, ExpectedTaskRevision: tv.Revision, ExecutionSettled: true, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"n":9007199254740993}`)}
|
||||
if _, err := api.ReviewTask(ctx, f.token, v.ID, string(executionID(t)), in); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("stale evidence accepted", err)
|
||||
}
|
||||
reviewWorkflowTask(t, f, v.ID, task, "SUCCEEDED")
|
||||
runtimeConsume(t, f, newRuntime(t, db), task)
|
||||
resolveWorkflowStep(t, f, v.ID, "task0", "SUCCEEDED", `{"n":9007199254740993}`)
|
||||
out := continueWorkflow(t, f, v.ID)
|
||||
if !out.Instance.Paused {
|
||||
t.Fatal("recovery cleared explicit pause")
|
||||
}
|
||||
stopStep(t, f, v.ID)
|
||||
if f.count("pulse_tasks", "1=1") != 1 {
|
||||
t.Fatal("paused recovery created successor")
|
||||
}
|
||||
applyControl(t, f, runtimeView(t, f, v.ID, "RUNNING"), "RESUME")
|
||||
runtimeAdvance(t, f, newRuntime(t, db), v.ID)
|
||||
if f.count("pulse_tasks", "1=1") != 2 {
|
||||
t.Fatal("resume did not continue reviewed output")
|
||||
}
|
||||
})
|
||||
t.Run("retry_backpressure_and_audit_failure_are_atomic", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
spec := runtimeSpec(f.resources.QueueID, 1)
|
||||
spec.Body.Steps[0].Task.Retry.MaxAttempts = 2
|
||||
spec.Body.Steps[0].Task.Retry.InitialDelayMS = 1000
|
||||
spec.Body.Steps[0].Task.Retry.MaxDelayMS = 1000
|
||||
v := runtimeStart(t, f, spec)
|
||||
grantWorkflowRecovery(f)
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "FAILED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
in := recovery.RetryStepInput{StepDecision: recoveryStep(t, f, v.ID, "task0"), SafeToRetry: true}
|
||||
key := string(executionID(t))
|
||||
f.exec(`UPDATE pulse_queues SET max_unfinished_tasks=1 WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, f.resources.QueueID)
|
||||
blocker := f.submit(1)
|
||||
if _, err := api.RetryStep(ctx, f.token, v.ID, key, in); fault.Code(err) != "QUEUE_FULL" {
|
||||
t.Fatal("retry bypassed queue capacity", err)
|
||||
}
|
||||
if f.count("weave_activations", "step_id=(SELECT id FROM weave_steps WHERE namespace_id=? AND workflow_id=? AND `key`='task0')", f.resources.NamespaceID, v.ID) != 1 {
|
||||
t.Fatal("backpressure left new activation")
|
||||
}
|
||||
_, err := f.tasks.CancelTask(ctx, f.token, blocker.ID, string(executionID(t)), service.CancelInput{ExpectedRevision: blocker.Revision, Reason: "release capacity"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
trigger := "retry_audit_reject_" + string(executionID(t))
|
||||
f.exec("CREATE TRIGGER " + trigger + " BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='controlled workflow retry audit failure'")
|
||||
_, err = api.RetryStep(ctx, f.token, v.ID, key, in)
|
||||
f.exec("DROP TRIGGER " + trigger)
|
||||
if err == nil || f.count("pulse_tasks", "retry_of=?", ref.TaskID) != 0 {
|
||||
t.Fatal("retry audit failure left partial task", err)
|
||||
}
|
||||
if _, err := api.RetryStep(ctx, f.token, v.ID, key, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
t.Run("task_deadline_is_not_extended_by_manual_retry", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
spec := runtimeSpec(f.resources.QueueID, 1)
|
||||
spec.TimeoutMS = 4000
|
||||
task := spec.Body.Steps[0].Task
|
||||
task.TimeoutMS = 700
|
||||
task.Retry.AttemptTimeoutMS = 200
|
||||
task.Retry.MaxAttempts = 2
|
||||
task.Retry.InitialDelayMS = 100
|
||||
task.Retry.MaxDelayMS = 100
|
||||
v := runtimeStart(t, f, spec)
|
||||
grantWorkflowRecovery(f)
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "FAILED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
time.Sleep(750 * time.Millisecond)
|
||||
if _, err := api.RetryStep(ctx, f.token, v.ID, string(executionID(t)), recovery.RetryStepInput{StepDecision: recoveryStep(t, f, v.ID, "task0"), SafeToRetry: true}); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("old task deadline extended", err)
|
||||
}
|
||||
if f.count("pulse_tasks", "1=1") != 1 {
|
||||
t.Fatal("expired task produced retry")
|
||||
}
|
||||
})
|
||||
t.Run("pending_and_quarantined_evidence_prevent_recovery", func(t *testing.T) {
|
||||
f, v, task := fixture(t, 1)
|
||||
tv := f.get(task)
|
||||
in := recovery.ReviewTaskInput{Decision: recoveryDecision(t, f, v.ID), TaskID: task, ExpectedTaskRevision: tv.Revision, ExecutionSettled: true, Outcome: "FAILED", Result: json.RawMessage(`{"known":true}`)}
|
||||
for _, state := range []string{"PENDING", "QUARANTINED"} {
|
||||
f.exec(`UPDATE weave_inbox SET state=? WHERE namespace_id=? AND source_task_id=?`, state, f.resources.NamespaceID, task)
|
||||
if _, err := api.ReviewTask(ctx, f.token, v.ID, string(executionID(t)), in); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("unprocessed evidence bypassed", state, err)
|
||||
}
|
||||
}
|
||||
f.exec(`UPDATE weave_inbox SET state='PROCESSED' WHERE namespace_id=? AND source_task_id=?`, f.resources.NamespaceID, task)
|
||||
if _, err := api.ReviewTask(ctx, f.token, v.ID, string(executionID(t)), in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
t.Run("wrong_task_scope_version_and_output_cannot_resolve", func(t *testing.T) {
|
||||
f, v, task := fixture(t, 1)
|
||||
other, _, foreign := fixture(t, 1)
|
||||
tv := f.get(task)
|
||||
in := recovery.ReviewTaskInput{Decision: recoveryDecision(t, f, v.ID), TaskID: foreign, ExpectedTaskRevision: other.get(foreign).Revision, ExecutionSettled: true, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"n":9007199254740993}`)}
|
||||
if _, err := api.ReviewTask(ctx, f.token, v.ID, string(executionID(t)), in); fault.Code(err) != "NOT_FOUND" {
|
||||
t.Fatal("foreign task scope accepted", err)
|
||||
}
|
||||
in.TaskID = task
|
||||
in.ExpectedTaskRevision = tv.Revision - 1
|
||||
if _, err := api.ReviewTask(ctx, f.token, v.ID, string(executionID(t)), in); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("old execution version accepted", err)
|
||||
}
|
||||
reviewWorkflowTask(t, f, v.ID, task, "SUCCEEDED")
|
||||
runtimeConsume(t, f, newRuntime(t, db), task)
|
||||
step := recoveryStep(t, f, v.ID, "task0")
|
||||
if _, err := api.ResolveStep(ctx, f.token, v.ID, string(executionID(t)), recovery.ResolveStepInput{StepDecision: step, ExecutionSettled: true, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"unexpected":true}`)}); fault.Code(err) != "INVALID_ARGUMENT" {
|
||||
t.Fatal("invalid fixed output accepted", err)
|
||||
}
|
||||
if _, err := api.Continue(ctx, f.token, v.ID, string(executionID(t)), recovery.ContinueInput{Decision: step.Decision, ReviewResolved: true}); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("unresolved step released", err)
|
||||
}
|
||||
})
|
||||
t.Run("expired_instance_can_settle_but_cannot_restart", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
spec := runtimeSpec(f.resources.QueueID, 2)
|
||||
spec.TimeoutMS = 800
|
||||
for _, s := range spec.Body.Steps {
|
||||
s.Task.TimeoutMS = 500
|
||||
s.Task.Retry.AttemptTimeoutMS = 200
|
||||
}
|
||||
v := runtimeStart(t, f, spec)
|
||||
grantWorkflowRecovery(f)
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "FAILED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
resolveWorkflowStep(t, f, v.ID, "task0", "SUCCEEDED", `{"n":9007199254740993}`)
|
||||
time.Sleep(850 * time.Millisecond)
|
||||
if _, err := api.Continue(ctx, f.token, v.ID, string(executionID(t)), recovery.ContinueInput{Decision: recoveryDecision(t, f, v.ID), ReviewResolved: true}); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("expired instance restarted", err)
|
||||
}
|
||||
applyControl(t, f, runtimeView(t, f, v.ID, "NEEDS_REVIEW"), "CANCEL")
|
||||
continueWorkflow(t, f, v.ID)
|
||||
drainStop(t, f, v.ID, "CANCELLED")
|
||||
if f.count("pulse_tasks", "1=1") != 1 {
|
||||
t.Fatal("expired recovery issued successor")
|
||||
}
|
||||
})
|
||||
t.Run("retry_rejects_corrupt_frozen_input_and_initial_deadline", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
spec := runtimeSpec(f.resources.QueueID, 1)
|
||||
spec.Body.Steps[0].Task.Retry.MaxAttempts = 2
|
||||
spec.Body.Steps[0].Task.Retry.InitialDelayMS = 1000
|
||||
spec.Body.Steps[0].Task.Retry.MaxDelayMS = 1000
|
||||
v := runtimeStart(t, f, spec)
|
||||
grantWorkflowRecovery(f)
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "FAILED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
in := recovery.RetryStepInput{StepDecision: recoveryStep(t, f, v.ID, "task0"), SafeToRetry: true}
|
||||
old := f.get(ref.TaskID)
|
||||
f.exec(`UPDATE pulse_tasks SET payload='{"wrong":true}' WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, ref.TaskID)
|
||||
f.exec(`UPDATE weave_activations SET input='{"wrong":true}' WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, in.ActivationID)
|
||||
if _, err := api.RetryStep(ctx, f.token, v.ID, string(executionID(t)), in); err == nil {
|
||||
t.Fatal("corrupt matching inputs bypassed fixed contract")
|
||||
}
|
||||
f.exec(`UPDATE pulse_tasks SET payload=? WHERE namespace_id=? AND id=?`, string(old.Payload), f.resources.NamespaceID, ref.TaskID)
|
||||
f.exec(`UPDATE weave_activations SET input=? WHERE namespace_id=? AND id=?`, string(old.Payload), f.resources.NamespaceID, in.ActivationID)
|
||||
f.exec(`UPDATE pulse_tasks SET deadline_at=DATE_ADD(deadline_at,INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, ref.TaskID)
|
||||
if _, err := api.RetryStep(ctx, f.token, v.ID, string(executionID(t)), in); err == nil {
|
||||
t.Fatal("mutated initial deadline accepted")
|
||||
}
|
||||
f.exec(`UPDATE pulse_tasks SET deadline_at=? WHERE namespace_id=? AND id=?`, old.DeadlineAt, f.resources.NamespaceID, ref.TaskID)
|
||||
if _, err := api.RetryStep(ctx, f.token, v.ID, string(executionID(t)), in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
t.Run("signal_deadline_review_preserves_expired_wait", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
spec := signalSpec(f.resources.QueueID, false)
|
||||
spec.Body.Steps[0].Signal.TimeoutMS = 100
|
||||
v := runtimeStart(t, f, spec)
|
||||
grantWorkflowRecovery(f)
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
runClock(t, f, clockCandidate(t, f, v.ID, "signal"))
|
||||
resolveWorkflowStep(t, f, v.ID, "signal", "SUCCEEDED", `{"n":9007199254740993}`)
|
||||
continueWorkflow(t, f, v.ID)
|
||||
runtimeAdvance(t, f, newRuntime(t, db), v.ID)
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
runtimeView(t, f, v.ID, "SUCCEEDED")
|
||||
if f.count("weave_waits", "workflow_id=? AND state='EXPIRED'", v.ID) != 1 || f.count("weave_signals", "1=1") != 0 {
|
||||
t.Fatal("manual resolution fabricated signal receipt")
|
||||
}
|
||||
})
|
||||
t.Run("timer_review_cannot_fulfil_original_delay_early", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
spec := runtimeSpec(f.resources.QueueID, 1)
|
||||
spec.TimeoutMS = 1500
|
||||
spec.Body.Steps[0].Task.TimeoutMS = 1000
|
||||
spec.Body.Steps[0].Task.Retry.AttemptTimeoutMS = 800
|
||||
spec.Body.Steps = append(spec.Body.Steps, definition.Step{Key: "timer", Kind: "TIMER", Timer: &definition.Timer{DelayMS: 1500}})
|
||||
spec.Body.OutputType = definition.Type{Kind: "NULL"}
|
||||
spec.Body.Output = definition.Value{Kind: "REFERENCE", Reference: &definition.Reference{Source: "timer"}}
|
||||
v := runtimeStart(t, f, spec)
|
||||
grantWorkflowRecovery(f)
|
||||
ref := retryStart(t, f)
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
c := clockCandidate(t, f, v.ID, "timer")
|
||||
waitClockDue(t, f, c)
|
||||
runClock(t, f, c)
|
||||
in := recovery.ResolveStepInput{StepDecision: recoveryStep(t, f, v.ID, "timer"), ExecutionSettled: true, Outcome: "SUCCEEDED", Result: json.RawMessage(`null`)}
|
||||
if _, err := api.ResolveStep(ctx, f.token, v.ID, string(executionID(t)), in); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("manual review shortened original timer", err)
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
resolveWorkflowStep(t, f, v.ID, "timer", "SUCCEEDED", `null`)
|
||||
if _, err := api.Continue(ctx, f.token, v.ID, string(executionID(t)), recovery.ContinueInput{Decision: recoveryDecision(t, f, v.ID), ReviewResolved: true}); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("elapsed timer extended workflow deadline", err)
|
||||
}
|
||||
applyControl(t, f, runtimeView(t, f, v.ID, "NEEDS_REVIEW"), "CANCEL")
|
||||
continueWorkflow(t, f, v.ID)
|
||||
drainStop(t, f, v.ID, "CANCELLED")
|
||||
})
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/instance"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/recovery"
|
||||
mysqldriver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
@@ -311,4 +312,67 @@ func TestIntegrationWorkflowRuntimeProcess(t *testing.T) {
|
||||
t.Fatal("unknown cancel leaked permit or started tail")
|
||||
}
|
||||
t.Log("parallel failure intent and running sibling survived another all-stop, settled FAILED only after result collection; user cancellation plus UNKNOWN recovered NEEDS_REVIEW with no successor or false terminal state")
|
||||
for _, p := range processes {
|
||||
stop(p)
|
||||
}
|
||||
grantWorkflowRecovery(f)
|
||||
reviewWorkflowTask(t, f, cancelled.ID, ref.TaskID, "FAILED")
|
||||
start("review-result-a")
|
||||
start("review-result-b")
|
||||
wait(func() bool { return f.count("weave_inbox", "workflow_id=? AND state='PENDING'", cancelled.ID) == 0 }, "manual review result consumption")
|
||||
for _, p := range processes {
|
||||
stop(p)
|
||||
}
|
||||
resolveWorkflowStep(t, f, cancelled.ID, "task0", "FAILED", `{"verified":true}`)
|
||||
continueWorkflow(t, f, cancelled.ID)
|
||||
start("review-stop-a")
|
||||
start("review-stop-b")
|
||||
wait(func() bool {
|
||||
current, e := api.Get(ctx, f.token, cancelled.ID)
|
||||
return e == nil && current.State == "CANCELLED"
|
||||
}, "reviewed cancellation resumes original stop after all-stop")
|
||||
for _, p := range processes {
|
||||
stop(p)
|
||||
}
|
||||
retrySpec := runtimeSpec(f.resources.QueueID, 1)
|
||||
retrySpec.Body.Steps[0].Task.Retry.MaxAttempts = 2
|
||||
retrySpec.Body.Steps[0].Task.Retry.InitialDelayMS = 1000
|
||||
retrySpec.Body.Steps[0].Task.Retry.MaxDelayMS = 1000
|
||||
retried := runtimeStart(t, f, retrySpec)
|
||||
grantWorkflowRecovery(f)
|
||||
ref = retryStart(t, f)
|
||||
retryFinish(t, f, ref, "FAILED", false)
|
||||
start("retry-review-a")
|
||||
start("retry-review-b")
|
||||
wait(func() bool {
|
||||
current, e := api.Get(ctx, f.token, retried.ID)
|
||||
return e == nil && current.State == "NEEDS_REVIEW"
|
||||
}, "failed step enters manual review")
|
||||
for _, p := range processes {
|
||||
stop(p)
|
||||
}
|
||||
old := f.get(ref.TaskID)
|
||||
retriedStep, err := recovery.New(NewWorkflowRecoveryStore(db)).RetryStep(ctx, f.token, retried.ID, string(executionID(t)), recovery.RetryStepInput{StepDecision: recoveryStep(t, f, retried.ID, "task0"), SafeToRetry: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
continueWorkflow(t, f, retried.ID)
|
||||
if retriedStep.Task.Retry.MaxAttempts != 1 || !retriedStep.Task.DeadlineAt.Equal(old.DeadlineAt) {
|
||||
t.Fatal("manual retry reset fixed budget")
|
||||
}
|
||||
start("manual-retry-a")
|
||||
start("manual-retry-b")
|
||||
ref = retryStart(t, f)
|
||||
if ref.TaskID != retriedStep.Task.ID {
|
||||
t.Fatal("restarted worker received wrong activation")
|
||||
}
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
wait(func() bool {
|
||||
current, e := api.Get(ctx, f.token, retried.ID)
|
||||
return e == nil && current.State == "SUCCEEDED"
|
||||
}, "new manual activation completed after restart")
|
||||
if f.count("pulse_tasks", "1=1") != 13 || f.used() != 0 || f.count("weave_activations", "workflow_id=? AND state='FAILED'", retried.ID) != 1 {
|
||||
t.Fatal("manual recovery duplicated work or rewrote old activation")
|
||||
}
|
||||
t.Log("manual execution review, explicit workflow release and new TASK activation survived all-stop and two fresh controllers; original cancellation, task deadline and remaining attempt budget preserved")
|
||||
}
|
||||
|
||||
157
internal/storage/mysql/workflow_step_retry.go
Normal file
157
internal/storage/mysql/workflow_step_retry.go
Normal file
@@ -0,0 +1,157 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/definition"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/engine"
|
||||
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/recovery"
|
||||
)
|
||||
|
||||
// RetryStep 为TASK建立新的执行激活；保留旧激活、原输入和绝对截止，跨激活扣除已用尝试。
|
||||
// POLL恢复需要同时保留原轮次身份，不在此处降级成新查询轮次。
|
||||
func (s *WorkflowRecoveryStore) RetryStep(ctx context.Context, t auth.Token, id core.ID, key string, in recovery.RetryStepInput) (recovery.Result, error) {
|
||||
return s.mutate(ctx, t, id, key, "workflow.step.retry", in, in.Decision, func(tx *sql.Tx, g *workflowGraph, _ core.ID, now time.Time, out *recovery.Result) (map[string]any, error) {
|
||||
row, err := workflowRecoveryStep(g, in.StepDecision)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if g.view.StopKind != nil || g.view.CancelRequestedAt != nil || !now.Before(g.view.DeadlineAt) {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
node, ok := g.program.Step(row.view.Key)
|
||||
if !ok || node.Task == nil {
|
||||
return nil, fault.Invalid("this retry operation currently supports TASK steps only")
|
||||
}
|
||||
if err := workflowStepSettled(ctx, tx, g, row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var sequence uint32
|
||||
if err := tx.QueryRowContext(ctx, `SELECT sequence FROM weave_activations WHERE namespace_id=? AND id=?`, t.NamespaceID, row.view.ActivationID).Scan(&sequence); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// 人工取消尚未开始的任务也不能无限创建激活；每个静态TASK的激活数不超过原max_attempts。
|
||||
if sequence == 0 || sequence >= node.Task.Retry.MaxAttempts {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
history, err := tx.QueryContext(ctx, `SELECT id,sequence,task_id FROM weave_activations WHERE namespace_id=? AND workflow_id=? AND step_id=? ORDER BY sequence LIMIT 101`, t.NamespaceID, id, row.view.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type prior struct {
|
||||
activation, task core.ID
|
||||
sequence uint32
|
||||
}
|
||||
var previous []prior
|
||||
for history.Next() {
|
||||
var v prior
|
||||
if err := history.Scan(&v.activation, &v.sequence, &v.task); err != nil {
|
||||
history.Close()
|
||||
return nil, err
|
||||
}
|
||||
previous = append(previous, v)
|
||||
}
|
||||
err = history.Err()
|
||||
history.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(previous) != int(sequence) || previous[len(previous)-1].activation != in.ActivationID {
|
||||
return nil, errors.New("workflow retry activation history incomplete")
|
||||
}
|
||||
var first, last service.TaskView
|
||||
var used uint32
|
||||
for i, prior := range previous {
|
||||
if prior.sequence != uint32(i+1) {
|
||||
return nil, errors.New("workflow retry sequence invalid")
|
||||
}
|
||||
v, err := readTask(ctx, tx, t.NamespaceID, prior.task, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if (v.State != "FAILED" && v.State != "CANCELLED") || v.EffectUncertain {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
if i == 0 {
|
||||
first = v
|
||||
if definition.ValidateData(node.Task.InputType, first.Payload) != nil || !first.DeadlineAt.Equal(minimumTime(first.CreatedAt.Add(time.Duration(node.Task.TimeoutMS)*time.Millisecond), g.view.DeadlineAt)) {
|
||||
return nil, errors.New("workflow retry initial contract or deadline mismatch")
|
||||
}
|
||||
}
|
||||
expectedPolicy := node.Task.Retry
|
||||
if used >= expectedPolicy.MaxAttempts {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
expectedPolicy.MaxAttempts -= used
|
||||
if v.Retry != expectedPolicy || v.AttemptCount > v.Retry.MaxAttempts || v.QueueID != node.Task.QueueID || v.Handler != node.Task.Handler || v.HandlerVersion != node.Task.HandlerVersion || v.PayloadVersion != node.Task.PayloadVersion || v.RecoverySafety != node.Task.RecoverySafety || v.Priority != node.Task.Priority || !sameWorkflowJSON(v.Payload, row.input) || !v.DeadlineAt.Equal(first.DeadlineAt) || v.BusinessKey != first.BusinessKey || v.ConcurrencyKey != first.ConcurrencyKey {
|
||||
return nil, errors.New("workflow retry frozen task history mismatch")
|
||||
}
|
||||
if i == 0 && v.RetryOf != nil || i > 0 && (v.RetryOf == nil || *v.RetryOf != last.ID) {
|
||||
return nil, errors.New("workflow retry task ancestry mismatch")
|
||||
}
|
||||
var attempts uint32
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_attempts WHERE namespace_id=? AND task_id=?`, t.NamespaceID, v.ID).Scan(&attempts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if attempts != v.AttemptCount {
|
||||
return nil, errors.New("workflow retry attempt history incomplete")
|
||||
}
|
||||
used += v.AttemptCount
|
||||
last = v
|
||||
}
|
||||
if used >= node.Task.Retry.MaxAttempts || !now.Before(first.DeadlineAt) || first.StartDeadlineAt != nil && !now.Before(*first.StartDeadlineAt) {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
policy := node.Task.Retry
|
||||
policy.MaxAttempts -= used
|
||||
input, err := service.NormalizeSubmit(service.SubmitInput{QueueID: first.QueueID, Handler: first.Handler, HandlerVersion: first.HandlerVersion, PayloadVersion: first.PayloadVersion, Payload: first.Payload, BusinessKey: first.BusinessKey, ConcurrencyKey: first.ConcurrencyKey, Priority: first.Priority, RecoverySafety: first.RecoverySafety, Retry: policy, TaskTimeoutMS: node.Task.TimeoutMS, StartDeadlineAt: first.StartDeadlineAt})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
activation, err := core.NewID()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
taskID, err := core.NewID()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
task, err := insertAcceptedTaskUntil(ctx, tx, t.NamespaceID, taskID, input, &last.ID, &first.DeadlineAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO weave_activations(namespace_id,id,workflow_id,step_id,sequence,state,task_id,input,created_at,updated_at) VALUES (?,?,?,?,?,'ACTIVE',?,?,?,?)`, t.NamespaceID, activation, id, row.view.ID, sequence+1, taskID, string(row.input), now, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET owner_kind=?,owner_id=?,gate_id=? WHERE namespace_id=? AND id=? AND owner_kind IS NULL`, engine.TaskOwnerKind, activation, g.gate, t.NamespaceID, taskID)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err = tx.ExecContext(ctx, `UPDATE weave_steps SET state='ACTIVE',current_activation_id=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=? AND current_activation_id=?`, activation, now, t.NamespaceID, row.view.ID, row.view.Revision, row.view.ActivationID)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.view.State, row.view.ActivationID, row.view.TaskID = weave.StepActive, &activation, &taskID
|
||||
row.view.Revision++
|
||||
row.output = nil
|
||||
if err := deleteWorkflowClock(ctx, tx, g, row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, t.NamespaceID, taskID, task.Revision, "task", "task.accepted", map[string]any{"task_id": taskID, "state": task.State, "workflow_id": id, "activation_id": activation, "retry_of": last.ID}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, t.NamespaceID, row.view.ID, row.view.Revision, "workflow_step", "workflow.step.changed", map[string]any{"workflow_id": id, "activation_id": activation, "state": row.view.State}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out.Step, out.Task = &row.view, &task
|
||||
return map[string]any{"step_id": row.view.ID, "previous_activation_id": in.ActivationID, "activation_id": activation, "sequence": sequence + 1, "retry_of": last.ID, "task_id": taskID, "safe_to_retry": true, "used_attempts": used, "remaining_attempts": policy.MaxAttempts, "deadline_at": first.DeadlineAt}, nil
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user