feat: add audited workflow review and bounded task recovery
This commit is contained in:
157
internal/storage/mysql/workflow_step_retry.go
Normal file
157
internal/storage/mysql/workflow_step_retry.go
Normal file
@@ -0,0 +1,157 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/definition"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/engine"
|
||||
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/recovery"
|
||||
)
|
||||
|
||||
// RetryStep 为TASK建立新的执行激活；保留旧激活、原输入和绝对截止，跨激活扣除已用尝试。
|
||||
// POLL恢复需要同时保留原轮次身份，不在此处降级成新查询轮次。
|
||||
func (s *WorkflowRecoveryStore) RetryStep(ctx context.Context, t auth.Token, id core.ID, key string, in recovery.RetryStepInput) (recovery.Result, error) {
|
||||
return s.mutate(ctx, t, id, key, "workflow.step.retry", in, in.Decision, func(tx *sql.Tx, g *workflowGraph, _ core.ID, now time.Time, out *recovery.Result) (map[string]any, error) {
|
||||
row, err := workflowRecoveryStep(g, in.StepDecision)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if g.view.StopKind != nil || g.view.CancelRequestedAt != nil || !now.Before(g.view.DeadlineAt) {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
node, ok := g.program.Step(row.view.Key)
|
||||
if !ok || node.Task == nil {
|
||||
return nil, fault.Invalid("this retry operation currently supports TASK steps only")
|
||||
}
|
||||
if err := workflowStepSettled(ctx, tx, g, row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var sequence uint32
|
||||
if err := tx.QueryRowContext(ctx, `SELECT sequence FROM weave_activations WHERE namespace_id=? AND id=?`, t.NamespaceID, row.view.ActivationID).Scan(&sequence); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// 人工取消尚未开始的任务也不能无限创建激活；每个静态TASK的激活数不超过原max_attempts。
|
||||
if sequence == 0 || sequence >= node.Task.Retry.MaxAttempts {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
history, err := tx.QueryContext(ctx, `SELECT id,sequence,task_id FROM weave_activations WHERE namespace_id=? AND workflow_id=? AND step_id=? ORDER BY sequence LIMIT 101`, t.NamespaceID, id, row.view.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type prior struct {
|
||||
activation, task core.ID
|
||||
sequence uint32
|
||||
}
|
||||
var previous []prior
|
||||
for history.Next() {
|
||||
var v prior
|
||||
if err := history.Scan(&v.activation, &v.sequence, &v.task); err != nil {
|
||||
history.Close()
|
||||
return nil, err
|
||||
}
|
||||
previous = append(previous, v)
|
||||
}
|
||||
err = history.Err()
|
||||
history.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(previous) != int(sequence) || previous[len(previous)-1].activation != in.ActivationID {
|
||||
return nil, errors.New("workflow retry activation history incomplete")
|
||||
}
|
||||
var first, last service.TaskView
|
||||
var used uint32
|
||||
for i, prior := range previous {
|
||||
if prior.sequence != uint32(i+1) {
|
||||
return nil, errors.New("workflow retry sequence invalid")
|
||||
}
|
||||
v, err := readTask(ctx, tx, t.NamespaceID, prior.task, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if (v.State != "FAILED" && v.State != "CANCELLED") || v.EffectUncertain {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
if i == 0 {
|
||||
first = v
|
||||
if definition.ValidateData(node.Task.InputType, first.Payload) != nil || !first.DeadlineAt.Equal(minimumTime(first.CreatedAt.Add(time.Duration(node.Task.TimeoutMS)*time.Millisecond), g.view.DeadlineAt)) {
|
||||
return nil, errors.New("workflow retry initial contract or deadline mismatch")
|
||||
}
|
||||
}
|
||||
expectedPolicy := node.Task.Retry
|
||||
if used >= expectedPolicy.MaxAttempts {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
expectedPolicy.MaxAttempts -= used
|
||||
if v.Retry != expectedPolicy || v.AttemptCount > v.Retry.MaxAttempts || v.QueueID != node.Task.QueueID || v.Handler != node.Task.Handler || v.HandlerVersion != node.Task.HandlerVersion || v.PayloadVersion != node.Task.PayloadVersion || v.RecoverySafety != node.Task.RecoverySafety || v.Priority != node.Task.Priority || !sameWorkflowJSON(v.Payload, row.input) || !v.DeadlineAt.Equal(first.DeadlineAt) || v.BusinessKey != first.BusinessKey || v.ConcurrencyKey != first.ConcurrencyKey {
|
||||
return nil, errors.New("workflow retry frozen task history mismatch")
|
||||
}
|
||||
if i == 0 && v.RetryOf != nil || i > 0 && (v.RetryOf == nil || *v.RetryOf != last.ID) {
|
||||
return nil, errors.New("workflow retry task ancestry mismatch")
|
||||
}
|
||||
var attempts uint32
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_attempts WHERE namespace_id=? AND task_id=?`, t.NamespaceID, v.ID).Scan(&attempts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if attempts != v.AttemptCount {
|
||||
return nil, errors.New("workflow retry attempt history incomplete")
|
||||
}
|
||||
used += v.AttemptCount
|
||||
last = v
|
||||
}
|
||||
if used >= node.Task.Retry.MaxAttempts || !now.Before(first.DeadlineAt) || first.StartDeadlineAt != nil && !now.Before(*first.StartDeadlineAt) {
|
||||
return nil, fault.Conflict
|
||||
}
|
||||
policy := node.Task.Retry
|
||||
policy.MaxAttempts -= used
|
||||
input, err := service.NormalizeSubmit(service.SubmitInput{QueueID: first.QueueID, Handler: first.Handler, HandlerVersion: first.HandlerVersion, PayloadVersion: first.PayloadVersion, Payload: first.Payload, BusinessKey: first.BusinessKey, ConcurrencyKey: first.ConcurrencyKey, Priority: first.Priority, RecoverySafety: first.RecoverySafety, Retry: policy, TaskTimeoutMS: node.Task.TimeoutMS, StartDeadlineAt: first.StartDeadlineAt})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
activation, err := core.NewID()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
taskID, err := core.NewID()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
task, err := insertAcceptedTaskUntil(ctx, tx, t.NamespaceID, taskID, input, &last.ID, &first.DeadlineAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO weave_activations(namespace_id,id,workflow_id,step_id,sequence,state,task_id,input,created_at,updated_at) VALUES (?,?,?,?,?,'ACTIVE',?,?,?,?)`, t.NamespaceID, activation, id, row.view.ID, sequence+1, taskID, string(row.input), now, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET owner_kind=?,owner_id=?,gate_id=? WHERE namespace_id=? AND id=? AND owner_kind IS NULL`, engine.TaskOwnerKind, activation, g.gate, t.NamespaceID, taskID)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err = tx.ExecContext(ctx, `UPDATE weave_steps SET state='ACTIVE',current_activation_id=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=? AND current_activation_id=?`, activation, now, t.NamespaceID, row.view.ID, row.view.Revision, row.view.ActivationID)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.view.State, row.view.ActivationID, row.view.TaskID = weave.StepActive, &activation, &taskID
|
||||
row.view.Revision++
|
||||
row.output = nil
|
||||
if err := deleteWorkflowClock(ctx, tx, g, row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, t.NamespaceID, taskID, task.Revision, "task", "task.accepted", map[string]any{"task_id": taskID, "state": task.State, "workflow_id": id, "activation_id": activation, "retry_of": last.ID}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, t.NamespaceID, row.view.ID, row.view.Revision, "workflow_step", "workflow.step.changed", map[string]any{"workflow_id": id, "activation_id": activation, "state": row.view.State}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out.Step, out.Task = &row.view, &task
|
||||
return map[string]any{"step_id": row.view.ID, "previous_activation_id": in.ActivationID, "activation_id": activation, "sequence": sequence + 1, "retry_of": last.ID, "task_id": taskID, "safe_to_retry": true, "used_attempts": used, "remaining_attempts": policy.MaxAttempts, "deadline_at": first.DeadlineAt}, nil
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user