feat: persist workflow signals and arbitrate poll completion
This commit is contained in:
315
internal/storage/mysql/workflow_signal_runtime.go
Normal file
315
internal/storage/mysql/workflow_signal_runtime.go
Normal file
@@ -0,0 +1,315 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/jsonvalue"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/engine"
|
||||
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
|
||||
)
|
||||
|
||||
// driveWorkflowSignal 同时用于等待时钟和当前查询结果；handled=false时继续原查询分类。
|
||||
// reason为空表示内部已消费Inbox并写入审计，调用方不能再次确认同一来源。
|
||||
func (r *WorkflowRuntime) driveWorkflowSignal(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, c workflowCandidate, source *workflowResult, allow bool, now time.Time) (handled, changed bool, reason string, err error) {
|
||||
// 来源矛盾单独转核查；任何SQL故障仍回滚整个事务并保留原来源。
|
||||
if _, err := tx.ExecContext(ctx, `SAVEPOINT workflow_signal_source`); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
before := cloneWorkflowGraph(g)
|
||||
handled, changed, reason, err = r.applyWorkflowSignal(ctx, tx, g, row, c, source, allow, now)
|
||||
var invalid workflowSourceError
|
||||
if !errors.As(err, &invalid) {
|
||||
return
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `ROLLBACK TO SAVEPOINT workflow_signal_source`); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
*g = *before
|
||||
row = g.steps[row.view.Key]
|
||||
if g.view.State != weave.InstanceNeedsReview {
|
||||
if err := holdWorkflow(ctx, tx, g, weave.InstanceNeedsReview, string(invalid), now); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
}
|
||||
var current *core.ID
|
||||
if row.view.Kind == "POLL" {
|
||||
if err := tx.QueryRowContext(ctx, `SELECT current_task_id FROM weave_poll_states WHERE namespace_id=? AND workflow_id=? AND activation_id=?`, g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(¤t); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if source != nil || current == nil {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE weave_poll_states SET current_task_id=NULL,next_poll_at=NULL,resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND activation_id=?`, now, now, g.view.NamespaceID, g.view.ID, row.view.ActivationID); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
}
|
||||
}
|
||||
// 已发出的查询仍由Pulse结清，不能因信号损坏而假设外部执行结束。
|
||||
if row.view.Kind != "POLL" || source != nil || current == nil {
|
||||
if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepNeedsReview, nil, now); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE weave_signals s JOIN weave_activations a ON a.namespace_id=s.namespace_id AND a.workflow_id=s.workflow_id AND a.sequence=s.target_activation_sequence SET s.state='REJECTED',s.disposition=?,s.revision=s.revision+1,s.updated_at=? WHERE s.namespace_id=? AND s.workflow_id=? AND s.target_step_key=? AND a.id=? AND a.step_id=? AND s.state='PENDING'`, string(invalid), now, g.view.NamespaceID, g.view.ID, row.view.Key, row.view.ActivationID, row.view.ID); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if err := deleteWorkflowClock(ctx, tx, g, row); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
return true, true, string(invalid), nil
|
||||
}
|
||||
|
||||
func (r *WorkflowRuntime) applyWorkflowSignal(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, c workflowCandidate, source *workflowResult, allow bool, now time.Time) (handled, changed bool, reason string, err error) {
|
||||
w, err := readWorkflowSignalWait(ctx, tx, g, row)
|
||||
if err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
node, _ := g.program.Step(row.view.Key)
|
||||
var poll weave.PollState
|
||||
if node.Poll != nil {
|
||||
var last core.ID
|
||||
poll, last, err = readWorkflowPollState(ctx, tx, g, row, w.activated)
|
||||
if err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if err := validatePollTaskOwner(ctx, tx, g, row, last); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if _, err := pollTaskInput(ctx, tx, g, row, node.Poll.Task); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if poll.CurrentTaskID != nil {
|
||||
if *poll.CurrentTaskID != last || row.view.State != weave.StepActive || poll.NextPollAt != nil {
|
||||
return false, false, "", errors.New("signal poll current round invalid")
|
||||
}
|
||||
if err := validatePollTaskOwner(ctx, tx, g, row, last); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if source != nil && source.task != last {
|
||||
return false, false, "", nil
|
||||
}
|
||||
} else {
|
||||
if source != nil {
|
||||
return false, false, "", nil
|
||||
}
|
||||
if row.view.State != weave.StepWaiting || poll.NextPollAt == nil {
|
||||
return false, false, "", errors.New("signal poll continuation missing")
|
||||
}
|
||||
var disposition string
|
||||
if err := tx.QueryRowContext(ctx, `SELECT error_code FROM weave_inbox WHERE namespace_id=? AND workflow_id=? AND source_task_id=? AND source_activation_id=? AND state='PROCESSED' ORDER BY source_revision DESC LIMIT 1`, g.view.NamespaceID, g.view.ID, last, row.view.ActivationID).Scan(&disposition); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if disposition != "POLL_PENDING" {
|
||||
return false, false, "", errors.New("signal poll previous round not pending")
|
||||
}
|
||||
}
|
||||
}
|
||||
v, err := selectedWorkflowSignal(ctx, tx, g, row, w, now)
|
||||
if err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if v == nil {
|
||||
if node.Poll != nil {
|
||||
return false, false, "", nil
|
||||
}
|
||||
if w.state != "WAITING" {
|
||||
return false, false, "", errors.New("signal step has settled wait without winner")
|
||||
}
|
||||
if now.Before(w.deadline) {
|
||||
return true, false, "", saveWorkflowClock(ctx, tx, g, row, "SIGNAL", row.view.Revision, w.deadline)
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_waits SET state='EXPIRED',resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND state='WAITING'`, now, now, g.view.NamespaceID, w.id)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if err := rejectOtherWorkflowSignals(ctx, tx, g, row, w.sequence, "SIGNAL_DEADLINE", now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepNeedsReview, nil, now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if err := deleteWorkflowClock(ctx, tx, g, row); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
return true, true, "SIGNAL_DEADLINE", holdWorkflow(ctx, tx, g, weave.InstanceNeedsReview, "SIGNAL_DEADLINE", now)
|
||||
}
|
||||
if node.Poll != nil && source == nil && poll.CurrentTaskID != nil {
|
||||
// 已落库查询结果必须按原事件时间参与竞争，不能由扫描先后决定赢家。
|
||||
task, err := readTask(ctx, tx, g.view.NamespaceID, *poll.CurrentTaskID, false)
|
||||
if err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if task.QueueID != node.Poll.Task.QueueID {
|
||||
return false, false, "", errors.New("signal query queue invalid")
|
||||
}
|
||||
if task.State.Terminal() || task.State == "NEEDS_REVIEW" {
|
||||
pending := workflowCandidate{kind: "INBOX", ns: g.view.NamespaceID, workflow: g.view.ID}
|
||||
var due time.Time
|
||||
if err := tx.QueryRowContext(ctx, `SELECT id,revision,available_at FROM weave_inbox WHERE namespace_id=? AND workflow_id=? AND source_task_id=? AND source_activation_id=? AND state='PENDING' ORDER BY source_revision DESC LIMIT 1`, g.view.NamespaceID, g.view.ID, poll.CurrentTaskID, row.view.ActivationID).Scan(&pending.id, &pending.revision, &due); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if due.After(now) {
|
||||
return true, false, "", saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision, due)
|
||||
}
|
||||
done, err := r.consume(ctx, tx, g, pending, allow, now)
|
||||
return true, done, "", err
|
||||
}
|
||||
}
|
||||
if node.Poll != nil && source != nil && w.winner == nil && source.snapshot.State == "SUCCEEDED" && !source.snapshot.EffectUncertain && source.occurred.Before(w.deadline) && !source.occurred.After(v.AcceptedAt) {
|
||||
outcome, err := g.program.ClassifyPoll(row.view.Key, source.snapshot.Result)
|
||||
if err == nil && outcome != engine.PollPending {
|
||||
return false, false, "", nil
|
||||
} // 较早的最终查询先胜出；PENDING不争夺终结权。
|
||||
}
|
||||
newWinner := w.winner == nil
|
||||
if err := claimWorkflowSignal(ctx, tx, g, &w, v, now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if err := rejectOtherWorkflowSignals(ctx, tx, g, row, w.sequence, "WAIT_SETTLED", now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if node.Poll != nil && poll.CurrentTaskID != nil && source == nil {
|
||||
task, err := readExecutionTask(ctx, tx, g.view.NamespaceID, *poll.CurrentTaskID)
|
||||
if err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if _, err := cancelExecutionTask(ctx, tx, g.view.NamespaceID, task, now, "WORKFLOW_SIGNAL_WON"); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if !now.Before(poll.DeadlineAt) && g.view.State != weave.InstanceNeedsReview {
|
||||
if err := holdWorkflow(ctx, tx, g, weave.InstanceNeedsReview, "POLL_SIGNAL_INFLIGHT_DEADLINE", now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
newWinner = true
|
||||
}
|
||||
if err := saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision, now.Add(time.Second)); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if newWinner {
|
||||
return true, true, "POLL_SIGNAL_SELECTED", nil
|
||||
}
|
||||
return true, false, "", nil
|
||||
}
|
||||
state, output, reason := weave.StepSucceeded, v.Payload, "SIGNAL_CONSUMED"
|
||||
if node.Poll != nil {
|
||||
outcome, err := g.program.ClassifyPoll(row.view.Key, v.Payload)
|
||||
if err != nil || outcome == engine.PollPending {
|
||||
return true, false, "", workflowSourceError("POLL_FINAL_SIGNAL_INVALID")
|
||||
}
|
||||
switch outcome {
|
||||
case engine.PollFailed:
|
||||
state, output, reason = weave.StepFailed, nil, "POLL_SIGNAL_FAILED"
|
||||
case engine.PollNeedsReview:
|
||||
state, output, reason = weave.StepNeedsReview, nil, "POLL_SIGNAL_REVIEW"
|
||||
default:
|
||||
reason = "POLL_SIGNAL_SUCCEEDED"
|
||||
}
|
||||
if source != nil {
|
||||
if source.snapshot.EffectUncertain || source.snapshot.State == "NEEDS_REVIEW" {
|
||||
state, output, reason = weave.StepNeedsReview, nil, "POLL_SIGNAL_QUERY_UNCERTAIN"
|
||||
} else if source.snapshot.State == "SUCCEEDED" {
|
||||
query, err := g.program.ClassifyPoll(row.view.Key, source.snapshot.Result)
|
||||
if err != nil {
|
||||
state, output, reason = weave.StepNeedsReview, nil, "POLL_SIGNAL_QUERY_INVALID"
|
||||
} else if query != engine.PollPending {
|
||||
left, err := jsonvalue.Canonical(source.snapshot.Result)
|
||||
if err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
right, err := jsonvalue.Canonical(v.Payload)
|
||||
if err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if !bytes.Equal(left, right) {
|
||||
state, output, reason = weave.StepNeedsReview, nil, "POLL_SIGNAL_QUERY_CONFLICT"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_poll_states SET current_task_id=NULL,next_poll_at=NULL,resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND resolved_at IS NULL`, now, now, g.view.NamespaceID, poll.ID)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
}
|
||||
if err := saveWorkflowResultStep(ctx, tx, g, row, state, output, now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if err := deleteWorkflowClock(ctx, tx, g, row); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if state == weave.StepNeedsReview || state == weave.StepFailed {
|
||||
next := weave.InstanceNeedsReview
|
||||
if state == weave.StepFailed && (g.version.Specification.FailurePolicy == "FAIL_AFTER_SETTLED" || g.view.CancelRequestedAt != nil) {
|
||||
next = weave.InstanceCancelling
|
||||
}
|
||||
if err := holdWorkflow(ctx, tx, g, next, reason, now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
} else if err := r.progress(ctx, tx, g, allow, now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
return true, true, reason, nil
|
||||
}
|
||||
|
||||
func (r *WorkflowRuntime) advanceSignalClock(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, c workflowCandidate, allow bool, now time.Time) (bool, error) {
|
||||
handled, changed, reason, err := r.driveWorkflowSignal(ctx, tx, g, row, c, nil, allow, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !handled {
|
||||
return false, errors.New("signal clock did not identify a wait")
|
||||
}
|
||||
if changed && reason != "" {
|
||||
return true, r.audit(ctx, tx, g, c.id, reason, now)
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// expireWorkflowSignal 对早到但未匹配的信号使用总截止索引；已按时匹配的信号仍先完成实际消费。
|
||||
func (r *WorkflowRuntime) expireWorkflowSignal(ctx context.Context, tx *sql.Tx, g *workflowGraph, c workflowCandidate, allow bool, now time.Time) (bool, error) {
|
||||
var due time.Time
|
||||
if err := tx.QueryRowContext(ctx, `SELECT available_at FROM weave_signals WHERE namespace_id=? AND workflow_id=? AND id=?`, c.ns, c.workflow, c.id).Scan(&due); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if due.After(now) {
|
||||
return false, nil
|
||||
}
|
||||
v, err := readWorkflowSignal(ctx, tx, c.ns, c.workflow, c.id)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if v.State != "PENDING" || now.Before(v.ExpiresAt) {
|
||||
return false, nil
|
||||
}
|
||||
row := g.steps[v.TargetStepKey]
|
||||
if row != nil && row.view.ActivationID != nil && !g.view.State.Terminal() && (row.view.State == weave.StepWaiting || row.view.State == weave.StepActive) {
|
||||
w, err := readWorkflowSignalWait(ctx, tx, g, row)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if w.sequence == v.TargetActivationSequence && w.correlation == v.CorrelationKey && w.contract.Name == v.Name && w.contract.PayloadVersion == v.PayloadVersion && v.AcceptedAt.Before(w.deadline) {
|
||||
handled, changed, reason, err := r.driveWorkflowSignal(ctx, tx, g, row, c, nil, allow, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if handled {
|
||||
// 若匹配到另一条更早的信号或正等待结果退避，避免过期索引持续抢占批次。
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE weave_signals SET available_at=DATE_ADD(?,INTERVAL 30 SECOND),revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND state='PENDING'`, now, now, c.ns, c.id); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if changed && reason != "" {
|
||||
return true, r.audit(ctx, tx, g, c.id, reason, now)
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_signals SET state='EXPIRED',disposition='UNMATCHED_DEADLINE',revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND state='PENDING' AND revision=?`, now, c.ns, c.id, v.Revision)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, r.audit(ctx, tx, g, c.id, "SIGNAL_EXPIRED", now)
|
||||
}
|
||||
Reference in New Issue
Block a user