feat: persist workflow signals and arbitrate poll completion
This commit is contained in:
@@ -64,6 +64,8 @@ func TestIntegrationApplicationResourcesUpgrade(t *testing.T) {
|
||||
retryFinish(t, g, finished, "SUCCEEDED", false)
|
||||
assertApplicationCounts(t, f, 4, 2, 1)
|
||||
// 还原后续新增表和字段，得到真实 v9 结构；现有任务内容保持不变。
|
||||
f.exec(`ALTER TABLE weave_signals DROP CHECK ck_signal_version_sequence, DROP CHECK ck_signal_pending_identity, DROP INDEX ix_signal_expiry, DROP INDEX ix_signal_activation, DROP COLUMN payload_version, DROP COLUMN target_activation_sequence, DROP COLUMN evidence_ref, DROP COLUMN disposition, DROP COLUMN available_at, MODIFY COLUMN type VARCHAR(64) NOT NULL COMMENT '事件或信号的稳定类型标识', MODIFY COLUMN correlation_key VARCHAR(191) NOT NULL COMMENT '外部业务操作关联键，防止旧信号误用于新工作'`)
|
||||
f.exec(`ALTER TABLE weave_waits DROP INDEX uq_wait_winner_signal, DROP COLUMN winner_signal_id`)
|
||||
f.exec(`ALTER TABLE weave_waits DROP COLUMN signal_name, DROP COLUMN payload_version, MODIFY COLUMN correlation_key VARCHAR(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_as_cs NOT NULL COMMENT '外部业务操作关联键，防止旧信号误用于新工作'`)
|
||||
f.exec(`ALTER TABLE weave_instances MODIFY COLUMN business_key VARCHAR(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_as_cs NULL COMMENT '业务侧关联键，用于检索追踪，不自动提供业务幂等'`)
|
||||
f.exec(`ALTER TABLE weave_inbox DROP CHECK ck_weave_inbox_source, DROP INDEX uq_weave_task_result, DROP COLUMN source_task_id, DROP COLUMN source_activation_id, DROP COLUMN source_revision, DROP COLUMN snapshot`)
|
||||
|
||||
82
internal/storage/mysql/execution_cancel.go
Normal file
82
internal/storage/mysql/execution_cancel.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
)
|
||||
|
||||
// cancelExecutionTask 由已验证所有权的调用方复用；调用前必须持有应用、队列和任务锁。
|
||||
// 仅已知未开始的任务立即结清，在途执行保留许可并等待结果，历史未知效果不能变成已取消。
|
||||
func cancelExecutionTask(ctx context.Context, tx *sql.Tx, ns core.ID, task executionTask, now time.Time, reason string) (service.TaskView, error) {
|
||||
var out service.TaskView
|
||||
view := task.view
|
||||
if view.State.Terminal() || view.CancelRequestedAt != nil {
|
||||
return view, nil
|
||||
}
|
||||
id := view.ID
|
||||
var err error
|
||||
var attempt *pulse.Attempt
|
||||
if view.State == pulse.TaskLeased || view.State == pulse.TaskRunning {
|
||||
if task.current == nil {
|
||||
return out, errors.New("active task lacks attempt")
|
||||
}
|
||||
a, err := readAttempt(ctx, tx, ns, *task.current)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if a.TaskID != id || a.Epoch != task.epoch || string(a.State) != string(view.State) {
|
||||
return out, errors.New("task attempt identity or state mismatch")
|
||||
}
|
||||
attempt = &a
|
||||
}
|
||||
next := view.State
|
||||
lease := task.lease
|
||||
var completed *time.Time
|
||||
if view.State == pulse.TaskWaiting || view.State == pulse.TaskLeased {
|
||||
next = pulse.TaskCancelled
|
||||
completed = &now
|
||||
lease = nil
|
||||
if view.EffectUncertain {
|
||||
next, completed = pulse.TaskNeedsReview, nil
|
||||
}
|
||||
if attempt != nil {
|
||||
if attempt.AuthorizedAt != nil {
|
||||
return out, errors.New("leased attempt already authorized")
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state='CANCELLED',ended_at=?,error_code='CANCELLED_BEFORE_START',revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, now, now, ns, attempt.ID, attempt.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if err := releasePermits(ctx, tx, ns, attempt.ID, now); err != nil {
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,cancel_requested_at=?,lease_until=?,completed_at=?,updated_at=?,revision=revision+1 WHERE namespace_id=? AND id=? AND revision=?`, next, now, lease, completed, now, ns, id, view.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if next.Terminal() {
|
||||
if err := releaseQueueTask(ctx, tx, ns, view.QueueID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
out, err = readTask(ctx, tx, ns, id, false)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
event := resultEvent(next)
|
||||
if next == view.State {
|
||||
event = "task.cancel_requested"
|
||||
}
|
||||
if err := emitEvent(ctx, tx, ns, id, out.Revision, "task", event, map[string]any{"task_id": id, "state": next, "reason": reason, "effect_uncertain": view.EffectUncertain}); err != nil {
|
||||
return out, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
-- 外部信号固定参数版本与目标激活代次；旧记录无此身份，不能推测为新等待的有效输入。
|
||||
ALTER TABLE weave_signals
|
||||
MODIFY COLUMN `type` VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL COMMENT '固定定义中的信号名称，新受理限制为128个ASCII字符',
|
||||
MODIFY COLUMN correlation_key VARBINARY(764) NOT NULL COMMENT '精确UTF8操作关联，保留尾部空格；新信号最多512字节，历史兼容764字节',
|
||||
ADD COLUMN payload_version VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL COMMENT '信号内容契约版本；为空仅表示未核实的历史记录',
|
||||
ADD COLUMN target_activation_sequence INT UNSIGNED NULL COMMENT '固定目标步骤激活序号，早到首轮为1；为空仅表示未核实历史',
|
||||
ADD COLUMN evidence_ref VARCHAR(512) NULL COMMENT '可选的受控证据引用；不保存秘密，也不由平台请求该地址',
|
||||
ADD COLUMN disposition VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NULL COMMENT '稳定消费或拒绝原因；PENDING时为空，不包含业务参数',
|
||||
ADD COLUMN available_at DATETIME(6) NULL COMMENT '未匹配信号下次到期检查时间；故障退避可延后检查，但不改变原截止',
|
||||
ADD KEY ix_signal_expiry (state,available_at,namespace_id,id),
|
||||
ADD KEY ix_signal_activation (namespace_id,workflow_id,target_step_key,target_activation_sequence,state,accepted_at,id);
|
||||
|
||||
UPDATE weave_signals SET state='REJECTED',disposition='LEGACY_UNVERIFIED',revision=revision+1,updated_at=UTC_TIMESTAMP(6) WHERE state='PENDING';
|
||||
UPDATE weave_signals SET available_at=expires_at;
|
||||
|
||||
ALTER TABLE weave_signals
|
||||
MODIFY COLUMN available_at DATETIME(6) NOT NULL COMMENT '未匹配信号下次到期检查时间；故障退避可延后检查，但不改变原截止',
|
||||
ADD CONSTRAINT ck_signal_version_sequence CHECK ((payload_version IS NULL AND target_activation_sequence IS NULL) OR (payload_version IS NOT NULL AND payload_version<>'' AND target_activation_sequence IS NOT NULL AND target_activation_sequence>0)),
|
||||
ADD CONSTRAINT ck_signal_pending_identity CHECK (state<>'PENDING' OR (payload_version IS NOT NULL AND target_activation_sequence IS NOT NULL AND disposition IS NULL));
|
||||
|
||||
ALTER TABLE weave_waits
|
||||
ADD COLUMN winner_signal_id BINARY(16) NULL COMMENT '胜出的外部信号标识；仅时间或查询结果结清时为空，所有者由同事务校验',
|
||||
ADD UNIQUE KEY uq_wait_winner_signal (namespace_id,winner_signal_id);
|
||||
|
||||
-- SIGNAL到期扫描与TIMER/POLL共用待续表；旧元数据缺失的等待交给运行器核查，不凭空授权。
|
||||
INSERT INTO weave_pending_advances(namespace_id,id,workflow_id,step_id,expected_step_revision,checkpoint,available_at)
|
||||
SELECT s.namespace_id,RANDOM_BYTES(16),s.workflow_id,s.id,s.revision,
|
||||
JSON_OBJECT('schema_version',1,'kind','SIGNAL','activation_id',LOWER(HEX(a.id))),w.deadline_at
|
||||
FROM weave_steps s JOIN weave_activations a ON a.namespace_id=s.namespace_id AND a.id=s.current_activation_id AND a.workflow_id=s.workflow_id AND a.step_id=s.id
|
||||
JOIN weave_instances i ON i.namespace_id=s.namespace_id AND i.id=s.workflow_id
|
||||
JOIN weave_waits w ON w.namespace_id=s.namespace_id AND w.workflow_id=s.workflow_id AND w.activation_id=a.id
|
||||
WHERE s.kind='SIGNAL' AND s.state='WAITING' AND a.state='WAITING' AND w.kind='SIGNAL' AND w.state='WAITING' AND i.state IN ('RUNNING','WAITING','CANCELLING','NEEDS_REVIEW')
|
||||
ON DUPLICATE KEY UPDATE id=weave_pending_advances.id;
|
||||
@@ -88,8 +88,8 @@ func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
|
||||
}
|
||||
indexes := func() []string {
|
||||
t.Helper()
|
||||
// v15替换retry_of索引、v17增加内部结果唯一索引，下方单独验证定义。
|
||||
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
|
||||
// v15替换retry_of索引、v17增加内部结果唯一索引、v20增加信号索引，下方单独验证定义。
|
||||
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result','ix_signal_expiry','ix_signal_activation','uq_wait_winner_signal') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -178,6 +178,16 @@ func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
|
||||
t.Fatalf("workflow result uniqueness=%q err=%v", resultIndex, err)
|
||||
}
|
||||
var collation string
|
||||
for _, index := range []struct{ table, name, want string }{
|
||||
{"weave_signals", "ix_signal_expiry", "state:1,available_at:1,namespace_id:1,id:1"},
|
||||
{"weave_signals", "ix_signal_activation", "namespace_id:1,workflow_id:1,target_step_key:1,target_activation_sequence:1,state:1,accepted_at:1,id:1"},
|
||||
{"weave_waits", "uq_wait_winner_signal", "namespace_id:0,winner_signal_id:0"},
|
||||
} {
|
||||
var got string
|
||||
if err := db.QueryRowContext(ctx, `SELECT GROUP_CONCAT(CONCAT(COLUMN_NAME,':',NON_UNIQUE) ORDER BY SEQ_IN_INDEX) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND INDEX_NAME=?`, index.table, index.name).Scan(&got); err != nil || got != index.want {
|
||||
t.Fatalf("signal index %s=%q err=%v", index.name, got, err)
|
||||
}
|
||||
}
|
||||
if err := db.QueryRowContext(ctx, `SELECT COLLATION_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='pulse_tasks' AND COLUMN_NAME='business_key'`).Scan(&collation); err != nil || collation != "utf8mb4_0900_bin" {
|
||||
t.Fatalf("business key collation=%q err=%v", collation, err)
|
||||
}
|
||||
|
||||
@@ -4,14 +4,11 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
)
|
||||
|
||||
@@ -56,68 +53,15 @@ func (s *TaskStore) CancelTask(ctx context.Context, token auth.Token, id core.ID
|
||||
if owner.Valid && owner.String != "schedule" {
|
||||
return fault.Conflict
|
||||
}
|
||||
var attempt *pulse.Attempt
|
||||
if view.State == pulse.TaskLeased || view.State == pulse.TaskRunning {
|
||||
if task.current == nil {
|
||||
return errors.New("active task lacks attempt")
|
||||
}
|
||||
a, err := readAttempt(ctx, tx, token.NamespaceID, *task.current)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if a.TaskID != id || a.Epoch != task.epoch || string(a.State) != string(view.State) {
|
||||
return errors.New("task attempt identity or state mismatch")
|
||||
}
|
||||
attempt = &a
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := view.State
|
||||
lease := task.lease
|
||||
var completed *time.Time
|
||||
if view.State == pulse.TaskWaiting || view.State == pulse.TaskLeased {
|
||||
next = pulse.TaskCancelled
|
||||
completed = &now
|
||||
lease = nil
|
||||
if view.EffectUncertain {
|
||||
next, completed = pulse.TaskNeedsReview, nil
|
||||
}
|
||||
if attempt != nil {
|
||||
if attempt.AuthorizedAt != nil {
|
||||
return errors.New("leased attempt already authorized")
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state='CANCELLED',ended_at=?,error_code='CANCELLED_BEFORE_START',revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, now, now, token.NamespaceID, attempt.ID, attempt.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := releasePermits(ctx, tx, token.NamespaceID, attempt.ID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,cancel_requested_at=?,lease_until=?,completed_at=?,updated_at=?,revision=revision+1 WHERE namespace_id=? AND id=? AND revision=?`, next, now, lease, completed, now, token.NamespaceID, id, in.ExpectedRevision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
if next.Terminal() {
|
||||
if err := releaseQueueTask(ctx, tx, token.NamespaceID, view.QueueID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
out.Task, err = readTask(ctx, tx, token.NamespaceID, id, false)
|
||||
out.Task, err = cancelExecutionTask(ctx, tx, token.NamespaceID, task, now, "CANCEL_REQUESTED")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
event := resultEvent(next)
|
||||
if next == view.State {
|
||||
event = "task.cancel_requested"
|
||||
}
|
||||
if err := emitEvent(ctx, tx, token.NamespaceID, id, out.Task.Revision, "task", event, map[string]any{"task_id": id, "state": next, "reason": "CANCEL_REQUESTED", "effect_uncertain": view.EffectUncertain}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := appendAudit(ctx, tx, token, identity, id, "task", "task.cancel", in.Reason, map[string]any{"state": view.State, "revision": fmt.Sprint(view.Revision)}, map[string]any{"state": next, "revision": fmt.Sprint(out.Task.Revision), "cancel_requested_at": now}); err != nil {
|
||||
if err := appendAudit(ctx, tx, token, identity, id, "task", "task.cancel", in.Reason, map[string]any{"state": view.State, "revision": fmt.Sprint(view.Revision)}, map[string]any{"state": out.Task.State, "revision": fmt.Sprint(out.Task.Revision), "cancel_requested_at": now}); err != nil {
|
||||
return err
|
||||
}
|
||||
return saveIdentity(ctx, tx, token.NamespaceID, identity, out)
|
||||
|
||||
@@ -255,6 +255,11 @@ func activateWorkflowStep(ctx context.Context, tx *sql.Tx, g *workflowGraph, row
|
||||
if err := saveWorkflowClock(ctx, tx, g, row, "TIMER", row.view.Revision+1, until); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if c.Wait.Kind == "SIGNAL" {
|
||||
// 先扫描早到信号；未匹配时运行器按原等待截止继续持久化。
|
||||
if err := saveWorkflowClock(ctx, tx, g, row, "SIGNAL", row.view.Revision+1, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if c.Task != nil {
|
||||
@@ -274,7 +279,11 @@ func activateWorkflowStep(ctx context.Context, tx *sql.Tx, g *workflowGraph, row
|
||||
}
|
||||
}
|
||||
if c.Kind == "POLL" {
|
||||
return saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision+1, deadline)
|
||||
due := deadline
|
||||
if c.Wait != nil {
|
||||
due = now
|
||||
}
|
||||
return saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision+1, due)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
// workflowCheckpoint 是待续表中版本化的内部唤醒身份；不保存内存执行栈或新的业务预算。
|
||||
type workflowCheckpoint struct {
|
||||
SchemaVersion uint32 `json:"schema_version"` // SchemaVersion 当前结构版本为1，未知版本禁止推进。
|
||||
Kind string `json:"kind"` // Kind 为ROOT、TIMER或POLL，决定实际权威记录的读取方式。
|
||||
Kind string `json:"kind"` // Kind 为ROOT、TIMER、SIGNAL或POLL，决定实际权威记录的读取方式。
|
||||
ActivationID *core.ID `json:"activation_id,omitempty"` // ActivationID 固定等待激活；ROOT不使用该字段。
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ func saveWorkflowClock(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *w
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if row.view.ActivationID == nil || kind != "TIMER" && kind != "POLL" {
|
||||
if row.view.ActivationID == nil || kind != "TIMER" && kind != "POLL" && kind != "SIGNAL" {
|
||||
return errors.New("workflow clock identity invalid")
|
||||
}
|
||||
raw, err := json.Marshal(workflowCheckpoint{1, kind, row.view.ActivationID})
|
||||
@@ -60,8 +60,24 @@ func (r *WorkflowRuntime) advanceClock(ctx context.Context, tx *sql.Tx, g *workf
|
||||
return false, err
|
||||
}
|
||||
if checkpoint.Kind == "POLL" {
|
||||
node, _ := g.program.Step(row.view.Key)
|
||||
if node.Poll.FinalSignal != nil {
|
||||
handled, changed, reason, err := r.driveWorkflowSignal(ctx, tx, g, row, c, nil, allow, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if handled {
|
||||
if changed && reason != "" {
|
||||
return true, r.audit(ctx, tx, g, c.id, reason, now)
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
}
|
||||
return r.advancePoll(ctx, tx, g, row, c, activated, allow, now)
|
||||
}
|
||||
if checkpoint.Kind == "SIGNAL" {
|
||||
return r.advanceSignalClock(ctx, tx, g, row, c, allow, now)
|
||||
}
|
||||
if checkpoint.Kind != "TIMER" || row.view.State != weave.StepWaiting {
|
||||
return false, errors.New("workflow timer state invalid")
|
||||
}
|
||||
|
||||
@@ -177,6 +177,15 @@ func (r *WorkflowRuntime) consume(ctx context.Context, tx *sql.Tx, g *workflowGr
|
||||
if resolved != nil || current == nil || *current != source.task {
|
||||
return true, r.confirmWorkflowResult(ctx, tx, g, c, "PROCESSED", "STALE_POLL_ROUND", now)
|
||||
}
|
||||
if node.Poll.FinalSignal != nil {
|
||||
handled, _, reason, err := r.driveWorkflowSignal(ctx, tx, g, row, c, source, allow, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if handled {
|
||||
return true, r.confirmWorkflowResult(ctx, tx, g, c, "PROCESSED", reason, now)
|
||||
}
|
||||
}
|
||||
// 单轮请求失败不能通过新建业务轮次绕过Pulse预算。
|
||||
if state != weave.StepSucceeded {
|
||||
state, reason = weave.StepNeedsReview, "POLL_REQUEST_REVIEW"
|
||||
@@ -216,6 +225,15 @@ func (r *WorkflowRuntime) consume(ctx context.Context, tx *sql.Tx, g *workflowGr
|
||||
return false, err
|
||||
}
|
||||
if settled != nil {
|
||||
if node.Poll.FinalSignal != nil {
|
||||
w, err := readWorkflowSignalWait(ctx, tx, g, row)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := rejectOtherWorkflowSignals(ctx, tx, g, row, w.sequence, "QUERY_SETTLED", now); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
waitState := "CANCELLED"
|
||||
if state == weave.StepSucceeded {
|
||||
waitState = "SATISFIED"
|
||||
|
||||
@@ -16,31 +16,42 @@ import (
|
||||
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
|
||||
)
|
||||
|
||||
func (r *WorkflowRuntime) advancePoll(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, c workflowCandidate, activated time.Time, allow bool, now time.Time) (bool, error) {
|
||||
// readWorkflowPollState 校验当前激活的完整轮次与原预算；时钟和最终信号共用相同检查。
|
||||
func readWorkflowPollState(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, activated time.Time) (weave.PollState, core.ID, error) {
|
||||
var p weave.PollState
|
||||
err := tx.QueryRowContext(ctx, `SELECT id,revision,max_rounds,used_rounds,max_attempts_per_round,interval_ms,deadline_at,next_poll_at,current_task_id,resolved_at FROM weave_poll_states WHERE namespace_id=? AND workflow_id=? AND activation_id=? FOR UPDATE`, c.ns, c.workflow, row.view.ActivationID).Scan(&p.ID, &p.Revision, &p.MaxRounds, &p.UsedRounds, &p.MaxAttemptsPerRound, &p.IntervalMS, &p.DeadlineAt, &p.NextPollAt, &p.CurrentTaskID, &p.ResolvedAt)
|
||||
err := tx.QueryRowContext(ctx, `SELECT id,revision,max_rounds,used_rounds,max_attempts_per_round,interval_ms,deadline_at,next_poll_at,current_task_id,resolved_at FROM weave_poll_states WHERE namespace_id=? AND workflow_id=? AND activation_id=? FOR UPDATE`, g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(&p.ID, &p.Revision, &p.MaxRounds, &p.UsedRounds, &p.MaxAttemptsPerRound, &p.IntervalMS, &p.DeadlineAt, &p.NextPollAt, &p.CurrentTaskID, &p.ResolvedAt)
|
||||
if err != nil {
|
||||
return p, "", err
|
||||
}
|
||||
node, _ := g.program.Step(row.view.Key)
|
||||
policy := node.Poll
|
||||
if policy == nil || p.MaxRounds != policy.MaxRounds || p.MaxAttemptsPerRound != policy.Task.Retry.MaxAttempts || p.IntervalMS != policy.IntervalMS || p.UsedRounds < 1 || !p.DeadlineAt.Equal(minimumTime(g.view.DeadlineAt, activated.Add(time.Duration(policy.TimeoutMS)*time.Millisecond))) {
|
||||
return p, "", errors.New("workflow poll policy mismatch")
|
||||
}
|
||||
if p.ResolvedAt != nil {
|
||||
return p, "", errors.New("unsettled poll step has resolved budget")
|
||||
}
|
||||
var count, minRound, maxRound uint32
|
||||
if err := tx.QueryRowContext(ctx, "SELECT COUNT(*),COALESCE(MIN(`round`),0),COALESCE(MAX(`round`),0) FROM weave_poll_rounds WHERE namespace_id=? AND workflow_id=? AND activation_id=?", g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(&count, &minRound, &maxRound); err != nil {
|
||||
return p, "", err
|
||||
}
|
||||
if count != p.UsedRounds || minRound != 1 || maxRound != p.UsedRounds {
|
||||
return p, "", errors.New("workflow poll rounds incomplete")
|
||||
}
|
||||
var lastTask core.ID
|
||||
if err := tx.QueryRowContext(ctx, "SELECT task_id FROM weave_poll_rounds WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND `round`=?", g.view.NamespaceID, g.view.ID, row.view.ActivationID, p.UsedRounds).Scan(&lastTask); err != nil {
|
||||
return p, "", err
|
||||
}
|
||||
return p, lastTask, nil
|
||||
}
|
||||
|
||||
func (r *WorkflowRuntime) advancePoll(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, c workflowCandidate, activated time.Time, allow bool, now time.Time) (bool, error) {
|
||||
p, lastTask, err := readWorkflowPollState(ctx, tx, g, row, activated)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
node, _ := g.program.Step(row.view.Key)
|
||||
policy := node.Poll
|
||||
if policy == nil || p.MaxRounds != policy.MaxRounds || p.MaxAttemptsPerRound != policy.Task.Retry.MaxAttempts || p.IntervalMS != policy.IntervalMS || p.UsedRounds < 1 || !p.DeadlineAt.Equal(minimumTime(g.view.DeadlineAt, activated.Add(time.Duration(policy.TimeoutMS)*time.Millisecond))) {
|
||||
return false, errors.New("workflow poll policy mismatch")
|
||||
}
|
||||
if p.ResolvedAt != nil {
|
||||
return false, errors.New("unsettled poll step has resolved budget")
|
||||
}
|
||||
var count, minRound, maxRound uint32
|
||||
if err := tx.QueryRowContext(ctx, "SELECT COUNT(*),COALESCE(MIN(`round`),0),COALESCE(MAX(`round`),0) FROM weave_poll_rounds WHERE namespace_id=? AND workflow_id=? AND activation_id=?", c.ns, c.workflow, row.view.ActivationID).Scan(&count, &minRound, &maxRound); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if count != p.UsedRounds || minRound != 1 || maxRound != p.UsedRounds {
|
||||
return false, errors.New("workflow poll rounds incomplete")
|
||||
}
|
||||
var lastTask core.ID
|
||||
if err := tx.QueryRowContext(ctx, "SELECT task_id FROM weave_poll_rounds WHERE namespace_id=? AND workflow_id=? AND activation_id=? AND `round`=?", c.ns, c.workflow, row.view.ActivationID, p.UsedRounds).Scan(&lastTask); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if p.CurrentTaskID != nil {
|
||||
if *p.CurrentTaskID != lastTask || p.NextPollAt != nil || row.view.State != weave.StepActive {
|
||||
return false, errors.New("workflow in-flight poll mismatch")
|
||||
|
||||
@@ -29,8 +29,8 @@ func NewWorkflowRuntime(db *sql.DB, node core.ID) (*WorkflowRuntime, error) {
|
||||
|
||||
type WorkflowRuntimeStats struct {
|
||||
Examined int // Examined 本批取出的持久化候选数，包含被其他节点先处理或跳过的记录。
|
||||
Consumed int // Consumed 已提交确认或隔离的内部结果数，事务回滚不计入。
|
||||
Advanced int // Advanced 已提交推进的根或等待检查点数，不等于业务任务执行次数。
|
||||
Consumed int // Consumed 以Inbox候选为入口提交确认或隔离的数量；时钟内消费计入Advanced，回滚不计入。
|
||||
Advanced int // Advanced 已提交推进的根、等待检查点或过期信号数，不等于业务任务执行次数。
|
||||
}
|
||||
|
||||
type workflowCandidate struct {
|
||||
@@ -39,7 +39,7 @@ type workflowCandidate struct {
|
||||
revision uint64
|
||||
}
|
||||
|
||||
// Tick 同时读取两种到期索引，各取最多limit项再全局选前limit项；不先锁Inbox再反锁父资源。
|
||||
// Tick 同时读取三种到期索引，各取最多limit项再全局选前limit项；不先锁Inbox或信号再反锁父资源。
|
||||
func (r *WorkflowRuntime) Tick(ctx context.Context, limit int) (WorkflowRuntimeStats, error) {
|
||||
var stats WorkflowRuntimeStats
|
||||
if limit < 1 || limit > 100 {
|
||||
@@ -47,7 +47,8 @@ func (r *WorkflowRuntime) Tick(ctx context.Context, limit int) (WorkflowRuntimeS
|
||||
}
|
||||
rows, err := r.db.QueryContext(ctx, `(SELECT 'INBOX' kind,namespace_id,id,workflow_id,revision,available_at FROM weave_inbox WHERE state='PENDING' AND available_at<=UTC_TIMESTAMP(6) ORDER BY available_at,namespace_id,id LIMIT ?)
|
||||
UNION ALL (SELECT 'ROOT' kind,namespace_id,id,workflow_id,revision,available_at FROM weave_pending_advances WHERE available_at<=UTC_TIMESTAMP(6) ORDER BY available_at,namespace_id,id LIMIT ?)
|
||||
ORDER BY available_at,namespace_id,id LIMIT ?`, limit, limit, limit)
|
||||
UNION ALL (SELECT 'SIGNAL' kind,namespace_id,id,workflow_id,revision,available_at FROM weave_signals WHERE state='PENDING' AND available_at<=UTC_TIMESTAMP(6) ORDER BY available_at,namespace_id,id LIMIT ?)
|
||||
ORDER BY available_at,namespace_id,id LIMIT ?`, limit, limit, limit, limit)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
@@ -95,6 +96,8 @@ func (r *WorkflowRuntime) deferCandidate(ctx context.Context, c workflowCandidat
|
||||
q := `UPDATE weave_pending_advances SET available_at=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 30 SECOND),revision=revision+1,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=? AND revision=?`
|
||||
if c.kind == "INBOX" {
|
||||
q = `UPDATE weave_inbox SET available_at=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 30 SECOND),error_code='RUNTIME_UNAVAILABLE',revision=revision+1,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=? AND revision=? AND state='PENDING'`
|
||||
} else if c.kind == "SIGNAL" {
|
||||
q = `UPDATE weave_signals SET available_at=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 30 SECOND),revision=revision+1,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=? AND revision=? AND state='PENDING'`
|
||||
}
|
||||
_, err := r.db.ExecContext(ctx, q, c.ns, c.id, c.revision)
|
||||
return err
|
||||
@@ -115,6 +118,10 @@ func (r *WorkflowRuntime) process(ctx context.Context, c workflowCandidate, skip
|
||||
done, err = r.consume(ctx, tx, g, c, allow, now)
|
||||
return err
|
||||
}
|
||||
if c.kind == "SIGNAL" {
|
||||
done, err = r.expireWorkflowSignal(ctx, tx, g, c, allow, now)
|
||||
return err
|
||||
}
|
||||
var step core.ID
|
||||
var expected uint64
|
||||
var raw []byte
|
||||
@@ -133,7 +140,7 @@ func (r *WorkflowRuntime) process(ctx context.Context, c workflowCandidate, skip
|
||||
if json.Unmarshal(raw, &checkpoint) != nil || checkpoint.SchemaVersion != 1 {
|
||||
return errors.New("workflow checkpoint invalid")
|
||||
}
|
||||
if checkpoint.Kind == "TIMER" || checkpoint.Kind == "POLL" {
|
||||
if checkpoint.Kind == "TIMER" || checkpoint.Kind == "POLL" || checkpoint.Kind == "SIGNAL" {
|
||||
done, err = r.advanceClock(ctx, tx, g, c, step, expected, checkpoint, allow, now)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -11,8 +11,8 @@ import (
|
||||
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
|
||||
)
|
||||
|
||||
// lockWorkflowGraph 只由内部运行器调用。身份→应用额度→排序队列→实例→门控，
|
||||
// 不用启动凭据继续授权；停用身份仍收集在途结果，但禁止创建新工作。
|
||||
// lockWorkflowGraph 由内部运行器和已经认证的信号受理事务复用。身份→应用额度→排序队列→实例→门控，
|
||||
// 运行器不用启动凭据继续授权；停用身份仍收集在途结果，但禁止创建新工作。
|
||||
func lockWorkflowGraph(ctx context.Context, tx *sql.Tx, ns, id core.ID, skip bool) (*workflowGraph, bool, error) {
|
||||
var enabled bool
|
||||
if err := tx.QueryRowContext(ctx, `SELECT n.enabled AND a.enabled FROM orbit_namespaces n JOIN orbit_applications a ON a.id=n.application_id WHERE n.id=? FOR SHARE`, ns).Scan(&enabled); err != nil {
|
||||
|
||||
@@ -211,4 +211,55 @@ func TestIntegrationWorkflowRuntimeProcess(t *testing.T) {
|
||||
t.Fatal("poll takeover lost budget or duplicated round")
|
||||
}
|
||||
t.Log("all controllers exited before timer expiry and poll PENDING delivery; two fresh processes preserved original wake/deadline and completed timer plus exactly two query rounds")
|
||||
for _, p := range processes {
|
||||
stop(p)
|
||||
}
|
||||
early := runtimeStart(t, f, signalSpec(f.resources.QueueID, true))
|
||||
grantWorkflowSignal(f)
|
||||
earlySignal := sendSignal(t, f, early, signalInput(early))
|
||||
ref = retryStart(t, f)
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
if f.count("weave_waits", "workflow_id=?", early.ID) != 0 || f.count("weave_signals", "id=? AND state='PENDING'", earlySignal.Signal.ID) != 1 {
|
||||
t.Fatal("early signal did not remain durable during all-stop")
|
||||
}
|
||||
start("signal-replacement-a")
|
||||
start("signal-replacement-b")
|
||||
wait(func() bool {
|
||||
current, err := api.Get(ctx, f.token, early.ID)
|
||||
return err == nil && current.State == "SUCCEEDED"
|
||||
}, "early signal after process restart")
|
||||
if f.count("weave_signals", "id=? AND state='CONSUMED'", earlySignal.Signal.ID) != 1 || f.count("pulse_tasks", "1=1") != 7 {
|
||||
t.Fatal("early signal restart duplicated work")
|
||||
}
|
||||
for _, p := range processes {
|
||||
stop(p)
|
||||
}
|
||||
final := runtimeStart(t, f, finalSignalSpec(f.resources.QueueID))
|
||||
grantWorkflowSignal(f)
|
||||
ref = retryStart(t, f)
|
||||
permits := f.used()
|
||||
sendSignal(t, f, final, pollSignalInput(final))
|
||||
start("final-signal-selection-a")
|
||||
start("final-signal-selection-b")
|
||||
wait(func() bool { return f.get(ref.TaskID).CancelRequestedAt != nil }, "signal cancellation intent")
|
||||
if f.used() != permits || f.get(ref.TaskID).State != "RUNNING" {
|
||||
t.Fatal("process released running query on final signal")
|
||||
}
|
||||
for _, p := range processes {
|
||||
stop(p)
|
||||
}
|
||||
finishQueryOnly(t, f, ref, "PENDING")
|
||||
if f.count("weave_inbox", "source_task_id=? AND state='PENDING'", ref.TaskID) != 1 {
|
||||
t.Fatal("query settlement missing with all controllers stopped")
|
||||
}
|
||||
start("final-signal-settlement-a")
|
||||
start("final-signal-settlement-b")
|
||||
wait(func() bool {
|
||||
current, err := api.Get(ctx, f.token, final.ID)
|
||||
return err == nil && current.State == "SUCCEEDED"
|
||||
}, "selected final signal and query settlement after restart")
|
||||
if f.used() != 0 || f.count("pulse_tasks", "1=1") != 8 || f.count("weave_poll_states", "workflow_id=? AND used_rounds=1 AND resolved_at IS NOT NULL", final.ID) != 1 || f.count("weave_pending_advances", "1=1") != 0 {
|
||||
t.Fatal("final signal restart lost winner or issued an extra round")
|
||||
}
|
||||
t.Log("early signal persisted before wait activation across all-stop; final signal selected with running query kept permits, then survived another all-stop and accepted PENDING settlement without issuing a second query")
|
||||
}
|
||||
|
||||
315
internal/storage/mysql/workflow_signal_runtime.go
Normal file
315
internal/storage/mysql/workflow_signal_runtime.go
Normal file
@@ -0,0 +1,315 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/jsonvalue"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/engine"
|
||||
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
|
||||
)
|
||||
|
||||
// driveWorkflowSignal 同时用于等待时钟和当前查询结果；handled=false时继续原查询分类。
|
||||
// reason为空表示内部已消费Inbox并写入审计，调用方不能再次确认同一来源。
|
||||
func (r *WorkflowRuntime) driveWorkflowSignal(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, c workflowCandidate, source *workflowResult, allow bool, now time.Time) (handled, changed bool, reason string, err error) {
|
||||
// 来源矛盾单独转核查；任何SQL故障仍回滚整个事务并保留原来源。
|
||||
if _, err := tx.ExecContext(ctx, `SAVEPOINT workflow_signal_source`); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
before := cloneWorkflowGraph(g)
|
||||
handled, changed, reason, err = r.applyWorkflowSignal(ctx, tx, g, row, c, source, allow, now)
|
||||
var invalid workflowSourceError
|
||||
if !errors.As(err, &invalid) {
|
||||
return
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `ROLLBACK TO SAVEPOINT workflow_signal_source`); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
*g = *before
|
||||
row = g.steps[row.view.Key]
|
||||
if g.view.State != weave.InstanceNeedsReview {
|
||||
if err := holdWorkflow(ctx, tx, g, weave.InstanceNeedsReview, string(invalid), now); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
}
|
||||
var current *core.ID
|
||||
if row.view.Kind == "POLL" {
|
||||
if err := tx.QueryRowContext(ctx, `SELECT current_task_id FROM weave_poll_states WHERE namespace_id=? AND workflow_id=? AND activation_id=?`, g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(¤t); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if source != nil || current == nil {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE weave_poll_states SET current_task_id=NULL,next_poll_at=NULL,resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND activation_id=?`, now, now, g.view.NamespaceID, g.view.ID, row.view.ActivationID); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
}
|
||||
}
|
||||
// 已发出的查询仍由Pulse结清，不能因信号损坏而假设外部执行结束。
|
||||
if row.view.Kind != "POLL" || source != nil || current == nil {
|
||||
if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepNeedsReview, nil, now); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE weave_signals s JOIN weave_activations a ON a.namespace_id=s.namespace_id AND a.workflow_id=s.workflow_id AND a.sequence=s.target_activation_sequence SET s.state='REJECTED',s.disposition=?,s.revision=s.revision+1,s.updated_at=? WHERE s.namespace_id=? AND s.workflow_id=? AND s.target_step_key=? AND a.id=? AND a.step_id=? AND s.state='PENDING'`, string(invalid), now, g.view.NamespaceID, g.view.ID, row.view.Key, row.view.ActivationID, row.view.ID); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if err := deleteWorkflowClock(ctx, tx, g, row); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
return true, true, string(invalid), nil
|
||||
}
|
||||
|
||||
func (r *WorkflowRuntime) applyWorkflowSignal(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, c workflowCandidate, source *workflowResult, allow bool, now time.Time) (handled, changed bool, reason string, err error) {
|
||||
w, err := readWorkflowSignalWait(ctx, tx, g, row)
|
||||
if err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
node, _ := g.program.Step(row.view.Key)
|
||||
var poll weave.PollState
|
||||
if node.Poll != nil {
|
||||
var last core.ID
|
||||
poll, last, err = readWorkflowPollState(ctx, tx, g, row, w.activated)
|
||||
if err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if err := validatePollTaskOwner(ctx, tx, g, row, last); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if _, err := pollTaskInput(ctx, tx, g, row, node.Poll.Task); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if poll.CurrentTaskID != nil {
|
||||
if *poll.CurrentTaskID != last || row.view.State != weave.StepActive || poll.NextPollAt != nil {
|
||||
return false, false, "", errors.New("signal poll current round invalid")
|
||||
}
|
||||
if err := validatePollTaskOwner(ctx, tx, g, row, last); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if source != nil && source.task != last {
|
||||
return false, false, "", nil
|
||||
}
|
||||
} else {
|
||||
if source != nil {
|
||||
return false, false, "", nil
|
||||
}
|
||||
if row.view.State != weave.StepWaiting || poll.NextPollAt == nil {
|
||||
return false, false, "", errors.New("signal poll continuation missing")
|
||||
}
|
||||
var disposition string
|
||||
if err := tx.QueryRowContext(ctx, `SELECT error_code FROM weave_inbox WHERE namespace_id=? AND workflow_id=? AND source_task_id=? AND source_activation_id=? AND state='PROCESSED' ORDER BY source_revision DESC LIMIT 1`, g.view.NamespaceID, g.view.ID, last, row.view.ActivationID).Scan(&disposition); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if disposition != "POLL_PENDING" {
|
||||
return false, false, "", errors.New("signal poll previous round not pending")
|
||||
}
|
||||
}
|
||||
}
|
||||
v, err := selectedWorkflowSignal(ctx, tx, g, row, w, now)
|
||||
if err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if v == nil {
|
||||
if node.Poll != nil {
|
||||
return false, false, "", nil
|
||||
}
|
||||
if w.state != "WAITING" {
|
||||
return false, false, "", errors.New("signal step has settled wait without winner")
|
||||
}
|
||||
if now.Before(w.deadline) {
|
||||
return true, false, "", saveWorkflowClock(ctx, tx, g, row, "SIGNAL", row.view.Revision, w.deadline)
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_waits SET state='EXPIRED',resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND state='WAITING'`, now, now, g.view.NamespaceID, w.id)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if err := rejectOtherWorkflowSignals(ctx, tx, g, row, w.sequence, "SIGNAL_DEADLINE", now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if err := saveWorkflowResultStep(ctx, tx, g, row, weave.StepNeedsReview, nil, now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if err := deleteWorkflowClock(ctx, tx, g, row); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
return true, true, "SIGNAL_DEADLINE", holdWorkflow(ctx, tx, g, weave.InstanceNeedsReview, "SIGNAL_DEADLINE", now)
|
||||
}
|
||||
if node.Poll != nil && source == nil && poll.CurrentTaskID != nil {
|
||||
// 已落库查询结果必须按原事件时间参与竞争，不能由扫描先后决定赢家。
|
||||
task, err := readTask(ctx, tx, g.view.NamespaceID, *poll.CurrentTaskID, false)
|
||||
if err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if task.QueueID != node.Poll.Task.QueueID {
|
||||
return false, false, "", errors.New("signal query queue invalid")
|
||||
}
|
||||
if task.State.Terminal() || task.State == "NEEDS_REVIEW" {
|
||||
pending := workflowCandidate{kind: "INBOX", ns: g.view.NamespaceID, workflow: g.view.ID}
|
||||
var due time.Time
|
||||
if err := tx.QueryRowContext(ctx, `SELECT id,revision,available_at FROM weave_inbox WHERE namespace_id=? AND workflow_id=? AND source_task_id=? AND source_activation_id=? AND state='PENDING' ORDER BY source_revision DESC LIMIT 1`, g.view.NamespaceID, g.view.ID, poll.CurrentTaskID, row.view.ActivationID).Scan(&pending.id, &pending.revision, &due); err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
if due.After(now) {
|
||||
return true, false, "", saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision, due)
|
||||
}
|
||||
done, err := r.consume(ctx, tx, g, pending, allow, now)
|
||||
return true, done, "", err
|
||||
}
|
||||
}
|
||||
if node.Poll != nil && source != nil && w.winner == nil && source.snapshot.State == "SUCCEEDED" && !source.snapshot.EffectUncertain && source.occurred.Before(w.deadline) && !source.occurred.After(v.AcceptedAt) {
|
||||
outcome, err := g.program.ClassifyPoll(row.view.Key, source.snapshot.Result)
|
||||
if err == nil && outcome != engine.PollPending {
|
||||
return false, false, "", nil
|
||||
} // 较早的最终查询先胜出；PENDING不争夺终结权。
|
||||
}
|
||||
newWinner := w.winner == nil
|
||||
if err := claimWorkflowSignal(ctx, tx, g, &w, v, now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if err := rejectOtherWorkflowSignals(ctx, tx, g, row, w.sequence, "WAIT_SETTLED", now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if node.Poll != nil && poll.CurrentTaskID != nil && source == nil {
|
||||
task, err := readExecutionTask(ctx, tx, g.view.NamespaceID, *poll.CurrentTaskID)
|
||||
if err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if _, err := cancelExecutionTask(ctx, tx, g.view.NamespaceID, task, now, "WORKFLOW_SIGNAL_WON"); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if !now.Before(poll.DeadlineAt) && g.view.State != weave.InstanceNeedsReview {
|
||||
if err := holdWorkflow(ctx, tx, g, weave.InstanceNeedsReview, "POLL_SIGNAL_INFLIGHT_DEADLINE", now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
newWinner = true
|
||||
}
|
||||
if err := saveWorkflowClock(ctx, tx, g, row, "POLL", row.view.Revision, now.Add(time.Second)); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if newWinner {
|
||||
return true, true, "POLL_SIGNAL_SELECTED", nil
|
||||
}
|
||||
return true, false, "", nil
|
||||
}
|
||||
state, output, reason := weave.StepSucceeded, v.Payload, "SIGNAL_CONSUMED"
|
||||
if node.Poll != nil {
|
||||
outcome, err := g.program.ClassifyPoll(row.view.Key, v.Payload)
|
||||
if err != nil || outcome == engine.PollPending {
|
||||
return true, false, "", workflowSourceError("POLL_FINAL_SIGNAL_INVALID")
|
||||
}
|
||||
switch outcome {
|
||||
case engine.PollFailed:
|
||||
state, output, reason = weave.StepFailed, nil, "POLL_SIGNAL_FAILED"
|
||||
case engine.PollNeedsReview:
|
||||
state, output, reason = weave.StepNeedsReview, nil, "POLL_SIGNAL_REVIEW"
|
||||
default:
|
||||
reason = "POLL_SIGNAL_SUCCEEDED"
|
||||
}
|
||||
if source != nil {
|
||||
if source.snapshot.EffectUncertain || source.snapshot.State == "NEEDS_REVIEW" {
|
||||
state, output, reason = weave.StepNeedsReview, nil, "POLL_SIGNAL_QUERY_UNCERTAIN"
|
||||
} else if source.snapshot.State == "SUCCEEDED" {
|
||||
query, err := g.program.ClassifyPoll(row.view.Key, source.snapshot.Result)
|
||||
if err != nil {
|
||||
state, output, reason = weave.StepNeedsReview, nil, "POLL_SIGNAL_QUERY_INVALID"
|
||||
} else if query != engine.PollPending {
|
||||
left, err := jsonvalue.Canonical(source.snapshot.Result)
|
||||
if err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
right, err := jsonvalue.Canonical(v.Payload)
|
||||
if err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if !bytes.Equal(left, right) {
|
||||
state, output, reason = weave.StepNeedsReview, nil, "POLL_SIGNAL_QUERY_CONFLICT"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_poll_states SET current_task_id=NULL,next_poll_at=NULL,resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND resolved_at IS NULL`, now, now, g.view.NamespaceID, poll.ID)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
}
|
||||
if err := saveWorkflowResultStep(ctx, tx, g, row, state, output, now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if err := deleteWorkflowClock(ctx, tx, g, row); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
if state == weave.StepNeedsReview || state == weave.StepFailed {
|
||||
next := weave.InstanceNeedsReview
|
||||
if state == weave.StepFailed && (g.version.Specification.FailurePolicy == "FAIL_AFTER_SETTLED" || g.view.CancelRequestedAt != nil) {
|
||||
next = weave.InstanceCancelling
|
||||
}
|
||||
if err := holdWorkflow(ctx, tx, g, next, reason, now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
} else if err := r.progress(ctx, tx, g, allow, now); err != nil {
|
||||
return true, false, "", err
|
||||
}
|
||||
return true, true, reason, nil
|
||||
}
|
||||
|
||||
func (r *WorkflowRuntime) advanceSignalClock(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, c workflowCandidate, allow bool, now time.Time) (bool, error) {
|
||||
handled, changed, reason, err := r.driveWorkflowSignal(ctx, tx, g, row, c, nil, allow, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !handled {
|
||||
return false, errors.New("signal clock did not identify a wait")
|
||||
}
|
||||
if changed && reason != "" {
|
||||
return true, r.audit(ctx, tx, g, c.id, reason, now)
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// expireWorkflowSignal 对早到但未匹配的信号使用总截止索引；已按时匹配的信号仍先完成实际消费。
|
||||
func (r *WorkflowRuntime) expireWorkflowSignal(ctx context.Context, tx *sql.Tx, g *workflowGraph, c workflowCandidate, allow bool, now time.Time) (bool, error) {
|
||||
var due time.Time
|
||||
if err := tx.QueryRowContext(ctx, `SELECT available_at FROM weave_signals WHERE namespace_id=? AND workflow_id=? AND id=?`, c.ns, c.workflow, c.id).Scan(&due); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if due.After(now) {
|
||||
return false, nil
|
||||
}
|
||||
v, err := readWorkflowSignal(ctx, tx, c.ns, c.workflow, c.id)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if v.State != "PENDING" || now.Before(v.ExpiresAt) {
|
||||
return false, nil
|
||||
}
|
||||
row := g.steps[v.TargetStepKey]
|
||||
if row != nil && row.view.ActivationID != nil && !g.view.State.Terminal() && (row.view.State == weave.StepWaiting || row.view.State == weave.StepActive) {
|
||||
w, err := readWorkflowSignalWait(ctx, tx, g, row)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if w.sequence == v.TargetActivationSequence && w.correlation == v.CorrelationKey && w.contract.Name == v.Name && w.contract.PayloadVersion == v.PayloadVersion && v.AcceptedAt.Before(w.deadline) {
|
||||
handled, changed, reason, err := r.driveWorkflowSignal(ctx, tx, g, row, c, nil, allow, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if handled {
|
||||
// 若匹配到另一条更早的信号或正等待结果退避，避免过期索引持续抢占批次。
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE weave_signals SET available_at=DATE_ADD(?,INTERVAL 30 SECOND),revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND state='PENDING'`, now, now, c.ns, c.id); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if changed && reason != "" {
|
||||
return true, r.audit(ctx, tx, g, c.id, reason, now)
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_signals SET state='EXPIRED',disposition='UNMATCHED_DEADLINE',revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND state='PENDING' AND revision=?`, now, c.ns, c.id, v.Revision)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, r.audit(ctx, tx, g, c.id, "SIGNAL_EXPIRED", now)
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/definition"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/instance"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/signal"
|
||||
)
|
||||
|
||||
func sendSignal(t *testing.T, f *executionFixture, v instance.View, in signal.Input) signal.Result {
|
||||
t.Helper()
|
||||
out, err := signal.New(NewWorkflowSignalStore(f.db)).Send(f.ctx, f.token, v.ID, string(executionID(t)), in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func finalSignalSpec(queue core.ID) definition.Specification {
|
||||
s := pollingSpec(queue)
|
||||
p := s.Body.Steps[0].Poll
|
||||
p.Task.ResultType.Enum = append(p.Task.ResultType.Enum, "FAILED")
|
||||
p.Failed = []string{"FAILED"}
|
||||
p.FinalSignal = &definition.Signal{Name: "external.done", PayloadVersion: "v1", Correlation: definition.Value{Kind: "REFERENCE", Reference: &definition.Reference{Source: "workflow", Path: []string{"business_key"}}}, PayloadType: p.Task.ResultType, TimeoutMS: p.TimeoutMS}
|
||||
s.Body.OutputType = p.Task.ResultType
|
||||
return s
|
||||
}
|
||||
func pollSignalInput(v instance.View) signal.Input {
|
||||
in := signalInput(v)
|
||||
in.TargetStepKey = "query"
|
||||
in.Payload = json.RawMessage(`"DONE"`)
|
||||
return in
|
||||
}
|
||||
func finishQueryOnly(t *testing.T, f *executionFixture, ref worker.AttemptRef, result string) {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(result)
|
||||
if _, err := f.a.Finish(f.ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: raw}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
func TestIntegrationWorkflowSignalRuntime(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, stop := context.WithTimeout(context.Background(), 150*time.Second)
|
||||
defer stop()
|
||||
db, err := Open(ctx, dsn, 32, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Run("wait_then_twelve_consumers_one_successor", func(t *testing.T) {
|
||||
f, v, api := signalStart(t, ctx, db, false)
|
||||
c := clockCandidate(t, f, v.ID, "signal")
|
||||
if runClock(t, f, c) || f.count("pulse_tasks", "1=1") != 0 {
|
||||
t.Fatal("empty wait advanced")
|
||||
}
|
||||
in := signalInput(v)
|
||||
key := string(executionID(t))
|
||||
out, err := api.Send(ctx, f.token, v.ID, key, in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other := sendSignal(t, f, v, in)
|
||||
c = clockCandidate(t, f, v.ID, "signal")
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 12)
|
||||
results := make(chan bool, 12)
|
||||
for i := 0; i < 12; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
done, err := newRuntime(t, db).process(ctx, c, false)
|
||||
errs <- err
|
||||
results <- done
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
close(results)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
changed := 0
|
||||
for done := range results {
|
||||
if done {
|
||||
changed++
|
||||
}
|
||||
}
|
||||
if changed != 1 || f.count("pulse_tasks", "1=1") != 1 || f.count("weave_signals", "state='CONSUMED'") != 1 || f.count("weave_signals", "state='REJECTED'") != 1 {
|
||||
t.Fatal("duplicate signal advancement", changed)
|
||||
}
|
||||
current, err := api.Get(ctx, f.token, v.ID, out.Signal.ID)
|
||||
if err != nil || current.State != "CONSUMED" || current.ConsumedWaitID == nil {
|
||||
t.Fatal("signal not consumed", err)
|
||||
}
|
||||
rejected, err := api.Get(ctx, f.token, v.ID, other.Signal.ID)
|
||||
if err != nil || rejected.Disposition != "WAIT_SETTLED" {
|
||||
t.Fatal("competing signal not retained", err)
|
||||
}
|
||||
replay, err := api.Send(ctx, f.token, v.ID, key, in)
|
||||
if err != nil || !replay.Deduplicated || replay.Signal.State != "PENDING" || replay.Signal.Revision != 1 {
|
||||
t.Fatal("original receipt was mutated", err)
|
||||
}
|
||||
var output string
|
||||
if err := db.QueryRowContext(ctx, `SELECT output FROM weave_activations WHERE namespace_id=? AND id=(SELECT current_activation_id FROM weave_steps WHERE namespace_id=? AND workflow_id=? AND `+"`key`"+`='signal')`, f.resources.NamespaceID, f.resources.NamespaceID, v.ID).Scan(&output); err != nil || !strings.Contains(output, "9007199254740993") {
|
||||
t.Fatal("signal number changed", err)
|
||||
}
|
||||
})
|
||||
t.Run("early_signal_survives_activation_and_fresh_runtime", func(t *testing.T) {
|
||||
f, v, _ := signalStart(t, ctx, db, true)
|
||||
sendSignal(t, f, v, signalInput(v))
|
||||
if f.count("weave_waits", "1=1") != 0 {
|
||||
t.Fatal("early signal activated blocked step")
|
||||
}
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
if !runClock(t, f, clockCandidate(t, f, v.ID, "signal")) {
|
||||
t.Fatal("early signal not resumed")
|
||||
}
|
||||
out := runtimeView(t, f, v.ID, "SUCCEEDED")
|
||||
if !strings.Contains(string(out.Output), "9007199254740993") || f.count("pulse_tasks", "1=1") != 1 {
|
||||
t.Fatal("early result lost or new work created")
|
||||
}
|
||||
})
|
||||
t.Run("paused_collects_and_persists_root", func(t *testing.T) {
|
||||
f, v, _ := signalStart(t, ctx, db, false)
|
||||
f.exec(`UPDATE weave_instances SET paused=TRUE WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||
f.exec(`UPDATE pulse_execution_gates SET paused=TRUE WHERE namespace_id=?`, f.resources.NamespaceID)
|
||||
sendSignal(t, f, v, signalInput(v))
|
||||
runClock(t, f, clockCandidate(t, f, v.ID, "signal"))
|
||||
if f.count("weave_signals", "state='CONSUMED'") != 1 || f.count("pulse_tasks", "1=1") != 0 || f.count("weave_pending_advances", "1=1") != 1 {
|
||||
t.Fatal("pause dropped evidence or started successor")
|
||||
}
|
||||
f.exec(`UPDATE weave_instances SET paused=FALSE WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||
f.exec(`UPDATE pulse_execution_gates SET paused=FALSE WHERE namespace_id=?`, f.resources.NamespaceID)
|
||||
runtimeAdvance(t, f, newRuntime(t, db), v.ID)
|
||||
if f.count("pulse_tasks", "1=1") != 1 {
|
||||
t.Fatal("root did not resume")
|
||||
}
|
||||
})
|
||||
for _, timely := range []bool{false, true} {
|
||||
name := "deadline_without_signal"
|
||||
if timely {
|
||||
name = "timely_acceptance_after_consumer_deadline"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
spec := signalSpec(f.resources.QueueID, false)
|
||||
spec.Body.Steps[0].Signal.TimeoutMS = 1000
|
||||
v := runtimeStart(t, f, spec)
|
||||
grantWorkflowSignal(f)
|
||||
if timely {
|
||||
sendSignal(t, f, v, signalInput(v))
|
||||
}
|
||||
c := clockCandidate(t, f, v.ID, "signal")
|
||||
if !timely {
|
||||
runClock(t, f, c)
|
||||
}
|
||||
var due time.Time
|
||||
if err := db.QueryRowContext(ctx, `SELECT deadline_at FROM weave_waits WHERE namespace_id=?`, f.resources.NamespaceID).Scan(&due); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for time.Now().Before(due.Add(20 * time.Millisecond)) {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
if !runClock(t, f, c) {
|
||||
t.Fatal("deadline not handled")
|
||||
}
|
||||
if timely {
|
||||
runtimeView(t, f, v.ID, "RUNNING")
|
||||
if f.count("pulse_tasks", "1=1") != 1 {
|
||||
t.Fatal("timely signal lost")
|
||||
}
|
||||
} else {
|
||||
runtimeView(t, f, v.ID, "NEEDS_REVIEW")
|
||||
if f.count("weave_waits", "state='EXPIRED'") != 1 || f.count("pulse_tasks", "1=1") != 0 {
|
||||
t.Fatal("expired wait progressed")
|
||||
}
|
||||
late := sendSignal(t, f, v, signalInput(v))
|
||||
if late.Signal.State != "REJECTED" {
|
||||
t.Fatal("settled wait accepted late signal")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("audit_failure_rolls_back_consumption_wait_successor", func(t *testing.T) {
|
||||
f, v, _ := signalStart(t, ctx, db, false)
|
||||
sendSignal(t, f, v, signalInput(v))
|
||||
c := clockCandidate(t, f, v.ID, "signal")
|
||||
f.exec(`CREATE TRIGGER test_signal_consume_failure BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected signal consume failure'`)
|
||||
defer f.exec(`DROP TRIGGER IF EXISTS test_signal_consume_failure`)
|
||||
if done, err := newRuntime(t, db).process(ctx, c, false); err == nil || done {
|
||||
t.Fatal("audit failure committed")
|
||||
}
|
||||
if f.count("weave_signals", "state='PENDING'") != 1 || f.count("weave_waits", "state='WAITING' AND winner_signal_id IS NULL") != 1 || f.count("pulse_tasks", "1=1") != 0 {
|
||||
t.Fatal("partial signal transaction")
|
||||
}
|
||||
f.exec(`DROP TRIGGER test_signal_consume_failure`)
|
||||
if !runClock(t, f, c) || f.count("pulse_tasks", "1=1") != 1 {
|
||||
t.Fatal("rollback not recoverable")
|
||||
}
|
||||
})
|
||||
for _, mutation := range []string{"payload", "actor", "accepted_at"} {
|
||||
t.Run("tampered_"+mutation+"_requires_review", func(t *testing.T) {
|
||||
f, v, _ := signalStart(t, ctx, db, false)
|
||||
out := sendSignal(t, f, v, signalInput(v))
|
||||
switch mutation {
|
||||
case "payload":
|
||||
f.exec(`UPDATE weave_signals SET payload='{"n":1}' WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, out.Signal.ID)
|
||||
case "actor":
|
||||
f.exec(`UPDATE weave_signals SET actor_id='spoofed' WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, out.Signal.ID)
|
||||
default:
|
||||
f.exec(`UPDATE weave_signals SET accepted_at=DATE_SUB(accepted_at,INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, out.Signal.ID)
|
||||
}
|
||||
if !runClock(t, f, clockCandidate(t, f, v.ID, "signal")) {
|
||||
t.Fatal("invalid source not handled")
|
||||
}
|
||||
runtimeView(t, f, v.ID, "NEEDS_REVIEW")
|
||||
if f.count("weave_signals", "state='REJECTED'") != 1 || f.count("weave_steps", "state='NEEDS_REVIEW'") != 1 || f.count("pulse_tasks", "1=1") != 0 {
|
||||
t.Fatal("mutable signal authorized work")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("early_mismatch_expires_without_changing_deadline_on_backoff", func(t *testing.T) {
|
||||
f, v, _ := signalStart(t, ctx, db, true)
|
||||
in := signalInput(v)
|
||||
in.CorrelationKey += "wrong"
|
||||
out := sendSignal(t, f, v, in)
|
||||
c := workflowCandidate{kind: "SIGNAL", ns: f.resources.NamespaceID, workflow: v.ID, id: out.Signal.ID, revision: 1}
|
||||
// 仅故障注入过期扫描条件，不把篡改过的信号用于匹配或推进。
|
||||
f.exec(`UPDATE weave_signals SET expires_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND),available_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?`, c.ns, c.id)
|
||||
var expires time.Time
|
||||
if err := db.QueryRowContext(ctx, `SELECT expires_at FROM weave_signals WHERE namespace_id=? AND id=?`, c.ns, c.id).Scan(&expires); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := newRuntime(t, db)
|
||||
if err := r.deferCandidate(ctx, c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var current, due time.Time
|
||||
if err := db.QueryRowContext(ctx, `SELECT expires_at,available_at FROM weave_signals WHERE namespace_id=? AND id=?`, c.ns, c.id).Scan(¤t, &due); err != nil || !current.Equal(expires) || !due.After(time.Now()) {
|
||||
t.Fatal("backoff extended deadline", err)
|
||||
}
|
||||
if done, err := r.process(ctx, c, false); err != nil || done {
|
||||
t.Fatal("ignored backoff", err)
|
||||
}
|
||||
f.exec(`UPDATE weave_signals SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, c.ns, c.id)
|
||||
if done, err := r.process(ctx, c, false); err != nil || !done {
|
||||
t.Fatal("unmatched expiry failed", err)
|
||||
}
|
||||
if f.count("weave_signals", "state='EXPIRED' AND disposition='UNMATCHED_DEADLINE'") != 1 || f.count("pulse_tasks", "1=1") != 1 {
|
||||
t.Fatal("expiry authorized work")
|
||||
}
|
||||
})
|
||||
t.Run("poll_signal_between_rounds_does_not_issue_query", func(t *testing.T) {
|
||||
f := newClockFixture(t, ctx, db)
|
||||
v := runtimeStart(t, f, finalSignalSpec(f.resources.QueueID))
|
||||
grantWorkflowSignal(f)
|
||||
ref := retryStart(t, f)
|
||||
finishPoll(t, f, ref, "PENDING")
|
||||
sendSignal(t, f, v, pollSignalInput(v))
|
||||
runClock(t, f, clockCandidate(t, f, v.ID, "query"))
|
||||
runtimeView(t, f, v.ID, "SUCCEEDED")
|
||||
if f.count("pulse_tasks", "1=1") != 1 || f.count("weave_poll_states", "used_rounds=1 AND resolved_at IS NOT NULL") != 1 || f.used() != 0 {
|
||||
t.Fatal("final signal consumed extra budget")
|
||||
}
|
||||
})
|
||||
t.Run("poll_signal_cancels_unstarted_query", func(t *testing.T) {
|
||||
f := newClockFixture(t, ctx, db)
|
||||
v := runtimeStart(t, f, finalSignalSpec(f.resources.QueueID))
|
||||
grantWorkflowSignal(f)
|
||||
sendSignal(t, f, v, pollSignalInput(v))
|
||||
runClock(t, f, clockCandidate(t, f, v.ID, "query"))
|
||||
var task core.ID
|
||||
if err := db.QueryRowContext(ctx, `SELECT id FROM pulse_tasks WHERE namespace_id=?`, f.resources.NamespaceID).Scan(&task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.get(task).State != "CANCELLED" || f.count("weave_steps", "`key`='query' AND state='ACTIVE'") != 1 {
|
||||
t.Fatal("unstarted query not safely cancelled")
|
||||
}
|
||||
runtimeConsume(t, f, newRuntime(t, db), task)
|
||||
runtimeView(t, f, v.ID, "SUCCEEDED")
|
||||
if f.count("weave_inbox", "state='PROCESSED'") != 1 || f.count("weave_pending_advances", "1=1") != 0 {
|
||||
t.Fatal("cancel result not collected")
|
||||
}
|
||||
})
|
||||
for _, outcome := range []string{"CANCELLED", "PENDING", "DONE", "UNKNOWN", "FAILED", "CONFLICT"} {
|
||||
t.Run("poll_running_signal_then_"+outcome, func(t *testing.T) {
|
||||
f := newClockFixture(t, ctx, db)
|
||||
v := runtimeStart(t, f, finalSignalSpec(f.resources.QueueID))
|
||||
grantWorkflowSignal(f)
|
||||
ref := retryStart(t, f)
|
||||
before := f.used()
|
||||
sendSignal(t, f, v, pollSignalInput(v))
|
||||
runClock(t, f, clockCandidate(t, f, v.ID, "query"))
|
||||
task := f.get(ref.TaskID)
|
||||
if task.State != "RUNNING" || task.CancelRequestedAt == nil || f.used() != before || f.count("weave_poll_states", "current_task_id IS NOT NULL AND resolved_at IS NULL") != 1 {
|
||||
t.Fatal("signal prematurely released execution")
|
||||
}
|
||||
switch outcome {
|
||||
case "PENDING", "DONE":
|
||||
finishQueryOnly(t, f, ref, outcome)
|
||||
case "CONFLICT":
|
||||
finishQueryOnly(t, f, ref, "FAILED")
|
||||
default:
|
||||
retryFinish(t, f, ref, outcome, false)
|
||||
}
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
want := "SUCCEEDED"
|
||||
if outcome == "UNKNOWN" || outcome == "CONFLICT" {
|
||||
want = "NEEDS_REVIEW"
|
||||
}
|
||||
runtimeView(t, f, v.ID, want)
|
||||
if f.used() != 0 || f.count("pulse_tasks", "1=1") != 1 || f.count("weave_poll_states", "current_task_id IS NULL AND resolved_at IS NOT NULL") != 1 {
|
||||
t.Fatal("query not settled exactly once")
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, clockFirst := range []bool{true, false} {
|
||||
name := "query_earlier_inbox_first"
|
||||
if clockFirst {
|
||||
name = "query_earlier_clock_first"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f := newClockFixture(t, ctx, db)
|
||||
v := runtimeStart(t, f, finalSignalSpec(f.resources.QueueID))
|
||||
grantWorkflowSignal(f)
|
||||
ref := retryStart(t, f)
|
||||
finishQueryOnly(t, f, ref, "DONE")
|
||||
in := pollSignalInput(v)
|
||||
in.Payload = json.RawMessage(`"FAILED"`)
|
||||
sendSignal(t, f, v, in)
|
||||
if clockFirst {
|
||||
runClock(t, f, clockCandidate(t, f, v.ID, "query"))
|
||||
} else {
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
}
|
||||
runtimeView(t, f, v.ID, "SUCCEEDED")
|
||||
if f.count("weave_signals", "state='REJECTED' AND disposition='QUERY_SETTLED'") != 1 || f.count("weave_waits", "state='SATISFIED' AND winner_signal_id IS NULL") != 1 {
|
||||
t.Fatal("scan order changed winner")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Log("signal wait/early/duplicate/deadline/pause/rollback/proof and POLL final-signal settlement verified on real MySQL")
|
||||
}
|
||||
166
internal/storage/mysql/workflow_signal_source.go
Normal file
166
internal/storage/mysql/workflow_signal_source.go
Normal file
@@ -0,0 +1,166 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/definition"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/signal"
|
||||
)
|
||||
|
||||
type workflowSignalWait struct {
|
||||
id core.ID
|
||||
sequence uint32
|
||||
state, correlation string
|
||||
deadline, activated time.Time
|
||||
winner *core.ID
|
||||
contract *definition.Signal
|
||||
}
|
||||
|
||||
// readWorkflowSignalWait 读取已经冻结的等待身份，不能把历史NULL元数据解释成有效信号契约。
|
||||
func readWorkflowSignalWait(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep) (workflowSignalWait, error) {
|
||||
var w workflowSignalWait
|
||||
node, ok := g.program.Step(row.view.Key)
|
||||
w.contract = signalContract(node)
|
||||
if !ok || w.contract == nil || row.view.ActivationID == nil {
|
||||
return w, errors.New("signal step definition invalid")
|
||||
}
|
||||
var count int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_waits WHERE namespace_id=? AND workflow_id=? AND activation_id=?`, g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(&count); err != nil {
|
||||
return w, err
|
||||
}
|
||||
if count != 1 {
|
||||
return w, errors.New("signal wait set invalid")
|
||||
}
|
||||
var name, version *string
|
||||
var kind string
|
||||
err := tx.QueryRowContext(ctx, `SELECT w.id,w.kind,w.state,w.signal_name,w.payload_version,w.correlation_key,w.deadline_at,w.winner_signal_id,a.sequence,a.created_at FROM weave_waits w JOIN weave_activations a ON a.namespace_id=w.namespace_id AND a.workflow_id=w.workflow_id AND a.id=w.activation_id AND a.step_id=? WHERE w.namespace_id=? AND w.workflow_id=? AND w.activation_id=? FOR UPDATE`, row.view.ID, g.view.NamespaceID, g.view.ID, row.view.ActivationID).Scan(&w.id, &kind, &w.state, &name, &version, &w.correlation, &w.deadline, &w.winner, &w.sequence, &w.activated)
|
||||
if err != nil {
|
||||
return w, err
|
||||
}
|
||||
if name == nil || version == nil {
|
||||
return w, workflowSourceError("SIGNAL_WAIT_UNVERIFIED")
|
||||
}
|
||||
if w.sequence == 0 || kind != string(row.view.Kind) || *name != w.contract.Name || *version != w.contract.PayloadVersion || !utf8.ValidString(w.correlation) || len(w.correlation) == 0 || len(w.correlation) > 512 || !w.deadline.Equal(minimumTime(g.view.DeadlineAt, w.activated.Add(time.Duration(w.contract.TimeoutMS)*time.Millisecond))) {
|
||||
return w, workflowSourceError("SIGNAL_WAIT_INVALID")
|
||||
}
|
||||
if node.Signal != nil {
|
||||
var input struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"payload_version"`
|
||||
Correlation string `json:"correlation"`
|
||||
}
|
||||
if json.Unmarshal(row.input, &input) != nil || input.Name != *name || input.Version != *version || input.Correlation != w.correlation {
|
||||
return w, workflowSourceError("SIGNAL_ACTIVATION_INPUT_INVALID")
|
||||
}
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// verifyWorkflowSignal 核对不可改写受理审计及请求摘要，防止可变状态读取替换原受理参数或时间。
|
||||
func verifyWorkflowSignal(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, w workflowSignalWait, v signal.View, now time.Time) error {
|
||||
if v.NamespaceID != g.view.NamespaceID || v.WorkflowID != g.view.ID || v.TargetStepKey != row.view.Key || v.TargetActivationSequence != w.sequence || v.CorrelationKey != w.correlation || v.Name != w.contract.Name || v.PayloadVersion != w.contract.PayloadVersion || v.AcceptedAt.After(now) || !v.AcceptedAt.Before(w.deadline) || !v.ExpiresAt.Equal(g.view.DeadlineAt) || definition.ValidateData(w.contract.PayloadType, v.Payload) != nil {
|
||||
return workflowSourceError("SIGNAL_IDENTITY_INVALID")
|
||||
}
|
||||
var digest core.Digest
|
||||
if err := tx.QueryRowContext(ctx, `SELECT request_digest FROM weave_signals WHERE namespace_id=? AND workflow_id=? AND id=?`, g.view.NamespaceID, g.view.ID, v.ID).Scan(&digest); err != nil {
|
||||
return err
|
||||
}
|
||||
var count int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM orbit_audit_events WHERE namespace_id=? AND resource_kind='workflow_signal' AND resource_id=? AND action='workflow.signal'`, g.view.NamespaceID, v.ID).Scan(&count); err != nil {
|
||||
return err
|
||||
}
|
||||
if count != 1 {
|
||||
return workflowSourceError("SIGNAL_ACCEPTANCE_PROOF_MISSING")
|
||||
}
|
||||
var actor, reason string
|
||||
var evidence sql.NullString
|
||||
var raw []byte
|
||||
if err := tx.QueryRowContext(ctx, "SELECT actor_id,reason,evidence_ref,`after` FROM orbit_audit_events WHERE namespace_id=? AND resource_kind='workflow_signal' AND resource_id=? AND action='workflow.signal'", g.view.NamespaceID, v.ID).Scan(&actor, &reason, &evidence, &raw); err != nil {
|
||||
return err
|
||||
}
|
||||
var meta struct {
|
||||
SignalID core.ID `json:"signal_id"`
|
||||
WorkflowID core.ID `json:"workflow_id"`
|
||||
Key string `json:"target_step_key"`
|
||||
Sequence uint32 `json:"target_activation_sequence"`
|
||||
Digest string `json:"request_digest"`
|
||||
AcceptedAt time.Time `json:"accepted_at"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
}
|
||||
if json.Unmarshal(raw, &meta) != nil || meta.SignalID != v.ID || meta.WorkflowID != g.view.ID || meta.Key != row.view.Key || meta.Sequence != w.sequence || meta.Digest != hex.EncodeToString(digest[:]) || !meta.AcceptedAt.Equal(v.AcceptedAt) || !meta.ExpiresAt.Equal(v.ExpiresAt) || actor != v.ActorID || evidence.String != v.EvidenceRef {
|
||||
return workflowSourceError("SIGNAL_ACCEPTANCE_PROOF_INVALID")
|
||||
}
|
||||
expected, err := requestDigest(signal.Input{Name: v.Name, PayloadVersion: v.PayloadVersion, TargetStepKey: v.TargetStepKey, TargetActivationSequence: v.TargetActivationSequence, CorrelationKey: v.CorrelationKey, Payload: v.Payload, Reason: reason, EvidenceRef: v.EvidenceRef})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if expected != digest {
|
||||
return workflowSourceError("SIGNAL_REQUEST_CHANGED")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// selectedWorkflowSignal 返回已胜出信号或最早按时受理的匹配信号；查询结果竞争由调用方在同一实例锁下裁决。
|
||||
func selectedWorkflowSignal(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, w workflowSignalWait, now time.Time) (*signal.View, error) {
|
||||
var id core.ID
|
||||
if w.winner != nil {
|
||||
if w.state != "SATISFIED" {
|
||||
return nil, workflowSourceError("SIGNAL_WINNER_WAIT_INVALID")
|
||||
}
|
||||
id = *w.winner
|
||||
} else {
|
||||
if w.state != "WAITING" {
|
||||
return nil, nil
|
||||
}
|
||||
err := tx.QueryRowContext(ctx, `SELECT id FROM weave_signals WHERE namespace_id=? AND workflow_id=? AND target_step_key=? AND target_activation_sequence=? AND state='PENDING' AND type=? AND payload_version=? AND correlation_key=? AND accepted_at<? ORDER BY accepted_at,id LIMIT 1 FOR UPDATE`, g.view.NamespaceID, g.view.ID, row.view.Key, w.sequence, w.contract.Name, w.contract.PayloadVersion, []byte(w.correlation), w.deadline).Scan(&id)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
v, err := readWorkflowSignal(ctx, tx, g.view.NamespaceID, g.view.ID, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if w.winner != nil && (v.State != "CONSUMED" || v.ConsumedWaitID == nil || *v.ConsumedWaitID != w.id) {
|
||||
return nil, workflowSourceError("SIGNAL_WINNER_ASSOCIATION_INVALID")
|
||||
}
|
||||
if err := verifyWorkflowSignal(ctx, tx, g, row, w, v, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &v, nil
|
||||
}
|
||||
|
||||
func claimWorkflowSignal(ctx context.Context, tx *sql.Tx, g *workflowGraph, w *workflowSignalWait, v *signal.View, now time.Time) error {
|
||||
if w.winner != nil {
|
||||
return nil
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_signals SET state='CONSUMED',disposition='WAIT_MATCHED',consumed_wait_id=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND workflow_id=? AND revision=? AND state='PENDING'`, w.id, now, g.view.NamespaceID, v.ID, g.view.ID, v.Revision)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return err
|
||||
}
|
||||
res, err = tx.ExecContext(ctx, `UPDATE weave_waits SET state='SATISFIED',winner_signal_id=?,resolved_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND workflow_id=? AND state='WAITING' AND winner_signal_id IS NULL`, v.ID, now, now, g.view.NamespaceID, w.id, g.view.ID)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return err
|
||||
}
|
||||
w.winner = &v.ID
|
||||
w.state = "SATISFIED"
|
||||
v.State = "CONSUMED"
|
||||
v.Revision++
|
||||
v.Disposition = "WAIT_MATCHED"
|
||||
v.ConsumedWaitID = &w.id
|
||||
return nil
|
||||
}
|
||||
func rejectOtherWorkflowSignals(ctx context.Context, tx *sql.Tx, g *workflowGraph, row *workflowStep, sequence uint32, reason string, now time.Time) error {
|
||||
_, err := tx.ExecContext(ctx, `UPDATE weave_signals SET state='REJECTED',disposition=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND workflow_id=? AND target_step_key=? AND target_activation_sequence=? AND state='PENDING'`, reason, now, g.view.NamespaceID, g.view.ID, row.view.Key, sequence)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
driver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// 用真实v19结构验证信号历史身份隔离和等待时钟回填，不用当前结构模拟旧数据库。
|
||||
func TestIntegrationWorkflowSignalUpgrade(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
admin, err := Open(ctx, dsn, 2, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer admin.Close()
|
||||
name := "orbit_signal_upgrade_" + string(executionID(t))
|
||||
if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+name+"`"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
cleanup, stop := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer stop()
|
||||
if _, err := admin.ExecContext(cleanup, "DROP DATABASE `"+name+"`"); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
cfg, err := driver.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
t.Fatal("invalid test DSN")
|
||||
}
|
||||
cfg.DBName = name
|
||||
db, err := Open(ctx, cfg.FormatDSN(), 4, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
exec := func(q string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := db.ExecContext(ctx, q, args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
exec(`CREATE TABLE orbit_schema_migrations (version BIGINT UNSIGNED PRIMARY KEY,name VARCHAR(191) NOT NULL,checksum CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,dirty BOOLEAN NOT NULL,applied_at DATETIME(6) NULL) ENGINE=InnoDB`)
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range migrations {
|
||||
if m.Version > 19 {
|
||||
break
|
||||
}
|
||||
statements, err := splitSQL(m.SQL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, q := range statements {
|
||||
exec(q)
|
||||
}
|
||||
exec(`INSERT INTO orbit_schema_migrations(version,name,checksum,dirty,applied_at) VALUES (?,?,?,FALSE,UTC_TIMESTAMP(6))`, m.Version, m.Name, m.Checksum)
|
||||
}
|
||||
var now time.Time
|
||||
if err := db.QueryRowContext(ctx, `SELECT UTC_TIMESTAMP(6)`).Scan(&now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ns := executionID(t)
|
||||
deadline := now.Add(time.Hour)
|
||||
type fixture struct{ workflow, step, activation core.ID }
|
||||
seed := func(key string, terminal, stale bool) fixture {
|
||||
t.Helper()
|
||||
f := fixture{executionID(t), executionID(t), executionID(t)}
|
||||
state := "WAITING"
|
||||
var completed *time.Time
|
||||
if terminal {
|
||||
state = "CANCELLED"
|
||||
completed = &now
|
||||
}
|
||||
exec(`INSERT INTO weave_instances(namespace_id,id,definition_id,definition_version_id,gate_id,state,input,policy_snapshot,deadline_at,completed_at) VALUES (?,?,?,?,?,?,'{}','{}',?,?)`, ns, f.workflow, executionID(t), executionID(t), executionID(t), state, deadline, completed)
|
||||
current := f.activation
|
||||
if stale {
|
||||
current = executionID(t)
|
||||
}
|
||||
exec("INSERT INTO weave_steps(namespace_id,id,workflow_id,`key`,kind,state,current_activation_id,revision) VALUES (?,?,?,?,'SIGNAL','WAITING',?,7)", ns, f.step, f.workflow, key, current)
|
||||
exec(`INSERT INTO weave_activations(namespace_id,id,workflow_id,step_id,sequence,state,input) VALUES (?,?,?,?,1,'WAITING','null')`, ns, f.activation, f.workflow, f.step)
|
||||
exec(`INSERT INTO weave_waits(namespace_id,id,workflow_id,activation_id,kind,correlation_key,deadline_at) VALUES (?,?,?,?,'SIGNAL','historical',?)`, ns, executionID(t), f.workflow, f.activation, deadline)
|
||||
return f
|
||||
}
|
||||
active := seed("signal", false, false)
|
||||
existing := seed("existing", false, false)
|
||||
seed("terminal", true, false)
|
||||
seed("stale", false, true)
|
||||
existingID := executionID(t)
|
||||
original := `{"schema_version":1,"kind":"SIGNAL","activation_id":"` + string(existing.activation) + `"}`
|
||||
exec(`INSERT INTO weave_pending_advances(namespace_id,id,workflow_id,step_id,expected_step_revision,checkpoint,available_at,revision) VALUES (?,?,?,?,7,?,?,11)`, ns, existingID, existing.workflow, existing.step, original, now)
|
||||
pending, consumed, wait := executionID(t), executionID(t), executionID(t)
|
||||
correlation := strings.Repeat("😀", 191)
|
||||
name64 := strings.Repeat("a", 64)
|
||||
exec(`INSERT INTO weave_signals(namespace_id,id,workflow_id,key_digest,request_digest,type,target_step_key,correlation_key,payload,actor_id,accepted_at,expires_at) VALUES (?,?,?,RANDOM_BYTES(32),RANDOM_BYTES(32),?,'signal',?,'{"n":9007199254740993}','legacy',?,?)`, ns, pending, active.workflow, name64, correlation, now, deadline)
|
||||
exec(`INSERT INTO weave_signals(namespace_id,id,workflow_id,key_digest,request_digest,type,target_step_key,correlation_key,payload,actor_id,state,consumed_wait_id,accepted_at,expires_at) VALUES (?,?,?,RANDOM_BYTES(32),RANDOM_BYTES(32),'done','signal','history ','null','legacy','CONSUMED',?,?,?)`, ns, consumed, active.workflow, wait, now, deadline)
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := CheckSchema(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var payload, corr, signalName, state, disposition string
|
||||
var sequence *uint32
|
||||
var version *string
|
||||
var revision uint64
|
||||
var expires, available time.Time
|
||||
if err := db.QueryRowContext(ctx, `SELECT payload,correlation_key,type,state,disposition,target_activation_sequence,payload_version,revision,expires_at,available_at FROM weave_signals WHERE namespace_id=? AND id=?`, ns, pending).Scan(&payload, &corr, &signalName, &state, &disposition, &sequence, &version, &revision, &expires, &available); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(payload, "9007199254740993") || corr != correlation || signalName != name64 || state != "REJECTED" || disposition != "LEGACY_UNVERIFIED" || sequence != nil || version != nil || revision != 2 || !expires.Equal(deadline) || !available.Equal(deadline) {
|
||||
t.Fatal("legacy signal rewritten or treated as verified")
|
||||
}
|
||||
var count int
|
||||
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_signals WHERE namespace_id=? AND id=? AND state='CONSUMED' AND consumed_wait_id=? AND revision=1 AND payload_version IS NULL AND target_activation_sequence IS NULL AND correlation_key=?`, ns, consumed, wait, []byte("history ")).Scan(&count); err != nil || count != 1 {
|
||||
t.Fatal("historical consumed evidence changed", err)
|
||||
}
|
||||
var raw []byte
|
||||
var due time.Time
|
||||
var expected uint64
|
||||
if err := db.QueryRowContext(ctx, `SELECT checkpoint,available_at,expected_step_revision FROM weave_pending_advances WHERE namespace_id=? AND workflow_id=? AND step_id=?`, ns, active.workflow, active.step).Scan(&raw, &due, &expected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var checkpoint workflowCheckpoint
|
||||
if err := json.Unmarshal(raw, &checkpoint); err != nil || checkpoint.Kind != "SIGNAL" || checkpoint.SchemaVersion != 1 || checkpoint.ActivationID == nil || *checkpoint.ActivationID != active.activation || !due.Equal(deadline) || expected != 7 {
|
||||
t.Fatal("signal checkpoint reset original wait deadline", err)
|
||||
}
|
||||
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_pending_advances`).Scan(&count); err != nil || count != 2 {
|
||||
t.Fatal("stale/terminal or duplicate checkpoint", count, err)
|
||||
}
|
||||
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM weave_pending_advances WHERE id=? AND revision=11 AND available_at=? AND checkpoint=?`, existingID, now, original).Scan(&count); err != nil || count != 1 {
|
||||
t.Fatal("existing checkpoint changed", err)
|
||||
}
|
||||
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatal("foreign keys introduced", err)
|
||||
}
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.QueryRowContext(ctx, `SELECT revision FROM weave_signals WHERE namespace_id=? AND id=?`, ns, pending).Scan(&revision); err != nil || revision != 2 {
|
||||
t.Fatal("migration replay modified history", err)
|
||||
}
|
||||
t.Log("real v19 upgrade preserves 764-byte legacy correlation, exact payload and consumed evidence; isolates unverifiable pending signals; backfills current SIGNAL at original deadline; zero foreign keys")
|
||||
}
|
||||
207
internal/storage/mysql/workflow_signals.go
Normal file
207
internal/storage/mysql/workflow_signals.go
Normal file
@@ -0,0 +1,207 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/definition"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/engine"
|
||||
weave "git.yuntuo.hk/k2safe/orbit/weave/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/signal"
|
||||
)
|
||||
|
||||
type WorkflowSignalStore struct{ *TaskStore }
|
||||
|
||||
func NewWorkflowSignalStore(db *sql.DB) *WorkflowSignalStore {
|
||||
return &WorkflowSignalStore{NewTaskStore(db)}
|
||||
}
|
||||
|
||||
var _ signal.Store = (*WorkflowSignalStore)(nil)
|
||||
|
||||
func signalContract(node definition.Step) *definition.Signal {
|
||||
if node.Signal != nil {
|
||||
return node.Signal
|
||||
}
|
||||
if node.Poll != nil {
|
||||
return node.Poll.FinalSignal
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const signalColumns = `id,namespace_id,workflow_id,revision,type,COALESCE(payload_version,''),target_step_key,COALESCE(target_activation_sequence,0),correlation_key,payload,actor_id,COALESCE(evidence_ref,''),state,COALESCE(disposition,''),accepted_at,expires_at,consumed_wait_id`
|
||||
|
||||
func readWorkflowSignal(ctx context.Context, tx *sql.Tx, ns, workflow, id core.ID) (signal.View, error) {
|
||||
var v signal.View
|
||||
var payload []byte
|
||||
err := tx.QueryRowContext(ctx, `SELECT `+signalColumns+` FROM weave_signals WHERE namespace_id=? AND workflow_id=? AND id=?`, ns, workflow, id).Scan(&v.ID, &v.NamespaceID, &v.WorkflowID, &v.Revision, &v.Name, &v.PayloadVersion, &v.TargetStepKey, &v.TargetActivationSequence, &v.CorrelationKey, &payload, &v.ActorID, &v.EvidenceRef, &v.State, &v.Disposition, &v.AcceptedAt, &v.ExpiresAt, &v.ConsumedWaitID)
|
||||
v.Payload = payload
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
err = fault.NotFound
|
||||
}
|
||||
return v, err
|
||||
}
|
||||
|
||||
func (s *WorkflowSignalStore) Send(ctx context.Context, t auth.Token, workflow core.ID, key string, in signal.Input) (signal.Result, error) {
|
||||
var out signal.Result
|
||||
err := s.transaction(ctx, t, signal.SendAction, func(tx *sql.Tx, p auth.Permissions) error {
|
||||
if _, err := lockApplicationResources(ctx, tx, t.NamespaceID, false); err != nil {
|
||||
return err
|
||||
}
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "workflow.signal", string(workflow), key, "workflow_signal", id, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g, _, err := lockWorkflowGraph(ctx, tx, t.NamespaceID, workflow, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, queue := range g.version.Analysis.Queues {
|
||||
if !p.AllowsQueue(queue) {
|
||||
return fault.NotFound
|
||||
}
|
||||
}
|
||||
node, ok := g.program.Step(in.TargetStepKey)
|
||||
contract := signalContract(node)
|
||||
if !ok || contract == nil || contract.Name != in.Name || contract.PayloadVersion != in.PayloadVersion || definition.ValidateData(contract.PayloadType, in.Payload) != nil {
|
||||
return fault.Invalid("signal does not match fixed workflow contract")
|
||||
}
|
||||
if node.Poll != nil {
|
||||
outcome, err := g.program.ClassifyPoll(in.TargetStepKey, in.Payload)
|
||||
if err != nil || outcome == engine.PollPending {
|
||||
return fault.Invalid("final signal requires a terminal business result")
|
||||
}
|
||||
}
|
||||
if cached != nil {
|
||||
if err := json.Unmarshal(cached, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
current, err := readWorkflowSignal(ctx, tx, t.NamespaceID, workflow, out.Signal.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if out.Signal.NamespaceID != t.NamespaceID || out.Signal.WorkflowID != workflow || !current.AcceptedAt.Equal(out.Signal.AcceptedAt) {
|
||||
return errors.New("workflow signal receipt mismatch")
|
||||
}
|
||||
out.Deduplicated = true
|
||||
return nil
|
||||
}
|
||||
var count int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM (SELECT id FROM weave_signals WHERE namespace_id=? AND workflow_id=? LIMIT ?) admitted_signals`, t.NamespaceID, workflow, signal.MaxPerWorkflow).Scan(&count); err != nil {
|
||||
return err
|
||||
}
|
||||
if count >= signal.MaxPerWorkflow {
|
||||
return fault.New("WORKFLOW_SIGNAL_LIMIT", "workflow signal limit reached")
|
||||
}
|
||||
row := g.steps[in.TargetStepKey]
|
||||
sequence := uint32(1)
|
||||
if row.view.ActivationID != nil {
|
||||
if err := tx.QueryRowContext(ctx, `SELECT sequence FROM weave_activations WHERE namespace_id=? AND workflow_id=? AND step_id=? AND id=?`, t.NamespaceID, workflow, row.view.ID, row.view.ActivationID).Scan(&sequence); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if in.TargetActivationSequence > sequence {
|
||||
return fault.Conflict
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
state, disposition := weave.SignalState("PENDING"), ""
|
||||
switch {
|
||||
case in.TargetActivationSequence < sequence:
|
||||
state, disposition = "REJECTED", "STALE_ACTIVATION"
|
||||
case g.view.State.Terminal():
|
||||
state, disposition = "REJECTED", "WORKFLOW_SETTLED"
|
||||
case !now.Before(g.view.DeadlineAt):
|
||||
state, disposition = "EXPIRED", "WORKFLOW_DEADLINE"
|
||||
case row.view.State == weave.StepSkipped || row.view.State == weave.StepSucceeded || row.view.State == weave.StepFailed || row.view.State == weave.StepCancelled || row.view.State == weave.StepNeedsReview:
|
||||
state, disposition = "REJECTED", "STEP_SETTLED"
|
||||
case row.view.State == weave.StepBlocked && g.view.CancelRequestedAt != nil:
|
||||
state, disposition = "REJECTED", "WORKFLOW_CANCELLING"
|
||||
}
|
||||
if disposition == "" && row.view.ActivationID != nil {
|
||||
var name, version *string
|
||||
var correlation, waitState string
|
||||
var deadline time.Time
|
||||
if err := tx.QueryRowContext(ctx, `SELECT signal_name,payload_version,correlation_key,state,deadline_at FROM weave_waits WHERE namespace_id=? AND workflow_id=? AND activation_id=? FOR UPDATE`, t.NamespaceID, workflow, row.view.ActivationID).Scan(&name, &version, &correlation, &waitState, &deadline); err != nil {
|
||||
return err
|
||||
}
|
||||
if name == nil || version == nil || *name != in.Name || *version != in.PayloadVersion {
|
||||
return errors.New("workflow signal wait contract invalid")
|
||||
}
|
||||
switch {
|
||||
case waitState != "WAITING":
|
||||
state, disposition = "REJECTED", "WAIT_SETTLED"
|
||||
case correlation != in.CorrelationKey:
|
||||
state, disposition = "REJECTED", "CORRELATION_MISMATCH"
|
||||
case !now.Before(deadline):
|
||||
state, disposition = "EXPIRED", "SIGNAL_DEADLINE"
|
||||
}
|
||||
}
|
||||
digest, err := requestDigest(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
keyDigest := core.Digest(sha256.Sum256([]byte(key)))
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO weave_signals(namespace_id,id,workflow_id,key_digest,request_digest,type,payload_version,target_step_key,target_activation_sequence,correlation_key,payload,actor_id,evidence_ref,state,disposition,accepted_at,expires_at,available_at,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, t.NamespaceID, id, workflow, keyDigest, digest, in.Name, in.PayloadVersion, in.TargetStepKey, in.TargetActivationSequence, []byte(in.CorrelationKey), string(in.Payload), string(t.CredentialID), businessKeyValue(in.EvidenceRef), state, businessKeyValue(disposition), now, g.view.DeadlineAt, g.view.DeadlineAt, now, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if state == "PENDING" && row.view.ActivationID != nil {
|
||||
if err := saveWorkflowClock(ctx, tx, g, row, string(row.view.Kind), row.view.Revision, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `UPDATE weave_instances SET revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, now, t.NamespaceID, workflow, g.view.Revision)
|
||||
if err := exactlyOne(res, err); err != nil {
|
||||
return err
|
||||
}
|
||||
g.view.Revision++
|
||||
meta := map[string]any{"signal_id": id, "workflow_id": workflow, "target_step_key": in.TargetStepKey, "target_activation_sequence": in.TargetActivationSequence, "state": state, "disposition": disposition, "request_digest": hex.EncodeToString(digest[:]), "accepted_at": now, "expires_at": g.view.DeadlineAt}
|
||||
if err := emitEvent(ctx, tx, t.NamespaceID, workflow, g.view.Revision, "workflow", "workflow.signal.accepted", meta); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := appendAuditWithEvidence(ctx, tx, t, identity, id, "workflow_signal", "workflow.signal", in.Reason, in.EvidenceRef, nil, meta); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Signal, err = readWorkflowSignal(ctx, tx, t.NamespaceID, workflow, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
func (s *WorkflowSignalStore) Get(ctx context.Context, t auth.Token, workflow, id core.ID) (signal.View, error) {
|
||||
var out signal.View
|
||||
err := s.transaction(ctx, t, signal.ReadAction, func(tx *sql.Tx, p auth.Permissions) error {
|
||||
v, _, err := readWorkflowInstance(ctx, tx, t.NamespaceID, workflow, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
version, err := instanceVersion(ctx, tx, v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, queue := range version.Analysis.Queues {
|
||||
if !p.AllowsQueue(queue) {
|
||||
return fault.NotFound
|
||||
}
|
||||
}
|
||||
out, err = readWorkflowSignal(ctx, tx, t.NamespaceID, workflow, id)
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
161
internal/storage/mysql/workflow_signals_integration_test.go
Normal file
161
internal/storage/mysql/workflow_signals_integration_test.go
Normal file
@@ -0,0 +1,161 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/definition"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/instance"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/signal"
|
||||
)
|
||||
|
||||
func signalSpec(queue core.ID, early bool) definition.Specification {
|
||||
spec := runtimeSpec(queue, 1)
|
||||
node := definition.Step{Key: "signal", Kind: "SIGNAL", Signal: &definition.Signal{Name: "external.done", PayloadVersion: "v1", Correlation: definition.Value{Kind: "REFERENCE", Reference: &definition.Reference{Source: "workflow", Path: []string{"business_key"}}}, PayloadType: spec.InputType, TimeoutMS: 30000}}
|
||||
if early {
|
||||
spec.Body.Steps = append(spec.Body.Steps, node)
|
||||
spec.Body.Output = definition.Value{Kind: "REFERENCE", Reference: &definition.Reference{Source: "signal"}}
|
||||
} else {
|
||||
spec.Body.Steps = append([]definition.Step{node}, spec.Body.Steps...)
|
||||
}
|
||||
return spec
|
||||
}
|
||||
func grantWorkflowSignal(f *executionFixture) {
|
||||
f.exec(`UPDATE orbit_credentials SET permissions=JSON_ARRAY_APPEND(permissions,'$.actions','workflows:signal') WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, f.resources.CredentialID)
|
||||
}
|
||||
func signalInput(v instance.View) signal.Input {
|
||||
return signal.Input{Name: "external.done", PayloadVersion: "v1", TargetStepKey: "signal", TargetActivationSequence: 1, CorrelationKey: v.BusinessKey, Payload: json.RawMessage(`{"n":9007199254740993}`), Reason: "verified external completion", EvidenceRef: "proof:event-1"}
|
||||
}
|
||||
func signalStart(t *testing.T, ctx context.Context, db *sql.DB, early bool) (*executionFixture, instance.View, *signal.Service) {
|
||||
t.Helper()
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
v := runtimeStart(t, f, signalSpec(f.resources.QueueID, early))
|
||||
grantWorkflowSignal(f)
|
||||
return f, v, signal.New(NewWorkflowSignalStore(db))
|
||||
}
|
||||
func TestIntegrationWorkflowSignalAdmission(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, stop := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer stop()
|
||||
db, err := Open(ctx, dsn, 32, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Run("concurrent_identity_exact_input_and_frozen_receipt", func(t *testing.T) {
|
||||
f, v, api := signalStart(t, ctx, db, false)
|
||||
in := signalInput(v)
|
||||
key := string(executionID(t))
|
||||
var wg sync.WaitGroup
|
||||
results := make(chan signal.Result, 12)
|
||||
errs := make(chan error, 12)
|
||||
for i := 0; i < 12; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
out, err := api.Send(ctx, f.token, v.ID, key, in)
|
||||
results <- out
|
||||
errs <- err
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var id core.ID
|
||||
fresh := 0
|
||||
for out := range results {
|
||||
if id == "" {
|
||||
id = out.Signal.ID
|
||||
}
|
||||
if out.Signal.ID != id || out.Signal.State != "PENDING" || out.Signal.CorrelationKey != in.CorrelationKey || string(out.Signal.Payload) != string(in.Payload) {
|
||||
t.Fatal("signal replay changed identity")
|
||||
}
|
||||
if !out.Deduplicated {
|
||||
fresh++
|
||||
}
|
||||
}
|
||||
if fresh != 1 || f.count("weave_signals", "1=1") != 1 || f.count("orbit_audit_events", "action='workflow.signal'") != 1 {
|
||||
t.Fatal("duplicate signal admission")
|
||||
}
|
||||
changed := in
|
||||
changed.Payload = json.RawMessage(`{"n":2}`)
|
||||
if _, err := api.Send(ctx, f.token, v.ID, key, changed); fault.Code(err) != "IDEMPOTENCY_CONFLICT" {
|
||||
t.Fatal("same key changed content", err)
|
||||
}
|
||||
current, err := api.Get(ctx, f.token, v.ID, id)
|
||||
if err != nil || current.ActorID != string(f.resources.CredentialID) || current.EvidenceRef != in.EvidenceRef {
|
||||
t.Fatal("signal query lost provenance", err)
|
||||
}
|
||||
})
|
||||
t.Run("early_signal_is_durable_without_premature_activation", func(t *testing.T) {
|
||||
f, v, api := signalStart(t, ctx, db, true)
|
||||
out, err := api.Send(ctx, f.token, v.ID, string(executionID(t)), signalInput(v))
|
||||
if err != nil || out.Signal.State != "PENDING" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.count("pulse_tasks", "1=1") != 1 || f.count("weave_steps", "kind='SIGNAL' AND state='BLOCKED' AND current_activation_id IS NULL") != 1 {
|
||||
t.Fatal("early signal activated successor")
|
||||
}
|
||||
})
|
||||
t.Run("contract_generation_and_scope_are_checked", func(t *testing.T) {
|
||||
f, v, api := signalStart(t, ctx, db, false)
|
||||
in := signalInput(v)
|
||||
for _, change := range []func(*signal.Input){func(in *signal.Input) { in.Name = "wrong" }, func(in *signal.Input) { in.PayloadVersion = "v2" }, func(in *signal.Input) { in.Payload = json.RawMessage(`"bad"`) }, func(in *signal.Input) { in.TargetActivationSequence = 2 }} {
|
||||
bad := in
|
||||
change(&bad)
|
||||
if _, err := api.Send(ctx, f.token, v.ID, string(executionID(t)), bad); err == nil {
|
||||
t.Fatal("invalid signal contract or generation admitted")
|
||||
}
|
||||
}
|
||||
if f.count("weave_signals", "1=1") != 0 {
|
||||
t.Fatal("rejected request was partially stored")
|
||||
}
|
||||
wrong := in
|
||||
wrong.CorrelationKey = "different"
|
||||
out, err := api.Send(ctx, f.token, v.ID, string(executionID(t)), wrong)
|
||||
if err != nil || out.Signal.State != "REJECTED" || out.Signal.Disposition != "CORRELATION_MISMATCH" {
|
||||
t.Fatal("mismatched correlation granted authority", err)
|
||||
}
|
||||
grantWorkflowInstance(t, f)
|
||||
if _, err := api.Send(ctx, f.token, v.ID, string(executionID(t)), in); fault.Code(err) != "PERMISSION_DENIED" {
|
||||
t.Fatal("signal action not enforced", err)
|
||||
}
|
||||
grantWorkflowSignal(f)
|
||||
grantWorkflowInstance(t, f, string(executionID(t)))
|
||||
grantWorkflowSignal(f)
|
||||
if _, err := api.Send(ctx, f.token, v.ID, string(executionID(t)), in); fault.Code(err) != "NOT_FOUND" {
|
||||
t.Fatal("signal queue scope not enforced", err)
|
||||
}
|
||||
})
|
||||
t.Run("audit_failure_rolls_back_signal_identity_and_clock", func(t *testing.T) {
|
||||
f, v, api := signalStart(t, ctx, db, false)
|
||||
key := string(executionID(t))
|
||||
f.exec(`CREATE TRIGGER test_signal_admission_failure BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected signal audit failure'`)
|
||||
_, err := api.Send(ctx, f.token, v.ID, key, signalInput(v))
|
||||
f.exec(`DROP TRIGGER test_signal_admission_failure`)
|
||||
if err == nil || f.count("weave_signals", "1=1") != 0 || f.count("orbit_idempotency_keys", "operation='workflow.signal'") != 0 {
|
||||
t.Fatal("signal admission partially committed")
|
||||
}
|
||||
if out, err := api.Send(ctx, f.token, v.ID, key, signalInput(v)); err != nil || out.Deduplicated {
|
||||
t.Fatal("original identity did not recover", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user