feat: add durable Worker execution and WSS server
This commit is contained in:
@@ -14,10 +14,11 @@ import (
|
||||
|
||||
// BootstrapInput 仅供持有数据库管理权限的初始化命令使用，不暴露为公共 HTTP。
|
||||
type BootstrapInput struct {
|
||||
Application string // Application 应用名称，每次初始化创建独立应用。
|
||||
Namespace string // Namespace 命名空间名称。
|
||||
Environment string // Environment 环境名称，必须显式指定。
|
||||
Queue string // Queue 初始队列名称。
|
||||
Application string // Application 应用名称，每次初始化创建独立应用。
|
||||
Namespace string // Namespace 命名空间名称。
|
||||
Environment string // Environment 环境名称，必须显式指定。
|
||||
Queue string // Queue 初始队列名称。
|
||||
WorkerAccess bool // WorkerAccess 显式增加开发用 Worker 接入及消费权限。
|
||||
}
|
||||
|
||||
// BootstrapResult 包含一次性交付的原始凭据，禁止记录到日志或提交到版本库。
|
||||
@@ -56,6 +57,9 @@ func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func(
|
||||
}
|
||||
out.Token = secret
|
||||
permissions := auth.Permissions{Actions: []string{"tasks:submit", "tasks:read", "tasks:cancel", "queues:read", "queues:manage"}, Queues: []string{"*"}}
|
||||
if in.WorkerAccess {
|
||||
permissions.Actions = append(permissions.Actions, "workers:connect", "workers:consume")
|
||||
}
|
||||
raw, err := json.Marshal(permissions)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -191,9 +191,11 @@ func checkModelColumns(t *testing.T, ctx context.Context, db *sql.DB) {
|
||||
pulse.Queue{},
|
||||
pulse.ExecutionGate{},
|
||||
pulse.WorkerSession{},
|
||||
pulse.WorkerInstance{},
|
||||
pulse.Task{},
|
||||
pulse.Attempt{},
|
||||
pulse.AttemptReceipt{},
|
||||
pulse.AttemptEvidence{},
|
||||
pulse.ConcurrencyLimit{},
|
||||
pulse.Permit{},
|
||||
pulse.Schedule{},
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
-- Worker 会话隔离和迟到结果证据。P2 之前没有受支持的 Worker 注册接口。
|
||||
ALTER TABLE pulse_worker_sessions ADD UNIQUE KEY uq_session_instance_id(namespace_id, worker_instance_id, id);
|
||||
|
||||
CREATE TABLE pulse_worker_instances (
|
||||
namespace_id BINARY(16) NOT NULL COMMENT '进程所属命名空间，参与授权和所有关联',
|
||||
id BINARY(16) NOT NULL COMMENT 'Worker 进程实例标识，进程重启必须换新值',
|
||||
revision BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '持久化变更版本，从一开始递增',
|
||||
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '首次注册时间，UTC 微秒精度',
|
||||
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '最近变更时间，不替代会话代次',
|
||||
credential_id BINARY(16) NOT NULL COMMENT '初始注册凭据，进程生命周期内固定',
|
||||
queue_id BINARY(16) NOT NULL COMMENT '绑定队列，切换队列需创建新进程实例',
|
||||
secret_digest BINARY(32) NOT NULL COMMENT '随机进程恢复秘密的 SHA-256 摘要，不保存原文',
|
||||
registration_digest BINARY(32) NOT NULL COMMENT '队列及处理器版本集合的语义摘要，重连必须匹配',
|
||||
session_epoch BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '最大已分配连接代次，注册前为零',
|
||||
current_session_id BINARY(16) NULL COMMENT '当前会话，注册事务建立会话前为空',
|
||||
draining BOOLEAN NOT NULL DEFAULT FALSE COMMENT '持久化排空标记，重连不能清除',
|
||||
PRIMARY KEY(namespace_id,id),
|
||||
UNIQUE KEY uq_worker_credential(namespace_id,id,credential_id),
|
||||
CONSTRAINT ck_worker_instance_revision CHECK (revision>0),
|
||||
CONSTRAINT ck_worker_instance_session CHECK ((session_epoch=0 AND current_session_id IS NULL) OR (session_epoch>0 AND current_session_id IS NOT NULL)),
|
||||
CONSTRAINT fk_worker_instance_credential FOREIGN KEY(namespace_id,credential_id) REFERENCES orbit_credentials(namespace_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT fk_worker_instance_queue FOREIGN KEY(namespace_id,queue_id) REFERENCES pulse_queues(namespace_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT fk_worker_instance_session FOREIGN KEY(namespace_id,id,current_session_id) REFERENCES pulse_worker_sessions(namespace_id,worker_instance_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='Worker 进程恢复身份和当前会话头，用于跨节点隔离旧连接';
|
||||
|
||||
ALTER TABLE pulse_worker_sessions ADD CONSTRAINT fk_worker_session_instance FOREIGN KEY(namespace_id,worker_instance_id,credential_id) REFERENCES pulse_worker_instances(namespace_id,id,credential_id) ON DELETE RESTRICT ON UPDATE RESTRICT;
|
||||
|
||||
ALTER TABLE pulse_worker_sessions ADD CONSTRAINT ck_worker_session_capacity CHECK (capacity<=256);
|
||||
|
||||
ALTER TABLE pulse_attempts ADD KEY ix_attempt_session_state(namespace_id,session_id,state);
|
||||
|
||||
ALTER TABLE pulse_attempts ADD CONSTRAINT ck_attempt_lifecycle CHECK ((state IN ('LEASED','RUNNING') AND ended_at IS NULL) OR (state NOT IN ('LEASED','RUNNING') AND ended_at IS NOT NULL));
|
||||
|
||||
ALTER TABLE pulse_attempts ADD CONSTRAINT ck_attempt_start CHECK ((state='LEASED' AND authorized_at IS NULL) OR (state='RUNNING' AND authorized_at IS NOT NULL) OR state NOT IN ('LEASED','RUNNING'));
|
||||
|
||||
ALTER TABLE pulse_tasks ADD KEY ix_task_deadline(state,deadline_at,namespace_id,id);
|
||||
|
||||
ALTER TABLE pulse_tasks ADD KEY ix_task_start_deadline(state,start_deadline_at,namespace_id,id);
|
||||
|
||||
CREATE TABLE pulse_attempt_evidence (
|
||||
namespace_id BINARY(16) NOT NULL COMMENT '证据所属命名空间，与尝试一致',
|
||||
id BINARY(16) NOT NULL COMMENT '证据记录标识',
|
||||
revision BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '记录版本；证据追加保存，不覆盖',
|
||||
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '首次收到证据的数据库时间，UTC',
|
||||
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '写入时间，追加记录与创建时间相同',
|
||||
attempt_id BINARY(16) NOT NULL COMMENT '证据所属执行尝试',
|
||||
message_id BINARY(16) NOT NULL COMMENT '结果消息身份，重报保持不变',
|
||||
outcome VARCHAR(16) NOT NULL COMMENT 'Worker 声明的结果：SUCCEEDED、FAILED 或 UNKNOWN',
|
||||
error_code VARCHAR(64) NULL COMMENT '稳定错误分类；无错误时为空，不保存原始异常正文',
|
||||
payload MEDIUMTEXT NOT NULL COMMENT '结果或证据引用的 JSON 文本，保留精确数字',
|
||||
request_digest BINARY(32) NOT NULL COMMENT '结果语义摘要，排除会话与秘密',
|
||||
reason VARCHAR(32) NOT NULL COMMENT '证据未改写权威结果的原因：LEASE_LOST、NOT_STARTED 或 RESULT_CONFLICT',
|
||||
PRIMARY KEY(namespace_id,id),
|
||||
UNIQUE KEY uq_attempt_evidence_message(namespace_id,attempt_id,message_id),
|
||||
CONSTRAINT ck_attempt_evidence_revision CHECK(revision>0),
|
||||
CONSTRAINT ck_attempt_evidence_json CHECK(JSON_VALID(payload)),
|
||||
CONSTRAINT ck_attempt_evidence_outcome CHECK(outcome IN ('SUCCEEDED','FAILED','UNKNOWN')),
|
||||
CONSTRAINT ck_attempt_evidence_reason CHECK(reason IN ('LEASE_LOST','NOT_STARTED','RESULT_CONFLICT')),
|
||||
CONSTRAINT fk_attempt_evidence_attempt FOREIGN KEY(namespace_id,attempt_id) REFERENCES pulse_attempts(namespace_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='迟到与冲突结果的追加证据，不自动改变任务终态';
|
||||
|
||||
CREATE TRIGGER immutable_attempt_evidence BEFORE UPDATE ON pulse_attempt_evidence FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='attempt evidence is immutable';
|
||||
521
internal/storage/mysql/worker_execution.go
Normal file
521
internal/storage/mysql/worker_execution.go
Normal file
@@ -0,0 +1,521 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
type executionTask struct {
|
||||
view service.TaskView
|
||||
current *core.ID
|
||||
epoch uint64
|
||||
lease *time.Time
|
||||
gate *core.ID
|
||||
maxAttempts uint32
|
||||
attemptTimeout uint64
|
||||
}
|
||||
type executionRows struct {
|
||||
task executionTask
|
||||
attempt pulse.Attempt
|
||||
paused bool
|
||||
session sessionState
|
||||
}
|
||||
|
||||
func readExecutionTask(ctx context.Context, tx *sql.Tx, ns, id core.ID) (executionTask, error) {
|
||||
var row executionTask
|
||||
view, err := readTask(ctx, tx, ns, id, true)
|
||||
if err != nil {
|
||||
return row, err
|
||||
}
|
||||
row.view = view
|
||||
err = tx.QueryRowContext(ctx, `SELECT current_attempt_id,lease_epoch,lease_until,gate_id,max_attempts,attempt_timeout_ms FROM pulse_tasks WHERE namespace_id=? AND id=?`, ns, id).Scan(&row.current, &row.epoch, &row.lease, &row.gate, &row.maxAttempts, &row.attemptTimeout)
|
||||
return row, err
|
||||
}
|
||||
func lockQueue(ctx context.Context, tx *sql.Tx, ns, id core.ID) (paused bool, max uint32, err error) {
|
||||
err = tx.QueryRowContext(ctx, "SELECT paused,max_concurrency FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE", ns, id).Scan(&paused, &max)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
err = fault.NotFound
|
||||
}
|
||||
return
|
||||
}
|
||||
func lockGate(ctx context.Context, tx *sql.Tx, ns core.ID, id *core.ID) (bool, error) {
|
||||
if id == nil {
|
||||
return false, nil
|
||||
}
|
||||
var paused bool
|
||||
err := tx.QueryRowContext(ctx, "SELECT paused FROM pulse_execution_gates WHERE namespace_id=? AND id=? FOR SHARE", ns, *id).Scan(&paused)
|
||||
return paused, err
|
||||
}
|
||||
func minimumTime(first time.Time, others ...time.Time) time.Time {
|
||||
for _, other := range others {
|
||||
if other.Before(first) {
|
||||
first = other
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
func queueLimit(ctx context.Context, tx *sql.Tx, ns, queue core.ID, max uint32) (core.ID, uint32, error) {
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
digest := core.Digest(sha256.Sum256([]byte(queue)))
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_concurrency_limits(namespace_id,id,scope_key,key_digest,max_permits) VALUES (?,?,'queue',?,?) ON DUPLICATE KEY UPDATE id=pulse_concurrency_limits.id`, ns, id, digest, max)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
var used uint32
|
||||
var configured uint32
|
||||
err = tx.QueryRowContext(ctx, `SELECT id,used_permits,max_permits FROM pulse_concurrency_limits WHERE namespace_id=? AND scope_key='queue' AND key_digest=? FOR UPDATE`, ns, digest).Scan(&id, &used, &configured)
|
||||
if err == nil && configured != max {
|
||||
err = errors.New("queue limit configuration requires coordinated update")
|
||||
}
|
||||
return id, used, err
|
||||
}
|
||||
func outstanding(ctx context.Context, tx *sql.Tx, ns, instance core.ID) (uint32, error) {
|
||||
var count uint32
|
||||
err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_attempts a JOIN pulse_worker_sessions s ON s.namespace_id=a.namespace_id AND s.id=a.session_id WHERE a.namespace_id=? AND s.worker_instance_id=? AND a.state IN ('LEASED','RUNNING')`, ns, instance).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.Binding) (*worker.Assignment, error) {
|
||||
var assignment *worker.Assignment
|
||||
err := s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
state, err := s.lockSession(ctx, tx, token, b, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if state.draining || state.capacity == 0 {
|
||||
return nil
|
||||
}
|
||||
count, err := outstanding(ctx, tx, token.NamespaceID, b.InstanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if count >= state.capacity {
|
||||
return nil
|
||||
}
|
||||
paused, max, err := lockQueue(ctx, tx, token.NamespaceID, state.queue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if paused {
|
||||
return nil
|
||||
}
|
||||
limit, used, err := queueLimit(ctx, tx, token.NamespaceID, state.queue, max)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if used >= max {
|
||||
return nil
|
||||
}
|
||||
args := []any{token.NamespaceID, state.queue}
|
||||
matches := make([]string, 0, len(state.capabilities))
|
||||
for _, c := range state.capabilities {
|
||||
matches = append(matches, "(handler=? AND handler_version=? AND payload_version=?)")
|
||||
args = append(args, c.Handler, c.HandlerVersion, c.PayloadVersion)
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, `SELECT id,gate_id FROM pulse_tasks WHERE namespace_id=? AND queue_id=? AND state='WAITING' AND available_at<=UTC_TIMESTAMP(6) AND deadline_at>UTC_TIMESTAMP(6) AND (start_deadline_at IS NULL OR start_deadline_at>UTC_TIMESTAMP(6)) AND attempt_count<max_attempts AND (gate_id IS NULL OR EXISTS (SELECT 1 FROM pulse_execution_gates g WHERE g.namespace_id=pulse_tasks.namespace_id AND g.id=pulse_tasks.gate_id AND g.paused=FALSE)) AND (`+strings.Join(matches, " OR ")+`) ORDER BY available_at,priority DESC,id LIMIT 16`, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type candidate struct {
|
||||
id core.ID
|
||||
gate *core.ID
|
||||
}
|
||||
candidates := []candidate{}
|
||||
for rows.Next() {
|
||||
var c candidate
|
||||
if err := rows.Scan(&c.id, &c.gate); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
candidates = append(candidates, c)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
gated, err := lockGate(ctx, tx, token.NamespaceID, candidate.gate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if gated {
|
||||
continue
|
||||
}
|
||||
var locked core.ID
|
||||
err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_tasks WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, token.NamespaceID, candidate.id).Scan(&locked)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task, err := readExecutionTask(ctx, tx, token.NamespaceID, locked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !state.expires.After(now) {
|
||||
return worker.StaleSession
|
||||
}
|
||||
if task.view.State != pulse.TaskWaiting || task.view.AvailableAt.After(now) || !task.view.DeadlineAt.After(now) || task.view.AttemptCount >= task.maxAttempts || (task.view.StartDeadlineAt != nil && !task.view.StartDeadlineAt.After(now)) {
|
||||
continue
|
||||
}
|
||||
if task.epoch == ^uint64(0) {
|
||||
return fault.Conflict
|
||||
}
|
||||
attempt, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
secret, digest, err := worker.NewSecret()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
until := minimumTime(now.Add(s.options.LeaseDuration), task.view.DeadlineAt, state.expires)
|
||||
if task.view.StartDeadlineAt != nil {
|
||||
until = minimumTime(until, *task.view.StartDeadlineAt)
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempts(namespace_id,id,task_id,epoch,session_id,token_digest,lease_until) VALUES (?,?,?,?,?,?,?)`, token.NamespaceID, attempt, locked, task.epoch+1, b.SessionID, digest, until)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
permit, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_permits(namespace_id,id,limit_id,attempt_id,lease_until) VALUES (?,?,?,?,?)`, token.NamespaceID, permit, limit, attempt, until)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err := tx.ExecContext(ctx, `UPDATE pulse_concurrency_limits SET used_permits=used_permits+1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND used_permits<max_permits`, now, token.NamespaceID, limit)
|
||||
if err := exactlyOne(changed, err); err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='LEASED',current_attempt_id=?,lease_epoch=lease_epoch+1,lease_until=?,attempt_count=attempt_count+1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND state='WAITING' AND revision=?`, attempt, until, now, token.NamespaceID, locked, task.view.Revision)
|
||||
if err := exactlyOne(changed, err); err != nil {
|
||||
return err
|
||||
}
|
||||
view, err := readTask(ctx, tx, token.NamespaceID, locked, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, token.NamespaceID, locked, view.Revision, "task", "task.leased", map[string]any{"attempt_id": attempt, "epoch": fmt.Sprint(task.epoch + 1), "session_id": b.SessionID, "lease_until": until}); err != nil {
|
||||
return err
|
||||
}
|
||||
assignment = &worker.Assignment{Task: view, AttemptID: attempt, Epoch: task.epoch + 1, LeaseSecret: secret, LeaseUntil: until, ServerTime: now}
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return assignment, nil
|
||||
}
|
||||
func exactlyOne(result sql.Result, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != 1 {
|
||||
return errors.New("execution invariant affected unexpected rows")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// lockExecution 的跨资源顺序为进程/会话 -> 队列 -> 可选门控 -> 任务 -> 尝试。
|
||||
// 恢复扫描也从队列锁开始，不反向锁定 Worker 进程。
|
||||
func (s *ExecutionStore) lockExecution(ctx context.Context, tx *sql.Tx, token auth.Token, b worker.Binding, p auth.Permissions, ref worker.AttemptRef, checkGate bool) (executionRows, error) {
|
||||
var out executionRows
|
||||
state, err := s.lockSession(ctx, tx, token, b, p)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.session = state
|
||||
var queue core.ID
|
||||
var gate *core.ID
|
||||
err = tx.QueryRowContext(ctx, `SELECT queue_id,gate_id FROM pulse_tasks WHERE namespace_id=? AND id=?`, token.NamespaceID, ref.TaskID).Scan(&queue, &gate)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return out, fault.NotFound
|
||||
}
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if queue != state.queue || !p.AllowsQueue(queue) {
|
||||
return out, fault.NotFound
|
||||
}
|
||||
out.paused, _, err = lockQueue(ctx, tx, token.NamespaceID, queue)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if checkGate {
|
||||
paused, err := lockGate(ctx, tx, token.NamespaceID, gate)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.paused = out.paused || paused
|
||||
}
|
||||
out.task, err = readExecutionTask(ctx, tx, token.NamespaceID, ref.TaskID)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.attempt, err = readAttempt(ctx, tx, token.NamespaceID, ref.AttemptID)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
proof, err := worker.SecretDigest(ref.LeaseSecret)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if out.attempt.TaskID != ref.TaskID || out.attempt.Epoch != ref.Epoch || subtle.ConstantTimeCompare(proof[:], out.attempt.TokenDigest[:]) != 1 {
|
||||
return out, fault.Forbidden
|
||||
}
|
||||
var instance core.ID
|
||||
if err := tx.QueryRowContext(ctx, `SELECT worker_instance_id FROM pulse_worker_sessions WHERE namespace_id=? AND id=?`, token.NamespaceID, out.attempt.SessionID).Scan(&instance); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if instance != b.InstanceID {
|
||||
return out, fault.Forbidden
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func readAttempt(ctx context.Context, tx *sql.Tx, ns, id core.ID) (pulse.Attempt, error) {
|
||||
var a pulse.Attempt
|
||||
a.NamespaceID = ns
|
||||
err := tx.QueryRowContext(ctx, `SELECT id,revision,task_id,epoch,session_id,token_digest,state,lease_until,authorized_at,ended_at,error_code,result,result_digest FROM pulse_attempts WHERE namespace_id=? AND id=? FOR UPDATE`, ns, id).Scan(&a.ID, &a.Revision, &a.TaskID, &a.Epoch, &a.SessionID, &a.TokenDigest, &a.State, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Result, &a.ResultDigest)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
err = fault.NotFound
|
||||
}
|
||||
return a, err
|
||||
}
|
||||
func leaseStatus(rows executionRows, b worker.Binding, now time.Time) worker.LeaseStatus {
|
||||
t, a := rows.task, rows.attempt
|
||||
until := minimumTime(a.LeaseUntil, rows.session.expires, t.view.DeadlineAt)
|
||||
if t.lease != nil {
|
||||
until = minimumTime(until, *t.lease)
|
||||
}
|
||||
active := rows.session.expires.After(now) && t.current != nil && *t.current == a.ID && t.epoch == a.Epoch && a.SessionID == b.SessionID && t.lease != nil && t.lease.After(now) && a.LeaseUntil.After(now) && t.view.DeadlineAt.After(now) && ((t.view.State == pulse.TaskLeased && a.State == pulse.AttemptLeased) || (t.view.State == pulse.TaskRunning && a.State == pulse.AttemptRunning))
|
||||
if a.AuthorizedAt != nil {
|
||||
deadline := a.AuthorizedAt.Add(time.Duration(t.attemptTimeout) * time.Millisecond)
|
||||
active = active && deadline.After(now)
|
||||
until = minimumTime(until, deadline)
|
||||
}
|
||||
return worker.LeaseStatus{Active: active, State: a.State, LeaseUntil: until, ServerTime: now}
|
||||
}
|
||||
func attemptIdentity(ref worker.AttemptRef) any {
|
||||
return struct {
|
||||
Task core.ID
|
||||
Attempt core.ID
|
||||
Epoch string
|
||||
}{ref.TaskID, ref.AttemptID, fmt.Sprint(ref.Epoch)}
|
||||
}
|
||||
func readReceipt(ctx context.Context, tx *sql.Tx, ns, attempt, message core.ID, operation string, digest core.Digest) (*worker.Receipt, error) {
|
||||
var saved core.Digest
|
||||
var raw []byte
|
||||
err := tx.QueryRowContext(ctx, `SELECT request_digest,response FROM pulse_attempt_receipts WHERE namespace_id=? AND attempt_id=? AND operation=? AND message_id=? FOR UPDATE`, ns, attempt, operation, message).Scan(&saved, &raw)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if saved != digest {
|
||||
return nil, fault.IdempotencyConflict
|
||||
}
|
||||
var receipt worker.Receipt
|
||||
if err := json.Unmarshal(raw, &receipt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &receipt, nil
|
||||
}
|
||||
func saveReceipt(ctx context.Context, tx *sql.Tx, ns, attempt, message core.ID, operation string, digest core.Digest, receipt worker.Receipt) error {
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := json.Marshal(receipt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempt_receipts(namespace_id,id,attempt_id,operation,message_id,request_digest,response,expires_at) VALUES (?,?,?,?,?,?,?,DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 90 DAY))`, ns, id, attempt, operation, message, digest, string(raw))
|
||||
return err
|
||||
}
|
||||
func renewPermits(ctx context.Context, tx *sql.Tx, ns, attempt core.ID, until, now time.Time) error {
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_permits SET lease_until=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND attempt_id=? AND released_at IS NULL`, until, now, ns, attempt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n < 1 {
|
||||
return errors.New("active attempt has no concurrency permit")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func releasePermits(ctx context.Context, tx *sql.Tx, ns, attempt core.ID, now time.Time) error {
|
||||
rows, err := tx.QueryContext(ctx, `SELECT id,limit_id FROM pulse_permits WHERE namespace_id=? AND attempt_id=? AND released_at IS NULL ORDER BY limit_id FOR UPDATE`, ns, attempt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type permit struct{ id, limit core.ID }
|
||||
permits := []permit{}
|
||||
for rows.Next() {
|
||||
var p permit
|
||||
if err := rows.Scan(&p.id, &p.limit); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
permits = append(permits, p)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(permits) == 0 {
|
||||
return errors.New("active attempt has no permit to release")
|
||||
}
|
||||
for _, p := range permits {
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_concurrency_limits SET used_permits=used_permits-1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND used_permits>0`, now, ns, p.limit)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err = tx.ExecContext(ctx, `UPDATE pulse_permits SET released_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND released_at IS NULL`, now, now, ns, p.id)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *ExecutionStore) Start(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, ref worker.AttemptRef) (worker.CommandResult, error) {
|
||||
return s.executionCommand(ctx, token, b, message, ref, "start")
|
||||
}
|
||||
func (s *ExecutionStore) Renew(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, ref worker.AttemptRef) (worker.CommandResult, error) {
|
||||
return s.executionCommand(ctx, token, b, message, ref, "renew")
|
||||
}
|
||||
func (s *ExecutionStore) executionCommand(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, ref worker.AttemptRef, operation string) (worker.CommandResult, error) {
|
||||
var out worker.CommandResult
|
||||
digest, err := requestDigest(attemptIdentity(ref))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
err = s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
rows, err := s.lockExecution(ctx, tx, token, b, p, ref, operation == "start")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
receipt, err := readReceipt(ctx, tx, token.NamespaceID, ref.AttemptID, message, operation, digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status := leaseStatus(rows, b, now)
|
||||
if receipt != nil {
|
||||
out = worker.CommandResult{Receipt: *receipt, Lease: status, Deduplicated: true}
|
||||
return nil
|
||||
}
|
||||
if !status.Active {
|
||||
return worker.LeaseLost
|
||||
}
|
||||
task, a := rows.task, rows.attempt
|
||||
authorized := a.AuthorizedAt
|
||||
if operation == "start" {
|
||||
if a.State != pulse.AttemptLeased || task.view.State != pulse.TaskLeased {
|
||||
return fault.Conflict
|
||||
}
|
||||
if rows.paused {
|
||||
return worker.GateClosed
|
||||
}
|
||||
if task.view.StartDeadlineAt != nil && !task.view.StartDeadlineAt.After(now) {
|
||||
return worker.LeaseLost
|
||||
}
|
||||
authorized = &now
|
||||
} else if a.State != pulse.AttemptRunning || authorized == nil {
|
||||
return fault.Conflict
|
||||
}
|
||||
until := minimumTime(now.Add(s.options.LeaseDuration), authorized.Add(time.Duration(task.attemptTimeout)*time.Millisecond), task.view.DeadlineAt, rows.session.expires)
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state='RUNNING',authorized_at=?,lease_until=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, authorized, until, now, token.NamespaceID, a.ID, a.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='RUNNING',lease_until=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, until, now, token.NamespaceID, ref.TaskID, task.view.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := renewPermits(ctx, tx, token.NamespaceID, a.ID, until, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if operation == "start" {
|
||||
if err := emitEvent(ctx, tx, token.NamespaceID, ref.TaskID, task.view.Revision+1, "task", "task.started", map[string]any{"attempt_id": a.ID, "epoch": fmt.Sprint(a.Epoch), "session_id": b.SessionID}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
receipt = &worker.Receipt{Operation: operation, TaskState: pulse.TaskRunning, AttemptState: pulse.AttemptRunning, TaskRevision: task.view.Revision + 1, LeaseUntil: until, Disposition: "APPLIED"}
|
||||
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, operation, digest, *receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
out = worker.CommandResult{Receipt: *receipt, Lease: worker.LeaseStatus{Active: true, State: pulse.AttemptRunning, LeaseUntil: until, ServerTime: now}}
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (s *ExecutionStore) Resume(ctx context.Context, token auth.Token, b worker.Binding, ref worker.AttemptRef) (worker.LeaseStatus, error) {
|
||||
var out worker.LeaseStatus
|
||||
err := s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
rows, err := s.lockExecution(ctx, tx, token, b, p, ref, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 只暂按新会话计算有效性；通过所有期限和当前尝试检查后才迁移归属，绝不延长期限。
|
||||
previous := rows.attempt.SessionID
|
||||
rows.attempt.SessionID = b.SessionID
|
||||
out = leaseStatus(rows, b, now)
|
||||
if !out.Active || previous == b.SessionID {
|
||||
return nil
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET session_id=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, b.SessionID, now, token.NamespaceID, ref.AttemptID, rows.attempt.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
return emitEvent(ctx, tx, token.NamespaceID, ref.AttemptID, rows.attempt.Revision+1, "attempt", "attempt.resumed", map[string]any{"from_session_id": previous, "to_session_id": b.SessionID, "node_id": s.node})
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
638
internal/storage/mysql/worker_integration_test.go
Normal file
638
internal/storage/mysql/worker_integration_test.go
Normal file
@@ -0,0 +1,638 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
type executionFixture struct {
|
||||
t *testing.T
|
||||
ctx context.Context
|
||||
db *sql.DB
|
||||
resources BootstrapResult
|
||||
token auth.Token
|
||||
tasks *service.Service
|
||||
storeA, storeB *ExecutionStore
|
||||
a, b *worker.Service
|
||||
hello worker.ConnectInput
|
||||
binding worker.Binding
|
||||
}
|
||||
|
||||
func executionID(t *testing.T) core.ID {
|
||||
t.Helper()
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
func newExecutionFixture(t *testing.T, ctx context.Context, db *sql.DB) *executionFixture {
|
||||
t.Helper()
|
||||
f := &executionFixture{t: t, ctx: ctx, db: db, tasks: service.New(NewTaskStore(db))}
|
||||
err := Bootstrap(ctx, db, BootstrapInput{Application: "worker-" + string(executionID(t)), Namespace: "default", Environment: "test", Queue: "work"}, func(v BootstrapResult) error { f.resources = v; return nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(auth.Permissions{Actions: []string{"tasks:submit", "tasks:read", "tasks:cancel", "queues:read", "queues:manage", "workers:connect", "workers:consume"}, Queues: []string{"*"}})
|
||||
f.exec("UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?", string(raw), f.resources.NamespaceID, f.resources.CredentialID)
|
||||
f.token, err = auth.Parse(f.resources.Token)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.storeA, err = NewExecutionStore(db, executionID(t), ExecutionOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.storeB, err = NewExecutionStore(db, executionID(t), ExecutionOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.a, f.b = worker.New(f.storeA), worker.New(f.storeB)
|
||||
secret, _, err := worker.NewSecret()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.hello = worker.ConnectInput{InstanceID: executionID(t), InstanceSecret: secret, QueueID: f.resources.QueueID, Capacity: 4, Capabilities: []worker.Capability{{Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1"}}}
|
||||
f.binding, _, err = f.a.Connect(ctx, f.token, f.hello)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
func (f *executionFixture) exec(q string, args ...any) {
|
||||
f.t.Helper()
|
||||
if _, err := f.db.ExecContext(f.ctx, q, args...); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
}
|
||||
func (f *executionFixture) submit(max uint32) service.TaskView {
|
||||
f.t.Helper()
|
||||
out, err := f.tasks.Submit(f.ctx, f.token, string(executionID(f.t)), service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`), Retry: pulse.RetryPolicy{MaxAttempts: max, AttemptTimeoutMS: 120000}})
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return out.Task
|
||||
}
|
||||
func (f *executionFixture) claim() (*worker.Assignment, worker.AttemptRef) {
|
||||
f.t.Helper()
|
||||
out, err := f.a.Claim(f.ctx, f.token, f.binding)
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
if out == nil {
|
||||
f.t.Fatal("expected assignment")
|
||||
}
|
||||
return out, worker.AttemptRef{TaskID: out.Task.ID, AttemptID: out.AttemptID, Epoch: out.Epoch, LeaseSecret: out.LeaseSecret}
|
||||
}
|
||||
func (f *executionFixture) get(id core.ID) service.TaskView {
|
||||
f.t.Helper()
|
||||
out, err := f.tasks.GetTask(f.ctx, f.token, id)
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func (f *executionFixture) expire(ref worker.AttemptRef) {
|
||||
f.t.Helper()
|
||||
f.exec("UPDATE pulse_tasks SET lease_until=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, ref.TaskID)
|
||||
f.exec("UPDATE pulse_attempts SET lease_until=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, ref.AttemptID)
|
||||
}
|
||||
func (f *executionFixture) count(table, where string, args ...any) int {
|
||||
f.t.Helper()
|
||||
var n int
|
||||
if err := f.db.QueryRowContext(f.ctx, "SELECT COUNT(*) FROM "+table+" WHERE namespace_id=? AND "+where, append([]any{f.resources.NamespaceID}, args...)...).Scan(&n); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
func (f *executionFixture) used() uint32 {
|
||||
f.t.Helper()
|
||||
var n uint32
|
||||
if err := f.db.QueryRowContext(f.ctx, "SELECT COALESCE(SUM(used_permits),0) FROM pulse_concurrency_limits WHERE namespace_id=?", f.resources.NamespaceID).Scan(&n); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
func expectWorkerCode(t *testing.T, err error, code string) {
|
||||
t.Helper()
|
||||
if fault.Code(err) != code {
|
||||
t.Fatalf("wanted %s got %v", code, err)
|
||||
}
|
||||
}
|
||||
func TestIntegrationWorkerExecution(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
db, err := Open(ctx, dsn, 32, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Run("durable_start_renew_and_exact_result_confirmation", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := f.submit(3)
|
||||
assignment, ref := f.claim()
|
||||
if assignment.Task.ID != task.ID || assignment.Task.State != pulse.TaskLeased || assignment.Task.AttemptCount != 1 || f.used() != 1 {
|
||||
t.Fatal("claim state/permit invariant")
|
||||
}
|
||||
startID := executionID(t)
|
||||
started, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
|
||||
if err != nil || !started.Lease.Active || started.Receipt.TaskState != pulse.TaskRunning {
|
||||
t.Fatalf("start: %v", err)
|
||||
}
|
||||
duplicate, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
|
||||
if err != nil || !duplicate.Deduplicated || duplicate.Receipt != started.Receipt {
|
||||
t.Fatalf("start replay: %v", err)
|
||||
}
|
||||
_, err = f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
renewID := executionID(t)
|
||||
renewed, err := f.a.Renew(ctx, f.token, f.binding, renewID, ref)
|
||||
if err != nil || !renewed.Lease.Active || renewed.Lease.LeaseUntil.Before(started.Lease.LeaseUntil) {
|
||||
t.Fatalf("renew: %v", err)
|
||||
}
|
||||
repeated, err := f.a.Renew(ctx, f.token, f.binding, renewID, ref)
|
||||
if err != nil || !repeated.Deduplicated || repeated.Receipt.LeaseUntil != renewed.Receipt.LeaseUntil {
|
||||
t.Fatalf("renew replay: %v", err)
|
||||
}
|
||||
result := worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"amount":12345678901234567890.1234567890123456789,"version":9007199254740993}`)}
|
||||
message := executionID(t)
|
||||
finished, err := f.a.Finish(ctx, f.token, f.binding, message, result)
|
||||
if err != nil || finished.Lease.Active || finished.Receipt.Disposition != "APPLIED" {
|
||||
t.Fatalf("finish: %v", err)
|
||||
}
|
||||
duplicate, err = f.a.Finish(ctx, f.token, f.binding, message, result)
|
||||
if err != nil || !duplicate.Deduplicated {
|
||||
t.Fatalf("finish replay %v", err)
|
||||
}
|
||||
duplicate, err = f.a.Finish(ctx, f.token, f.binding, executionID(t), result)
|
||||
if err != nil || !duplicate.Deduplicated {
|
||||
t.Fatalf("new message equivalent result %v", err)
|
||||
}
|
||||
current := f.get(task.ID)
|
||||
if current.State != pulse.TaskSucceeded || current.CompletedAt == nil || !strings.Contains(string(current.Result), "12345678901234567890.1234567890123456789") || f.used() != 0 {
|
||||
t.Fatal("finish lost state, precision or permit")
|
||||
}
|
||||
if f.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 1 || f.count("orbit_audit_events", "resource_id=? AND action='task.result'", task.ID) != 1 {
|
||||
t.Fatal("duplicated completion history")
|
||||
}
|
||||
_, err = f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
||||
expectWorkerCode(t, err, "LEASE_LOST")
|
||||
oldAck, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
|
||||
if err != nil || !oldAck.Deduplicated || oldAck.Lease.Active {
|
||||
t.Fatalf("old start acknowledgement resurrected execution %v", err)
|
||||
}
|
||||
changed := result
|
||||
changed.Result = json.RawMessage(`{"changed":true}`)
|
||||
_, err = f.a.Finish(ctx, f.token, f.binding, message, changed)
|
||||
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
|
||||
conflict, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), changed)
|
||||
if err != nil || conflict.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(task.ID).State != pulse.TaskSucceeded {
|
||||
t.Fatalf("conflict rewrote outcome %v", err)
|
||||
}
|
||||
if f.count("pulse_attempt_evidence", "attempt_id=? AND reason='RESULT_CONFLICT'", ref.AttemptID) != 1 {
|
||||
t.Fatal("conflict evidence missing")
|
||||
}
|
||||
})
|
||||
t.Run("cross_node_resume_fences_old_session_and_requires_secrets", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
f.submit(3)
|
||||
_, ref := f.claim()
|
||||
start := executionID(t)
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, start, ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := f.get(ref.TaskID)
|
||||
newBinding, welcome, err := f.b.Connect(ctx, f.token, f.hello)
|
||||
if err != nil || welcome.Epoch != 2 {
|
||||
t.Fatalf("reconnect %v", err)
|
||||
}
|
||||
_, err = f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
||||
expectWorkerCode(t, err, "STALE_SESSION")
|
||||
_, err = f.b.Renew(ctx, f.token, newBinding, executionID(t), ref)
|
||||
expectWorkerCode(t, err, "LEASE_LOST")
|
||||
wrong := ref
|
||||
wrong.LeaseSecret = strings.Repeat("A", 43)
|
||||
_, err = f.b.Resume(ctx, f.token, newBinding, wrong)
|
||||
expectWorkerCode(t, err, "PERMISSION_DENIED")
|
||||
resumed, err := f.b.Resume(ctx, f.token, newBinding, ref)
|
||||
if err != nil || !resumed.Active || resumed.State != pulse.AttemptRunning {
|
||||
t.Fatalf("resume %v", err)
|
||||
}
|
||||
duplicate, err := f.b.Start(ctx, f.token, newBinding, start, ref)
|
||||
if err != nil || !duplicate.Deduplicated || !duplicate.Lease.Active {
|
||||
t.Fatalf("start ack recovery %v", err)
|
||||
}
|
||||
if f.get(ref.TaskID).AttemptCount != before.AttemptCount || f.used() != 1 {
|
||||
t.Fatal("resume created attempt or duplicated permit")
|
||||
}
|
||||
if _, err := f.b.Resume(ctx, f.token, newBinding, ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.count("orbit_events", "aggregate_id=? AND type='attempt.resumed'", ref.AttemptID) != 1 {
|
||||
t.Fatal("resume event duplicated")
|
||||
}
|
||||
badHello := f.hello
|
||||
badHello.InstanceSecret = strings.Repeat("A", 43)
|
||||
_, _, err = f.b.Connect(ctx, f.token, badHello)
|
||||
expectWorkerCode(t, err, "PERMISSION_DENIED")
|
||||
badHello = f.hello
|
||||
badHello.Capabilities = []worker.Capability{{Handler: "another", HandlerVersion: "v1", PayloadVersion: "v1"}}
|
||||
_, _, err = f.b.Connect(ctx, f.token, badHello)
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
f.expire(ref)
|
||||
expired, err := f.b.Resume(ctx, f.token, newBinding, ref)
|
||||
if err != nil || expired.Active {
|
||||
t.Fatalf("expired resume %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("multi_node_claim_obeys_queue_and_process_capacity", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
f.exec("UPDATE pulse_queues SET max_concurrency=2 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.QueueID)
|
||||
for i := 0; i < 8; i++ {
|
||||
f.submit(3)
|
||||
}
|
||||
secret, _, err := worker.NewSecret()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hello := f.hello
|
||||
hello.InstanceID = executionID(t)
|
||||
hello.InstanceSecret = secret
|
||||
hello.Capacity = 1
|
||||
bindingB, _, err := f.b.Connect(ctx, f.token, hello)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.a.Capacity(ctx, f.token, f.binding, worker.CapacityInput{Sequence: 1, Capacity: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assignments := make(chan *worker.Assignment, 16)
|
||||
errs := make(chan error, 16)
|
||||
var wg sync.WaitGroup
|
||||
start := make(chan struct{})
|
||||
for i := 0; i < 16; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
var a *worker.Assignment
|
||||
var err error
|
||||
if i%2 == 0 {
|
||||
a, err = f.a.Claim(ctx, f.token, f.binding)
|
||||
} else {
|
||||
a, err = f.b.Claim(ctx, f.token, bindingB)
|
||||
}
|
||||
assignments <- a
|
||||
errs <- err
|
||||
}(i)
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
close(assignments)
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ids := map[core.ID]bool{}
|
||||
for a := range assignments {
|
||||
if a == nil {
|
||||
continue
|
||||
}
|
||||
if ids[a.Task.ID] {
|
||||
t.Fatal("task dispatched twice")
|
||||
}
|
||||
ids[a.Task.ID] = true
|
||||
}
|
||||
if len(ids) != 2 || f.used() != 2 {
|
||||
t.Fatalf("concurrency overrun: assignments=%d used=%d", len(ids), f.used())
|
||||
}
|
||||
b2, _, err := f.b.Connect(ctx, f.token, hello)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, err := f.b.Claim(ctx, f.token, b2)
|
||||
if err != nil || a != nil {
|
||||
t.Fatalf("reconnect reset capacity %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("drain_is_durable_and_capacity_sequence_is_not_additive", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
f.submit(3)
|
||||
_, ref := f.claim()
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input := worker.CapacityInput{Sequence: 1, Capacity: 0, Drain: true}
|
||||
if _, err := f.a.Capacity(ctx, f.token, f.binding, input); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.a.Capacity(ctx, f.token, f.binding, input); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input.Capacity = 2
|
||||
_, err := f.a.Capacity(ctx, f.token, f.binding, input)
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
input.Sequence = 2
|
||||
input.Drain = false
|
||||
_, err = f.a.Capacity(ctx, f.token, f.binding, input)
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
f.submit(3)
|
||||
a, err := f.a.Claim(ctx, f.token, f.binding)
|
||||
if err != nil || a != nil {
|
||||
t.Fatalf("draining worker received work %v", err)
|
||||
}
|
||||
if f.used() != 1 {
|
||||
t.Fatal("capacity zero released in-flight permit")
|
||||
}
|
||||
newBinding, welcome, err := f.b.Connect(ctx, f.token, f.hello)
|
||||
if err != nil || !welcome.Draining {
|
||||
t.Fatalf("drain lost on reconnect %v", err)
|
||||
}
|
||||
if _, err := f.b.Resume(ctx, f.token, newBinding, ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.b.Finish(ctx, f.token, newBinding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.used() != 0 {
|
||||
t.Fatal("drained execution did not release capacity")
|
||||
}
|
||||
})
|
||||
t.Run("paused_gate_blocks_start_but_not_saved_result", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
gate := executionID(t)
|
||||
f.exec("INSERT INTO pulse_execution_gates(namespace_id,id,paused) VALUES (?,?,TRUE)", f.resources.NamespaceID, gate)
|
||||
// 超过一次候选页的暂停任务不能饿死后面的普通任务。
|
||||
for i := 0; i < 18; i++ {
|
||||
task := f.submit(3)
|
||||
f.exec("UPDATE pulse_tasks SET gate_id=? WHERE namespace_id=? AND id=?", gate, f.resources.NamespaceID, task.ID)
|
||||
}
|
||||
ready := f.submit(3)
|
||||
a, ref := f.claim()
|
||||
if a.Task.ID != ready.ID {
|
||||
t.Fatal("paused candidate starvation")
|
||||
}
|
||||
f.exec("UPDATE pulse_tasks SET gate_id=? WHERE namespace_id=? AND id=?", gate, f.resources.NamespaceID, ready.ID)
|
||||
start := executionID(t)
|
||||
_, err := f.a.Start(ctx, f.token, f.binding, start, ref)
|
||||
expectWorkerCode(t, err, "EXECUTION_PAUSED")
|
||||
f.exec("UPDATE pulse_execution_gates SET paused=FALSE WHERE namespace_id=? AND id=?", f.resources.NamespaceID, gate)
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, start, ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.exec("UPDATE pulse_execution_gates SET paused=TRUE WHERE namespace_id=? AND id=?", f.resources.NamespaceID, gate)
|
||||
duplicate, err := f.a.Start(ctx, f.token, f.binding, start, ref)
|
||||
if err != nil || !duplicate.Deduplicated || !duplicate.Lease.Active {
|
||||
t.Fatalf("pause blocked original start receipt %v", err)
|
||||
}
|
||||
if _, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
t.Run("recovery_is_bounded_and_unknown_effects_are_not_replayed", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := f.submit(2)
|
||||
_, ref := f.claim()
|
||||
f.expire(ref)
|
||||
stats, err := f.storeB.Recover(ctx, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current := f.get(task.ID)
|
||||
if current.State != pulse.TaskWaiting || current.AttemptCount != 1 || f.used() != 0 || stats.Requeued < 1 {
|
||||
t.Fatal("pre-start lease recovery failed")
|
||||
}
|
||||
f.exec("UPDATE pulse_tasks SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID)
|
||||
_, newRef := f.claim()
|
||||
if newRef.Epoch != 2 || newRef.AttemptID == ref.AttemptID {
|
||||
t.Fatal("attempt epoch not advanced")
|
||||
}
|
||||
_, err = f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
||||
expectWorkerCode(t, err, "LEASE_LOST")
|
||||
start := executionID(t)
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, start, newRef); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.expire(newRef)
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 2)
|
||||
for _, store := range []*ExecutionStore{f.storeA, f.storeB} {
|
||||
wg.Add(1)
|
||||
go func(store *ExecutionStore) { defer wg.Done(); _, err := store.Recover(ctx, 100); errs <- err }(store)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
current = f.get(task.ID)
|
||||
if current.State != pulse.TaskNeedsReview || current.AttemptCount != 2 || current.CompletedAt != nil || f.used() != 0 {
|
||||
t.Fatal("running expiry must become review without replay")
|
||||
}
|
||||
if f.count("orbit_events", "aggregate_id=? AND type='task.needs_review'", task.ID) != 1 {
|
||||
t.Fatal("recovery event duplicated")
|
||||
}
|
||||
late := worker.FinishInput{Ref: newRef, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"external_ref":"late-result"}`)}
|
||||
message := executionID(t)
|
||||
evidence, err := f.a.Finish(ctx, f.token, f.binding, message, late)
|
||||
if err != nil || evidence.Receipt.Disposition != "EVIDENCE_ONLY" || evidence.Lease.Active {
|
||||
t.Fatalf("late evidence %v", err)
|
||||
}
|
||||
evidence, err = f.a.Finish(ctx, f.token, f.binding, message, late)
|
||||
if err != nil || !evidence.Deduplicated {
|
||||
t.Fatalf("late replay %v", err)
|
||||
}
|
||||
if f.get(task.ID).State != pulse.TaskNeedsReview || f.count("pulse_attempt_evidence", "attempt_id=?", newRef.AttemptID) != 1 {
|
||||
t.Fatal("late result rewrote authority or duplicated evidence")
|
||||
}
|
||||
duplicate, err := f.a.Start(ctx, f.token, f.binding, start, newRef)
|
||||
if err != nil || !duplicate.Deduplicated || duplicate.Lease.Active {
|
||||
t.Fatalf("old acknowledgement restored lease %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("attempt_timeout_and_session_expiry_cap_renewal", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := f.submit(3)
|
||||
_, ref := f.claim()
|
||||
f.exec("UPDATE pulse_tasks SET attempt_timeout_ms=5000 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID)
|
||||
begun, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
renewed, err := f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
||||
if err != nil || renewed.Lease.LeaseUntil.After(begun.Lease.LeaseUntil) {
|
||||
t.Fatalf("renew bypassed absolute attempt deadline %v", err)
|
||||
}
|
||||
f.exec("UPDATE pulse_worker_sessions SET expires_at=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.binding.SessionID)
|
||||
renewed, err = f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
||||
if err != nil || renewed.Lease.LeaseUntil.Sub(renewed.Lease.ServerTime) > time.Second {
|
||||
t.Fatalf("renew exceeded session expiration %v", err)
|
||||
}
|
||||
f.exec("UPDATE pulse_worker_sessions SET expires_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.binding.SessionID)
|
||||
_, err = f.a.Capacity(ctx, f.token, f.binding, worker.CapacityInput{Sequence: 1, Capacity: 4})
|
||||
expectWorkerCode(t, err, "STALE_SESSION")
|
||||
})
|
||||
t.Run("result_event_failure_rolls_back_task_receipt_and_permit", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := f.submit(3)
|
||||
_, ref := f.claim()
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.exec("CREATE TRIGGER orbit_test_reject_worker_audit BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='worker audit failure'")
|
||||
message := executionID(t)
|
||||
in := worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}
|
||||
_, err := f.a.Finish(ctx, f.token, f.binding, message, in)
|
||||
f.exec("DROP TRIGGER orbit_test_reject_worker_audit")
|
||||
if err == nil {
|
||||
t.Fatal("expected injected failure")
|
||||
}
|
||||
if f.get(task.ID).State != pulse.TaskRunning || f.used() != 1 || f.count("pulse_attempt_receipts", "attempt_id=? AND operation='finish'", ref.AttemptID) != 0 || f.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 0 {
|
||||
t.Fatal("partial result transaction committed")
|
||||
}
|
||||
if _, err := f.a.Finish(ctx, f.token, f.binding, message, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
t.Run("finish_without_start_is_only_evidence_and_cannot_free_capacity", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := f.submit(1)
|
||||
_, ref := f.claim()
|
||||
result, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)})
|
||||
if err != nil || result.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(task.ID).State != pulse.TaskLeased || f.used() != 1 {
|
||||
t.Fatalf("unstarted result accepted %v", err)
|
||||
}
|
||||
f.expire(ref)
|
||||
if _, err := f.storeA.Recover(ctx, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.get(task.ID).State != pulse.TaskFailed || f.used() != 0 {
|
||||
t.Fatal("exhausted pre-start attempt did not terminate")
|
||||
}
|
||||
})
|
||||
t.Run("known_failure_unknown_and_waiting_deadlines", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
for _, outcome := range []string{"FAILED", "UNKNOWN"} {
|
||||
task := f.submit(3)
|
||||
_, ref := f.claim()
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: outcome, ErrorCode: "EXTERNAL_RESULT", Result: json.RawMessage(`{"evidence_ref":"object/result"}`)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := pulse.TaskFailed
|
||||
if outcome == "UNKNOWN" {
|
||||
want = pulse.TaskNeedsReview
|
||||
}
|
||||
if out.Receipt.TaskState != want || f.get(task.ID).State != want {
|
||||
t.Fatal("outcome classification lost")
|
||||
}
|
||||
}
|
||||
task := f.submit(3)
|
||||
f.exec("UPDATE pulse_tasks SET available_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 2 HOUR),deadline_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 HOUR) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID)
|
||||
if _, err := f.storeA.Recover(ctx, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.get(task.ID).State != pulse.TaskFailed {
|
||||
t.Fatal("waiting deadline not enforced")
|
||||
}
|
||||
})
|
||||
t.Run("authority_is_checked_on_every_execution_command", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
f.submit(3)
|
||||
_, ref := f.claim()
|
||||
f.exec("UPDATE orbit_credentials SET revoked_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.CredentialID)
|
||||
_, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
||||
expectWorkerCode(t, err, "UNAUTHENTICATED")
|
||||
f.exec("UPDATE orbit_credentials SET revoked_at=NULL WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.CredentialID)
|
||||
other := newExecutionFixture(t, ctx, db)
|
||||
_, err = other.a.Resume(ctx, other.token, other.binding, ref)
|
||||
expectWorkerCode(t, err, "NOT_FOUND")
|
||||
copied := f.binding
|
||||
copied.Proof = core.Digest{}
|
||||
_, err = f.a.Claim(ctx, f.token, copied)
|
||||
expectWorkerCode(t, err, "STALE_SESSION")
|
||||
cancelled, stop := context.WithCancel(ctx)
|
||||
stop()
|
||||
_, err = f.a.Start(cancelled, f.token, f.binding, executionID(t), ref)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(fmt.Sprint(err))
|
||||
}
|
||||
})
|
||||
t.Run("claim_event_failure_does_not_leave_an_invisible_lease", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := f.submit(3)
|
||||
f.exec("CREATE TRIGGER orbit_test_reject_claim_event BEFORE INSERT ON orbit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='claim event failure'")
|
||||
assignment, err := f.a.Claim(ctx, f.token, f.binding)
|
||||
f.exec("DROP TRIGGER orbit_test_reject_claim_event")
|
||||
if err == nil || assignment != nil {
|
||||
t.Fatal("expected failed claim without usable assignment")
|
||||
}
|
||||
if f.get(task.ID).State != pulse.TaskWaiting || f.used() != 0 || f.count("pulse_attempts", "task_id=?", task.ID) != 0 || f.count("pulse_permits", "TRUE") != 0 {
|
||||
t.Fatal("claim partially committed")
|
||||
}
|
||||
if _, err := f.a.Claim(ctx, f.token, f.binding); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
t.Run("lock_wait_does_not_reuse_an_earlier_clock_sample", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
f.submit(3)
|
||||
_, ref := f.claim()
|
||||
holder, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer holder.Rollback()
|
||||
if _, err := holder.ExecContext(ctx, "UPDATE pulse_queues SET revision=revision+1 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.QueueID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, table := range []string{"pulse_tasks", "pulse_attempts"} {
|
||||
id := ref.TaskID
|
||||
if table == "pulse_attempts" {
|
||||
id = ref.AttemptID
|
||||
}
|
||||
if _, err := holder.ExecContext(ctx, "UPDATE "+table+" SET lease_until=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 150000 MICROSECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
result := make(chan error, 1)
|
||||
go func() { _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); result <- err }()
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
if err := holder.Commit(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expectWorkerCode(t, <-result, "LEASE_LOST")
|
||||
if f.get(ref.TaskID).State != pulse.TaskLeased {
|
||||
t.Fatal("stale clock authorized start")
|
||||
}
|
||||
})
|
||||
}
|
||||
166
internal/storage/mysql/worker_recovery.go
Normal file
166
internal/storage/mysql/worker_recovery.go
Normal file
@@ -0,0 +1,166 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
// Recover 是受控后台扫描入口，每批最多 100 项、每项一个短事务，可由多个节点竞争。
|
||||
// 当前已授权执行的失联一律转核查，自动安全重试在 P3 接入冻结策略后实现。
|
||||
func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.RecoveryStats, error) {
|
||||
var out worker.RecoveryStats
|
||||
if limit < 1 || limit > 100 {
|
||||
return out, fault.Invalid("recovery batch limit must be 1 to 100")
|
||||
}
|
||||
type candidate struct{ ns, id, queue core.ID }
|
||||
candidates := []candidate{}
|
||||
seen := map[string]bool{}
|
||||
// 分别使用租约、总截止及开始截止索引，避免对整个 WAITING 队列做 COALESCE 扫描。
|
||||
for _, q := range []string{
|
||||
`SELECT namespace_id,id,queue_id FROM pulse_tasks WHERE state IN ('LEASED','RUNNING') AND lease_until<=UTC_TIMESTAMP(6) ORDER BY lease_until,namespace_id,id LIMIT ?`,
|
||||
`SELECT namespace_id,id,queue_id FROM pulse_tasks WHERE state='WAITING' AND deadline_at<=UTC_TIMESTAMP(6) ORDER BY deadline_at,namespace_id,id LIMIT ?`,
|
||||
`SELECT namespace_id,id,queue_id FROM pulse_tasks WHERE state='WAITING' AND start_deadline_at<=UTC_TIMESTAMP(6) ORDER BY start_deadline_at,namespace_id,id LIMIT ?`,
|
||||
} {
|
||||
if len(candidates) >= limit {
|
||||
break
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, q, limit-len(candidates))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var c candidate
|
||||
if err := rows.Scan(&c.ns, &c.id, &c.queue); err != nil {
|
||||
rows.Close()
|
||||
return out, err
|
||||
}
|
||||
key := string(c.ns) + string(c.id)
|
||||
if !seen[key] {
|
||||
candidates = append(candidates, c)
|
||||
seen[key] = true
|
||||
}
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
for _, c := range candidates {
|
||||
out.Examined++
|
||||
var changed pulse.TaskState
|
||||
err := WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
||||
// 与 Worker 命令一致先锁队列；有其他节点正在处理时跳过，本批不占用等待线程。
|
||||
var queue core.ID
|
||||
err := tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, c.ns, c.queue).Scan(&queue)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var taskID core.ID
|
||||
err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_tasks WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, c.ns, c.id).Scan(&taskID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task, err := readExecutionTask(ctx, tx, c.ns, c.id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if task.view.State == pulse.TaskWaiting {
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if task.view.DeadlineAt.After(now) && (task.view.StartDeadlineAt == nil || task.view.StartDeadlineAt.After(now)) {
|
||||
return nil
|
||||
}
|
||||
changed = pulse.TaskFailed
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='FAILED',completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, now, now, c.ns, c.id, task.view.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", "task.failed", map[string]any{"reason": "START_DEADLINE_EXCEEDED", "node_id": s.node})
|
||||
}
|
||||
if task.view.State != pulse.TaskLeased && task.view.State != pulse.TaskRunning {
|
||||
return nil
|
||||
}
|
||||
if task.current == nil || task.lease == nil {
|
||||
return errors.New("active task lacks attempt or lease")
|
||||
}
|
||||
attempt, err := readAttempt(ctx, tx, c.ns, *task.current)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if task.lease.After(now) || attempt.LeaseUntil.After(now) {
|
||||
return nil
|
||||
}
|
||||
if attempt.Epoch != task.epoch || attempt.TaskID != c.id {
|
||||
return errors.New("task attempt identity mismatch")
|
||||
}
|
||||
if attempt.State != pulse.AttemptLeased && attempt.State != pulse.AttemptRunning {
|
||||
return errors.New("active task references ended attempt")
|
||||
}
|
||||
attemptState := pulse.AttemptExpired
|
||||
next := pulse.TaskFailed
|
||||
reason := "LEASE_EXPIRED_BEFORE_START"
|
||||
var completed *time.Time = &now
|
||||
available := task.view.AvailableAt
|
||||
if attempt.AuthorizedAt != nil {
|
||||
attemptState = pulse.AttemptUnknown
|
||||
next = pulse.TaskNeedsReview
|
||||
reason = "EXECUTION_OUTCOME_UNKNOWN"
|
||||
completed = nil
|
||||
} else if task.view.AttemptCount < task.maxAttempts {
|
||||
// 未批准开始的尝试可以有界重派，次数不重置；开始截止与总截止仍有效。
|
||||
nextAt := now.Add(time.Second)
|
||||
if nextAt.Before(task.view.DeadlineAt) && (task.view.StartDeadlineAt == nil || nextAt.Before(*task.view.StartDeadlineAt)) {
|
||||
next = pulse.TaskWaiting
|
||||
available = nextAt
|
||||
completed = nil
|
||||
}
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, reason, now, c.ns, attempt.ID, attempt.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,lease_until=NULL,available_at=?,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, next, available, completed, now, c.ns, c.id, task.view.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := releasePermits(ctx, tx, c.ns, attempt.ID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
changed = next
|
||||
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", resultEvent(next), map[string]any{"attempt_id": attempt.ID, "epoch": fmt.Sprint(attempt.Epoch), "reason": reason, "node_id": s.node})
|
||||
})
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
switch changed {
|
||||
case pulse.TaskWaiting:
|
||||
out.Requeued++
|
||||
case pulse.TaskFailed:
|
||||
out.Failed++
|
||||
case pulse.TaskNeedsReview:
|
||||
out.NeedsReview++
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
132
internal/storage/mysql/worker_results.go
Normal file
132
internal/storage/mysql/worker_results.go
Normal file
@@ -0,0 +1,132 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
func (s *ExecutionStore) Finish(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, in worker.FinishInput) (worker.CommandResult, error) {
|
||||
var out worker.CommandResult
|
||||
digest, err := requestDigest(struct {
|
||||
Identity any
|
||||
Outcome string
|
||||
Result core.JSON
|
||||
ErrorCode string
|
||||
}{attemptIdentity(in.Ref), in.Outcome, core.JSON(in.Result), in.ErrorCode})
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
err = s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
rows, err := s.lockExecution(ctx, tx, token, b, p, in.Ref, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cached, err := readReceipt(ctx, tx, token.NamespaceID, in.Ref.AttemptID, message, "finish", digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status := leaseStatus(rows, b, now)
|
||||
if cached != nil {
|
||||
out = worker.CommandResult{Receipt: *cached, Lease: status, Deduplicated: true}
|
||||
return nil
|
||||
}
|
||||
task, a := rows.task, rows.attempt
|
||||
receipt := worker.Receipt{Operation: "finish", TaskState: task.view.State, AttemptState: a.State, TaskRevision: task.view.Revision, LeaseUntil: a.LeaseUntil, Disposition: "APPLIED"}
|
||||
if a.ResultDigest != nil && *a.ResultDigest == digest {
|
||||
// 新消息 ID 重报已保存的同一结果，也不能重复事件、许可释放或状态推进。
|
||||
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
out = worker.CommandResult{Receipt: receipt, Lease: status, Deduplicated: true}
|
||||
return nil
|
||||
}
|
||||
reason := ""
|
||||
if a.ResultDigest != nil {
|
||||
reason = "RESULT_CONFLICT"
|
||||
} else if a.AuthorizedAt == nil {
|
||||
reason = "NOT_STARTED"
|
||||
} else if !status.Active || a.State != pulse.AttemptRunning {
|
||||
reason = "LEASE_LOST"
|
||||
}
|
||||
if reason != "" {
|
||||
if err := saveEvidence(ctx, tx, token.NamespaceID, message, in, digest, reason); err != nil {
|
||||
return err
|
||||
}
|
||||
receipt.Disposition = "EVIDENCE_ONLY"
|
||||
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
out = worker.CommandResult{Receipt: receipt, Lease: status}
|
||||
return nil
|
||||
}
|
||||
attemptState, taskState := pulse.AttemptState(in.Outcome), pulse.TaskState(in.Outcome)
|
||||
var completed *time.Time = &now
|
||||
if in.Outcome == "UNKNOWN" {
|
||||
taskState = pulse.TaskNeedsReview
|
||||
completed = nil
|
||||
}
|
||||
changed, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=NULLIF(?,''),result=?,result_digest=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, in.ErrorCode, string(in.Result), digest, now, token.NamespaceID, a.ID, a.Revision)
|
||||
if err := exactlyOne(changed, err); err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,result=?,lease_until=NULL,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, taskState, string(in.Result), completed, now, token.NamespaceID, in.Ref.TaskID, task.view.Revision)
|
||||
if err := exactlyOne(changed, err); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := releasePermits(ctx, tx, token.NamespaceID, a.ID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
event := map[string]any{"attempt_id": a.ID, "epoch": fmt.Sprint(a.Epoch), "state": taskState, "outcome": in.Outcome, "error_code": in.ErrorCode}
|
||||
if err := emitEvent(ctx, tx, token.NamespaceID, in.Ref.TaskID, task.view.Revision+1, "task", resultEvent(taskState), event); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := appendAudit(ctx, tx, token, message, in.Ref.TaskID, "task", "task.result", "worker result", map[string]any{"state": task.view.State}, event); err != nil {
|
||||
return err
|
||||
}
|
||||
receipt.TaskState = taskState
|
||||
receipt.AttemptState = attemptState
|
||||
receipt.TaskRevision++
|
||||
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
out = worker.CommandResult{Receipt: receipt, Lease: worker.LeaseStatus{State: attemptState, LeaseUntil: a.LeaseUntil, ServerTime: now}}
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
func resultEvent(state pulse.TaskState) string {
|
||||
switch state {
|
||||
case pulse.TaskSucceeded:
|
||||
return "task.succeeded"
|
||||
case pulse.TaskFailed:
|
||||
return "task.failed"
|
||||
case pulse.TaskNeedsReview:
|
||||
return "task.needs_review"
|
||||
case pulse.TaskWaiting:
|
||||
return "task.requeued"
|
||||
default:
|
||||
return "task.cancelled"
|
||||
}
|
||||
}
|
||||
func saveEvidence(ctx context.Context, tx *sql.Tx, ns, message core.ID, in worker.FinishInput, digest core.Digest, reason string) error {
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempt_evidence(namespace_id,id,attempt_id,message_id,outcome,error_code,payload,request_digest,reason) VALUES (?,?,?,?,?,NULLIF(?,''),?,?,?)`, ns, id, in.Ref.AttemptID, message, in.Outcome, in.ErrorCode, string(in.Result), digest, reason)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return emitEvent(ctx, tx, ns, id, 1, "attempt_evidence", "attempt.evidence_received", map[string]any{"attempt_id": in.Ref.AttemptID, "task_id": in.Ref.TaskID, "reason": reason, "outcome": in.Outcome})
|
||||
}
|
||||
211
internal/storage/mysql/worker_sessions.go
Normal file
211
internal/storage/mysql/worker_sessions.go
Normal file
@@ -0,0 +1,211 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
// ExecutionOptions 是服务端执行窗口，不由 Worker 任意指定。
|
||||
type ExecutionOptions struct {
|
||||
LeaseDuration time.Duration // LeaseDuration 每次领取/续租的最大窗口。
|
||||
SessionTTL time.Duration // SessionTTL 会话心跳有效期，不代替执行租约。
|
||||
}
|
||||
|
||||
// ExecutionStore 所有执行权以 MySQL 为准，节点内存不持有可绕过数据库的授权。
|
||||
type ExecutionStore struct {
|
||||
db *sql.DB
|
||||
node core.ID
|
||||
options ExecutionOptions
|
||||
}
|
||||
|
||||
var _ worker.Store = (*ExecutionStore)(nil)
|
||||
|
||||
func (s *ExecutionStore) Authorize(ctx context.Context, token auth.Token) error {
|
||||
return s.transaction(ctx, token, "workers:connect", func(*sql.Tx, auth.Permissions) error { return nil })
|
||||
}
|
||||
|
||||
func NewExecutionStore(db *sql.DB, node core.ID, options ExecutionOptions) (*ExecutionStore, error) {
|
||||
if options.LeaseDuration == 0 {
|
||||
options.LeaseDuration = 30 * time.Second
|
||||
}
|
||||
if options.SessionTTL == 0 {
|
||||
options.SessionTTL = 45 * time.Second
|
||||
}
|
||||
if node.Validate() != nil || options.LeaseDuration < time.Second || options.LeaseDuration > 2*time.Minute || options.SessionTTL < options.LeaseDuration || options.SessionTTL > 5*time.Minute {
|
||||
return nil, fault.Invalid("invalid execution windows or node identity")
|
||||
}
|
||||
return &ExecutionStore{db: db, node: node, options: options}, nil
|
||||
}
|
||||
func (s *ExecutionStore) transaction(ctx context.Context, token auth.Token, action string, fn func(*sql.Tx, auth.Permissions) error) error {
|
||||
return (&TaskStore{db: s.db}).transaction(ctx, token, action, fn)
|
||||
}
|
||||
|
||||
type sessionState struct {
|
||||
queue core.ID
|
||||
capacity uint32
|
||||
sequence uint64
|
||||
draining bool
|
||||
expires time.Time
|
||||
capabilities []worker.Capability
|
||||
}
|
||||
|
||||
func (s *ExecutionStore) welcome(b worker.Binding, state sessionState, now time.Time) worker.Welcome {
|
||||
return worker.Welcome{SessionID: b.SessionID, Epoch: b.Epoch, ServerTime: now, ExpiresAt: state.expires, LeaseDurationMS: uint64(s.options.LeaseDuration / time.Millisecond), Draining: state.draining}
|
||||
}
|
||||
func (s *ExecutionStore) Connect(ctx context.Context, token auth.Token, in worker.ConnectInput) (worker.Binding, worker.Welcome, error) {
|
||||
var binding worker.Binding
|
||||
var out worker.Welcome
|
||||
proof, err := worker.SecretDigest(in.InstanceSecret)
|
||||
if err != nil {
|
||||
return binding, out, err
|
||||
}
|
||||
registration, err := requestDigest(struct {
|
||||
Queue core.ID
|
||||
Capabilities []worker.Capability
|
||||
}{in.QueueID, in.Capabilities})
|
||||
if err != nil {
|
||||
return binding, out, err
|
||||
}
|
||||
err = s.transaction(ctx, token, "workers:connect", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
if !p.AllowsQueue(in.QueueID) {
|
||||
return fault.Forbidden
|
||||
}
|
||||
_, err := tx.ExecContext(ctx, `INSERT INTO pulse_worker_instances(namespace_id,id,credential_id,queue_id,secret_digest,registration_digest) VALUES (?,?,?,?,?,?) ON DUPLICATE KEY UPDATE id=pulse_worker_instances.id`, token.NamespaceID, in.InstanceID, token.CredentialID, in.QueueID, proof, registration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var credential core.ID
|
||||
var storedProof, storedRegistration core.Digest
|
||||
var epoch uint64
|
||||
var old *core.ID
|
||||
var draining bool
|
||||
err = tx.QueryRowContext(ctx, `SELECT credential_id,secret_digest,registration_digest,session_epoch,current_session_id,draining FROM pulse_worker_instances WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, in.InstanceID).Scan(&credential, &storedProof, &storedRegistration, &epoch, &old, &draining)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if credential != token.CredentialID || subtle.ConstantTimeCompare(proof[:], storedProof[:]) != 1 {
|
||||
return fault.Forbidden
|
||||
}
|
||||
if registration != storedRegistration || epoch == ^uint64(0) {
|
||||
return fault.Conflict
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
session, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
binding = worker.Binding{InstanceID: in.InstanceID, SessionID: session, Epoch: epoch + 1, Proof: proof}
|
||||
state := sessionState{queue: in.QueueID, capacity: in.Capacity, draining: draining, expires: now.Add(s.options.SessionTTL), capabilities: in.Capabilities}
|
||||
caps, err := json.Marshal(in.Capabilities)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_worker_sessions(namespace_id,id,worker_instance_id,credential_id,node_id,session_epoch,capabilities,capacity,draining,expires_at) VALUES (?,?,?,?,?,?,?,?,?,?)`, token.NamespaceID, session, in.InstanceID, token.CredentialID, s.node, binding.Epoch, string(caps), in.Capacity, draining, state.expires)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if old != nil {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE pulse_worker_sessions SET expires_at=?,draining=TRUE,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, now, now, token.NamespaceID, *old); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE pulse_worker_instances SET current_session_id=?,session_epoch=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, session, binding.Epoch, now, token.NamespaceID, in.InstanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, token.NamespaceID, session, 1, "worker_session", "worker.connected", map[string]any{"instance_id": in.InstanceID, "node_id": s.node, "epoch": binding.Epoch}); err != nil {
|
||||
return err
|
||||
}
|
||||
out = s.welcome(binding, state, now)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return worker.Binding{}, worker.Welcome{}, err
|
||||
}
|
||||
return binding, out, nil
|
||||
}
|
||||
|
||||
// lockSession 的顺序为凭据 -> 进程头 -> 当前会话；新连接和旧连接命令在同一行上串行化。
|
||||
func (s *ExecutionStore) lockSession(ctx context.Context, tx *sql.Tx, token auth.Token, b worker.Binding, p auth.Permissions) (sessionState, error) {
|
||||
var state sessionState
|
||||
var credential core.ID
|
||||
var proof core.Digest
|
||||
var current *core.ID
|
||||
var epoch uint64
|
||||
if b.InstanceID.Validate() != nil || b.SessionID.Validate() != nil || b.Epoch == 0 {
|
||||
return state, worker.StaleSession
|
||||
}
|
||||
err := tx.QueryRowContext(ctx, `SELECT credential_id,queue_id,secret_digest,current_session_id,session_epoch,draining FROM pulse_worker_instances WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, b.InstanceID).Scan(&credential, &state.queue, &proof, ¤t, &epoch, &state.draining)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return state, worker.StaleSession
|
||||
}
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
if credential != token.CredentialID || current == nil || *current != b.SessionID || epoch != b.Epoch || subtle.ConstantTimeCompare(proof[:], b.Proof[:]) != 1 {
|
||||
return state, worker.StaleSession
|
||||
}
|
||||
if !p.AllowsQueue(state.queue) {
|
||||
return state, fault.Forbidden
|
||||
}
|
||||
var node core.ID
|
||||
var raw []byte
|
||||
err = tx.QueryRowContext(ctx, `SELECT node_id,capacity,capacity_sequence,capabilities,expires_at FROM pulse_worker_sessions WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, b.SessionID).Scan(&node, &state.capacity, &state.sequence, &raw, &state.expires)
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
if node != s.node || !state.expires.After(now) {
|
||||
return state, worker.StaleSession
|
||||
}
|
||||
if err := json.Unmarshal(raw, &state.capabilities); err != nil {
|
||||
return state, err
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
func (s *ExecutionStore) Capacity(ctx context.Context, token auth.Token, b worker.Binding, in worker.CapacityInput) (worker.Welcome, error) {
|
||||
var out worker.Welcome
|
||||
err := s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
state, err := s.lockSession(ctx, tx, token, b, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if in.Sequence < state.sequence || (in.Sequence == state.sequence && (in.Capacity != state.capacity || in.Drain != state.draining)) || (state.draining && !in.Drain) {
|
||||
return fault.Conflict
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
state.expires = now.Add(s.options.SessionTTL)
|
||||
state.capacity = in.Capacity
|
||||
state.draining = state.draining || in.Drain
|
||||
_, err = tx.ExecContext(ctx, `UPDATE pulse_worker_sessions SET capacity=?,capacity_sequence=?,draining=?,expires_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, state.capacity, in.Sequence, state.draining, state.expires, now, token.NamespaceID, b.SessionID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if in.Drain {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE pulse_worker_instances SET draining=TRUE,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND draining=FALSE`, now, token.NamespaceID, b.InstanceID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
out = s.welcome(b, state, now)
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
244
internal/storage/mysql/worker_websocket_integration_test.go
Normal file
244
internal/storage/mysql/worker_websocket_integration_test.go
Normal file
@@ -0,0 +1,244 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/transport/httpserver"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
type testWorkerFrame struct {
|
||||
Version string `json:"protocol_version"`
|
||||
Type string `json:"type"`
|
||||
MessageID core.ID `json:"message_id"`
|
||||
ReplyTo core.ID `json:"reply_to,omitempty"`
|
||||
SessionID core.ID `json:"session_id,omitempty"`
|
||||
Body json.RawMessage `json:"body,omitempty"`
|
||||
Error *struct {
|
||||
Code string `json:"code"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
type testWorkerSocket struct {
|
||||
t *testing.T
|
||||
ctx context.Context
|
||||
conn *websocket.Conn
|
||||
session core.ID
|
||||
welcome worker.Welcome
|
||||
}
|
||||
|
||||
func connectTestWorker(t *testing.T, ctx context.Context, server *httptest.Server, f *executionFixture) *testWorkerSocket {
|
||||
t.Helper()
|
||||
header := http.Header{}
|
||||
header.Set("Authorization", "Bearer "+f.resources.Token)
|
||||
conn, response, err := websocket.Dial(ctx, strings.Replace(server.URL, "https://", "wss://", 1)+"/v1/worker/connect", &websocket.DialOptions{HTTPClient: server.Client(), HTTPHeader: header, Subprotocols: []string{httpserver.WorkerSubprotocol}})
|
||||
if err != nil {
|
||||
status := 0
|
||||
if response != nil {
|
||||
status = response.StatusCode
|
||||
}
|
||||
t.Fatalf("Worker dial failed HTTP %d: %v", status, err)
|
||||
}
|
||||
t.Cleanup(func() { conn.CloseNow() })
|
||||
client := &testWorkerSocket{t: t, ctx: ctx, conn: conn}
|
||||
conn.SetReadLimit(512 << 10)
|
||||
hello := client.send("worker.hello", f.hello, "")
|
||||
reply := client.read()
|
||||
if reply.Type != "worker.welcome" || reply.ReplyTo != hello || reply.Error != nil {
|
||||
t.Fatal("missing welcome")
|
||||
}
|
||||
if err := json.Unmarshal(reply.Body, &client.welcome); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client.session = client.welcome.SessionID
|
||||
return client
|
||||
}
|
||||
func (c *testWorkerSocket) send(kind string, body any, id core.ID) core.ID {
|
||||
c.t.Helper()
|
||||
if id == "" {
|
||||
id = executionID(c.t)
|
||||
}
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
c.t.Fatal(err)
|
||||
}
|
||||
frame, err := json.Marshal(testWorkerFrame{Version: "1", Type: kind, MessageID: id, SessionID: c.session, Body: raw})
|
||||
if err != nil {
|
||||
c.t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(c.ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
if err := c.conn.Write(ctx, websocket.MessageText, frame); err != nil {
|
||||
c.t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
func (c *testWorkerSocket) read() testWorkerFrame {
|
||||
c.t.Helper()
|
||||
ctx, cancel := context.WithTimeout(c.ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
kind, raw, err := c.conn.Read(ctx)
|
||||
if err != nil {
|
||||
c.t.Fatal(err)
|
||||
}
|
||||
var frame testWorkerFrame
|
||||
if kind != websocket.MessageText || json.Unmarshal(raw, &frame) != nil || frame.Version != "1" || frame.MessageID.Validate() != nil {
|
||||
c.t.Fatal("invalid server frame")
|
||||
}
|
||||
return frame
|
||||
}
|
||||
func (c *testWorkerSocket) ack(id core.ID) json.RawMessage {
|
||||
c.t.Helper()
|
||||
frame := c.read()
|
||||
if frame.Type != "ack" || frame.ReplyTo != id || frame.SessionID != c.session || frame.Error != nil {
|
||||
code := ""
|
||||
if frame.Error != nil {
|
||||
code = frame.Error.Code
|
||||
}
|
||||
c.t.Fatalf("expected ack, got %s/%s", frame.Type, code)
|
||||
}
|
||||
return frame.Body
|
||||
}
|
||||
func TestIntegrationWorkerWebSocket(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
db, err := Open(ctx, dsn, 16, 4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fixture := newExecutionFixture(t, ctx, db)
|
||||
start := func(svc *worker.Service) (*httptest.Server, *httpserver.WorkerHub) {
|
||||
t.Helper()
|
||||
hub := httpserver.NewWorkerHub(ctx, svc)
|
||||
server := httpserver.New("", func(context.Context) error { return nil })
|
||||
httpserver.RegisterWorkers(server, hub)
|
||||
tls := httptest.NewTLSServer(server.Handler)
|
||||
t.Cleanup(func() {
|
||||
bounded, stop := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer stop()
|
||||
if err := hub.Shutdown(bounded); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tls.Close()
|
||||
})
|
||||
return tls, hub
|
||||
}
|
||||
serverA, hubA := start(fixture.a)
|
||||
serverB, _ := start(fixture.b)
|
||||
t.Run("handshake_auth_protocol_origin_and_URL", func(t *testing.T) {
|
||||
cases := []struct {
|
||||
token, protocol, path, origin string
|
||||
status int
|
||||
}{
|
||||
{"", httpserver.WorkerSubprotocol, "/v1/worker/connect", "", 401},
|
||||
{fixture.resources.Token, "unknown.v1", "/v1/worker/connect", "", 426},
|
||||
{fixture.resources.Token, httpserver.WorkerSubprotocol, "/v1/worker/connect?token=forbidden", "", 400},
|
||||
{fixture.resources.Token, httpserver.WorkerSubprotocol, "/v1/worker/connect", "https://foreign.example", 403},
|
||||
}
|
||||
for _, test := range cases {
|
||||
headers := http.Header{}
|
||||
if test.token != "" {
|
||||
headers.Set("Authorization", "Bearer "+test.token)
|
||||
}
|
||||
if test.origin != "" {
|
||||
headers.Set("Origin", test.origin)
|
||||
}
|
||||
conn, response, err := websocket.Dial(ctx, strings.Replace(serverA.URL, "https://", "wss://", 1)+test.path, &websocket.DialOptions{HTTPClient: serverA.Client(), HTTPHeader: headers, Subprotocols: []string{test.protocol}})
|
||||
if conn != nil {
|
||||
conn.CloseNow()
|
||||
}
|
||||
if err == nil || response == nil || response.StatusCode != test.status {
|
||||
t.Fatalf("handshake expected %d", test.status)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("WSS_assignment_start_and_cross_node_ack_recovery", func(t *testing.T) {
|
||||
task := fixture.submit(3)
|
||||
first := connectTestWorker(t, ctx, serverA, fixture)
|
||||
// 只有显式容量确认后才启用新派发，留出恢复核对阶段。
|
||||
first.ack(first.send("worker.capacity", worker.CapacityInput{Sequence: 1, Capacity: 1}, ""))
|
||||
frame := first.read()
|
||||
if frame.Type != "task.assign" {
|
||||
t.Fatalf("wanted assignment got %s", frame.Type)
|
||||
}
|
||||
var assignment worker.Assignment
|
||||
if err := json.Unmarshal(frame.Body, &assignment); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if assignment.Task.ID != task.ID || assignment.LeaseSecret == "" {
|
||||
t.Fatal("wrong assignment")
|
||||
}
|
||||
ref := worker.AttemptRef{TaskID: task.ID, AttemptID: assignment.AttemptID, Epoch: assignment.Epoch, LeaseSecret: assignment.LeaseSecret}
|
||||
startMessage := first.send("task.start", ref, "")
|
||||
var begun worker.CommandResult
|
||||
if err := json.Unmarshal(first.ack(startMessage), &begun); err != nil || !begun.Lease.Active || begun.Lease.State != pulse.AttemptRunning {
|
||||
t.Fatal("missing durable start")
|
||||
}
|
||||
// 模拟旧节点连接断开；新的 WSS 连接进入不同节点并原子迁移有效尝试。
|
||||
_ = first.conn.CloseNow()
|
||||
second := connectTestWorker(t, ctx, serverB, fixture)
|
||||
var resumed worker.LeaseStatus
|
||||
if err := json.Unmarshal(second.ack(second.send("worker.resume", ref, "")), &resumed); err != nil || !resumed.Active {
|
||||
t.Fatal("WSS resume failed")
|
||||
}
|
||||
var original worker.CommandResult
|
||||
if err := json.Unmarshal(second.ack(second.send("task.start", ref, startMessage)), &original); err != nil || !original.Deduplicated || !original.Lease.Active {
|
||||
t.Fatal("original start receipt not recovered")
|
||||
}
|
||||
finishMessage := second.send("task.finish", worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"output_ref":"objects/result","n":9007199254740993}`)}, "")
|
||||
var finish worker.CommandResult
|
||||
if err := json.Unmarshal(second.ack(finishMessage), &finish); err != nil || finish.Receipt.Disposition != "APPLIED" || finish.Lease.Active {
|
||||
t.Fatal("WSS finish failed")
|
||||
}
|
||||
// 丢弃该连接的操作确认上下文，再用稳定身份取回同一结果。
|
||||
_ = second.conn.CloseNow()
|
||||
third := connectTestWorker(t, ctx, serverA, fixture)
|
||||
var duplicate worker.CommandResult
|
||||
if err := json.Unmarshal(third.ack(third.send("task.finish", worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"output_ref":"objects/result","n":9007199254740993}`)}, finishMessage)), &duplicate); err != nil || !duplicate.Deduplicated {
|
||||
t.Fatal("finish receipt not recovered")
|
||||
}
|
||||
if fixture.get(task.ID).State != pulse.TaskSucceeded || fixture.used() != 0 || fixture.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 1 {
|
||||
t.Fatal("WSS result effects duplicated")
|
||||
}
|
||||
})
|
||||
t.Run("strict_worker_frame_and_hub_shutdown", func(t *testing.T) {
|
||||
conn := connectTestWorker(t, ctx, serverB, fixture)
|
||||
raw := []byte(`{"protocol_version":"1","protocol_version":"1","type":"worker.capacity"}`)
|
||||
if err := conn.conn.Write(ctx, websocket.MessageText, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
readCtx, stop := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer stop()
|
||||
_, _, err := conn.conn.Read(readCtx)
|
||||
if websocket.CloseStatus(err) != websocket.StatusPolicyViolation {
|
||||
t.Fatalf("duplicate frame not rejected: %v", err)
|
||||
}
|
||||
active := connectTestWorker(t, ctx, serverA, fixture)
|
||||
shutdown, stopShutdown := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer stopShutdown()
|
||||
if err := hubA.Shutdown(shutdown); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err = active.conn.Read(readCtx)
|
||||
if err == nil {
|
||||
t.Fatal("shutdown left a hijacked connection alive")
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user