diff --git a/README.md b/README.md index 9e46ddb..8358c62 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ Orbit 是面向多个业务项目的通用任务调度与流程编排平台， ## 当前进度 -已实现 30 个数据模型、版本化迁移、凭据及范围校验、队列创建查询、任务幂等提交/查询/分页/待执行取消，以及独立 Go SDK 和可运行示例。WSS Worker、调度器和流程执行引擎按 [持续开发计划](docs/08-development-plan.md) 继续实现。 +已实现 32 个数据模型、版本化迁移、凭据及范围校验、队列创建查询、任务幂等提交/查询/分页/待执行取消，以及对应 Go SDK 和示例。P2 服务端执行内核和 WSS 已通过阶段内回归；Worker SDK 自动运行器、调度器和流程引擎按 [持续开发计划](docs/08-development-plan.md) 继续实现。 -readiness 只表示数据库与迁移记录可用，不表示完整平台已具备生产服务能力。验证证据见 [P0](docs/reports/phase-0.md) 与 [P1 报告](docs/reports/phase-1.md)。 +readiness 只表示数据库与迁移记录可用，不表示完整平台已具备生产服务能力。验证证据见 [P0](docs/reports/phase-0.md)、[P1](docs/reports/phase-1.md) 与 [P2 服务端记录](docs/reports/phase-2-server.md)。 ## 目录 @@ -23,6 +23,7 @@ platform/model/ 应用、命名空间、幂等、事件和审计 platform/auth/ 凭据格式与动作/队列授权范围 pulse/model/ 任务、尝试、队列、计划和执行许可 pulse/service/ 任务与队列应用契约、输入校验 +pulse/worker/ 会话、开始、续租、结果和恢复契约 weave/model/ 定义、实例、步骤、信号和查询轮次 internal/config/ 配置校验 internal/storage/mysql/ 连接、事务、迁移及集成测试 diff --git a/cmd/orbit-admin/main.go b/cmd/orbit-admin/main.go index 6792c66..5a51896 100644 --- a/cmd/orbit-admin/main.go +++ b/cmd/orbit-admin/main.go @@ -32,6 +32,7 @@ func run() error { flags.StringVar(&in.Environment, "environment", "", "environment name") flags.StringVar(&in.Queue, "queue", "", "initial queue name") flags.StringVar(&output, "output", "", "new private credentials JSON file") + flags.BoolVar(&in.WorkerAccess, "worker-access", false, "also grant Worker connect and consume actions for development") if err := flags.Parse(os.Args[2:]); err != nil { return err } diff --git a/cmd/orbit/main.go b/cmd/orbit/main.go index df85c91..2148fa4 100644 --- a/cmd/orbit/main.go +++ b/cmd/orbit/main.go @@ -14,7 +14,9 @@ import ( "git.yuntuo.hk/k2safe/orbit/internal/config" "git.yuntuo.hk/k2safe/orbit/internal/storage/mysql" "git.yuntuo.hk/k2safe/orbit/internal/transport/httpserver" + core "git.yuntuo.hk/k2safe/orbit/model" "git.yuntuo.hk/k2safe/orbit/pulse/service" + "git.yuntuo.hk/k2safe/orbit/pulse/worker" ) func main() { @@ -49,13 +51,49 @@ func run() error { } return mysql.CheckSchema(ctx, db) }, service.New(mysql.NewTaskStore(db))) + node, err := core.NewID() + if err != nil { + return err + } + executions, err := mysql.NewExecutionStore(db, node, mysql.ExecutionOptions{}) + if err != nil { + return err + } + hub := httpserver.NewWorkerHub(root, worker.New(executions)) + httpserver.RegisterWorkers(server, hub) + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = hub.Shutdown(ctx) + }() + recoveryDone := make(chan struct{}) + recoveryCtx, stopRecovery := context.WithCancel(root) + go func() { + defer close(recoveryDone) + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for { + select { + case <-recoveryCtx.Done(): + return + case <-ticker.C: + call, cancel := context.WithTimeout(recoveryCtx, 5*time.Second) + _, err := executions.Recover(call, 100) + cancel() + if err != nil && recoveryCtx.Err() == nil { + slog.Error("execution recovery batch failed", "node_id", node) + } + } + } + }() + defer func() { stopRecovery(); <-recoveryDone }() listener, err := net.Listen("tcp", cfg.HTTPAddr) if err != nil { return err } done := make(chan error, 1) go func() { done <- server.Serve(listener) }() - slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task and queue management") + slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task, queue and Worker APIs", "node_id", node) select { case err := <-done: if errors.Is(err, http.ErrServerClosed) { diff --git a/docs/03-http-websocket-protocol.md b/docs/03-http-websocket-protocol.md index ccc0225..8907f9a 100644 --- a/docs/03-http-websocket-protocol.md +++ b/docs/03-http-websocket-protocol.md @@ -2,6 +2,8 @@ > 版本 0.2 · 2026-09-17。HTTP + WS 已确认。本文的具体路径、消息名、字段及状态码是首版设计建议，业务协议尚未实现，需评审后冻结；健康接口的已实现契约见接口手册。 +实现更新：HTTP 任务/队列接口已完成 P1；Worker 服务端已进入 P2，实际帧名、字段、回执与边界见 [Worker 契约](api/worker.md)。本文保留早期完整设计，未实现项目不能据此视为已交付。 + ## 1. 通用契约 - 生产使用 HTTPS / WSS；HTTP 承担任务提交、流程启动与信号、查询、管理，WS 承担 Worker 双向通信与结果订阅。 diff --git a/docs/07-data-model.md b/docs/07-data-model.md index e70af12..28f3abb 100644 --- a/docs/07-data-model.md +++ b/docs/07-data-model.md @@ -6,7 +6,7 @@ - `model/`：ID、摘要、JSON、命名空间记录等公共原语。 - `platform/model/`：应用、命名空间、凭据、幂等、事件、订阅交付及审计。 -- `pulse/model/`：队列、执行门控、会话、任务、尝试、回执、并发许可和调度计划。 +- `pulse/model/`：队列、执行门控、进程/会话、任务、尝试、回执、迟到证据、并发许可和调度计划。 - `weave/model/`：定义及不可变版本、实例、步骤激活、等待、信号、查询轮次与恢复记录。 - `internal/storage/mysql/`：连接、事务与显式迁移，不向 SDK 暴露。 - `sdk/go/`：独立 Go module；对外 DTO 与服务端模型分别维护。 @@ -34,7 +34,7 @@ ## 3. 实体清单 -下表与初始迁移中的 30 个实体一致；迁移自身另有 `orbit_schema_migrations` 记录。 +当前共 32 个实体：初始迁移包含 30 个，000003 增加进程会话头和迟到证据。迁移自身另有 `orbit_schema_migrations` 记录。 | 模块 | Go 模型 | 数据表 | | --- | --- | --- | @@ -49,9 +49,11 @@ | pulse | `Queue` | `pulse_queues` | | pulse | `ExecutionGate` | `pulse_execution_gates` | | pulse | `WorkerSession` | `pulse_worker_sessions` | +| pulse | `WorkerInstance` | `pulse_worker_instances` | | pulse | `Task` | `pulse_tasks` | | pulse | `Attempt` | `pulse_attempts` | | pulse | `AttemptReceipt` | `pulse_attempt_receipts` | +| pulse | `AttemptEvidence` | `pulse_attempt_evidence` | | pulse | `ConcurrencyLimit` | `pulse_concurrency_limits` | | pulse | `Permit` | `pulse_permits` | | pulse | `Schedule` | `pulse_schedules` | @@ -86,6 +88,8 @@ 外键与 `CHECK` 无法代替执行引擎。P1 已完成 HTTP 动作/队列授权、幂等摘要与原回执、受理事件，以及待执行取消的 CAS/审计事务。执行相关的以下条件继续分阶段实现： +P2 服务端已新增会话隔离、两步派发、租约、并发许可、结果回执/证据和恢复扫描，见 [执行协议](api/worker.md) 与 [阶段记录](reports/phase-2-server.md)。Worker SDK 和完整 P2 验收仍在推进。 + 1. 权限验证、凭据校验、命名空间过滤和队列消费授权。 2. 同一次事务更新任务、尝试、租约、并发许可、事件与流程交付记录。 3. CAS 竞争、数据库时间校验、门控与开始授权串行化。 diff --git a/docs/08-development-plan.md b/docs/08-development-plan.md index eef14ff..f0a2fc8 100644 --- a/docs/08-development-plan.md +++ b/docs/08-development-plan.md @@ -26,7 +26,7 @@ | --- | --- | --- | --- | | P0 基础与数据模型 | Go workspace、独立 SDK module、公共/Pulse/Weave 模型、MySQL 迁移、配置、健康检查 | 编译/竞态/静态检查；真实 MySQL 建表、隔离、唯一性、预算、不可变版本、迁移失败防护；SDK 健康调用 | 已完成并通过阶段回归（见 P0 报告） | | P1 任务受理闭环 | 应用凭据与作用域权限、队列、幂等提交、查询、待执行取消、受理事件；SDK 客户端 | API/SDK 创建查询闭环；重复键与内容冲突、响应未确认重报、跨命名空间拒绝、参数边界 | 已完成并通过阶段回归（见 P1 报告；真实断路/提交 ACK 丢失注入留后续故障阶段） | -| P2 Worker 执行闭环 | WSS 接入、能力/容量、领取与开始、租约、续租、结果确认、排空与恢复；Worker SDK | 真 Worker 执行；失联/重连/重复确认/旧尝试拒绝；业务效果未知转核查；节点故障接管 | 待开发 | +| P2 Worker 执行闭环 | WSS 接入、能力/容量、领取与开始、租约、续租、结果确认、排空与恢复；Worker SDK | 真 Worker 执行；失联/重连/重复确认/旧尝试拒绝；业务效果未知转核查；节点故障接管 | 开发中：服务端执行事务和 WSS 已通过阶段内回归；Worker SDK 自动运行/重连/退出及完整验收待完成 | | P3 Pulse 完整能力 | 安全重试、延时、Cron/固定间隔、漏跑策略、计划版本、并发许可、配额/背压、可靠结果订阅 | 多实例领取/触发竞争、重试上限、取消竞争、队列隔离、积压恢复、事件重复交付 | 待开发 | | P4 Weave 主流程 | SDK 定义构建与发布校验、实例、顺序/分支/结构化并行汇合、信号/定时等待、版本绑定 | 重启接续、早到/重复信号、并行竞争、内部事件重放、发布版本兼容；端到端通用流程 | 待开发 | | P5 Weave 恢复与查询 | 有界查询、暂停继续、取消、人工核查恢复、控制门控与审计 | 轮次与尝试独立预算；暂停/开始/回执/超时竞争；恢复保留预算；未知结果不盲目重复 | 待开发 | @@ -73,3 +73,4 @@ SDK、文档与示例随 P1–P5 同步开发，P7 是全量核对，不是最 - [生产验收总清单](05-production-acceptance.md) - [P0 验证记录](reports/phase-0.md) - [P1 验证记录](reports/phase-1.md) +- [P2 服务端阶段记录](reports/phase-2-server.md) diff --git a/docs/README.md b/docs/README.md index 6d8a6f1..cef0b53 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,7 +2,7 @@ - 文档版本：0.2 - 整理日期：2026-09-17 -- 阶段：P0 模型基础与 P1 任务受理/SDK 已实现，平台完整功能按持续计划开发；尚未完成生产验收 +- 阶段：P0/P1 已完成，P2 服务端执行/WSS 已通过阶段内回归，Worker SDK 继续开发；尚未完成生产验收 - 适用范围：Orbit 统一平台，包含 Pulse 任务调度模块、Weave 流程编排模块与公共能力。平台面向多个业务系统复用。 ## 阅读顺序 @@ -79,4 +79,4 @@ Orbit 不内置特定行业规则；Pulse 管理执行，Weave 管理步骤依 2026-09-17，文档更新至 0.2：用户确认研发定义发布流程及 Weave 首版功能，明确纳入有界重复查询和暂停/继续等生产管理能力。技术细则与参数继续按建议或待定标注，未将功能确认等同于实现或验收通过。 -持续开发目标已建立。当前实现和验收进度以 [开发计划](08-development-plan.md)、[P0 报告](reports/phase-0.md)、[P1 报告](reports/phase-1.md) 为准；早期设计稿中的“未实现”描述保留其编写阶段语境。 +持续开发目标已建立。当前实现和验收进度以 [开发计划](08-development-plan.md)、[P0 报告](reports/phase-0.md)、[P1 报告](reports/phase-1.md)、[P2 服务端记录](reports/phase-2-server.md) 为准；早期设计稿中的“未实现”描述保留其编写阶段语境。 diff --git a/docs/api/README.md b/docs/api/README.md index c0ea42c..adcecf8 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -11,7 +11,7 @@ | `/v1/tasks` 提交、查询、分页与待执行取消 | P1 已实现 | [任务接口](tasks.md) | | `/v1/queues` 创建与查询 | P1 已实现 | [队列接口](queues.md) | | 凭据校验与初始化 CLI | P1 已实现 | [鉴权](authentication.md) | -| `/v1/worker/connect` | 计划 P2，尚未实现 | [协议草案](../03-http-websocket-protocol.md) | +| `/v1/worker/connect` | P2 服务端已实现，Worker SDK 仍在开发 | [Worker 契约](worker.md) | | `/v1/schedules` 与计划管理 | 计划 P3，尚未实现 | [协议草案](../03-http-websocket-protocol.md) | | `/v1/events/connect` | 计划 P3，尚未实现 | [协议草案](../03-http-websocket-protocol.md) | | `/v1/workflows` 与流程管理 | 计划 P4–P5，尚未实现 | [流程接口规划](workflows.md) | diff --git a/docs/api/authentication.md b/docs/api/authentication.md index 845f396..1a48053 100644 --- a/docs/api/authentication.md +++ b/docs/api/authentication.md @@ -51,3 +51,5 @@ actions 不支持通配符；queues 可列出队列 ID，星号仅表示当前 原始秘密错误、凭据撤销/过期、应用或命名空间停用统一返回 401。合法身份缺少动作权限返回 403。读取不在范围内的单个资源返回 404，避免披露资源内容。 SDK 使用 `orbit.Options{Token: token}`。健康查询不发送凭据。当前不启用业务权限缓存，撤销校验不依赖缓存过期。 + +P2 增加 workers:connect 和 workers:consume。bootstrap 只有显式提供 --worker-access 时才同时授予这两项开发权限；现有凭据不自动变化。Worker 还需独立的进程恢复秘密和尝试秘密，见 [Worker 契约](worker.md)。 diff --git a/docs/api/tasks.md b/docs/api/tasks.md index a6707a9..2ceccc2 100644 --- a/docs/api/tasks.md +++ b/docs/api/tasks.md @@ -2,6 +2,8 @@ 状态：P1 已实现并通过真实 MySQL 与 SDK 回归。当前提供受理、查询、分页和待执行取消；Worker 执行与自动重试在 P2–P3 实现。 +P2 进展：服务端执行内核和 WSS 已可产生 LEASED、RUNNING、SUCCEEDED、FAILED、NEEDS_REVIEW 等状态，见 [Worker 契约](worker.md)；独立 Worker SDK 仍在开发，自动策略重试尚未完成。 + ## 公共约定 业务接口需要 `Authorization: Bearer `；写入需要 `Content-Type: application/json` 和稳定的 `Idempotency-Key`。凭据决定命名空间，请求体不能覆盖它。详见 [鉴权与初始化](authentication.md)。 diff --git a/docs/api/worker.md b/docs/api/worker.md new file mode 100644 index 0000000..657e5a0 --- /dev/null +++ b/docs/api/worker.md @@ -0,0 +1,146 @@ +# Worker 通信与执行契约 v1（开发中） + +已实现服务端执行事务、WebSocket 接入和真实 WSS 回归。**P2 尚未完成**：独立 Go SDK 的 Worker 运行器、自动重连/续租、处理器生命周期与优雅退出还在开发。本页记录当前可验证的服务端契约，不把测试客户端当作可交付 Worker SDK。 + +## 1. 接入与身份 + +入口 `GET /v1/worker/connect`，子协议必须为 `pulse.worker.v1`。Authorization 使用 Bearer 凭据；禁止 URL 查询参数。握手先校验 workers:connect，消费控制操作再次校验 workers:consume、队列范围和凭据有效性。 + +初始化开发凭据可显式增加 Worker 权限： + +```sh +go run ./cmd/orbit-admin bootstrap \ + --application worker-demo --namespace default --environment development \ + --queue objects --worker-access --output .local/worker.credentials.json +``` + +默认 bootstrap 仍只有 P1 的五项权限，现有凭据不会自动扩权。生产细粒度凭据管理在 P6 完成。 + +当前 orbit 命令监听 HTTP，开发可使用 ws；生产需要受控的 TLS 终止。WSS 回归使用真实 TLS 测试服务器及受信任测试证书，没有跳过证书校验。传输库锁定 github.com/coder/websocket v1.8.15，使用上下文、读写期限、子协议和默认 Origin 校验，关闭压缩。[上游版本记录](https://github.com/coder/websocket/releases/tag/v1.8.15)、[上游接口文档](https://pkg.go.dev/github.com/coder/websocket@v1.8.15)。 + +## 2. 帧格式 + +只接受 JSON 文本帧；单帧最多 512 KiB，业务结果最多 256 KiB。重复键、未知字段、无效 Unicode 和非法数值按 HTTP 相同规则拒绝。 + +```json +{ + "protocol_version":"1", + "type":"task.start", + "message_id":"44444444444444444444444444444444", + "session_id":"33333333333333333333333333333333", + "body":{ + "task_id":"11111111111111111111111111111111", + "attempt_id":"22222222222222222222222222222222", + "epoch":"1", + "lease_secret":"<执行秘密，仅通过受控连接传输>" + } +} +``` + +所有 ID 为 32 位小写十六进制；epoch、sequence、revision 用十进制字符串。message_id 标识一次操作，重连重报保持不变；session_id 必须换为当前连接身份。服务端响应以 reply_to 关联请求： + +```json +{ + "protocol_version":"1", + "type":"error", + "message_id":"55555555555555555555555555555555", + "reply_to":"44444444444444444444444444444444", + "session_id":"33333333333333333333333333333333", + "error":{"code":"LEASE_LOST","message":"attempt no longer grants execution authority"} +} +``` + +普通成功响应 type=ack，body 为下表对应结果。服务端主动派发 type=task.assign，无 reply_to。错误仅包含稳定分类和公开消息，不包含 SQL、业务参数或秘密。 + +## 3. 注册、恢复与容量 + +连接后 5 秒内必须发送 worker.hello，hello 不携带 session_id。 + +| hello 字段 | 含义 | +| --- | --- | +| instance_id | Worker 进程实例 ID；进程重启必须换新值 | +| instance_secret | 32 字节随机秘密的无填充 base64url 编码，共 43 字符；同进程重连保留 | +| queue_id | 当前进程绑定队列 | +| capabilities | 1–32 个 handler / handler_version / payload_version 精确版本组合，不可重复 | +| capacity | 总执行槽数量，1–256，不是可用槽的累加值 | + +重连必须保持凭据、队列、处理器集合和进程秘密，处理器集合的排列顺序不影响身份。新连接递增会话代次并使旧会话失效。知道 instance_id 或 task_id 不能接管执行。 + +worker.welcome 返回 session_id、epoch、server_time、expires_at、lease_duration_ms、draining。当前窗口为会话 45 秒、执行租约最多 30 秒；注册及能力声明不等于开始授权。 + +**派发在收到第一次 worker.capacity 确认后才启用**，便于客户端先恢复旧尝试。 + +| 类型 | body | 返回 | +| --- | --- | --- | +| worker.resume | AttemptRef，字段见上方 task.start 示例 | 当前 LeaseStatus；成功迁移有效尝试到新会话，不创建尝试或延长期限 | +| worker.capacity | sequence 字符串、capacity 0–256、drain bool | 更新后的 welcome 字段 | +| task.assign | 服务端主动发送 | task 完整任务、attempt_id、epoch、lease_secret、lease_until、server_time | + +capacity 使用单调递增序号和覆盖值。同序号同内容可重复，同序号不同内容或旧序号冲突；capacity=0 不释放已有任务的并发许可。drain=true 是进程级持久化排空，重连保留，不能通过后续消息清除；新进程使用新实例身份。 + +一个 Worker 进程当前绑定一个队列。可以运行多个 Worker 进程或后续 SDK 实例消费不同队列；不能通过重连静默切换队列。 + +## 4. 开始、续租与结果 + +| 类型 | body | 语义 | +| --- | --- | --- | +| task.start | AttemptRef | 领取后再次检查当前会话、尝试、租约、队列暂停及流程门控，保存 RUNNING 后确认 | +| task.renew | AttemptRef | 只续期当前 RUNNING；新消息可续租，重复原消息只取回原回执 | +| task.finish | ref: AttemptRef、outcome、result、error_code | 保存明确结果或未知效果；返回操作回执和当前授权 | + +outcome 为 SUCCEEDED / FAILED / UNKNOWN。result 为非 null JSON 或证据引用。FAILED/UNKNOWN 必须带 1–64 字符的稳定 error_code；SUCCEEDED 不带错误分类。当前 FAILED 成为任务失败终态，UNKNOWN 转 NEEDS_REVIEW；策略化自动重试在 P3 实现。 + +开始、续租和结束返回 CommandResult： + +```json +{ + "receipt":{ + "operation":"start", + "task_state":"RUNNING", + "attempt_state":"RUNNING", + "task_revision":"3", + "lease_until":"2026-09-17T08:00:30Z", + "disposition":"APPLIED" + }, + "lease":{ + "active":true, + "state":"RUNNING", + "lease_until":"2026-09-17T08:00:30Z", + "server_time":"2026-09-17T08:00:00Z" + }, + "deduplicated":false +} +``` + +receipt 是原操作的不可变确认；lease 是本次重新核验的执行权，两者不能混用： + +- 只有 lease.active=true、lease.state=RUNNING，且仍在保守的本地有效期内，才允许处理器推进。LEASED 的 active 只表示预占有效。 +- 原 start 回执可能仍表示 RUNNING，但当前 lease.active=false；旧 ACK 不能让已过期、已结束或已被替换的尝试复活。 +- 有效期不超过租约窗口、会话期限、任务总截止和从 authorized_at 起计算的单次尝试时限。 +- 客户端需要扣除请求总耗时及安全余量，用本地单调时钟停止执行，不能假定本地墙上时钟与数据库一致。 +- 同结果重报不重复释放许可或产生成功事件。同消息不同内容返回 IDEMPOTENCY_CONFLICT。 +- 迟到、未开始或与已保存结果冲突的内容，经过身份和秘密验证后保存为 EVIDENCE_ONLY；不改变任务权威结果，不当作新的执行授权。 +- 断线后原结果仍保留稳定消息 ID，可在新会话恢复/核对后重报。不会因重连创建第二次业务执行。 + +## 5. 领取、并发与回收 + +平台在同一事务中分配队列并发许可、创建尝试、递增任务执行代次和次数、保存 LEASED 与派发事件。领取本身消耗一次 max_attempts 预算，包括最终未获开始授权的尝试，预算不会因重连或过期恢复重置。 + +运行中的进程总容量跨其所有旧/新会话统计。声明容量减少、断开连接或重复结果，都不能重复释放额度。当前已实施队列 max_concurrency；rate_per_second、其他配额和调度策略在 P3 完成。 + +每个节点每 2 秒发起一批恢复扫描，每批最多 100 项： + +- 未批准开始即到期：有剩余次数和截止时间时，延后 1 秒重派；否则 FAILED。 +- 已批准开始后失联：NEEDS_REVIEW，尝试 UNKNOWN，不能因网络超时断定业务未生效。 +- WAITING 超过总截止或最晚开始时间：FAILED。 +- 回收再次锁定队列、任务、尝试并读取数据库时间，与续租/完成串行化；多个节点不能重复释放许可或重复终结事件。 + +此处的安全回收不是完整重试引擎；IDEMPOTENT 的自动重试和策略退避仍是 P3 工作。 + +## 6. 连接保护与当前边界 + +当前节点上限 128 条 Worker 连接；每连接入站队列 4 帧、出站队列 8 帧，派发保留 ACK 缓冲空间，每秒最多发起 4 次新派发尝试。读空闲期限 60 秒、写期限 5 秒、单个数据库控制操作期限 8 秒。慢消费者导致连接关闭时，已提交状态仍由稳定身份恢复。 + +这些是有界的开发基线，不代表经过容量压测的生产参数。当前同队列操作使用较粗的行锁协调，吞吐与公平性需要 P6 按负载验证。服务退出会显式关闭被 HTTP Upgrade 接管的连接；断开不会直接撤销外部已发出的请求。 + +运行中取消、SDK 自动重连/续租、非协作处理器处理、完整 WS JSON Schema、凭据轮换和多节点进程级故障演练尚未全部完成。实际证据见 [P2 服务端阶段记录](../reports/phase-2-server.md)。 diff --git a/docs/examples/README.md b/docs/examples/README.md index ab75cf2..4fe216d 100644 --- a/docs/examples/README.md +++ b/docs/examples/README.md @@ -13,7 +13,7 @@ ## 已可运行：文件转换任务的受理与取消 - 源码：[SDK 任务受理示例](../../sdk/go/examples/tasks/main.go)。 -- 前置：完成迁移、初始化凭据、启动服务，详见 [SDK 接入](../sdk/README.md)。 +- 前置：完成迁移、初始化凭据、启动服务，在没有 Worker 消费的专用队列运行此受理/取消示例，详见 [SDK 接入](../sdk/README.md)。 - 输入：对象引用、格式与精确整数；提交两次同一身份，再查询、分页并带版本取消。 - 预期：重复请求返回同一任务，最终 CANCELLED/revision=2。当前不启动 Worker，不实际生成文件。 - 异常：凭据失效、权限不足、内容冲突、版本竞争和未知网络结果，见 [任务接口](../api/tasks.md)。 diff --git a/docs/reports/phase-2-server.md b/docs/reports/phase-2-server.md new file mode 100644 index 0000000..07a2922 --- /dev/null +++ b/docs/reports/phase-2-server.md @@ -0,0 +1,57 @@ +# P2 服务端执行与 WSS 阶段记录 + +日期：2026-09-17。**这是 P2 的阶段内进展，不是 P2 或生产验收完成。** + +## 实现 + +新增 WorkerInstance 与 AttemptEvidence，当前共 32 个持久化实体。字段、模型职责、单位、可空/秘密/代次语义及 SQL COMMENT 均为中文。新增前向迁移 000003_worker_execution，旧迁移不改写。 + +服务端现有能力包括进程恢复秘密、会话头与连接代次、能力绑定、容量覆盖和排空、两步领取/开始、尝试秘密、续租、结果回执、迟到证据、跨节点恢复，以及到期扫描。队列并发许可随执行事务领取、续期和释放。 + +HTTP 进程已注册 Worker WebSocket 入口，并启动受控恢复扫描；使用有界网络队列、消息/连接上限和明确的关闭流程。默认 bootstrap 不扩权，新增 --worker-access 显式为开发凭据增加 Worker 权限。 + +## 实际回归 + +环境：macOS arm64，Go 1.27.1，Docker 隔离 MySQL 8.4.10。入口为 make test、make vet、make build、make integration；数据库临时创建并清理，不使用已有业务数据库。 + +| 场景 | 结果 | +| --- | --- | +| 新迁移与全部 Go 模型字段匹配、前序 P0/P1 回归 | 通过 | +| 开始/续租/结果确认的原消息重报 | 原回执可查，不重复执行状态变更或延长旧续租 | +| 多消息提交同一已保存结果 | 只产生一次结果事件与审计、只释放一次许可 | +| 精确整数与小数结果 | 保存与查询不损失数值内容 | +| 不同节点重连与恢复同一尝试 | 原子迁移会话；次数和许可不增加 | +| 旧会话、错误尝试秘密、错误进程证明、注册内容改变 | 拒绝 | +| 两个实例、两个节点、16 个并发领取请求 | 队列并发 2、进程容量各 1 均不被突破 | +| 重连时旧会话已有执行 | 不重置容量计数 | +| 排空、容量零、重复/冲突序号 | 不给新工作，不误释放已有许可，排空跨重连保留 | +| 前 18 个任务门控暂停，后面有普通任务 | 普通任务仍能被领取，无候选页饥饿 | +| 暂停与开始、已授权开始回执、结果保存 | 未开始被阻止；已开始的原回执和结果仍可保存 | +| 未开始到期、已开始失联、两个恢复节点竞争 | 有限重派/FAILED/NEEDS_REVIEW 正确；无重复释放 | +| 迟到成功、未开始提交结果、冲突结果 | EVIDENCE_ONLY，保留证据，不改写权威结果 | +| 原 start ACK 在结束或过期后重报 | receipt 保留，当前 lease.active=false | +| 单次绝对时限与会话过期 | 续租不能越过限制，过期会话不能靠心跳复活 | +| 获取锁之前进入请求、锁等待期间租约过期 | 不使用早先时钟样本批准开始 | +| 领取事件或完成审计 INSERT 被注入失败 | 任务、尝试、许可、回执、事件整体回滚 | +| 凭据撤销、跨命名空间、已取消 context | 执行控制拒绝 | +| WSS 真 TLS 握手，鉴权/子协议/Origin/URL 规则 | 通过；使用受信任测试证书，不跳过验证 | +| WSS 节点 A 派发开始 → 断开 → 节点 B 恢复 → 结果确认重报 | 同一尝试、一次成功事件、许可正确释放 | +| 重复 JSON 字段、服务关闭与已接管长连接 | 非法帧关闭；节点关闭无残留长连接 | + +WSS 测试客户端直接发送协议帧，并显式重连到另一个测试服务器。当前没有把这项测试说成 SDK 自动重连或真实主机宕机演练。两个 ExecutionStore/Hub 使用不同节点 ID、同一个真实数据库；它们位于同一测试进程。 + +## 事务与恢复约束 + +每条命令重新校验当前凭据。执行路径先锁进程头/会话，再锁队列、可选门控、任务、尝试；回收从队列开始，不反向锁 Worker。拿到相关锁后重新读取数据库时间。状态、回执、额度和事件在短事务内提交，处理器不会在数据库事务中运行。 + +会话秘密、执行秘密仅存摘要，普通任务查询与日志不输出原文。保存迟到证据不构成执行授权。粗粒度队列锁是当前正确性基线，需要后续执行计划、压力与公平性验证；没有声称支持某个生产吞吐数字。 + +## 剩余工作 + +- Worker SDK 的处理器注册与运行循环、保守租约截止、自动续租、重连、结果重报、优雅退出。 +- API/SDK 执行历史与证据查询，WS JSON Schema 和兼容性验证。 +- 运行中取消、客户端慢消费及更多网络故障、结果未知/不协作处理器的端到端用例。 +- P3 的策略化自动重试、速率和其他配额、调度及结果订阅。 +- P4–P7 的流程、运维、性能和最终交付验证。 + +当前已实现的是可以经 WSS 操作的服务端执行内核。尚未完成独立 Worker 接入交付，也未完成生产高可用和容量验收。 diff --git a/go.mod b/go.mod index e4e5454..775b9f1 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,9 @@ module git.yuntuo.hk/k2safe/orbit go 1.27.1 -require github.com/go-sql-driver/mysql v1.10.1 +require ( + github.com/coder/websocket v1.8.15 + github.com/go-sql-driver/mysql v1.10.1 +) require filippo.io/edwards25519 v1.2.0 // indirect diff --git a/go.sum b/go.sum index 5a95e9a..f808e3b 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= +github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= diff --git a/internal/storage/mysql/bootstrap.go b/internal/storage/mysql/bootstrap.go index a5be27e..899b5c5 100644 --- a/internal/storage/mysql/bootstrap.go +++ b/internal/storage/mysql/bootstrap.go @@ -14,10 +14,11 @@ import ( // BootstrapInput 仅供持有数据库管理权限的初始化命令使用，不暴露为公共 HTTP。 type BootstrapInput struct { - Application string // Application 应用名称，每次初始化创建独立应用。 - Namespace string // Namespace 命名空间名称。 - Environment string // Environment 环境名称，必须显式指定。 - Queue string // Queue 初始队列名称。 + Application string // Application 应用名称，每次初始化创建独立应用。 + Namespace string // Namespace 命名空间名称。 + Environment string // Environment 环境名称，必须显式指定。 + Queue string // Queue 初始队列名称。 + WorkerAccess bool // WorkerAccess 显式增加开发用 Worker 接入及消费权限。 } // BootstrapResult 包含一次性交付的原始凭据，禁止记录到日志或提交到版本库。 @@ -56,6 +57,9 @@ func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func( } out.Token = secret permissions := auth.Permissions{Actions: []string{"tasks:submit", "tasks:read", "tasks:cancel", "queues:read", "queues:manage"}, Queues: []string{"*"}} + if in.WorkerAccess { + permissions.Actions = append(permissions.Actions, "workers:connect", "workers:consume") + } raw, err := json.Marshal(permissions) if err != nil { return err diff --git a/internal/storage/mysql/integration_test.go b/internal/storage/mysql/integration_test.go index c5d3065..a6b3c47 100644 --- a/internal/storage/mysql/integration_test.go +++ b/internal/storage/mysql/integration_test.go @@ -191,9 +191,11 @@ func checkModelColumns(t *testing.T, ctx context.Context, db *sql.DB) { pulse.Queue{}, pulse.ExecutionGate{}, pulse.WorkerSession{}, + pulse.WorkerInstance{}, pulse.Task{}, pulse.Attempt{}, pulse.AttemptReceipt{}, + pulse.AttemptEvidence{}, pulse.ConcurrencyLimit{}, pulse.Permit{}, pulse.Schedule{}, diff --git a/internal/storage/mysql/migrations/000003_worker_execution.up.sql b/internal/storage/mysql/migrations/000003_worker_execution.up.sql new file mode 100644 index 0000000..384255e --- /dev/null +++ b/internal/storage/mysql/migrations/000003_worker_execution.up.sql @@ -0,0 +1,62 @@ +-- Worker 会话隔离和迟到结果证据。P2 之前没有受支持的 Worker 注册接口。 +ALTER TABLE pulse_worker_sessions ADD UNIQUE KEY uq_session_instance_id(namespace_id, worker_instance_id, id); + +CREATE TABLE pulse_worker_instances ( + namespace_id BINARY(16) NOT NULL COMMENT '进程所属命名空间，参与授权和所有关联', + id BINARY(16) NOT NULL COMMENT 'Worker 进程实例标识，进程重启必须换新值', + revision BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '持久化变更版本，从一开始递增', + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '首次注册时间，UTC 微秒精度', + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '最近变更时间，不替代会话代次', + credential_id BINARY(16) NOT NULL COMMENT '初始注册凭据，进程生命周期内固定', + queue_id BINARY(16) NOT NULL COMMENT '绑定队列，切换队列需创建新进程实例', + secret_digest BINARY(32) NOT NULL COMMENT '随机进程恢复秘密的 SHA-256 摘要，不保存原文', + registration_digest BINARY(32) NOT NULL COMMENT '队列及处理器版本集合的语义摘要，重连必须匹配', + session_epoch BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '最大已分配连接代次，注册前为零', + current_session_id BINARY(16) NULL COMMENT '当前会话，注册事务建立会话前为空', + draining BOOLEAN NOT NULL DEFAULT FALSE COMMENT '持久化排空标记，重连不能清除', + PRIMARY KEY(namespace_id,id), + UNIQUE KEY uq_worker_credential(namespace_id,id,credential_id), + CONSTRAINT ck_worker_instance_revision CHECK (revision>0), + CONSTRAINT ck_worker_instance_session CHECK ((session_epoch=0 AND current_session_id IS NULL) OR (session_epoch>0 AND current_session_id IS NOT NULL)), + CONSTRAINT fk_worker_instance_credential FOREIGN KEY(namespace_id,credential_id) REFERENCES orbit_credentials(namespace_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT, + CONSTRAINT fk_worker_instance_queue FOREIGN KEY(namespace_id,queue_id) REFERENCES pulse_queues(namespace_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT, + CONSTRAINT fk_worker_instance_session FOREIGN KEY(namespace_id,id,current_session_id) REFERENCES pulse_worker_sessions(namespace_id,worker_instance_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='Worker 进程恢复身份和当前会话头，用于跨节点隔离旧连接'; + +ALTER TABLE pulse_worker_sessions ADD CONSTRAINT fk_worker_session_instance FOREIGN KEY(namespace_id,worker_instance_id,credential_id) REFERENCES pulse_worker_instances(namespace_id,id,credential_id) ON DELETE RESTRICT ON UPDATE RESTRICT; + +ALTER TABLE pulse_worker_sessions ADD CONSTRAINT ck_worker_session_capacity CHECK (capacity<=256); + +ALTER TABLE pulse_attempts ADD KEY ix_attempt_session_state(namespace_id,session_id,state); + +ALTER TABLE pulse_attempts ADD CONSTRAINT ck_attempt_lifecycle CHECK ((state IN ('LEASED','RUNNING') AND ended_at IS NULL) OR (state NOT IN ('LEASED','RUNNING') AND ended_at IS NOT NULL)); + +ALTER TABLE pulse_attempts ADD CONSTRAINT ck_attempt_start CHECK ((state='LEASED' AND authorized_at IS NULL) OR (state='RUNNING' AND authorized_at IS NOT NULL) OR state NOT IN ('LEASED','RUNNING')); + +ALTER TABLE pulse_tasks ADD KEY ix_task_deadline(state,deadline_at,namespace_id,id); + +ALTER TABLE pulse_tasks ADD KEY ix_task_start_deadline(state,start_deadline_at,namespace_id,id); + +CREATE TABLE pulse_attempt_evidence ( + namespace_id BINARY(16) NOT NULL COMMENT '证据所属命名空间，与尝试一致', + id BINARY(16) NOT NULL COMMENT '证据记录标识', + revision BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '记录版本；证据追加保存，不覆盖', + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '首次收到证据的数据库时间，UTC', + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '写入时间，追加记录与创建时间相同', + attempt_id BINARY(16) NOT NULL COMMENT '证据所属执行尝试', + message_id BINARY(16) NOT NULL COMMENT '结果消息身份，重报保持不变', + outcome VARCHAR(16) NOT NULL COMMENT 'Worker 声明的结果：SUCCEEDED、FAILED 或 UNKNOWN', + error_code VARCHAR(64) NULL COMMENT '稳定错误分类；无错误时为空，不保存原始异常正文', + payload MEDIUMTEXT NOT NULL COMMENT '结果或证据引用的 JSON 文本，保留精确数字', + request_digest BINARY(32) NOT NULL COMMENT '结果语义摘要，排除会话与秘密', + reason VARCHAR(32) NOT NULL COMMENT '证据未改写权威结果的原因：LEASE_LOST、NOT_STARTED 或 RESULT_CONFLICT', + PRIMARY KEY(namespace_id,id), + UNIQUE KEY uq_attempt_evidence_message(namespace_id,attempt_id,message_id), + CONSTRAINT ck_attempt_evidence_revision CHECK(revision>0), + CONSTRAINT ck_attempt_evidence_json CHECK(JSON_VALID(payload)), + CONSTRAINT ck_attempt_evidence_outcome CHECK(outcome IN ('SUCCEEDED','FAILED','UNKNOWN')), + CONSTRAINT ck_attempt_evidence_reason CHECK(reason IN ('LEASE_LOST','NOT_STARTED','RESULT_CONFLICT')), + CONSTRAINT fk_attempt_evidence_attempt FOREIGN KEY(namespace_id,attempt_id) REFERENCES pulse_attempts(namespace_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='迟到与冲突结果的追加证据，不自动改变任务终态'; + +CREATE TRIGGER immutable_attempt_evidence BEFORE UPDATE ON pulse_attempt_evidence FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='attempt evidence is immutable'; diff --git a/internal/storage/mysql/worker_execution.go b/internal/storage/mysql/worker_execution.go new file mode 100644 index 0000000..1b1d0fb --- /dev/null +++ b/internal/storage/mysql/worker_execution.go @@ -0,0 +1,521 @@ +package mysql + +import ( + "context" + "crypto/sha256" + "crypto/subtle" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "git.yuntuo.hk/k2safe/orbit/internal/fault" + core "git.yuntuo.hk/k2safe/orbit/model" + "git.yuntuo.hk/k2safe/orbit/platform/auth" + pulse "git.yuntuo.hk/k2safe/orbit/pulse/model" + "git.yuntuo.hk/k2safe/orbit/pulse/service" + "git.yuntuo.hk/k2safe/orbit/pulse/worker" +) + +type executionTask struct { + view service.TaskView + current *core.ID + epoch uint64 + lease *time.Time + gate *core.ID + maxAttempts uint32 + attemptTimeout uint64 +} +type executionRows struct { + task executionTask + attempt pulse.Attempt + paused bool + session sessionState +} + +func readExecutionTask(ctx context.Context, tx *sql.Tx, ns, id core.ID) (executionTask, error) { + var row executionTask + view, err := readTask(ctx, tx, ns, id, true) + if err != nil { + return row, err + } + row.view = view + err = tx.QueryRowContext(ctx, `SELECT current_attempt_id,lease_epoch,lease_until,gate_id,max_attempts,attempt_timeout_ms FROM pulse_tasks WHERE namespace_id=? AND id=?`, ns, id).Scan(&row.current, &row.epoch, &row.lease, &row.gate, &row.maxAttempts, &row.attemptTimeout) + return row, err +} +func lockQueue(ctx context.Context, tx *sql.Tx, ns, id core.ID) (paused bool, max uint32, err error) { + err = tx.QueryRowContext(ctx, "SELECT paused,max_concurrency FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE", ns, id).Scan(&paused, &max) + if errors.Is(err, sql.ErrNoRows) { + err = fault.NotFound + } + return +} +func lockGate(ctx context.Context, tx *sql.Tx, ns core.ID, id *core.ID) (bool, error) { + if id == nil { + return false, nil + } + var paused bool + err := tx.QueryRowContext(ctx, "SELECT paused FROM pulse_execution_gates WHERE namespace_id=? AND id=? FOR SHARE", ns, *id).Scan(&paused) + return paused, err +} +func minimumTime(first time.Time, others ...time.Time) time.Time { + for _, other := range others { + if other.Before(first) { + first = other + } + } + return first +} +func queueLimit(ctx context.Context, tx *sql.Tx, ns, queue core.ID, max uint32) (core.ID, uint32, error) { + id, err := core.NewID() + if err != nil { + return "", 0, err + } + digest := core.Digest(sha256.Sum256([]byte(queue))) + _, err = tx.ExecContext(ctx, `INSERT INTO pulse_concurrency_limits(namespace_id,id,scope_key,key_digest,max_permits) VALUES (?,?,'queue',?,?) ON DUPLICATE KEY UPDATE id=pulse_concurrency_limits.id`, ns, id, digest, max) + if err != nil { + return "", 0, err + } + var used uint32 + var configured uint32 + err = tx.QueryRowContext(ctx, `SELECT id,used_permits,max_permits FROM pulse_concurrency_limits WHERE namespace_id=? AND scope_key='queue' AND key_digest=? FOR UPDATE`, ns, digest).Scan(&id, &used, &configured) + if err == nil && configured != max { + err = errors.New("queue limit configuration requires coordinated update") + } + return id, used, err +} +func outstanding(ctx context.Context, tx *sql.Tx, ns, instance core.ID) (uint32, error) { + var count uint32 + err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_attempts a JOIN pulse_worker_sessions s ON s.namespace_id=a.namespace_id AND s.id=a.session_id WHERE a.namespace_id=? AND s.worker_instance_id=? AND a.state IN ('LEASED','RUNNING')`, ns, instance).Scan(&count) + return count, err +} + +func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.Binding) (*worker.Assignment, error) { + var assignment *worker.Assignment + err := s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error { + state, err := s.lockSession(ctx, tx, token, b, p) + if err != nil { + return err + } + if state.draining || state.capacity == 0 { + return nil + } + count, err := outstanding(ctx, tx, token.NamespaceID, b.InstanceID) + if err != nil { + return err + } + if count >= state.capacity { + return nil + } + paused, max, err := lockQueue(ctx, tx, token.NamespaceID, state.queue) + if err != nil { + return err + } + if paused { + return nil + } + limit, used, err := queueLimit(ctx, tx, token.NamespaceID, state.queue, max) + if err != nil { + return err + } + if used >= max { + return nil + } + args := []any{token.NamespaceID, state.queue} + matches := make([]string, 0, len(state.capabilities)) + for _, c := range state.capabilities { + matches = append(matches, "(handler=? AND handler_version=? AND payload_version=?)") + args = append(args, c.Handler, c.HandlerVersion, c.PayloadVersion) + } + if len(matches) == 0 { + return nil + } + rows, err := tx.QueryContext(ctx, `SELECT id,gate_id FROM pulse_tasks WHERE namespace_id=? AND queue_id=? AND state='WAITING' AND available_at<=UTC_TIMESTAMP(6) AND deadline_at>UTC_TIMESTAMP(6) AND (start_deadline_at IS NULL OR start_deadline_at>UTC_TIMESTAMP(6)) AND attempt_count= task.maxAttempts || (task.view.StartDeadlineAt != nil && !task.view.StartDeadlineAt.After(now)) { + continue + } + if task.epoch == ^uint64(0) { + return fault.Conflict + } + attempt, err := core.NewID() + if err != nil { + return err + } + secret, digest, err := worker.NewSecret() + if err != nil { + return err + } + until := minimumTime(now.Add(s.options.LeaseDuration), task.view.DeadlineAt, state.expires) + if task.view.StartDeadlineAt != nil { + until = minimumTime(until, *task.view.StartDeadlineAt) + } + _, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempts(namespace_id,id,task_id,epoch,session_id,token_digest,lease_until) VALUES (?,?,?,?,?,?,?)`, token.NamespaceID, attempt, locked, task.epoch+1, b.SessionID, digest, until) + if err != nil { + return err + } + permit, err := core.NewID() + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, `INSERT INTO pulse_permits(namespace_id,id,limit_id,attempt_id,lease_until) VALUES (?,?,?,?,?)`, token.NamespaceID, permit, limit, attempt, until) + if err != nil { + return err + } + changed, err := tx.ExecContext(ctx, `UPDATE pulse_concurrency_limits SET used_permits=used_permits+1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND used_permits 队列 -> 可选门控 -> 任务 -> 尝试。 +// 恢复扫描也从队列锁开始，不反向锁定 Worker 进程。 +func (s *ExecutionStore) lockExecution(ctx context.Context, tx *sql.Tx, token auth.Token, b worker.Binding, p auth.Permissions, ref worker.AttemptRef, checkGate bool) (executionRows, error) { + var out executionRows + state, err := s.lockSession(ctx, tx, token, b, p) + if err != nil { + return out, err + } + out.session = state + var queue core.ID + var gate *core.ID + err = tx.QueryRowContext(ctx, `SELECT queue_id,gate_id FROM pulse_tasks WHERE namespace_id=? AND id=?`, token.NamespaceID, ref.TaskID).Scan(&queue, &gate) + if errors.Is(err, sql.ErrNoRows) { + return out, fault.NotFound + } + if err != nil { + return out, err + } + if queue != state.queue || !p.AllowsQueue(queue) { + return out, fault.NotFound + } + out.paused, _, err = lockQueue(ctx, tx, token.NamespaceID, queue) + if err != nil { + return out, err + } + if checkGate { + paused, err := lockGate(ctx, tx, token.NamespaceID, gate) + if err != nil { + return out, err + } + out.paused = out.paused || paused + } + out.task, err = readExecutionTask(ctx, tx, token.NamespaceID, ref.TaskID) + if err != nil { + return out, err + } + out.attempt, err = readAttempt(ctx, tx, token.NamespaceID, ref.AttemptID) + if err != nil { + return out, err + } + proof, err := worker.SecretDigest(ref.LeaseSecret) + if err != nil { + return out, err + } + if out.attempt.TaskID != ref.TaskID || out.attempt.Epoch != ref.Epoch || subtle.ConstantTimeCompare(proof[:], out.attempt.TokenDigest[:]) != 1 { + return out, fault.Forbidden + } + var instance core.ID + if err := tx.QueryRowContext(ctx, `SELECT worker_instance_id FROM pulse_worker_sessions WHERE namespace_id=? AND id=?`, token.NamespaceID, out.attempt.SessionID).Scan(&instance); err != nil { + return out, err + } + if instance != b.InstanceID { + return out, fault.Forbidden + } + return out, nil +} +func readAttempt(ctx context.Context, tx *sql.Tx, ns, id core.ID) (pulse.Attempt, error) { + var a pulse.Attempt + a.NamespaceID = ns + err := tx.QueryRowContext(ctx, `SELECT id,revision,task_id,epoch,session_id,token_digest,state,lease_until,authorized_at,ended_at,error_code,result,result_digest FROM pulse_attempts WHERE namespace_id=? AND id=? FOR UPDATE`, ns, id).Scan(&a.ID, &a.Revision, &a.TaskID, &a.Epoch, &a.SessionID, &a.TokenDigest, &a.State, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Result, &a.ResultDigest) + if errors.Is(err, sql.ErrNoRows) { + err = fault.NotFound + } + return a, err +} +func leaseStatus(rows executionRows, b worker.Binding, now time.Time) worker.LeaseStatus { + t, a := rows.task, rows.attempt + until := minimumTime(a.LeaseUntil, rows.session.expires, t.view.DeadlineAt) + if t.lease != nil { + until = minimumTime(until, *t.lease) + } + active := rows.session.expires.After(now) && t.current != nil && *t.current == a.ID && t.epoch == a.Epoch && a.SessionID == b.SessionID && t.lease != nil && t.lease.After(now) && a.LeaseUntil.After(now) && t.view.DeadlineAt.After(now) && ((t.view.State == pulse.TaskLeased && a.State == pulse.AttemptLeased) || (t.view.State == pulse.TaskRunning && a.State == pulse.AttemptRunning)) + if a.AuthorizedAt != nil { + deadline := a.AuthorizedAt.Add(time.Duration(t.attemptTimeout) * time.Millisecond) + active = active && deadline.After(now) + until = minimumTime(until, deadline) + } + return worker.LeaseStatus{Active: active, State: a.State, LeaseUntil: until, ServerTime: now} +} +func attemptIdentity(ref worker.AttemptRef) any { + return struct { + Task core.ID + Attempt core.ID + Epoch string + }{ref.TaskID, ref.AttemptID, fmt.Sprint(ref.Epoch)} +} +func readReceipt(ctx context.Context, tx *sql.Tx, ns, attempt, message core.ID, operation string, digest core.Digest) (*worker.Receipt, error) { + var saved core.Digest + var raw []byte + err := tx.QueryRowContext(ctx, `SELECT request_digest,response FROM pulse_attempt_receipts WHERE namespace_id=? AND attempt_id=? AND operation=? AND message_id=? FOR UPDATE`, ns, attempt, operation, message).Scan(&saved, &raw) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if saved != digest { + return nil, fault.IdempotencyConflict + } + var receipt worker.Receipt + if err := json.Unmarshal(raw, &receipt); err != nil { + return nil, err + } + return &receipt, nil +} +func saveReceipt(ctx context.Context, tx *sql.Tx, ns, attempt, message core.ID, operation string, digest core.Digest, receipt worker.Receipt) error { + id, err := core.NewID() + if err != nil { + return err + } + raw, err := json.Marshal(receipt) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempt_receipts(namespace_id,id,attempt_id,operation,message_id,request_digest,response,expires_at) VALUES (?,?,?,?,?,?,?,DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 90 DAY))`, ns, id, attempt, operation, message, digest, string(raw)) + return err +} +func renewPermits(ctx context.Context, tx *sql.Tx, ns, attempt core.ID, until, now time.Time) error { + result, err := tx.ExecContext(ctx, `UPDATE pulse_permits SET lease_until=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND attempt_id=? AND released_at IS NULL`, until, now, ns, attempt) + if err != nil { + return err + } + n, err := result.RowsAffected() + if err != nil { + return err + } + if n < 1 { + return errors.New("active attempt has no concurrency permit") + } + return nil +} +func releasePermits(ctx context.Context, tx *sql.Tx, ns, attempt core.ID, now time.Time) error { + rows, err := tx.QueryContext(ctx, `SELECT id,limit_id FROM pulse_permits WHERE namespace_id=? AND attempt_id=? AND released_at IS NULL ORDER BY limit_id FOR UPDATE`, ns, attempt) + if err != nil { + return err + } + type permit struct{ id, limit core.ID } + permits := []permit{} + for rows.Next() { + var p permit + if err := rows.Scan(&p.id, &p.limit); err != nil { + rows.Close() + return err + } + permits = append(permits, p) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + if len(permits) == 0 { + return errors.New("active attempt has no permit to release") + } + for _, p := range permits { + result, err := tx.ExecContext(ctx, `UPDATE pulse_concurrency_limits SET used_permits=used_permits-1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND used_permits>0`, now, ns, p.limit) + if err := exactlyOne(result, err); err != nil { + return err + } + result, err = tx.ExecContext(ctx, `UPDATE pulse_permits SET released_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND released_at IS NULL`, now, now, ns, p.id) + if err := exactlyOne(result, err); err != nil { + return err + } + } + return nil +} + +func (s *ExecutionStore) Start(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, ref worker.AttemptRef) (worker.CommandResult, error) { + return s.executionCommand(ctx, token, b, message, ref, "start") +} +func (s *ExecutionStore) Renew(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, ref worker.AttemptRef) (worker.CommandResult, error) { + return s.executionCommand(ctx, token, b, message, ref, "renew") +} +func (s *ExecutionStore) executionCommand(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, ref worker.AttemptRef, operation string) (worker.CommandResult, error) { + var out worker.CommandResult + digest, err := requestDigest(attemptIdentity(ref)) + if err != nil { + return out, err + } + err = s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error { + rows, err := s.lockExecution(ctx, tx, token, b, p, ref, operation == "start") + if err != nil { + return err + } + receipt, err := readReceipt(ctx, tx, token.NamespaceID, ref.AttemptID, message, operation, digest) + if err != nil { + return err + } + now, err := dbNow(ctx, tx) + if err != nil { + return err + } + status := leaseStatus(rows, b, now) + if receipt != nil { + out = worker.CommandResult{Receipt: *receipt, Lease: status, Deduplicated: true} + return nil + } + if !status.Active { + return worker.LeaseLost + } + task, a := rows.task, rows.attempt + authorized := a.AuthorizedAt + if operation == "start" { + if a.State != pulse.AttemptLeased || task.view.State != pulse.TaskLeased { + return fault.Conflict + } + if rows.paused { + return worker.GateClosed + } + if task.view.StartDeadlineAt != nil && !task.view.StartDeadlineAt.After(now) { + return worker.LeaseLost + } + authorized = &now + } else if a.State != pulse.AttemptRunning || authorized == nil { + return fault.Conflict + } + until := minimumTime(now.Add(s.options.LeaseDuration), authorized.Add(time.Duration(task.attemptTimeout)*time.Millisecond), task.view.DeadlineAt, rows.session.expires) + result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state='RUNNING',authorized_at=?,lease_until=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, authorized, until, now, token.NamespaceID, a.ID, a.Revision) + if err := exactlyOne(result, err); err != nil { + return err + } + result, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='RUNNING',lease_until=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, until, now, token.NamespaceID, ref.TaskID, task.view.Revision) + if err := exactlyOne(result, err); err != nil { + return err + } + if err := renewPermits(ctx, tx, token.NamespaceID, a.ID, until, now); err != nil { + return err + } + if operation == "start" { + if err := emitEvent(ctx, tx, token.NamespaceID, ref.TaskID, task.view.Revision+1, "task", "task.started", map[string]any{"attempt_id": a.ID, "epoch": fmt.Sprint(a.Epoch), "session_id": b.SessionID}); err != nil { + return err + } + } + receipt = &worker.Receipt{Operation: operation, TaskState: pulse.TaskRunning, AttemptState: pulse.AttemptRunning, TaskRevision: task.view.Revision + 1, LeaseUntil: until, Disposition: "APPLIED"} + if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, operation, digest, *receipt); err != nil { + return err + } + out = worker.CommandResult{Receipt: *receipt, Lease: worker.LeaseStatus{Active: true, State: pulse.AttemptRunning, LeaseUntil: until, ServerTime: now}} + return nil + }) + return out, err +} + +func (s *ExecutionStore) Resume(ctx context.Context, token auth.Token, b worker.Binding, ref worker.AttemptRef) (worker.LeaseStatus, error) { + var out worker.LeaseStatus + err := s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error { + rows, err := s.lockExecution(ctx, tx, token, b, p, ref, false) + if err != nil { + return err + } + now, err := dbNow(ctx, tx) + if err != nil { + return err + } + // 只暂按新会话计算有效性；通过所有期限和当前尝试检查后才迁移归属，绝不延长期限。 + previous := rows.attempt.SessionID + rows.attempt.SessionID = b.SessionID + out = leaseStatus(rows, b, now) + if !out.Active || previous == b.SessionID { + return nil + } + result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET session_id=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, b.SessionID, now, token.NamespaceID, ref.AttemptID, rows.attempt.Revision) + if err := exactlyOne(result, err); err != nil { + return err + } + return emitEvent(ctx, tx, token.NamespaceID, ref.AttemptID, rows.attempt.Revision+1, "attempt", "attempt.resumed", map[string]any{"from_session_id": previous, "to_session_id": b.SessionID, "node_id": s.node}) + }) + return out, err +} diff --git a/internal/storage/mysql/worker_integration_test.go b/internal/storage/mysql/worker_integration_test.go new file mode 100644 index 0000000..850cdcc --- /dev/null +++ b/internal/storage/mysql/worker_integration_test.go @@ -0,0 +1,638 @@ +package mysql + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "sync" + "testing" + "time" + + "git.yuntuo.hk/k2safe/orbit/internal/fault" + core "git.yuntuo.hk/k2safe/orbit/model" + "git.yuntuo.hk/k2safe/orbit/platform/auth" + pulse "git.yuntuo.hk/k2safe/orbit/pulse/model" + "git.yuntuo.hk/k2safe/orbit/pulse/service" + "git.yuntuo.hk/k2safe/orbit/pulse/worker" +) + +type executionFixture struct { + t *testing.T + ctx context.Context + db *sql.DB + resources BootstrapResult + token auth.Token + tasks *service.Service + storeA, storeB *ExecutionStore + a, b *worker.Service + hello worker.ConnectInput + binding worker.Binding +} + +func executionID(t *testing.T) core.ID { + t.Helper() + id, err := core.NewID() + if err != nil { + t.Fatal(err) + } + return id +} +func newExecutionFixture(t *testing.T, ctx context.Context, db *sql.DB) *executionFixture { + t.Helper() + f := &executionFixture{t: t, ctx: ctx, db: db, tasks: service.New(NewTaskStore(db))} + err := Bootstrap(ctx, db, BootstrapInput{Application: "worker-" + string(executionID(t)), Namespace: "default", Environment: "test", Queue: "work"}, func(v BootstrapResult) error { f.resources = v; return nil }) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(auth.Permissions{Actions: []string{"tasks:submit", "tasks:read", "tasks:cancel", "queues:read", "queues:manage", "workers:connect", "workers:consume"}, Queues: []string{"*"}}) + f.exec("UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?", string(raw), f.resources.NamespaceID, f.resources.CredentialID) + f.token, err = auth.Parse(f.resources.Token) + if err != nil { + t.Fatal(err) + } + f.storeA, err = NewExecutionStore(db, executionID(t), ExecutionOptions{}) + if err != nil { + t.Fatal(err) + } + f.storeB, err = NewExecutionStore(db, executionID(t), ExecutionOptions{}) + if err != nil { + t.Fatal(err) + } + f.a, f.b = worker.New(f.storeA), worker.New(f.storeB) + secret, _, err := worker.NewSecret() + if err != nil { + t.Fatal(err) + } + f.hello = worker.ConnectInput{InstanceID: executionID(t), InstanceSecret: secret, QueueID: f.resources.QueueID, Capacity: 4, Capabilities: []worker.Capability{{Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1"}}} + f.binding, _, err = f.a.Connect(ctx, f.token, f.hello) + if err != nil { + t.Fatal(err) + } + return f +} +func (f *executionFixture) exec(q string, args ...any) { + f.t.Helper() + if _, err := f.db.ExecContext(f.ctx, q, args...); err != nil { + f.t.Fatal(err) + } +} +func (f *executionFixture) submit(max uint32) service.TaskView { + f.t.Helper() + out, err := f.tasks.Submit(f.ctx, f.token, string(executionID(f.t)), service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`), Retry: pulse.RetryPolicy{MaxAttempts: max, AttemptTimeoutMS: 120000}}) + if err != nil { + f.t.Fatal(err) + } + return out.Task +} +func (f *executionFixture) claim() (*worker.Assignment, worker.AttemptRef) { + f.t.Helper() + out, err := f.a.Claim(f.ctx, f.token, f.binding) + if err != nil { + f.t.Fatal(err) + } + if out == nil { + f.t.Fatal("expected assignment") + } + return out, worker.AttemptRef{TaskID: out.Task.ID, AttemptID: out.AttemptID, Epoch: out.Epoch, LeaseSecret: out.LeaseSecret} +} +func (f *executionFixture) get(id core.ID) service.TaskView { + f.t.Helper() + out, err := f.tasks.GetTask(f.ctx, f.token, id) + if err != nil { + f.t.Fatal(err) + } + return out +} +func (f *executionFixture) expire(ref worker.AttemptRef) { + f.t.Helper() + f.exec("UPDATE pulse_tasks SET lease_until=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, ref.TaskID) + f.exec("UPDATE pulse_attempts SET lease_until=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, ref.AttemptID) +} +func (f *executionFixture) count(table, where string, args ...any) int { + f.t.Helper() + var n int + if err := f.db.QueryRowContext(f.ctx, "SELECT COUNT(*) FROM "+table+" WHERE namespace_id=? AND "+where, append([]any{f.resources.NamespaceID}, args...)...).Scan(&n); err != nil { + f.t.Fatal(err) + } + return n +} +func (f *executionFixture) used() uint32 { + f.t.Helper() + var n uint32 + if err := f.db.QueryRowContext(f.ctx, "SELECT COALESCE(SUM(used_permits),0) FROM pulse_concurrency_limits WHERE namespace_id=?", f.resources.NamespaceID).Scan(&n); err != nil { + f.t.Fatal(err) + } + return n +} +func expectWorkerCode(t *testing.T, err error, code string) { + t.Helper() + if fault.Code(err) != code { + t.Fatalf("wanted %s got %v", code, err) + } +} +func TestIntegrationWorkerExecution(t *testing.T) { + dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN") + if dsn == "" { + t.Skip("requires disposable MySQL") + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + db, err := Open(ctx, dsn, 32, 8) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := Migrate(ctx, db); err != nil { + t.Fatal(err) + } + t.Run("durable_start_renew_and_exact_result_confirmation", func(t *testing.T) { + f := newExecutionFixture(t, ctx, db) + task := f.submit(3) + assignment, ref := f.claim() + if assignment.Task.ID != task.ID || assignment.Task.State != pulse.TaskLeased || assignment.Task.AttemptCount != 1 || f.used() != 1 { + t.Fatal("claim state/permit invariant") + } + startID := executionID(t) + started, err := f.a.Start(ctx, f.token, f.binding, startID, ref) + if err != nil || !started.Lease.Active || started.Receipt.TaskState != pulse.TaskRunning { + t.Fatalf("start: %v", err) + } + duplicate, err := f.a.Start(ctx, f.token, f.binding, startID, ref) + if err != nil || !duplicate.Deduplicated || duplicate.Receipt != started.Receipt { + t.Fatalf("start replay: %v", err) + } + _, err = f.a.Start(ctx, f.token, f.binding, executionID(t), ref) + expectWorkerCode(t, err, "STATE_CONFLICT") + renewID := executionID(t) + renewed, err := f.a.Renew(ctx, f.token, f.binding, renewID, ref) + if err != nil || !renewed.Lease.Active || renewed.Lease.LeaseUntil.Before(started.Lease.LeaseUntil) { + t.Fatalf("renew: %v", err) + } + repeated, err := f.a.Renew(ctx, f.token, f.binding, renewID, ref) + if err != nil || !repeated.Deduplicated || repeated.Receipt.LeaseUntil != renewed.Receipt.LeaseUntil { + t.Fatalf("renew replay: %v", err) + } + result := worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"amount":12345678901234567890.1234567890123456789,"version":9007199254740993}`)} + message := executionID(t) + finished, err := f.a.Finish(ctx, f.token, f.binding, message, result) + if err != nil || finished.Lease.Active || finished.Receipt.Disposition != "APPLIED" { + t.Fatalf("finish: %v", err) + } + duplicate, err = f.a.Finish(ctx, f.token, f.binding, message, result) + if err != nil || !duplicate.Deduplicated { + t.Fatalf("finish replay %v", err) + } + duplicate, err = f.a.Finish(ctx, f.token, f.binding, executionID(t), result) + if err != nil || !duplicate.Deduplicated { + t.Fatalf("new message equivalent result %v", err) + } + current := f.get(task.ID) + if current.State != pulse.TaskSucceeded || current.CompletedAt == nil || !strings.Contains(string(current.Result), "12345678901234567890.1234567890123456789") || f.used() != 0 { + t.Fatal("finish lost state, precision or permit") + } + if f.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 1 || f.count("orbit_audit_events", "resource_id=? AND action='task.result'", task.ID) != 1 { + t.Fatal("duplicated completion history") + } + _, err = f.a.Renew(ctx, f.token, f.binding, executionID(t), ref) + expectWorkerCode(t, err, "LEASE_LOST") + oldAck, err := f.a.Start(ctx, f.token, f.binding, startID, ref) + if err != nil || !oldAck.Deduplicated || oldAck.Lease.Active { + t.Fatalf("old start acknowledgement resurrected execution %v", err) + } + changed := result + changed.Result = json.RawMessage(`{"changed":true}`) + _, err = f.a.Finish(ctx, f.token, f.binding, message, changed) + expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT") + conflict, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), changed) + if err != nil || conflict.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(task.ID).State != pulse.TaskSucceeded { + t.Fatalf("conflict rewrote outcome %v", err) + } + if f.count("pulse_attempt_evidence", "attempt_id=? AND reason='RESULT_CONFLICT'", ref.AttemptID) != 1 { + t.Fatal("conflict evidence missing") + } + }) + t.Run("cross_node_resume_fences_old_session_and_requires_secrets", func(t *testing.T) { + f := newExecutionFixture(t, ctx, db) + f.submit(3) + _, ref := f.claim() + start := executionID(t) + if _, err := f.a.Start(ctx, f.token, f.binding, start, ref); err != nil { + t.Fatal(err) + } + before := f.get(ref.TaskID) + newBinding, welcome, err := f.b.Connect(ctx, f.token, f.hello) + if err != nil || welcome.Epoch != 2 { + t.Fatalf("reconnect %v", err) + } + _, err = f.a.Renew(ctx, f.token, f.binding, executionID(t), ref) + expectWorkerCode(t, err, "STALE_SESSION") + _, err = f.b.Renew(ctx, f.token, newBinding, executionID(t), ref) + expectWorkerCode(t, err, "LEASE_LOST") + wrong := ref + wrong.LeaseSecret = strings.Repeat("A", 43) + _, err = f.b.Resume(ctx, f.token, newBinding, wrong) + expectWorkerCode(t, err, "PERMISSION_DENIED") + resumed, err := f.b.Resume(ctx, f.token, newBinding, ref) + if err != nil || !resumed.Active || resumed.State != pulse.AttemptRunning { + t.Fatalf("resume %v", err) + } + duplicate, err := f.b.Start(ctx, f.token, newBinding, start, ref) + if err != nil || !duplicate.Deduplicated || !duplicate.Lease.Active { + t.Fatalf("start ack recovery %v", err) + } + if f.get(ref.TaskID).AttemptCount != before.AttemptCount || f.used() != 1 { + t.Fatal("resume created attempt or duplicated permit") + } + if _, err := f.b.Resume(ctx, f.token, newBinding, ref); err != nil { + t.Fatal(err) + } + if f.count("orbit_events", "aggregate_id=? AND type='attempt.resumed'", ref.AttemptID) != 1 { + t.Fatal("resume event duplicated") + } + badHello := f.hello + badHello.InstanceSecret = strings.Repeat("A", 43) + _, _, err = f.b.Connect(ctx, f.token, badHello) + expectWorkerCode(t, err, "PERMISSION_DENIED") + badHello = f.hello + badHello.Capabilities = []worker.Capability{{Handler: "another", HandlerVersion: "v1", PayloadVersion: "v1"}} + _, _, err = f.b.Connect(ctx, f.token, badHello) + expectWorkerCode(t, err, "STATE_CONFLICT") + f.expire(ref) + expired, err := f.b.Resume(ctx, f.token, newBinding, ref) + if err != nil || expired.Active { + t.Fatalf("expired resume %v", err) + } + }) + t.Run("multi_node_claim_obeys_queue_and_process_capacity", func(t *testing.T) { + f := newExecutionFixture(t, ctx, db) + f.exec("UPDATE pulse_queues SET max_concurrency=2 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.QueueID) + for i := 0; i < 8; i++ { + f.submit(3) + } + secret, _, err := worker.NewSecret() + if err != nil { + t.Fatal(err) + } + hello := f.hello + hello.InstanceID = executionID(t) + hello.InstanceSecret = secret + hello.Capacity = 1 + bindingB, _, err := f.b.Connect(ctx, f.token, hello) + if err != nil { + t.Fatal(err) + } + if _, err := f.a.Capacity(ctx, f.token, f.binding, worker.CapacityInput{Sequence: 1, Capacity: 1}); err != nil { + t.Fatal(err) + } + assignments := make(chan *worker.Assignment, 16) + errs := make(chan error, 16) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := 0; i < 16; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + var a *worker.Assignment + var err error + if i%2 == 0 { + a, err = f.a.Claim(ctx, f.token, f.binding) + } else { + a, err = f.b.Claim(ctx, f.token, bindingB) + } + assignments <- a + errs <- err + }(i) + } + close(start) + wg.Wait() + close(assignments) + close(errs) + for err := range errs { + if err != nil { + t.Fatal(err) + } + } + ids := map[core.ID]bool{} + for a := range assignments { + if a == nil { + continue + } + if ids[a.Task.ID] { + t.Fatal("task dispatched twice") + } + ids[a.Task.ID] = true + } + if len(ids) != 2 || f.used() != 2 { + t.Fatalf("concurrency overrun: assignments=%d used=%d", len(ids), f.used()) + } + b2, _, err := f.b.Connect(ctx, f.token, hello) + if err != nil { + t.Fatal(err) + } + a, err := f.b.Claim(ctx, f.token, b2) + if err != nil || a != nil { + t.Fatalf("reconnect reset capacity %v", err) + } + }) + t.Run("drain_is_durable_and_capacity_sequence_is_not_additive", func(t *testing.T) { + f := newExecutionFixture(t, ctx, db) + f.submit(3) + _, ref := f.claim() + if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil { + t.Fatal(err) + } + input := worker.CapacityInput{Sequence: 1, Capacity: 0, Drain: true} + if _, err := f.a.Capacity(ctx, f.token, f.binding, input); err != nil { + t.Fatal(err) + } + if _, err := f.a.Capacity(ctx, f.token, f.binding, input); err != nil { + t.Fatal(err) + } + input.Capacity = 2 + _, err := f.a.Capacity(ctx, f.token, f.binding, input) + expectWorkerCode(t, err, "STATE_CONFLICT") + input.Sequence = 2 + input.Drain = false + _, err = f.a.Capacity(ctx, f.token, f.binding, input) + expectWorkerCode(t, err, "STATE_CONFLICT") + f.submit(3) + a, err := f.a.Claim(ctx, f.token, f.binding) + if err != nil || a != nil { + t.Fatalf("draining worker received work %v", err) + } + if f.used() != 1 { + t.Fatal("capacity zero released in-flight permit") + } + newBinding, welcome, err := f.b.Connect(ctx, f.token, f.hello) + if err != nil || !welcome.Draining { + t.Fatalf("drain lost on reconnect %v", err) + } + if _, err := f.b.Resume(ctx, f.token, newBinding, ref); err != nil { + t.Fatal(err) + } + if _, err := f.b.Finish(ctx, f.token, newBinding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}); err != nil { + t.Fatal(err) + } + if f.used() != 0 { + t.Fatal("drained execution did not release capacity") + } + }) + t.Run("paused_gate_blocks_start_but_not_saved_result", func(t *testing.T) { + f := newExecutionFixture(t, ctx, db) + gate := executionID(t) + f.exec("INSERT INTO pulse_execution_gates(namespace_id,id,paused) VALUES (?,?,TRUE)", f.resources.NamespaceID, gate) + // 超过一次候选页的暂停任务不能饿死后面的普通任务。 + for i := 0; i < 18; i++ { + task := f.submit(3) + f.exec("UPDATE pulse_tasks SET gate_id=? WHERE namespace_id=? AND id=?", gate, f.resources.NamespaceID, task.ID) + } + ready := f.submit(3) + a, ref := f.claim() + if a.Task.ID != ready.ID { + t.Fatal("paused candidate starvation") + } + f.exec("UPDATE pulse_tasks SET gate_id=? WHERE namespace_id=? AND id=?", gate, f.resources.NamespaceID, ready.ID) + start := executionID(t) + _, err := f.a.Start(ctx, f.token, f.binding, start, ref) + expectWorkerCode(t, err, "EXECUTION_PAUSED") + f.exec("UPDATE pulse_execution_gates SET paused=FALSE WHERE namespace_id=? AND id=?", f.resources.NamespaceID, gate) + if _, err := f.a.Start(ctx, f.token, f.binding, start, ref); err != nil { + t.Fatal(err) + } + f.exec("UPDATE pulse_execution_gates SET paused=TRUE WHERE namespace_id=? AND id=?", f.resources.NamespaceID, gate) + duplicate, err := f.a.Start(ctx, f.token, f.binding, start, ref) + if err != nil || !duplicate.Deduplicated || !duplicate.Lease.Active { + t.Fatalf("pause blocked original start receipt %v", err) + } + if _, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}); err != nil { + t.Fatal(err) + } + }) + t.Run("recovery_is_bounded_and_unknown_effects_are_not_replayed", func(t *testing.T) { + f := newExecutionFixture(t, ctx, db) + task := f.submit(2) + _, ref := f.claim() + f.expire(ref) + stats, err := f.storeB.Recover(ctx, 100) + if err != nil { + t.Fatal(err) + } + current := f.get(task.ID) + if current.State != pulse.TaskWaiting || current.AttemptCount != 1 || f.used() != 0 || stats.Requeued < 1 { + t.Fatal("pre-start lease recovery failed") + } + f.exec("UPDATE pulse_tasks SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID) + _, newRef := f.claim() + if newRef.Epoch != 2 || newRef.AttemptID == ref.AttemptID { + t.Fatal("attempt epoch not advanced") + } + _, err = f.a.Start(ctx, f.token, f.binding, executionID(t), ref) + expectWorkerCode(t, err, "LEASE_LOST") + start := executionID(t) + if _, err := f.a.Start(ctx, f.token, f.binding, start, newRef); err != nil { + t.Fatal(err) + } + f.expire(newRef) + var wg sync.WaitGroup + errs := make(chan error, 2) + for _, store := range []*ExecutionStore{f.storeA, f.storeB} { + wg.Add(1) + go func(store *ExecutionStore) { defer wg.Done(); _, err := store.Recover(ctx, 100); errs <- err }(store) + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatal(err) + } + } + current = f.get(task.ID) + if current.State != pulse.TaskNeedsReview || current.AttemptCount != 2 || current.CompletedAt != nil || f.used() != 0 { + t.Fatal("running expiry must become review without replay") + } + if f.count("orbit_events", "aggregate_id=? AND type='task.needs_review'", task.ID) != 1 { + t.Fatal("recovery event duplicated") + } + late := worker.FinishInput{Ref: newRef, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"external_ref":"late-result"}`)} + message := executionID(t) + evidence, err := f.a.Finish(ctx, f.token, f.binding, message, late) + if err != nil || evidence.Receipt.Disposition != "EVIDENCE_ONLY" || evidence.Lease.Active { + t.Fatalf("late evidence %v", err) + } + evidence, err = f.a.Finish(ctx, f.token, f.binding, message, late) + if err != nil || !evidence.Deduplicated { + t.Fatalf("late replay %v", err) + } + if f.get(task.ID).State != pulse.TaskNeedsReview || f.count("pulse_attempt_evidence", "attempt_id=?", newRef.AttemptID) != 1 { + t.Fatal("late result rewrote authority or duplicated evidence") + } + duplicate, err := f.a.Start(ctx, f.token, f.binding, start, newRef) + if err != nil || !duplicate.Deduplicated || duplicate.Lease.Active { + t.Fatalf("old acknowledgement restored lease %v", err) + } + }) + t.Run("attempt_timeout_and_session_expiry_cap_renewal", func(t *testing.T) { + f := newExecutionFixture(t, ctx, db) + task := f.submit(3) + _, ref := f.claim() + f.exec("UPDATE pulse_tasks SET attempt_timeout_ms=5000 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID) + begun, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref) + if err != nil { + t.Fatal(err) + } + renewed, err := f.a.Renew(ctx, f.token, f.binding, executionID(t), ref) + if err != nil || renewed.Lease.LeaseUntil.After(begun.Lease.LeaseUntil) { + t.Fatalf("renew bypassed absolute attempt deadline %v", err) + } + f.exec("UPDATE pulse_worker_sessions SET expires_at=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.binding.SessionID) + renewed, err = f.a.Renew(ctx, f.token, f.binding, executionID(t), ref) + if err != nil || renewed.Lease.LeaseUntil.Sub(renewed.Lease.ServerTime) > time.Second { + t.Fatalf("renew exceeded session expiration %v", err) + } + f.exec("UPDATE pulse_worker_sessions SET expires_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.binding.SessionID) + _, err = f.a.Capacity(ctx, f.token, f.binding, worker.CapacityInput{Sequence: 1, Capacity: 4}) + expectWorkerCode(t, err, "STALE_SESSION") + }) + t.Run("result_event_failure_rolls_back_task_receipt_and_permit", func(t *testing.T) { + f := newExecutionFixture(t, ctx, db) + task := f.submit(3) + _, ref := f.claim() + if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil { + t.Fatal(err) + } + f.exec("CREATE TRIGGER orbit_test_reject_worker_audit BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='worker audit failure'") + message := executionID(t) + in := worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)} + _, err := f.a.Finish(ctx, f.token, f.binding, message, in) + f.exec("DROP TRIGGER orbit_test_reject_worker_audit") + if err == nil { + t.Fatal("expected injected failure") + } + if f.get(task.ID).State != pulse.TaskRunning || f.used() != 1 || f.count("pulse_attempt_receipts", "attempt_id=? AND operation='finish'", ref.AttemptID) != 0 || f.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 0 { + t.Fatal("partial result transaction committed") + } + if _, err := f.a.Finish(ctx, f.token, f.binding, message, in); err != nil { + t.Fatal(err) + } + }) + t.Run("finish_without_start_is_only_evidence_and_cannot_free_capacity", func(t *testing.T) { + f := newExecutionFixture(t, ctx, db) + task := f.submit(1) + _, ref := f.claim() + result, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}) + if err != nil || result.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(task.ID).State != pulse.TaskLeased || f.used() != 1 { + t.Fatalf("unstarted result accepted %v", err) + } + f.expire(ref) + if _, err := f.storeA.Recover(ctx, 100); err != nil { + t.Fatal(err) + } + if f.get(task.ID).State != pulse.TaskFailed || f.used() != 0 { + t.Fatal("exhausted pre-start attempt did not terminate") + } + }) + t.Run("known_failure_unknown_and_waiting_deadlines", func(t *testing.T) { + f := newExecutionFixture(t, ctx, db) + for _, outcome := range []string{"FAILED", "UNKNOWN"} { + task := f.submit(3) + _, ref := f.claim() + if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil { + t.Fatal(err) + } + out, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: outcome, ErrorCode: "EXTERNAL_RESULT", Result: json.RawMessage(`{"evidence_ref":"object/result"}`)}) + if err != nil { + t.Fatal(err) + } + want := pulse.TaskFailed + if outcome == "UNKNOWN" { + want = pulse.TaskNeedsReview + } + if out.Receipt.TaskState != want || f.get(task.ID).State != want { + t.Fatal("outcome classification lost") + } + } + task := f.submit(3) + f.exec("UPDATE pulse_tasks SET available_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 2 HOUR),deadline_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 HOUR) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID) + if _, err := f.storeA.Recover(ctx, 100); err != nil { + t.Fatal(err) + } + if f.get(task.ID).State != pulse.TaskFailed { + t.Fatal("waiting deadline not enforced") + } + }) + t.Run("authority_is_checked_on_every_execution_command", func(t *testing.T) { + f := newExecutionFixture(t, ctx, db) + f.submit(3) + _, ref := f.claim() + f.exec("UPDATE orbit_credentials SET revoked_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.CredentialID) + _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref) + expectWorkerCode(t, err, "UNAUTHENTICATED") + f.exec("UPDATE orbit_credentials SET revoked_at=NULL WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.CredentialID) + other := newExecutionFixture(t, ctx, db) + _, err = other.a.Resume(ctx, other.token, other.binding, ref) + expectWorkerCode(t, err, "NOT_FOUND") + copied := f.binding + copied.Proof = core.Digest{} + _, err = f.a.Claim(ctx, f.token, copied) + expectWorkerCode(t, err, "STALE_SESSION") + cancelled, stop := context.WithCancel(ctx) + stop() + _, err = f.a.Start(cancelled, f.token, f.binding, executionID(t), ref) + if !errors.Is(err, context.Canceled) { + t.Fatal(fmt.Sprint(err)) + } + }) + t.Run("claim_event_failure_does_not_leave_an_invisible_lease", func(t *testing.T) { + f := newExecutionFixture(t, ctx, db) + task := f.submit(3) + f.exec("CREATE TRIGGER orbit_test_reject_claim_event BEFORE INSERT ON orbit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='claim event failure'") + assignment, err := f.a.Claim(ctx, f.token, f.binding) + f.exec("DROP TRIGGER orbit_test_reject_claim_event") + if err == nil || assignment != nil { + t.Fatal("expected failed claim without usable assignment") + } + if f.get(task.ID).State != pulse.TaskWaiting || f.used() != 0 || f.count("pulse_attempts", "task_id=?", task.ID) != 0 || f.count("pulse_permits", "TRUE") != 0 { + t.Fatal("claim partially committed") + } + if _, err := f.a.Claim(ctx, f.token, f.binding); err != nil { + t.Fatal(err) + } + }) + t.Run("lock_wait_does_not_reuse_an_earlier_clock_sample", func(t *testing.T) { + f := newExecutionFixture(t, ctx, db) + f.submit(3) + _, ref := f.claim() + holder, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer holder.Rollback() + if _, err := holder.ExecContext(ctx, "UPDATE pulse_queues SET revision=revision+1 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.QueueID); err != nil { + t.Fatal(err) + } + for _, table := range []string{"pulse_tasks", "pulse_attempts"} { + id := ref.TaskID + if table == "pulse_attempts" { + id = ref.AttemptID + } + if _, err := holder.ExecContext(ctx, "UPDATE "+table+" SET lease_until=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 150000 MICROSECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, id); err != nil { + t.Fatal(err) + } + } + result := make(chan error, 1) + go func() { _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); result <- err }() + time.Sleep(250 * time.Millisecond) + if err := holder.Commit(); err != nil { + t.Fatal(err) + } + expectWorkerCode(t, <-result, "LEASE_LOST") + if f.get(ref.TaskID).State != pulse.TaskLeased { + t.Fatal("stale clock authorized start") + } + }) +} diff --git a/internal/storage/mysql/worker_recovery.go b/internal/storage/mysql/worker_recovery.go new file mode 100644 index 0000000..42d4e6a --- /dev/null +++ b/internal/storage/mysql/worker_recovery.go @@ -0,0 +1,166 @@ +package mysql + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "git.yuntuo.hk/k2safe/orbit/internal/fault" + core "git.yuntuo.hk/k2safe/orbit/model" + pulse "git.yuntuo.hk/k2safe/orbit/pulse/model" + "git.yuntuo.hk/k2safe/orbit/pulse/worker" +) + +// Recover 是受控后台扫描入口，每批最多 100 项、每项一个短事务，可由多个节点竞争。 +// 当前已授权执行的失联一律转核查，自动安全重试在 P3 接入冻结策略后实现。 +func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.RecoveryStats, error) { + var out worker.RecoveryStats + if limit < 1 || limit > 100 { + return out, fault.Invalid("recovery batch limit must be 1 to 100") + } + type candidate struct{ ns, id, queue core.ID } + candidates := []candidate{} + seen := map[string]bool{} + // 分别使用租约、总截止及开始截止索引，避免对整个 WAITING 队列做 COALESCE 扫描。 + for _, q := range []string{ + `SELECT namespace_id,id,queue_id FROM pulse_tasks WHERE state IN ('LEASED','RUNNING') AND lease_until<=UTC_TIMESTAMP(6) ORDER BY lease_until,namespace_id,id LIMIT ?`, + `SELECT namespace_id,id,queue_id FROM pulse_tasks WHERE state='WAITING' AND deadline_at<=UTC_TIMESTAMP(6) ORDER BY deadline_at,namespace_id,id LIMIT ?`, + `SELECT namespace_id,id,queue_id FROM pulse_tasks WHERE state='WAITING' AND start_deadline_at<=UTC_TIMESTAMP(6) ORDER BY start_deadline_at,namespace_id,id LIMIT ?`, + } { + if len(candidates) >= limit { + break + } + rows, err := s.db.QueryContext(ctx, q, limit-len(candidates)) + if err != nil { + return out, err + } + for rows.Next() { + var c candidate + if err := rows.Scan(&c.ns, &c.id, &c.queue); err != nil { + rows.Close() + return out, err + } + key := string(c.ns) + string(c.id) + if !seen[key] { + candidates = append(candidates, c) + seen[key] = true + } + } + err = rows.Err() + rows.Close() + if err != nil { + return out, err + } + } + for _, c := range candidates { + out.Examined++ + var changed pulse.TaskState + err := WithTx(ctx, s.db, func(tx *sql.Tx) error { + // 与 Worker 命令一致先锁队列；有其他节点正在处理时跳过，本批不占用等待线程。 + var queue core.ID + err := tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, c.ns, c.queue).Scan(&queue) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return err + } + var taskID core.ID + err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_tasks WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, c.ns, c.id).Scan(&taskID) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return err + } + task, err := readExecutionTask(ctx, tx, c.ns, c.id) + if err != nil { + return err + } + if task.view.State == pulse.TaskWaiting { + now, err := dbNow(ctx, tx) + if err != nil { + return err + } + if task.view.DeadlineAt.After(now) && (task.view.StartDeadlineAt == nil || task.view.StartDeadlineAt.After(now)) { + return nil + } + changed = pulse.TaskFailed + result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='FAILED',completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, now, now, c.ns, c.id, task.view.Revision) + if err := exactlyOne(result, err); err != nil { + return err + } + return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", "task.failed", map[string]any{"reason": "START_DEADLINE_EXCEEDED", "node_id": s.node}) + } + if task.view.State != pulse.TaskLeased && task.view.State != pulse.TaskRunning { + return nil + } + if task.current == nil || task.lease == nil { + return errors.New("active task lacks attempt or lease") + } + attempt, err := readAttempt(ctx, tx, c.ns, *task.current) + if err != nil { + return err + } + now, err := dbNow(ctx, tx) + if err != nil { + return err + } + if task.lease.After(now) || attempt.LeaseUntil.After(now) { + return nil + } + if attempt.Epoch != task.epoch || attempt.TaskID != c.id { + return errors.New("task attempt identity mismatch") + } + if attempt.State != pulse.AttemptLeased && attempt.State != pulse.AttemptRunning { + return errors.New("active task references ended attempt") + } + attemptState := pulse.AttemptExpired + next := pulse.TaskFailed + reason := "LEASE_EXPIRED_BEFORE_START" + var completed *time.Time = &now + available := task.view.AvailableAt + if attempt.AuthorizedAt != nil { + attemptState = pulse.AttemptUnknown + next = pulse.TaskNeedsReview + reason = "EXECUTION_OUTCOME_UNKNOWN" + completed = nil + } else if task.view.AttemptCount < task.maxAttempts { + // 未批准开始的尝试可以有界重派，次数不重置；开始截止与总截止仍有效。 + nextAt := now.Add(time.Second) + if nextAt.Before(task.view.DeadlineAt) && (task.view.StartDeadlineAt == nil || nextAt.Before(*task.view.StartDeadlineAt)) { + next = pulse.TaskWaiting + available = nextAt + completed = nil + } + } + result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, reason, now, c.ns, attempt.ID, attempt.Revision) + if err := exactlyOne(result, err); err != nil { + return err + } + result, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,lease_until=NULL,available_at=?,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, next, available, completed, now, c.ns, c.id, task.view.Revision) + if err := exactlyOne(result, err); err != nil { + return err + } + if err := releasePermits(ctx, tx, c.ns, attempt.ID, now); err != nil { + return err + } + changed = next + return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", resultEvent(next), map[string]any{"attempt_id": attempt.ID, "epoch": fmt.Sprint(attempt.Epoch), "reason": reason, "node_id": s.node}) + }) + if err != nil { + return out, err + } + switch changed { + case pulse.TaskWaiting: + out.Requeued++ + case pulse.TaskFailed: + out.Failed++ + case pulse.TaskNeedsReview: + out.NeedsReview++ + } + } + return out, nil +} diff --git a/internal/storage/mysql/worker_results.go b/internal/storage/mysql/worker_results.go new file mode 100644 index 0000000..e11c6c4 --- /dev/null +++ b/internal/storage/mysql/worker_results.go @@ -0,0 +1,132 @@ +package mysql + +import ( + "context" + "database/sql" + "fmt" + "time" + + core "git.yuntuo.hk/k2safe/orbit/model" + "git.yuntuo.hk/k2safe/orbit/platform/auth" + pulse "git.yuntuo.hk/k2safe/orbit/pulse/model" + "git.yuntuo.hk/k2safe/orbit/pulse/worker" +) + +func (s *ExecutionStore) Finish(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, in worker.FinishInput) (worker.CommandResult, error) { + var out worker.CommandResult + digest, err := requestDigest(struct { + Identity any + Outcome string + Result core.JSON + ErrorCode string + }{attemptIdentity(in.Ref), in.Outcome, core.JSON(in.Result), in.ErrorCode}) + if err != nil { + return out, err + } + err = s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error { + rows, err := s.lockExecution(ctx, tx, token, b, p, in.Ref, false) + if err != nil { + return err + } + cached, err := readReceipt(ctx, tx, token.NamespaceID, in.Ref.AttemptID, message, "finish", digest) + if err != nil { + return err + } + now, err := dbNow(ctx, tx) + if err != nil { + return err + } + status := leaseStatus(rows, b, now) + if cached != nil { + out = worker.CommandResult{Receipt: *cached, Lease: status, Deduplicated: true} + return nil + } + task, a := rows.task, rows.attempt + receipt := worker.Receipt{Operation: "finish", TaskState: task.view.State, AttemptState: a.State, TaskRevision: task.view.Revision, LeaseUntil: a.LeaseUntil, Disposition: "APPLIED"} + if a.ResultDigest != nil && *a.ResultDigest == digest { + // 新消息 ID 重报已保存的同一结果，也不能重复事件、许可释放或状态推进。 + if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil { + return err + } + out = worker.CommandResult{Receipt: receipt, Lease: status, Deduplicated: true} + return nil + } + reason := "" + if a.ResultDigest != nil { + reason = "RESULT_CONFLICT" + } else if a.AuthorizedAt == nil { + reason = "NOT_STARTED" + } else if !status.Active || a.State != pulse.AttemptRunning { + reason = "LEASE_LOST" + } + if reason != "" { + if err := saveEvidence(ctx, tx, token.NamespaceID, message, in, digest, reason); err != nil { + return err + } + receipt.Disposition = "EVIDENCE_ONLY" + if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil { + return err + } + out = worker.CommandResult{Receipt: receipt, Lease: status} + return nil + } + attemptState, taskState := pulse.AttemptState(in.Outcome), pulse.TaskState(in.Outcome) + var completed *time.Time = &now + if in.Outcome == "UNKNOWN" { + taskState = pulse.TaskNeedsReview + completed = nil + } + changed, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=NULLIF(?,''),result=?,result_digest=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, in.ErrorCode, string(in.Result), digest, now, token.NamespaceID, a.ID, a.Revision) + if err := exactlyOne(changed, err); err != nil { + return err + } + changed, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,result=?,lease_until=NULL,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, taskState, string(in.Result), completed, now, token.NamespaceID, in.Ref.TaskID, task.view.Revision) + if err := exactlyOne(changed, err); err != nil { + return err + } + if err := releasePermits(ctx, tx, token.NamespaceID, a.ID, now); err != nil { + return err + } + event := map[string]any{"attempt_id": a.ID, "epoch": fmt.Sprint(a.Epoch), "state": taskState, "outcome": in.Outcome, "error_code": in.ErrorCode} + if err := emitEvent(ctx, tx, token.NamespaceID, in.Ref.TaskID, task.view.Revision+1, "task", resultEvent(taskState), event); err != nil { + return err + } + if err := appendAudit(ctx, tx, token, message, in.Ref.TaskID, "task", "task.result", "worker result", map[string]any{"state": task.view.State}, event); err != nil { + return err + } + receipt.TaskState = taskState + receipt.AttemptState = attemptState + receipt.TaskRevision++ + if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil { + return err + } + out = worker.CommandResult{Receipt: receipt, Lease: worker.LeaseStatus{State: attemptState, LeaseUntil: a.LeaseUntil, ServerTime: now}} + return nil + }) + return out, err +} +func resultEvent(state pulse.TaskState) string { + switch state { + case pulse.TaskSucceeded: + return "task.succeeded" + case pulse.TaskFailed: + return "task.failed" + case pulse.TaskNeedsReview: + return "task.needs_review" + case pulse.TaskWaiting: + return "task.requeued" + default: + return "task.cancelled" + } +} +func saveEvidence(ctx context.Context, tx *sql.Tx, ns, message core.ID, in worker.FinishInput, digest core.Digest, reason string) error { + id, err := core.NewID() + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempt_evidence(namespace_id,id,attempt_id,message_id,outcome,error_code,payload,request_digest,reason) VALUES (?,?,?,?,?,NULLIF(?,''),?,?,?)`, ns, id, in.Ref.AttemptID, message, in.Outcome, in.ErrorCode, string(in.Result), digest, reason) + if err != nil { + return err + } + return emitEvent(ctx, tx, ns, id, 1, "attempt_evidence", "attempt.evidence_received", map[string]any{"attempt_id": in.Ref.AttemptID, "task_id": in.Ref.TaskID, "reason": reason, "outcome": in.Outcome}) +} diff --git a/internal/storage/mysql/worker_sessions.go b/internal/storage/mysql/worker_sessions.go new file mode 100644 index 0000000..beba416 --- /dev/null +++ b/internal/storage/mysql/worker_sessions.go @@ -0,0 +1,211 @@ +package mysql + +import ( + "context" + "crypto/subtle" + "database/sql" + "encoding/json" + "errors" + "time" + + "git.yuntuo.hk/k2safe/orbit/internal/fault" + core "git.yuntuo.hk/k2safe/orbit/model" + "git.yuntuo.hk/k2safe/orbit/platform/auth" + "git.yuntuo.hk/k2safe/orbit/pulse/worker" +) + +// ExecutionOptions 是服务端执行窗口，不由 Worker 任意指定。 +type ExecutionOptions struct { + LeaseDuration time.Duration // LeaseDuration 每次领取/续租的最大窗口。 + SessionTTL time.Duration // SessionTTL 会话心跳有效期，不代替执行租约。 +} + +// ExecutionStore 所有执行权以 MySQL 为准，节点内存不持有可绕过数据库的授权。 +type ExecutionStore struct { + db *sql.DB + node core.ID + options ExecutionOptions +} + +var _ worker.Store = (*ExecutionStore)(nil) + +func (s *ExecutionStore) Authorize(ctx context.Context, token auth.Token) error { + return s.transaction(ctx, token, "workers:connect", func(*sql.Tx, auth.Permissions) error { return nil }) +} + +func NewExecutionStore(db *sql.DB, node core.ID, options ExecutionOptions) (*ExecutionStore, error) { + if options.LeaseDuration == 0 { + options.LeaseDuration = 30 * time.Second + } + if options.SessionTTL == 0 { + options.SessionTTL = 45 * time.Second + } + if node.Validate() != nil || options.LeaseDuration < time.Second || options.LeaseDuration > 2*time.Minute || options.SessionTTL < options.LeaseDuration || options.SessionTTL > 5*time.Minute { + return nil, fault.Invalid("invalid execution windows or node identity") + } + return &ExecutionStore{db: db, node: node, options: options}, nil +} +func (s *ExecutionStore) transaction(ctx context.Context, token auth.Token, action string, fn func(*sql.Tx, auth.Permissions) error) error { + return (&TaskStore{db: s.db}).transaction(ctx, token, action, fn) +} + +type sessionState struct { + queue core.ID + capacity uint32 + sequence uint64 + draining bool + expires time.Time + capabilities []worker.Capability +} + +func (s *ExecutionStore) welcome(b worker.Binding, state sessionState, now time.Time) worker.Welcome { + return worker.Welcome{SessionID: b.SessionID, Epoch: b.Epoch, ServerTime: now, ExpiresAt: state.expires, LeaseDurationMS: uint64(s.options.LeaseDuration / time.Millisecond), Draining: state.draining} +} +func (s *ExecutionStore) Connect(ctx context.Context, token auth.Token, in worker.ConnectInput) (worker.Binding, worker.Welcome, error) { + var binding worker.Binding + var out worker.Welcome + proof, err := worker.SecretDigest(in.InstanceSecret) + if err != nil { + return binding, out, err + } + registration, err := requestDigest(struct { + Queue core.ID + Capabilities []worker.Capability + }{in.QueueID, in.Capabilities}) + if err != nil { + return binding, out, err + } + err = s.transaction(ctx, token, "workers:connect", func(tx *sql.Tx, p auth.Permissions) error { + if !p.AllowsQueue(in.QueueID) { + return fault.Forbidden + } + _, err := tx.ExecContext(ctx, `INSERT INTO pulse_worker_instances(namespace_id,id,credential_id,queue_id,secret_digest,registration_digest) VALUES (?,?,?,?,?,?) ON DUPLICATE KEY UPDATE id=pulse_worker_instances.id`, token.NamespaceID, in.InstanceID, token.CredentialID, in.QueueID, proof, registration) + if err != nil { + return err + } + var credential core.ID + var storedProof, storedRegistration core.Digest + var epoch uint64 + var old *core.ID + var draining bool + err = tx.QueryRowContext(ctx, `SELECT credential_id,secret_digest,registration_digest,session_epoch,current_session_id,draining FROM pulse_worker_instances WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, in.InstanceID).Scan(&credential, &storedProof, &storedRegistration, &epoch, &old, &draining) + if err != nil { + return err + } + if credential != token.CredentialID || subtle.ConstantTimeCompare(proof[:], storedProof[:]) != 1 { + return fault.Forbidden + } + if registration != storedRegistration || epoch == ^uint64(0) { + return fault.Conflict + } + now, err := dbNow(ctx, tx) + if err != nil { + return err + } + session, err := core.NewID() + if err != nil { + return err + } + binding = worker.Binding{InstanceID: in.InstanceID, SessionID: session, Epoch: epoch + 1, Proof: proof} + state := sessionState{queue: in.QueueID, capacity: in.Capacity, draining: draining, expires: now.Add(s.options.SessionTTL), capabilities: in.Capabilities} + caps, err := json.Marshal(in.Capabilities) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, `INSERT INTO pulse_worker_sessions(namespace_id,id,worker_instance_id,credential_id,node_id,session_epoch,capabilities,capacity,draining,expires_at) VALUES (?,?,?,?,?,?,?,?,?,?)`, token.NamespaceID, session, in.InstanceID, token.CredentialID, s.node, binding.Epoch, string(caps), in.Capacity, draining, state.expires) + if err != nil { + return err + } + if old != nil { + if _, err := tx.ExecContext(ctx, `UPDATE pulse_worker_sessions SET expires_at=?,draining=TRUE,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, now, now, token.NamespaceID, *old); err != nil { + return err + } + } + _, err = tx.ExecContext(ctx, `UPDATE pulse_worker_instances SET current_session_id=?,session_epoch=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, session, binding.Epoch, now, token.NamespaceID, in.InstanceID) + if err != nil { + return err + } + if err := emitEvent(ctx, tx, token.NamespaceID, session, 1, "worker_session", "worker.connected", map[string]any{"instance_id": in.InstanceID, "node_id": s.node, "epoch": binding.Epoch}); err != nil { + return err + } + out = s.welcome(binding, state, now) + return nil + }) + if err != nil { + return worker.Binding{}, worker.Welcome{}, err + } + return binding, out, nil +} + +// lockSession 的顺序为凭据 -> 进程头 -> 当前会话；新连接和旧连接命令在同一行上串行化。 +func (s *ExecutionStore) lockSession(ctx context.Context, tx *sql.Tx, token auth.Token, b worker.Binding, p auth.Permissions) (sessionState, error) { + var state sessionState + var credential core.ID + var proof core.Digest + var current *core.ID + var epoch uint64 + if b.InstanceID.Validate() != nil || b.SessionID.Validate() != nil || b.Epoch == 0 { + return state, worker.StaleSession + } + err := tx.QueryRowContext(ctx, `SELECT credential_id,queue_id,secret_digest,current_session_id,session_epoch,draining FROM pulse_worker_instances WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, b.InstanceID).Scan(&credential, &state.queue, &proof, ¤t, &epoch, &state.draining) + if errors.Is(err, sql.ErrNoRows) { + return state, worker.StaleSession + } + if err != nil { + return state, err + } + if credential != token.CredentialID || current == nil || *current != b.SessionID || epoch != b.Epoch || subtle.ConstantTimeCompare(proof[:], b.Proof[:]) != 1 { + return state, worker.StaleSession + } + if !p.AllowsQueue(state.queue) { + return state, fault.Forbidden + } + var node core.ID + var raw []byte + err = tx.QueryRowContext(ctx, `SELECT node_id,capacity,capacity_sequence,capabilities,expires_at FROM pulse_worker_sessions WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, b.SessionID).Scan(&node, &state.capacity, &state.sequence, &raw, &state.expires) + if err != nil { + return state, err + } + now, err := dbNow(ctx, tx) + if err != nil { + return state, err + } + if node != s.node || !state.expires.After(now) { + return state, worker.StaleSession + } + if err := json.Unmarshal(raw, &state.capabilities); err != nil { + return state, err + } + return state, nil +} +func (s *ExecutionStore) Capacity(ctx context.Context, token auth.Token, b worker.Binding, in worker.CapacityInput) (worker.Welcome, error) { + var out worker.Welcome + err := s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error { + state, err := s.lockSession(ctx, tx, token, b, p) + if err != nil { + return err + } + if in.Sequence < state.sequence || (in.Sequence == state.sequence && (in.Capacity != state.capacity || in.Drain != state.draining)) || (state.draining && !in.Drain) { + return fault.Conflict + } + now, err := dbNow(ctx, tx) + if err != nil { + return err + } + state.expires = now.Add(s.options.SessionTTL) + state.capacity = in.Capacity + state.draining = state.draining || in.Drain + _, err = tx.ExecContext(ctx, `UPDATE pulse_worker_sessions SET capacity=?,capacity_sequence=?,draining=?,expires_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, state.capacity, in.Sequence, state.draining, state.expires, now, token.NamespaceID, b.SessionID) + if err != nil { + return err + } + if in.Drain { + if _, err := tx.ExecContext(ctx, `UPDATE pulse_worker_instances SET draining=TRUE,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND draining=FALSE`, now, token.NamespaceID, b.InstanceID); err != nil { + return err + } + } + out = s.welcome(b, state, now) + return nil + }) + return out, err +} diff --git a/internal/storage/mysql/worker_websocket_integration_test.go b/internal/storage/mysql/worker_websocket_integration_test.go new file mode 100644 index 0000000..5a7d86e --- /dev/null +++ b/internal/storage/mysql/worker_websocket_integration_test.go @@ -0,0 +1,244 @@ +package mysql + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "git.yuntuo.hk/k2safe/orbit/internal/transport/httpserver" + core "git.yuntuo.hk/k2safe/orbit/model" + pulse "git.yuntuo.hk/k2safe/orbit/pulse/model" + "git.yuntuo.hk/k2safe/orbit/pulse/worker" + "github.com/coder/websocket" +) + +type testWorkerFrame struct { + Version string `json:"protocol_version"` + Type string `json:"type"` + MessageID core.ID `json:"message_id"` + ReplyTo core.ID `json:"reply_to,omitempty"` + SessionID core.ID `json:"session_id,omitempty"` + Body json.RawMessage `json:"body,omitempty"` + Error *struct { + Code string `json:"code"` + } `json:"error,omitempty"` +} +type testWorkerSocket struct { + t *testing.T + ctx context.Context + conn *websocket.Conn + session core.ID + welcome worker.Welcome +} + +func connectTestWorker(t *testing.T, ctx context.Context, server *httptest.Server, f *executionFixture) *testWorkerSocket { + t.Helper() + header := http.Header{} + header.Set("Authorization", "Bearer "+f.resources.Token) + conn, response, err := websocket.Dial(ctx, strings.Replace(server.URL, "https://", "wss://", 1)+"/v1/worker/connect", &websocket.DialOptions{HTTPClient: server.Client(), HTTPHeader: header, Subprotocols: []string{httpserver.WorkerSubprotocol}}) + if err != nil { + status := 0 + if response != nil { + status = response.StatusCode + } + t.Fatalf("Worker dial failed HTTP %d: %v", status, err) + } + t.Cleanup(func() { conn.CloseNow() }) + client := &testWorkerSocket{t: t, ctx: ctx, conn: conn} + conn.SetReadLimit(512 << 10) + hello := client.send("worker.hello", f.hello, "") + reply := client.read() + if reply.Type != "worker.welcome" || reply.ReplyTo != hello || reply.Error != nil { + t.Fatal("missing welcome") + } + if err := json.Unmarshal(reply.Body, &client.welcome); err != nil { + t.Fatal(err) + } + client.session = client.welcome.SessionID + return client +} +func (c *testWorkerSocket) send(kind string, body any, id core.ID) core.ID { + c.t.Helper() + if id == "" { + id = executionID(c.t) + } + raw, err := json.Marshal(body) + if err != nil { + c.t.Fatal(err) + } + frame, err := json.Marshal(testWorkerFrame{Version: "1", Type: kind, MessageID: id, SessionID: c.session, Body: raw}) + if err != nil { + c.t.Fatal(err) + } + ctx, cancel := context.WithTimeout(c.ctx, 5*time.Second) + defer cancel() + if err := c.conn.Write(ctx, websocket.MessageText, frame); err != nil { + c.t.Fatal(err) + } + return id +} +func (c *testWorkerSocket) read() testWorkerFrame { + c.t.Helper() + ctx, cancel := context.WithTimeout(c.ctx, 5*time.Second) + defer cancel() + kind, raw, err := c.conn.Read(ctx) + if err != nil { + c.t.Fatal(err) + } + var frame testWorkerFrame + if kind != websocket.MessageText || json.Unmarshal(raw, &frame) != nil || frame.Version != "1" || frame.MessageID.Validate() != nil { + c.t.Fatal("invalid server frame") + } + return frame +} +func (c *testWorkerSocket) ack(id core.ID) json.RawMessage { + c.t.Helper() + frame := c.read() + if frame.Type != "ack" || frame.ReplyTo != id || frame.SessionID != c.session || frame.Error != nil { + code := "" + if frame.Error != nil { + code = frame.Error.Code + } + c.t.Fatalf("expected ack, got %s/%s", frame.Type, code) + } + return frame.Body +} +func TestIntegrationWorkerWebSocket(t *testing.T) { + dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN") + if dsn == "" { + t.Skip("requires disposable MySQL") + } + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + db, err := Open(ctx, dsn, 16, 4) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := Migrate(ctx, db); err != nil { + t.Fatal(err) + } + fixture := newExecutionFixture(t, ctx, db) + start := func(svc *worker.Service) (*httptest.Server, *httpserver.WorkerHub) { + t.Helper() + hub := httpserver.NewWorkerHub(ctx, svc) + server := httpserver.New("", func(context.Context) error { return nil }) + httpserver.RegisterWorkers(server, hub) + tls := httptest.NewTLSServer(server.Handler) + t.Cleanup(func() { + bounded, stop := context.WithTimeout(context.Background(), 3*time.Second) + defer stop() + if err := hub.Shutdown(bounded); err != nil { + t.Error(err) + } + tls.Close() + }) + return tls, hub + } + serverA, hubA := start(fixture.a) + serverB, _ := start(fixture.b) + t.Run("handshake_auth_protocol_origin_and_URL", func(t *testing.T) { + cases := []struct { + token, protocol, path, origin string + status int + }{ + {"", httpserver.WorkerSubprotocol, "/v1/worker/connect", "", 401}, + {fixture.resources.Token, "unknown.v1", "/v1/worker/connect", "", 426}, + {fixture.resources.Token, httpserver.WorkerSubprotocol, "/v1/worker/connect?token=forbidden", "", 400}, + {fixture.resources.Token, httpserver.WorkerSubprotocol, "/v1/worker/connect", "https://foreign.example", 403}, + } + for _, test := range cases { + headers := http.Header{} + if test.token != "" { + headers.Set("Authorization", "Bearer "+test.token) + } + if test.origin != "" { + headers.Set("Origin", test.origin) + } + conn, response, err := websocket.Dial(ctx, strings.Replace(serverA.URL, "https://", "wss://", 1)+test.path, &websocket.DialOptions{HTTPClient: serverA.Client(), HTTPHeader: headers, Subprotocols: []string{test.protocol}}) + if conn != nil { + conn.CloseNow() + } + if err == nil || response == nil || response.StatusCode != test.status { + t.Fatalf("handshake expected %d", test.status) + } + } + }) + t.Run("WSS_assignment_start_and_cross_node_ack_recovery", func(t *testing.T) { + task := fixture.submit(3) + first := connectTestWorker(t, ctx, serverA, fixture) + // 只有显式容量确认后才启用新派发，留出恢复核对阶段。 + first.ack(first.send("worker.capacity", worker.CapacityInput{Sequence: 1, Capacity: 1}, "")) + frame := first.read() + if frame.Type != "task.assign" { + t.Fatalf("wanted assignment got %s", frame.Type) + } + var assignment worker.Assignment + if err := json.Unmarshal(frame.Body, &assignment); err != nil { + t.Fatal(err) + } + if assignment.Task.ID != task.ID || assignment.LeaseSecret == "" { + t.Fatal("wrong assignment") + } + ref := worker.AttemptRef{TaskID: task.ID, AttemptID: assignment.AttemptID, Epoch: assignment.Epoch, LeaseSecret: assignment.LeaseSecret} + startMessage := first.send("task.start", ref, "") + var begun worker.CommandResult + if err := json.Unmarshal(first.ack(startMessage), &begun); err != nil || !begun.Lease.Active || begun.Lease.State != pulse.AttemptRunning { + t.Fatal("missing durable start") + } + // 模拟旧节点连接断开；新的 WSS 连接进入不同节点并原子迁移有效尝试。 + _ = first.conn.CloseNow() + second := connectTestWorker(t, ctx, serverB, fixture) + var resumed worker.LeaseStatus + if err := json.Unmarshal(second.ack(second.send("worker.resume", ref, "")), &resumed); err != nil || !resumed.Active { + t.Fatal("WSS resume failed") + } + var original worker.CommandResult + if err := json.Unmarshal(second.ack(second.send("task.start", ref, startMessage)), &original); err != nil || !original.Deduplicated || !original.Lease.Active { + t.Fatal("original start receipt not recovered") + } + finishMessage := second.send("task.finish", worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"output_ref":"objects/result","n":9007199254740993}`)}, "") + var finish worker.CommandResult + if err := json.Unmarshal(second.ack(finishMessage), &finish); err != nil || finish.Receipt.Disposition != "APPLIED" || finish.Lease.Active { + t.Fatal("WSS finish failed") + } + // 丢弃该连接的操作确认上下文，再用稳定身份取回同一结果。 + _ = second.conn.CloseNow() + third := connectTestWorker(t, ctx, serverA, fixture) + var duplicate worker.CommandResult + if err := json.Unmarshal(third.ack(third.send("task.finish", worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"output_ref":"objects/result","n":9007199254740993}`)}, finishMessage)), &duplicate); err != nil || !duplicate.Deduplicated { + t.Fatal("finish receipt not recovered") + } + if fixture.get(task.ID).State != pulse.TaskSucceeded || fixture.used() != 0 || fixture.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 1 { + t.Fatal("WSS result effects duplicated") + } + }) + t.Run("strict_worker_frame_and_hub_shutdown", func(t *testing.T) { + conn := connectTestWorker(t, ctx, serverB, fixture) + raw := []byte(`{"protocol_version":"1","protocol_version":"1","type":"worker.capacity"}`) + if err := conn.conn.Write(ctx, websocket.MessageText, raw); err != nil { + t.Fatal(err) + } + readCtx, stop := context.WithTimeout(ctx, 3*time.Second) + defer stop() + _, _, err := conn.conn.Read(readCtx) + if websocket.CloseStatus(err) != websocket.StatusPolicyViolation { + t.Fatalf("duplicate frame not rejected: %v", err) + } + active := connectTestWorker(t, ctx, serverA, fixture) + shutdown, stopShutdown := context.WithTimeout(ctx, 3*time.Second) + defer stopShutdown() + if err := hubA.Shutdown(shutdown); err != nil { + t.Fatal(err) + } + _, _, err = active.conn.Read(readCtx) + if err == nil { + t.Fatal("shutdown left a hijacked connection alive") + } + }) +} diff --git a/internal/transport/httpserver/worker.go b/internal/transport/httpserver/worker.go new file mode 100644 index 0000000..c5178f3 --- /dev/null +++ b/internal/transport/httpserver/worker.go @@ -0,0 +1,401 @@ +package httpserver + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "reflect" + "strings" + "sync" + "time" + + "git.yuntuo.hk/k2safe/orbit/internal/fault" + "git.yuntuo.hk/k2safe/orbit/internal/jsonvalue" + core "git.yuntuo.hk/k2safe/orbit/model" + "git.yuntuo.hk/k2safe/orbit/platform/auth" + "git.yuntuo.hk/k2safe/orbit/pulse/worker" + "github.com/coder/websocket" +) + +const WorkerSubprotocol = "pulse.worker.v1" + +// WorkerHub 管理节点内连接生命周期；任务归属和容量仍由 MySQL 决定。 +type WorkerHub struct { + service *worker.Service + ctx context.Context + cancel context.CancelFunc + slots chan struct{} + mu sync.Mutex + closing bool + wg sync.WaitGroup + poll time.Duration +} + +func NewWorkerHub(parent context.Context, service *worker.Service) *WorkerHub { + ctx, cancel := context.WithCancel(parent) + return &WorkerHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, 128), poll: time.Second} +} +func RegisterWorkers(server *http.Server, hub *WorkerHub) { + server.Handler.(*http.ServeMux).Handle("GET /v1/worker/connect", hub) +} + +// Shutdown 停止派发并关闭本节点连接；不清除持久化租约，由重連核对或到期恢复处理。 +func (h *WorkerHub) Shutdown(ctx context.Context) error { + h.mu.Lock() + h.closing = true + h.cancel() + h.mu.Unlock() + done := make(chan struct{}) + go func() { h.wg.Wait(); close(done) }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +type workerFrame struct { + Version string `json:"protocol_version"` + Type string `json:"type"` + MessageID core.ID `json:"message_id"` + ReplyTo core.ID `json:"reply_to,omitempty"` + SessionID core.ID `json:"session_id,omitempty"` + Body json.RawMessage `json:"body,omitempty"` + Error *fault.Error `json:"error,omitempty"` +} +type outboundFrame struct { + data []byte + done chan error +} + +func (h *WorkerHub) ServeHTTP(w http.ResponseWriter, r *http.Request) { + requestID, _ := core.NewID() + w.Header().Set("X-Request-ID", string(requestID)) + w.Header().Set("Cache-Control", "no-store") + if r.URL.RawQuery != "" { + writeError(w, string(requestID), fault.Invalid("Worker credentials and options must not be placed in URL")) + return + } + values := r.Header.Values("Authorization") + if len(values) != 1 { + writeError(w, string(requestID), fault.Unauthorized) + return + } + scheme, value, ok := strings.Cut(values[0], " ") + if !ok || !strings.EqualFold(scheme, "Bearer") { + writeError(w, string(requestID), fault.Unauthorized) + return + } + token, err := auth.Parse(value) + if err != nil { + writeError(w, string(requestID), err) + return + } + supported := false + for _, header := range r.Header.Values("Sec-WebSocket-Protocol") { + for _, protocol := range strings.Split(header, ",") { + supported = supported || strings.TrimSpace(protocol) == WorkerSubprotocol + } + } + if !supported { + writeJSON(w, http.StatusUpgradeRequired, map[string]any{"request_id": requestID, "error": fault.New("PROTOCOL_REQUIRED", "pulse.worker.v1 subprotocol is required")}) + return + } + h.mu.Lock() + if h.closing || h.ctx.Err() != nil { + h.mu.Unlock() + writeError(w, string(requestID), fault.Unavailable) + return + } + select { + case h.slots <- struct{}{}: + h.wg.Add(1) + default: + h.mu.Unlock() + writeJSON(w, 429, map[string]any{"request_id": requestID, "error": fault.New("CAPACITY_EXCEEDED", "node connection capacity exceeded")}) + return + } + h.mu.Unlock() + defer func() { <-h.slots; h.wg.Done() }() + authorize, cancel := context.WithTimeout(r.Context(), 5*time.Second) + err = h.service.Authorize(authorize, token) + cancel() + if err != nil { + writeError(w, string(requestID), err) + return + } + conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{WorkerSubprotocol}, CompressionMode: websocket.CompressionDisabled}) + if err != nil { + return + } + defer conn.CloseNow() + conn.SetReadLimit(512 << 10) + ctx, cancel := context.WithCancel(h.ctx) + defer cancel() + helloCtx, helloCancel := context.WithTimeout(ctx, 5*time.Second) + kind, raw, err := conn.Read(helloCtx) + helloCancel() + if err != nil { + return + } + var hello workerFrame + var input worker.ConnectInput + if kind != websocket.MessageText || decodeWorkerJSON(raw, &hello) != nil || hello.Version != "1" || hello.Type != "worker.hello" || hello.MessageID.Validate() != nil || hello.SessionID != "" || hello.ReplyTo != "" || hello.Error != nil || decodeWorkerJSON(hello.Body, &input) != nil { + _ = conn.Close(websocket.StatusPolicyViolation, "invalid worker hello") + return + } + call, cancelCall := context.WithTimeout(ctx, 8*time.Second) + binding, welcome, err := h.service.Connect(call, token, input) + cancelCall() + if err != nil { + _ = writeWorkerFrame(ctx, conn, workerFrame{Version: "1", Type: "error", MessageID: requestID, ReplyTo: hello.MessageID, Error: publicWorkerError(err)}) + return + } + welcomeRaw, err := json.Marshal(welcome) + if err != nil { + return + } + if err := writeWorkerFrame(ctx, conn, workerFrame{Version: "1", Type: "worker.welcome", MessageID: requestID, ReplyTo: hello.MessageID, SessionID: binding.SessionID, Body: welcomeRaw}); err != nil { + return + } + // 有界读写队列与 SQL 控制循环分开，处理器绝不运行在这里。 + incoming := make(chan workerFrame, 4) + outgoing := make(chan outboundFrame, 8) + var loops sync.WaitGroup + loops.Add(2) + go func() { + defer loops.Done() + defer cancel() + for { + readCtx, stop := context.WithTimeout(ctx, 60*time.Second) + kind, raw, err := conn.Read(readCtx) + stop() + if err != nil { + return + } + var frame workerFrame + if kind != websocket.MessageText || decodeWorkerJSON(raw, &frame) != nil || frame.Version != "1" || frame.MessageID.Validate() != nil || frame.SessionID != binding.SessionID || frame.ReplyTo != "" || frame.Error != nil { + _ = conn.Close(websocket.StatusPolicyViolation, "invalid worker frame") + return + } + select { + case incoming <- frame: + case <-ctx.Done(): + return + } + } + }() + go func() { + defer loops.Done() + defer cancel() + for { + select { + case <-ctx.Done(): + return + case frame := <-outgoing: + writeCtx, stop := context.WithTimeout(ctx, 5*time.Second) + err := conn.Write(writeCtx, websocket.MessageText, frame.data) + stop() + if frame.done != nil { + frame.done <- err + } + if err != nil { + return + } + } + } + }() + defer func() { cancel(); _ = conn.CloseNow(); loops.Wait() }() + send := func(frame workerFrame, wait bool) bool { + frame.Version = "1" + frame.SessionID = binding.SessionID + if frame.MessageID == "" { + var err error + frame.MessageID, err = core.NewID() + if err != nil { + return false + } + } + raw, err := json.Marshal(frame) + if err != nil { + return false + } + out := outboundFrame{data: raw} + if wait { + out.done = make(chan error, 1) + } + select { + case outgoing <- out: + default: + return false + } + if !wait { + return true + } + select { + case err := <-out.done: + return err == nil + case <-ctx.Done(): + return false + } + } + capacity := input.Capacity + draining := welcome.Draining + eligible := false + leases := map[core.ID]time.Time{} + ticker := time.NewTicker(h.poll) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case frame := <-incoming: + call, stop := context.WithTimeout(ctx, 8*time.Second) + var body any + var commandErr error + switch frame.Type { + case "worker.capacity": + var in worker.CapacityInput + if commandErr = decodeWorkerJSON(frame.Body, &in); commandErr == nil { + var out worker.Welcome + out, commandErr = h.service.Capacity(call, token, binding, in) + body = out + if commandErr == nil { + capacity = in.Capacity + draining = out.Draining + eligible = true + } + } + case "worker.resume": + var ref worker.AttemptRef + if commandErr = decodeWorkerJSON(frame.Body, &ref); commandErr == nil { + var out worker.LeaseStatus + out, commandErr = h.service.Resume(call, token, binding, ref) + body = out + if commandErr == nil { + if out.Active { + leases[ref.AttemptID] = out.LeaseUntil + } else { + delete(leases, ref.AttemptID) + } + } + } + case "task.start", "task.renew": + var ref worker.AttemptRef + if commandErr = decodeWorkerJSON(frame.Body, &ref); commandErr == nil { + var out worker.CommandResult + if frame.Type == "task.start" { + out, commandErr = h.service.Start(call, token, binding, frame.MessageID, ref) + } else { + out, commandErr = h.service.Renew(call, token, binding, frame.MessageID, ref) + } + body = out + if commandErr == nil { + if out.Lease.Active { + leases[ref.AttemptID] = out.Lease.LeaseUntil + } else { + delete(leases, ref.AttemptID) + } + } + } + case "task.finish": + var in worker.FinishInput + if commandErr = decodeWorkerJSON(frame.Body, &in); commandErr == nil { + var out worker.CommandResult + out, commandErr = h.service.Finish(call, token, binding, frame.MessageID, in) + body = out + if commandErr == nil && !out.Lease.Active { + delete(leases, in.Ref.AttemptID) + } + } + default: + commandErr = fault.Invalid("unsupported worker message type") + } + stop() + reply := workerFrame{Type: "ack", ReplyTo: frame.MessageID} + if commandErr != nil { + reply.Type = "error" + reply.Error = publicWorkerError(commandErr) + } else { + reply.Body, commandErr = json.Marshal(body) + if commandErr != nil { + return + } + } + fatal := commandErr != nil && (fault.Code(commandErr) == "STALE_SESSION" || fault.Code(commandErr) == "UNAUTHENTICATED" || fault.Code(commandErr) == "PERMISSION_DENIED") + if !send(reply, fatal) || fatal { + return + } + case <-ticker.C: + // 本地记录仅减少空轮询；SQL 仍原子验证总容量、队列额度和有效会话。 + now := time.Now() + for attempt, until := range leases { + if !until.After(now) { + delete(leases, attempt) + } + } + if !eligible || draining || capacity == 0 || len(leases) >= int(capacity) || len(outgoing) >= 4 { + continue + } + for burst := 0; burst < 4 && len(leases) < int(capacity) && len(outgoing) < 4; burst++ { + call, stop := context.WithTimeout(ctx, 8*time.Second) + assignment, err := h.service.Claim(call, token, binding) + stop() + if err != nil { + if fault.Code(err) == "STALE_SESSION" || fault.Code(err) == "UNAUTHENTICATED" || fault.Code(err) == "PERMISSION_DENIED" { + send(workerFrame{Type: "error", Error: publicWorkerError(err)}, true) + return + } + break + } + if assignment == nil { + break + } + raw, err := json.Marshal(assignment) + if err != nil { + return + } + leases[assignment.AttemptID] = assignment.LeaseUntil + if !send(workerFrame{Type: "task.assign", Body: raw}, false) { + return + } + } + } + } +} +func publicWorkerError(err error) *fault.Error { + var known *fault.Error + if errors.As(err, &known) { + return known + } + return fault.Unavailable +} +func writeWorkerFrame(ctx context.Context, conn *websocket.Conn, frame workerFrame) error { + raw, err := json.Marshal(frame) + if err != nil { + return err + } + bounded, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + return conn.Write(bounded, websocket.MessageText, raw) +} +func decodeWorkerJSON(raw []byte, out any) error { + if len(raw) == 0 || len(raw) > 512<<10 { + return fault.Invalid("invalid worker message size") + } + if _, err := jsonvalue.Canonical(raw); err != nil { + return fault.Invalid("invalid worker JSON") + } + if err := exactFields(raw, reflect.TypeOf(out).Elem()); err != nil { + return fault.Invalid("unknown or invalid worker field") + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(out); err != nil { + return fault.Invalid("invalid worker field value") + } + return nil +} diff --git a/platform/auth/token.go b/platform/auth/token.go index 1106195..00f54da 100644 --- a/platform/auth/token.go +++ b/platform/auth/token.go @@ -58,7 +58,7 @@ type Permissions struct { Queues []string `json:"queues"` } -var supported = map[string]bool{"tasks:submit": true, "tasks:read": true, "tasks:cancel": true, "queues:read": true, "queues:manage": true} +var supported = map[string]bool{"tasks:submit": true, "tasks:read": true, "tasks:cancel": true, "queues:read": true, "queues:manage": true, "workers:connect": true, "workers:consume": true} func (p Permissions) Validate() error { if len(p.Actions) == 0 || len(p.Actions) > 32 || len(p.Queues) == 0 || len(p.Queues) > 128 { diff --git a/pulse/model/attempt_evidence.go b/pulse/model/attempt_evidence.go new file mode 100644 index 0000000..846d9c9 --- /dev/null +++ b/pulse/model/attempt_evidence.go @@ -0,0 +1,25 @@ +package model + +import core "git.yuntuo.hk/k2safe/orbit/model" + +// AttemptEvidence 保留迟到或冲突结果，作为核查证据；保存证据不会改写任务结果。 +type AttemptEvidence struct { + // Record 包含命名空间、证据标识、并发版本与首次收到时间。 + core.Record + // AttemptID 证据所属尝试，提交者仍需证明原执行秘密与进程身份。 + AttemptID core.ID `db:"attempt_id"` + // MessageID 稳定结果消息标识，用于同一证据重报去重。 + MessageID core.ID `db:"message_id"` + // Outcome Worker 报告的结果分类，不代表平台认可为任务终态。 + Outcome string `db:"outcome"` + // ErrorCode 可选稳定错误分类；无错误分类时为空。 + ErrorCode *string `db:"error_code"` + // Payload 结果或证据引用 JSON 文本，保留数字精度。 + Payload core.JSON `db:"payload"` + // RequestDigest 结果语义摘要，排除会话身份与原始秘密。 + RequestDigest core.Digest `db:"request_digest"` + // Reason 未用于更新权威状态的原因，例如授权失效或与已保存结果冲突。 + Reason string `db:"reason"` +} + +func (AttemptEvidence) TableName() string { return "pulse_attempt_evidence" } diff --git a/pulse/model/worker_instance.go b/pulse/model/worker_instance.go new file mode 100644 index 0000000..8acd504 --- /dev/null +++ b/pulse/model/worker_instance.go @@ -0,0 +1,25 @@ +package model + +import core "git.yuntuo.hk/k2safe/orbit/model" + +// WorkerInstance 保存进程实例的持久化会话头；重连推进代次，不能仅凭实例 ID 接管。 +type WorkerInstance struct { + // Record 包含命名空间、标识、并发版本与 UTC 时间。 + core.Record + // CredentialID 初次注册的凭据，当前进程生命周期内不可替换。 + CredentialID core.ID `db:"credential_id"` + // QueueID 进程绑定的消费队列；切换队列应创建新实例。 + QueueID core.ID `db:"queue_id"` + // SecretDigest 进程持有的随机恢复秘密摘要；知道实例 ID 不构成恢复授权。 + SecretDigest core.Digest `db:"secret_digest"` + // RegistrationDigest 队列与处理器版本集合的摘要，重连不得改变注册语义。 + RegistrationDigest core.Digest `db:"registration_digest"` + // SessionEpoch 已分配的最大连接代次，初始为零，重连递增。 + SessionEpoch uint64 `db:"session_epoch"` + // CurrentSessionID 当前连接；注册事务创建会话之前为空。 + CurrentSessionID *core.ID `db:"current_session_id"` + // Draining 持久化排空意图，重连保留；排空后不再给该进程新工作。 + Draining bool `db:"draining"` +} + +func (WorkerInstance) TableName() string { return "pulse_worker_instances" } diff --git a/pulse/worker/service.go b/pulse/worker/service.go new file mode 100644 index 0000000..399fefc --- /dev/null +++ b/pulse/worker/service.go @@ -0,0 +1,139 @@ +package worker + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "sort" + + "git.yuntuo.hk/k2safe/orbit/internal/fault" + "git.yuntuo.hk/k2safe/orbit/internal/jsonvalue" + core "git.yuntuo.hk/k2safe/orbit/model" + "git.yuntuo.hk/k2safe/orbit/platform/auth" + "git.yuntuo.hk/k2safe/orbit/pulse/service" +) + +var StaleSession = fault.New("STALE_SESSION", "connection is expired or superseded") +var LeaseLost = fault.New("LEASE_LOST", "attempt no longer grants execution authority") +var GateClosed = fault.New("EXECUTION_PAUSED", "queue or execution gate is paused") + +type Service struct{ store Store } + +func New(store Store) *Service { return &Service{store: store} } +func (s *Service) Authorize(ctx context.Context, t auth.Token) error { + return s.store.Authorize(ctx, t) +} +func NewSecret() (string, core.Digest, error) { + var raw [32]byte + if _, err := rand.Read(raw[:]); err != nil { + return "", core.Digest{}, err + } + return base64.RawURLEncoding.EncodeToString(raw[:]), core.Digest(sha256.Sum256(raw[:])), nil +} +func SecretDigest(secret string) (core.Digest, error) { + if len(secret) != 43 { + return core.Digest{}, fault.Unauthorized + } + raw, err := base64.RawURLEncoding.Strict().DecodeString(secret) + if err != nil || len(raw) != 32 { + return core.Digest{}, fault.Unauthorized + } + return core.Digest(sha256.Sum256(raw)), nil +} +func NormalizeConnect(in ConnectInput) (ConnectInput, error) { + if in.InstanceID.Validate() != nil || in.QueueID.Validate() != nil || in.Capacity < 1 || in.Capacity > 256 || len(in.Capabilities) < 1 || len(in.Capabilities) > 32 { + return in, fault.Invalid("invalid worker identity, queue, capacity or capability count") + } + if _, err := SecretDigest(in.InstanceSecret); err != nil { + return in, fault.Invalid("instance_secret must encode 32 random bytes") + } + in.Capabilities = append([]Capability(nil), in.Capabilities...) + for _, c := range in.Capabilities { + if !service.ValidName(c.Handler, 128) || !service.ValidName(c.HandlerVersion, 64) || !service.ValidName(c.PayloadVersion, 32) { + return in, fault.Invalid("invalid worker capability") + } + } + sort.Slice(in.Capabilities, func(i, j int) bool { + a, b := in.Capabilities[i], in.Capabilities[j] + if a.Handler != b.Handler { + return a.Handler < b.Handler + } + if a.HandlerVersion != b.HandlerVersion { + return a.HandlerVersion < b.HandlerVersion + } + return a.PayloadVersion < b.PayloadVersion + }) + for i := 1; i < len(in.Capabilities); i++ { + if in.Capabilities[i] == in.Capabilities[i-1] { + return in, fault.Invalid("duplicate worker capability") + } + } + return in, nil +} +func (s *Service) Connect(ctx context.Context, t auth.Token, in ConnectInput) (Binding, Welcome, error) { + in, err := NormalizeConnect(in) + if err != nil { + return Binding{}, Welcome{}, err + } + return s.store.Connect(ctx, t, in) +} +func (s *Service) Capacity(ctx context.Context, t auth.Token, b Binding, in CapacityInput) (Welcome, error) { + if in.Sequence == 0 || in.Capacity > 256 { + return Welcome{}, fault.Invalid("invalid capacity sequence or value") + } + return s.store.Capacity(ctx, t, b, in) +} +func (s *Service) Claim(ctx context.Context, t auth.Token, b Binding) (*Assignment, error) { + return s.store.Claim(ctx, t, b) +} +func validRef(ref AttemptRef) error { + if ref.TaskID.Validate() != nil || ref.AttemptID.Validate() != nil || ref.Epoch == 0 { + return fault.Invalid("invalid attempt identity") + } + _, err := SecretDigest(ref.LeaseSecret) + return err +} +func validCommand(message core.ID, ref AttemptRef) error { + if message.Validate() != nil { + return fault.Invalid("invalid operation message id") + } + return validRef(ref) +} +func (s *Service) Start(ctx context.Context, t auth.Token, b Binding, message core.ID, ref AttemptRef) (CommandResult, error) { + if err := validCommand(message, ref); err != nil { + return CommandResult{}, err + } + return s.store.Start(ctx, t, b, message, ref) +} +func (s *Service) Renew(ctx context.Context, t auth.Token, b Binding, message core.ID, ref AttemptRef) (CommandResult, error) { + if err := validCommand(message, ref); err != nil { + return CommandResult{}, err + } + return s.store.Renew(ctx, t, b, message, ref) +} +func (s *Service) Finish(ctx context.Context, t auth.Token, b Binding, message core.ID, in FinishInput) (CommandResult, error) { + if err := validCommand(message, in.Ref); err != nil { + return CommandResult{}, err + } + if in.Outcome != "SUCCEEDED" && in.Outcome != "FAILED" && in.Outcome != "UNKNOWN" { + return CommandResult{}, fault.Invalid("invalid result outcome") + } + if len(in.Result) == 0 || len(in.Result) > 256<<10 || bytes.Equal(bytes.TrimSpace(in.Result), []byte("null")) { + return CommandResult{}, fault.Invalid("result must be non-null JSON up to 256 KiB") + } + if _, err := jsonvalue.Canonical(in.Result); err != nil { + return CommandResult{}, fault.Invalid("invalid result JSON") + } + if (in.Outcome == "SUCCEEDED" && in.ErrorCode != "") || (in.Outcome != "SUCCEEDED" && !service.ValidName(in.ErrorCode, 64)) { + return CommandResult{}, fault.Invalid("error_code must match the outcome") + } + return s.store.Finish(ctx, t, b, message, in) +} +func (s *Service) Resume(ctx context.Context, t auth.Token, b Binding, ref AttemptRef) (LeaseStatus, error) { + if err := validRef(ref); err != nil { + return LeaseStatus{}, err + } + return s.store.Resume(ctx, t, b, ref) +} diff --git a/pulse/worker/service_test.go b/pulse/worker/service_test.go new file mode 100644 index 0000000..0b4d9cd --- /dev/null +++ b/pulse/worker/service_test.go @@ -0,0 +1,38 @@ +package worker + +import ( + "strings" + "testing" + + core "git.yuntuo.hk/k2safe/orbit/model" +) + +func TestRegistrationAndSecretBoundaries(t *testing.T) { + secret, digest, err := NewSecret() + if err != nil { + t.Fatal(err) + } + parsed, err := SecretDigest(secret) + if err != nil || parsed != digest { + t.Fatal("secret round trip") + } + for _, bad := range []string{secret + "=", secret + "\n", strings.Repeat("!", 43), secret[:42]} { + if _, err := SecretDigest(bad); err == nil { + t.Fatal("malformed secret accepted") + } + } + input := ConnectInput{InstanceID: core.ID(strings.Repeat("1", 32)), InstanceSecret: secret, QueueID: core.ID(strings.Repeat("2", 32)), Capacity: 1, Capabilities: []Capability{{"z", "v1", "v1"}, {"a", "v1", "v1"}}} + normalized, err := NormalizeConnect(input) + if err != nil || normalized.Capabilities[0].Handler != "a" || input.Capabilities[0].Handler != "z" { + t.Fatalf("normalization changed caller input: %v", err) + } + input.Capabilities = append(input.Capabilities, input.Capabilities[0]) + if _, err := NormalizeConnect(input); err == nil { + t.Fatal("duplicate capability accepted") + } + input = normalized + input.Capacity = 257 + if _, err := NormalizeConnect(input); err == nil { + t.Fatal("excess capacity accepted") + } +} diff --git a/pulse/worker/types.go b/pulse/worker/types.go new file mode 100644 index 0000000..ebc04dc --- /dev/null +++ b/pulse/worker/types.go @@ -0,0 +1,124 @@ +// Package worker 定义执行控制契约；处理器本身在外部 Worker 进程执行。 +package worker + +import ( + "context" + "encoding/json" + "time" + + core "git.yuntuo.hk/k2safe/orbit/model" + "git.yuntuo.hk/k2safe/orbit/platform/auth" + pulse "git.yuntuo.hk/k2safe/orbit/pulse/model" + "git.yuntuo.hk/k2safe/orbit/pulse/service" +) + +// Capability 精确声明可处理的版本，不做静默版本降级。 +type Capability struct { + Handler string `json:"handler"` // Handler 处理器名称。 + HandlerVersion string `json:"handler_version"` // HandlerVersion 处理器版本。 + PayloadVersion string `json:"payload_version"` // PayloadVersion 参数版本。 +} + +// ConnectInput 在每次连接时提交；同一进程恢复时保留实例和秘密。 +type ConnectInput struct { + InstanceID core.ID `json:"instance_id"` // InstanceID 进程标识，重启换新值。 + InstanceSecret string `json:"instance_secret"` // InstanceSecret 32 字节随机秘密，只保存摘要。 + QueueID core.ID `json:"queue_id"` // QueueID 当前进程绑定的消费队列。 + Capabilities []Capability `json:"capabilities"` // Capabilities 1–32 个精确版本组合，重连固定。 + Capacity uint32 `json:"capacity"` // Capacity 总执行槽数量，1–256。 +} + +// Binding 是接入层持有的认证结果，不作为公开请求字段接受或写入日志。 +type Binding struct { + InstanceID core.ID // InstanceID 已证明秘密的进程标识。 + SessionID core.ID // SessionID 当前连接标识。 + Epoch uint64 // Epoch 连接代次。 + Proof core.Digest // Proof 进程秘密摘要，持久化验证旧会话隔离。 +} + +// Welcome 给出数据库时钟与控制参数；时间用于 SDK 计算保守的单调时钟期限。 +type Welcome struct { + SessionID core.ID `json:"session_id"` // SessionID 本次连接标识。 + Epoch uint64 `json:"epoch,string"` // Epoch 当前连接代次。 + ServerTime time.Time `json:"server_time"` // ServerTime 本次响应对应的数据库时间。 + ExpiresAt time.Time `json:"expires_at"` // ExpiresAt 会话有效期限，不能代替任务租约。 + LeaseDurationMS uint64 `json:"lease_duration_ms"` // LeaseDurationMS 单次续租窗口，毫秒。 + Draining bool `json:"draining"` // Draining 排空标记，跨重连保留。 +} + +// CapacityInput 使用覆盖值和序号，重复消息不会重复增加执行容量。 +type CapacityInput struct { + Sequence uint64 `json:"sequence,string"` // Sequence 本会话内从一开始递增。 + Capacity uint32 `json:"capacity"` // Capacity 总容量，零表示暂不领取；不释放已有授权。 + Drain bool `json:"drain"` // Drain 为真后持久化排空，不能通过后续消息清除。 +} + +// Assignment 仅通过已认证执行连接交付，包含敏感尝试秘密。 +type Assignment struct { + Task service.TaskView `json:"task"` // Task 受理快照及业务参数。 + AttemptID core.ID `json:"attempt_id"` // AttemptID 本次尝试标识。 + Epoch uint64 `json:"epoch,string"` // Epoch 任务执行代次，隔离旧尝试。 + LeaseSecret string `json:"lease_secret"` // LeaseSecret 执行秘密，禁止普通日志及查询返回。 + LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 尚未开始时的预占有效期。 + ServerTime time.Time `json:"server_time"` // ServerTime 领取时数据库时间。 +} + +// AttemptRef 证明对一次尝试的控制权；连接代次仍由 Binding 单独验证。 +type AttemptRef struct { + TaskID core.ID `json:"task_id"` // TaskID 所属任务。 + AttemptID core.ID `json:"attempt_id"` // AttemptID 目标尝试。 + Epoch uint64 `json:"epoch,string"` // Epoch 目标执行代次。 + LeaseSecret string `json:"lease_secret"` // LeaseSecret 原始执行秘密，只在受控连接传输。 +} + +// FinishInput 区分明确结果与未知效果，未知不能被解释为可以盲目重跑。 +type FinishInput struct { + Ref AttemptRef `json:"ref"` // Ref 当前尝试授权证明。 + Outcome string `json:"outcome"` // Outcome 为 SUCCEEDED、FAILED 或 UNKNOWN。 + Result json.RawMessage `json:"result"` // Result 非 null JSON 结果或证据引用，最多 256 KiB。 + ErrorCode string `json:"error_code,omitempty"` // ErrorCode FAILED/UNKNOWN 必填稳定分类，不包含原始异常正文。 +} + +// Receipt 是不可变的操作回执，不代表当前仍有执行权。 +type Receipt struct { + Operation string `json:"operation"` // Operation start、renew 或 finish。 + TaskState pulse.TaskState `json:"task_state"` // TaskState 提交该操作时的任务状态。 + AttemptState pulse.AttemptState `json:"attempt_state"` // AttemptState 操作提交时的尝试状态。 + TaskRevision uint64 `json:"task_revision,string"` // TaskRevision 操作提交时的任务版本。 + LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 操作提交时租约期限，历史值不能授权新执行。 + Disposition string `json:"disposition"` // Disposition 为 APPLIED 或 EVIDENCE_ONLY。 +} + +// LeaseStatus 每次响应重新计算；处理器仅在 Active 为真、State 为 RUNNING 且期限内推进。 +// LEASED 的 Active 只代表预占仍有效，不能代替开始授权。 +type LeaseStatus struct { + Active bool `json:"active"` // Active 当前会话仍持有未过期的有效尝试。 + State pulse.AttemptState `json:"state"` // State 当前尝试状态。 + LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 当前租约，不因重放旧回执自动延长。 + ServerTime time.Time `json:"server_time"` // ServerTime 完成锁定后重新读取的数据库时间。 +} + +// CommandResult 将原回执与当前执行权分开，避免旧 start ACK 使过期尝试复活。 +type CommandResult struct { + Receipt Receipt `json:"receipt"` // Receipt 对应稳定消息身份的原回执。 + Lease LeaseStatus `json:"lease"` // Lease 当前授权判断。 + Deduplicated bool `json:"deduplicated"` // Deduplicated 是否返回已保存的回执。 +} + +// RecoveryStats 统计本批恢复，不作为持久化成功率或容量指标。 +type RecoveryStats struct { + Examined int + Requeued int + Failed int + NeedsReview int +} +type Store interface { + Authorize(context.Context, auth.Token) error + Connect(context.Context, auth.Token, ConnectInput) (Binding, Welcome, error) + Capacity(context.Context, auth.Token, Binding, CapacityInput) (Welcome, error) + Claim(context.Context, auth.Token, Binding) (*Assignment, error) + Start(context.Context, auth.Token, Binding, core.ID, AttemptRef) (CommandResult, error) + Renew(context.Context, auth.Token, Binding, core.ID, AttemptRef) (CommandResult, error) + Finish(context.Context, auth.Token, Binding, core.ID, FinishInput) (CommandResult, error) + Resume(context.Context, auth.Token, Binding, AttemptRef) (LeaseStatus, error) +}