feat: add durable Worker execution and WSS server
This commit is contained in:
@@ -8,9 +8,9 @@ Orbit 是面向多个业务项目的通用任务调度与流程编排平台，
|
||||
|
||||
## 当前进度
|
||||
|
||||
已实现 30 个数据模型、版本化迁移、凭据及范围校验、队列创建查询、任务幂等提交/查询/分页/待执行取消，以及独立 Go SDK 和可运行示例。WSS Worker、调度器和流程执行引擎按 [持续开发计划](docs/08-development-plan.md) 继续实现。
|
||||
已实现 32 个数据模型、版本化迁移、凭据及范围校验、队列创建查询、任务幂等提交/查询/分页/待执行取消，以及对应 Go SDK 和示例。P2 服务端执行内核和 WSS 已通过阶段内回归；Worker SDK 自动运行器、调度器和流程引擎按 [持续开发计划](docs/08-development-plan.md) 继续实现。
|
||||
|
||||
readiness 只表示数据库与迁移记录可用，不表示完整平台已具备生产服务能力。验证证据见 [P0](docs/reports/phase-0.md) 与 [P1 报告](docs/reports/phase-1.md)。
|
||||
readiness 只表示数据库与迁移记录可用，不表示完整平台已具备生产服务能力。验证证据见 [P0](docs/reports/phase-0.md)、[P1](docs/reports/phase-1.md) 与 [P2 服务端记录](docs/reports/phase-2-server.md)。
|
||||
|
||||
## 目录
|
||||
|
||||
@@ -23,6 +23,7 @@ platform/model/ 应用、命名空间、幂等、事件和审计
|
||||
platform/auth/ 凭据格式与动作/队列授权范围
|
||||
pulse/model/ 任务、尝试、队列、计划和执行许可
|
||||
pulse/service/ 任务与队列应用契约、输入校验
|
||||
pulse/worker/ 会话、开始、续租、结果和恢复契约
|
||||
weave/model/ 定义、实例、步骤、信号和查询轮次
|
||||
internal/config/ 配置校验
|
||||
internal/storage/mysql/ 连接、事务、迁移及集成测试
|
||||
|
||||
@@ -32,6 +32,7 @@ func run() error {
|
||||
flags.StringVar(&in.Environment, "environment", "", "environment name")
|
||||
flags.StringVar(&in.Queue, "queue", "", "initial queue name")
|
||||
flags.StringVar(&output, "output", "", "new private credentials JSON file")
|
||||
flags.BoolVar(&in.WorkerAccess, "worker-access", false, "also grant Worker connect and consume actions for development")
|
||||
if err := flags.Parse(os.Args[2:]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -14,7 +14,9 @@ import (
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/config"
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/storage/mysql"
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/transport/httpserver"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -49,13 +51,49 @@ func run() error {
|
||||
}
|
||||
return mysql.CheckSchema(ctx, db)
|
||||
}, service.New(mysql.NewTaskStore(db)))
|
||||
node, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
executions, err := mysql.NewExecutionStore(db, node, mysql.ExecutionOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hub := httpserver.NewWorkerHub(root, worker.New(executions))
|
||||
httpserver.RegisterWorkers(server, hub)
|
||||
defer func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = hub.Shutdown(ctx)
|
||||
}()
|
||||
recoveryDone := make(chan struct{})
|
||||
recoveryCtx, stopRecovery := context.WithCancel(root)
|
||||
go func() {
|
||||
defer close(recoveryDone)
|
||||
ticker := time.NewTicker(2 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-recoveryCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
call, cancel := context.WithTimeout(recoveryCtx, 5*time.Second)
|
||||
_, err := executions.Recover(call, 100)
|
||||
cancel()
|
||||
if err != nil && recoveryCtx.Err() == nil {
|
||||
slog.Error("execution recovery batch failed", "node_id", node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
defer func() { stopRecovery(); <-recoveryDone }()
|
||||
listener, err := net.Listen("tcp", cfg.HTTPAddr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- server.Serve(listener) }()
|
||||
slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task and queue management")
|
||||
slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task, queue and Worker APIs", "node_id", node)
|
||||
select {
|
||||
case err := <-done:
|
||||
if errors.Is(err, http.ErrServerClosed) {
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
> 版本 0.2 · 2026-09-17。HTTP + WS 已确认。本文的具体路径、消息名、字段及状态码是首版设计建议，业务协议尚未实现，需评审后冻结；健康接口的已实现契约见接口手册。
|
||||
|
||||
实现更新：HTTP 任务/队列接口已完成 P1；Worker 服务端已进入 P2，实际帧名、字段、回执与边界见 [Worker 契约](api/worker.md)。本文保留早期完整设计，未实现项目不能据此视为已交付。
|
||||
|
||||
## 1. 通用契约
|
||||
|
||||
- 生产使用 HTTPS / WSS；HTTP 承担任务提交、流程启动与信号、查询、管理，WS 承担 Worker 双向通信与结果订阅。
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
- `model/`：ID、摘要、JSON、命名空间记录等公共原语。
|
||||
- `platform/model/`：应用、命名空间、凭据、幂等、事件、订阅交付及审计。
|
||||
- `pulse/model/`：队列、执行门控、会话、任务、尝试、回执、并发许可和调度计划。
|
||||
- `pulse/model/`：队列、执行门控、进程/会话、任务、尝试、回执、迟到证据、并发许可和调度计划。
|
||||
- `weave/model/`：定义及不可变版本、实例、步骤激活、等待、信号、查询轮次与恢复记录。
|
||||
- `internal/storage/mysql/`：连接、事务与显式迁移，不向 SDK 暴露。
|
||||
- `sdk/go/`：独立 Go module；对外 DTO 与服务端模型分别维护。
|
||||
@@ -34,7 +34,7 @@
|
||||
|
||||
## 3. 实体清单
|
||||
|
||||
下表与初始迁移中的 30 个实体一致；迁移自身另有 `orbit_schema_migrations` 记录。
|
||||
当前共 32 个实体：初始迁移包含 30 个，000003 增加进程会话头和迟到证据。迁移自身另有 `orbit_schema_migrations` 记录。
|
||||
|
||||
| 模块 | Go 模型 | 数据表 |
|
||||
| --- | --- | --- |
|
||||
@@ -49,9 +49,11 @@
|
||||
| pulse | `Queue` | `pulse_queues` |
|
||||
| pulse | `ExecutionGate` | `pulse_execution_gates` |
|
||||
| pulse | `WorkerSession` | `pulse_worker_sessions` |
|
||||
| pulse | `WorkerInstance` | `pulse_worker_instances` |
|
||||
| pulse | `Task` | `pulse_tasks` |
|
||||
| pulse | `Attempt` | `pulse_attempts` |
|
||||
| pulse | `AttemptReceipt` | `pulse_attempt_receipts` |
|
||||
| pulse | `AttemptEvidence` | `pulse_attempt_evidence` |
|
||||
| pulse | `ConcurrencyLimit` | `pulse_concurrency_limits` |
|
||||
| pulse | `Permit` | `pulse_permits` |
|
||||
| pulse | `Schedule` | `pulse_schedules` |
|
||||
@@ -86,6 +88,8 @@
|
||||
|
||||
外键与 `CHECK` 无法代替执行引擎。P1 已完成 HTTP 动作/队列授权、幂等摘要与原回执、受理事件，以及待执行取消的 CAS/审计事务。执行相关的以下条件继续分阶段实现：
|
||||
|
||||
P2 服务端已新增会话隔离、两步派发、租约、并发许可、结果回执/证据和恢复扫描，见 [执行协议](api/worker.md) 与 [阶段记录](reports/phase-2-server.md)。Worker SDK 和完整 P2 验收仍在推进。
|
||||
|
||||
1. 权限验证、凭据校验、命名空间过滤和队列消费授权。
|
||||
2. 同一次事务更新任务、尝试、租约、并发许可、事件与流程交付记录。
|
||||
3. CAS 竞争、数据库时间校验、门控与开始授权串行化。
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
| --- | --- | --- | --- |
|
||||
| P0 基础与数据模型 | Go workspace、独立 SDK module、公共/Pulse/Weave 模型、MySQL 迁移、配置、健康检查 | 编译/竞态/静态检查；真实 MySQL 建表、隔离、唯一性、预算、不可变版本、迁移失败防护；SDK 健康调用 | 已完成并通过阶段回归（见 P0 报告） |
|
||||
| P1 任务受理闭环 | 应用凭据与作用域权限、队列、幂等提交、查询、待执行取消、受理事件；SDK 客户端 | API/SDK 创建查询闭环；重复键与内容冲突、响应未确认重报、跨命名空间拒绝、参数边界 | 已完成并通过阶段回归（见 P1 报告；真实断路/提交 ACK 丢失注入留后续故障阶段） |
|
||||
| P2 Worker 执行闭环 | WSS 接入、能力/容量、领取与开始、租约、续租、结果确认、排空与恢复；Worker SDK | 真 Worker 执行；失联/重连/重复确认/旧尝试拒绝；业务效果未知转核查；节点故障接管 | 待开发 |
|
||||
| P2 Worker 执行闭环 | WSS 接入、能力/容量、领取与开始、租约、续租、结果确认、排空与恢复；Worker SDK | 真 Worker 执行；失联/重连/重复确认/旧尝试拒绝；业务效果未知转核查；节点故障接管 | 开发中：服务端执行事务和 WSS 已通过阶段内回归；Worker SDK 自动运行/重连/退出及完整验收待完成 |
|
||||
| P3 Pulse 完整能力 | 安全重试、延时、Cron/固定间隔、漏跑策略、计划版本、并发许可、配额/背压、可靠结果订阅 | 多实例领取/触发竞争、重试上限、取消竞争、队列隔离、积压恢复、事件重复交付 | 待开发 |
|
||||
| P4 Weave 主流程 | SDK 定义构建与发布校验、实例、顺序/分支/结构化并行汇合、信号/定时等待、版本绑定 | 重启接续、早到/重复信号、并行竞争、内部事件重放、发布版本兼容；端到端通用流程 | 待开发 |
|
||||
| P5 Weave 恢复与查询 | 有界查询、暂停继续、取消、人工核查恢复、控制门控与审计 | 轮次与尝试独立预算；暂停/开始/回执/超时竞争；恢复保留预算；未知结果不盲目重复 | 待开发 |
|
||||
@@ -73,3 +73,4 @@ SDK、文档与示例随 P1–P5 同步开发，P7 是全量核对，不是最
|
||||
- [生产验收总清单](05-production-acceptance.md)
|
||||
- [P0 验证记录](reports/phase-0.md)
|
||||
- [P1 验证记录](reports/phase-1.md)
|
||||
- [P2 服务端阶段记录](reports/phase-2-server.md)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
- 文档版本：0.2
|
||||
- 整理日期：2026-09-17
|
||||
- 阶段：P0 模型基础与 P1 任务受理/SDK 已实现，平台完整功能按持续计划开发；尚未完成生产验收
|
||||
- 阶段：P0/P1 已完成，P2 服务端执行/WSS 已通过阶段内回归，Worker SDK 继续开发；尚未完成生产验收
|
||||
- 适用范围：Orbit 统一平台，包含 Pulse 任务调度模块、Weave 流程编排模块与公共能力。平台面向多个业务系统复用。
|
||||
|
||||
## 阅读顺序
|
||||
@@ -79,4 +79,4 @@ Orbit 不内置特定行业规则；Pulse 管理执行，Weave 管理步骤依
|
||||
|
||||
2026-09-17，文档更新至 0.2：用户确认研发定义发布流程及 Weave 首版功能，明确纳入有界重复查询和暂停/继续等生产管理能力。技术细则与参数继续按建议或待定标注，未将功能确认等同于实现或验收通过。
|
||||
|
||||
持续开发目标已建立。当前实现和验收进度以 [开发计划](08-development-plan.md)、[P0 报告](reports/phase-0.md)、[P1 报告](reports/phase-1.md) 为准；早期设计稿中的“未实现”描述保留其编写阶段语境。
|
||||
持续开发目标已建立。当前实现和验收进度以 [开发计划](08-development-plan.md)、[P0 报告](reports/phase-0.md)、[P1 报告](reports/phase-1.md)、[P2 服务端记录](reports/phase-2-server.md) 为准；早期设计稿中的“未实现”描述保留其编写阶段语境。
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
| `/v1/tasks` 提交、查询、分页与待执行取消 | P1 已实现 | [任务接口](tasks.md) |
|
||||
| `/v1/queues` 创建与查询 | P1 已实现 | [队列接口](queues.md) |
|
||||
| 凭据校验与初始化 CLI | P1 已实现 | [鉴权](authentication.md) |
|
||||
| `/v1/worker/connect` | 计划 P2，尚未实现 | [协议草案](../03-http-websocket-protocol.md) |
|
||||
| `/v1/worker/connect` | P2 服务端已实现，Worker SDK 仍在开发 | [Worker 契约](worker.md) |
|
||||
| `/v1/schedules` 与计划管理 | 计划 P3，尚未实现 | [协议草案](../03-http-websocket-protocol.md) |
|
||||
| `/v1/events/connect` | 计划 P3，尚未实现 | [协议草案](../03-http-websocket-protocol.md) |
|
||||
| `/v1/workflows` 与流程管理 | 计划 P4–P5，尚未实现 | [流程接口规划](workflows.md) |
|
||||
|
||||
@@ -51,3 +51,5 @@ actions 不支持通配符；queues 可列出队列 ID，星号仅表示当前
|
||||
原始秘密错误、凭据撤销/过期、应用或命名空间停用统一返回 401。合法身份缺少动作权限返回 403。读取不在范围内的单个资源返回 404，避免披露资源内容。
|
||||
|
||||
SDK 使用 `orbit.Options{Token: token}`。健康查询不发送凭据。当前不启用业务权限缓存，撤销校验不依赖缓存过期。
|
||||
|
||||
P2 增加 workers:connect 和 workers:consume。bootstrap 只有显式提供 --worker-access 时才同时授予这两项开发权限；现有凭据不自动变化。Worker 还需独立的进程恢复秘密和尝试秘密，见 [Worker 契约](worker.md)。
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
状态：P1 已实现并通过真实 MySQL 与 SDK 回归。当前提供受理、查询、分页和待执行取消；Worker 执行与自动重试在 P2–P3 实现。
|
||||
|
||||
P2 进展：服务端执行内核和 WSS 已可产生 LEASED、RUNNING、SUCCEEDED、FAILED、NEEDS_REVIEW 等状态，见 [Worker 契约](worker.md)；独立 Worker SDK 仍在开发，自动策略重试尚未完成。
|
||||
|
||||
## 公共约定
|
||||
|
||||
业务接口需要 `Authorization: Bearer <token>`；写入需要 `Content-Type: application/json` 和稳定的 `Idempotency-Key`。凭据决定命名空间，请求体不能覆盖它。详见 [鉴权与初始化](authentication.md)。
|
||||
|
||||
146
docs/api/worker.md
Normal file
146
docs/api/worker.md
Normal file
@@ -0,0 +1,146 @@
|
||||
# Worker 通信与执行契约 v1（开发中）
|
||||
|
||||
已实现服务端执行事务、WebSocket 接入和真实 WSS 回归。**P2 尚未完成**：独立 Go SDK 的 Worker 运行器、自动重连/续租、处理器生命周期与优雅退出还在开发。本页记录当前可验证的服务端契约，不把测试客户端当作可交付 Worker SDK。
|
||||
|
||||
## 1. 接入与身份
|
||||
|
||||
入口 `GET /v1/worker/connect`，子协议必须为 `pulse.worker.v1`。Authorization 使用 Bearer 凭据；禁止 URL 查询参数。握手先校验 workers:connect，消费控制操作再次校验 workers:consume、队列范围和凭据有效性。
|
||||
|
||||
初始化开发凭据可显式增加 Worker 权限：
|
||||
|
||||
```sh
|
||||
go run ./cmd/orbit-admin bootstrap \
|
||||
--application worker-demo --namespace default --environment development \
|
||||
--queue objects --worker-access --output .local/worker.credentials.json
|
||||
```
|
||||
|
||||
默认 bootstrap 仍只有 P1 的五项权限，现有凭据不会自动扩权。生产细粒度凭据管理在 P6 完成。
|
||||
|
||||
当前 orbit 命令监听 HTTP，开发可使用 ws；生产需要受控的 TLS 终止。WSS 回归使用真实 TLS 测试服务器及受信任测试证书，没有跳过证书校验。传输库锁定 github.com/coder/websocket v1.8.15，使用上下文、读写期限、子协议和默认 Origin 校验，关闭压缩。[上游版本记录](https://github.com/coder/websocket/releases/tag/v1.8.15)、[上游接口文档](https://pkg.go.dev/github.com/coder/websocket@v1.8.15)。
|
||||
|
||||
## 2. 帧格式
|
||||
|
||||
只接受 JSON 文本帧；单帧最多 512 KiB，业务结果最多 256 KiB。重复键、未知字段、无效 Unicode 和非法数值按 HTTP 相同规则拒绝。
|
||||
|
||||
```json
|
||||
{
|
||||
"protocol_version":"1",
|
||||
"type":"task.start",
|
||||
"message_id":"44444444444444444444444444444444",
|
||||
"session_id":"33333333333333333333333333333333",
|
||||
"body":{
|
||||
"task_id":"11111111111111111111111111111111",
|
||||
"attempt_id":"22222222222222222222222222222222",
|
||||
"epoch":"1",
|
||||
"lease_secret":"<执行秘密，仅通过受控连接传输>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
所有 ID 为 32 位小写十六进制；epoch、sequence、revision 用十进制字符串。message_id 标识一次操作，重连重报保持不变；session_id 必须换为当前连接身份。服务端响应以 reply_to 关联请求：
|
||||
|
||||
```json
|
||||
{
|
||||
"protocol_version":"1",
|
||||
"type":"error",
|
||||
"message_id":"55555555555555555555555555555555",
|
||||
"reply_to":"44444444444444444444444444444444",
|
||||
"session_id":"33333333333333333333333333333333",
|
||||
"error":{"code":"LEASE_LOST","message":"attempt no longer grants execution authority"}
|
||||
}
|
||||
```
|
||||
|
||||
普通成功响应 type=ack，body 为下表对应结果。服务端主动派发 type=task.assign，无 reply_to。错误仅包含稳定分类和公开消息，不包含 SQL、业务参数或秘密。
|
||||
|
||||
## 3. 注册、恢复与容量
|
||||
|
||||
连接后 5 秒内必须发送 worker.hello，hello 不携带 session_id。
|
||||
|
||||
| hello 字段 | 含义 |
|
||||
| --- | --- |
|
||||
| instance_id | Worker 进程实例 ID；进程重启必须换新值 |
|
||||
| instance_secret | 32 字节随机秘密的无填充 base64url 编码，共 43 字符；同进程重连保留 |
|
||||
| queue_id | 当前进程绑定队列 |
|
||||
| capabilities | 1–32 个 handler / handler_version / payload_version 精确版本组合，不可重复 |
|
||||
| capacity | 总执行槽数量，1–256，不是可用槽的累加值 |
|
||||
|
||||
重连必须保持凭据、队列、处理器集合和进程秘密，处理器集合的排列顺序不影响身份。新连接递增会话代次并使旧会话失效。知道 instance_id 或 task_id 不能接管执行。
|
||||
|
||||
worker.welcome 返回 session_id、epoch、server_time、expires_at、lease_duration_ms、draining。当前窗口为会话 45 秒、执行租约最多 30 秒；注册及能力声明不等于开始授权。
|
||||
|
||||
**派发在收到第一次 worker.capacity 确认后才启用**，便于客户端先恢复旧尝试。
|
||||
|
||||
| 类型 | body | 返回 |
|
||||
| --- | --- | --- |
|
||||
| worker.resume | AttemptRef，字段见上方 task.start 示例 | 当前 LeaseStatus；成功迁移有效尝试到新会话，不创建尝试或延长期限 |
|
||||
| worker.capacity | sequence 字符串、capacity 0–256、drain bool | 更新后的 welcome 字段 |
|
||||
| task.assign | 服务端主动发送 | task 完整任务、attempt_id、epoch、lease_secret、lease_until、server_time |
|
||||
|
||||
capacity 使用单调递增序号和覆盖值。同序号同内容可重复，同序号不同内容或旧序号冲突；capacity=0 不释放已有任务的并发许可。drain=true 是进程级持久化排空，重连保留，不能通过后续消息清除；新进程使用新实例身份。
|
||||
|
||||
一个 Worker 进程当前绑定一个队列。可以运行多个 Worker 进程或后续 SDK 实例消费不同队列；不能通过重连静默切换队列。
|
||||
|
||||
## 4. 开始、续租与结果
|
||||
|
||||
| 类型 | body | 语义 |
|
||||
| --- | --- | --- |
|
||||
| task.start | AttemptRef | 领取后再次检查当前会话、尝试、租约、队列暂停及流程门控，保存 RUNNING 后确认 |
|
||||
| task.renew | AttemptRef | 只续期当前 RUNNING；新消息可续租，重复原消息只取回原回执 |
|
||||
| task.finish | ref: AttemptRef、outcome、result、error_code | 保存明确结果或未知效果；返回操作回执和当前授权 |
|
||||
|
||||
outcome 为 SUCCEEDED / FAILED / UNKNOWN。result 为非 null JSON 或证据引用。FAILED/UNKNOWN 必须带 1–64 字符的稳定 error_code；SUCCEEDED 不带错误分类。当前 FAILED 成为任务失败终态，UNKNOWN 转 NEEDS_REVIEW；策略化自动重试在 P3 实现。
|
||||
|
||||
开始、续租和结束返回 CommandResult：
|
||||
|
||||
```json
|
||||
{
|
||||
"receipt":{
|
||||
"operation":"start",
|
||||
"task_state":"RUNNING",
|
||||
"attempt_state":"RUNNING",
|
||||
"task_revision":"3",
|
||||
"lease_until":"2026-09-17T08:00:30Z",
|
||||
"disposition":"APPLIED"
|
||||
},
|
||||
"lease":{
|
||||
"active":true,
|
||||
"state":"RUNNING",
|
||||
"lease_until":"2026-09-17T08:00:30Z",
|
||||
"server_time":"2026-09-17T08:00:00Z"
|
||||
},
|
||||
"deduplicated":false
|
||||
}
|
||||
```
|
||||
|
||||
receipt 是原操作的不可变确认；lease 是本次重新核验的执行权，两者不能混用：
|
||||
|
||||
- 只有 lease.active=true、lease.state=RUNNING，且仍在保守的本地有效期内，才允许处理器推进。LEASED 的 active 只表示预占有效。
|
||||
- 原 start 回执可能仍表示 RUNNING，但当前 lease.active=false；旧 ACK 不能让已过期、已结束或已被替换的尝试复活。
|
||||
- 有效期不超过租约窗口、会话期限、任务总截止和从 authorized_at 起计算的单次尝试时限。
|
||||
- 客户端需要扣除请求总耗时及安全余量，用本地单调时钟停止执行，不能假定本地墙上时钟与数据库一致。
|
||||
- 同结果重报不重复释放许可或产生成功事件。同消息不同内容返回 IDEMPOTENCY_CONFLICT。
|
||||
- 迟到、未开始或与已保存结果冲突的内容，经过身份和秘密验证后保存为 EVIDENCE_ONLY；不改变任务权威结果，不当作新的执行授权。
|
||||
- 断线后原结果仍保留稳定消息 ID，可在新会话恢复/核对后重报。不会因重连创建第二次业务执行。
|
||||
|
||||
## 5. 领取、并发与回收
|
||||
|
||||
平台在同一事务中分配队列并发许可、创建尝试、递增任务执行代次和次数、保存 LEASED 与派发事件。领取本身消耗一次 max_attempts 预算，包括最终未获开始授权的尝试，预算不会因重连或过期恢复重置。
|
||||
|
||||
运行中的进程总容量跨其所有旧/新会话统计。声明容量减少、断开连接或重复结果，都不能重复释放额度。当前已实施队列 max_concurrency；rate_per_second、其他配额和调度策略在 P3 完成。
|
||||
|
||||
每个节点每 2 秒发起一批恢复扫描，每批最多 100 项：
|
||||
|
||||
- 未批准开始即到期：有剩余次数和截止时间时，延后 1 秒重派；否则 FAILED。
|
||||
- 已批准开始后失联：NEEDS_REVIEW，尝试 UNKNOWN，不能因网络超时断定业务未生效。
|
||||
- WAITING 超过总截止或最晚开始时间：FAILED。
|
||||
- 回收再次锁定队列、任务、尝试并读取数据库时间，与续租/完成串行化；多个节点不能重复释放许可或重复终结事件。
|
||||
|
||||
此处的安全回收不是完整重试引擎；IDEMPOTENT 的自动重试和策略退避仍是 P3 工作。
|
||||
|
||||
## 6. 连接保护与当前边界
|
||||
|
||||
当前节点上限 128 条 Worker 连接；每连接入站队列 4 帧、出站队列 8 帧，派发保留 ACK 缓冲空间，每秒最多发起 4 次新派发尝试。读空闲期限 60 秒、写期限 5 秒、单个数据库控制操作期限 8 秒。慢消费者导致连接关闭时，已提交状态仍由稳定身份恢复。
|
||||
|
||||
这些是有界的开发基线，不代表经过容量压测的生产参数。当前同队列操作使用较粗的行锁协调，吞吐与公平性需要 P6 按负载验证。服务退出会显式关闭被 HTTP Upgrade 接管的连接；断开不会直接撤销外部已发出的请求。
|
||||
|
||||
运行中取消、SDK 自动重连/续租、非协作处理器处理、完整 WS JSON Schema、凭据轮换和多节点进程级故障演练尚未全部完成。实际证据见 [P2 服务端阶段记录](../reports/phase-2-server.md)。
|
||||
@@ -13,7 +13,7 @@
|
||||
## 已可运行：文件转换任务的受理与取消
|
||||
|
||||
- 源码：[SDK 任务受理示例](../../sdk/go/examples/tasks/main.go)。
|
||||
- 前置：完成迁移、初始化凭据、启动服务，详见 [SDK 接入](../sdk/README.md)。
|
||||
- 前置：完成迁移、初始化凭据、启动服务，在没有 Worker 消费的专用队列运行此受理/取消示例，详见 [SDK 接入](../sdk/README.md)。
|
||||
- 输入：对象引用、格式与精确整数；提交两次同一身份，再查询、分页并带版本取消。
|
||||
- 预期：重复请求返回同一任务，最终 CANCELLED/revision=2。当前不启动 Worker，不实际生成文件。
|
||||
- 异常：凭据失效、权限不足、内容冲突、版本竞争和未知网络结果，见 [任务接口](../api/tasks.md)。
|
||||
|
||||
57
docs/reports/phase-2-server.md
Normal file
57
docs/reports/phase-2-server.md
Normal file
@@ -0,0 +1,57 @@
|
||||
# P2 服务端执行与 WSS 阶段记录
|
||||
|
||||
日期：2026-09-17。**这是 P2 的阶段内进展，不是 P2 或生产验收完成。**
|
||||
|
||||
## 实现
|
||||
|
||||
新增 WorkerInstance 与 AttemptEvidence，当前共 32 个持久化实体。字段、模型职责、单位、可空/秘密/代次语义及 SQL COMMENT 均为中文。新增前向迁移 000003_worker_execution，旧迁移不改写。
|
||||
|
||||
服务端现有能力包括进程恢复秘密、会话头与连接代次、能力绑定、容量覆盖和排空、两步领取/开始、尝试秘密、续租、结果回执、迟到证据、跨节点恢复，以及到期扫描。队列并发许可随执行事务领取、续期和释放。
|
||||
|
||||
HTTP 进程已注册 Worker WebSocket 入口，并启动受控恢复扫描；使用有界网络队列、消息/连接上限和明确的关闭流程。默认 bootstrap 不扩权，新增 --worker-access 显式为开发凭据增加 Worker 权限。
|
||||
|
||||
## 实际回归
|
||||
|
||||
环境：macOS arm64，Go 1.27.1，Docker 隔离 MySQL 8.4.10。入口为 make test、make vet、make build、make integration；数据库临时创建并清理，不使用已有业务数据库。
|
||||
|
||||
| 场景 | 结果 |
|
||||
| --- | --- |
|
||||
| 新迁移与全部 Go 模型字段匹配、前序 P0/P1 回归 | 通过 |
|
||||
| 开始/续租/结果确认的原消息重报 | 原回执可查，不重复执行状态变更或延长旧续租 |
|
||||
| 多消息提交同一已保存结果 | 只产生一次结果事件与审计、只释放一次许可 |
|
||||
| 精确整数与小数结果 | 保存与查询不损失数值内容 |
|
||||
| 不同节点重连与恢复同一尝试 | 原子迁移会话；次数和许可不增加 |
|
||||
| 旧会话、错误尝试秘密、错误进程证明、注册内容改变 | 拒绝 |
|
||||
| 两个实例、两个节点、16 个并发领取请求 | 队列并发 2、进程容量各 1 均不被突破 |
|
||||
| 重连时旧会话已有执行 | 不重置容量计数 |
|
||||
| 排空、容量零、重复/冲突序号 | 不给新工作，不误释放已有许可，排空跨重连保留 |
|
||||
| 前 18 个任务门控暂停，后面有普通任务 | 普通任务仍能被领取，无候选页饥饿 |
|
||||
| 暂停与开始、已授权开始回执、结果保存 | 未开始被阻止；已开始的原回执和结果仍可保存 |
|
||||
| 未开始到期、已开始失联、两个恢复节点竞争 | 有限重派/FAILED/NEEDS_REVIEW 正确；无重复释放 |
|
||||
| 迟到成功、未开始提交结果、冲突结果 | EVIDENCE_ONLY，保留证据，不改写权威结果 |
|
||||
| 原 start ACK 在结束或过期后重报 | receipt 保留，当前 lease.active=false |
|
||||
| 单次绝对时限与会话过期 | 续租不能越过限制，过期会话不能靠心跳复活 |
|
||||
| 获取锁之前进入请求、锁等待期间租约过期 | 不使用早先时钟样本批准开始 |
|
||||
| 领取事件或完成审计 INSERT 被注入失败 | 任务、尝试、许可、回执、事件整体回滚 |
|
||||
| 凭据撤销、跨命名空间、已取消 context | 执行控制拒绝 |
|
||||
| WSS 真 TLS 握手，鉴权/子协议/Origin/URL 规则 | 通过；使用受信任测试证书，不跳过验证 |
|
||||
| WSS 节点 A 派发开始 → 断开 → 节点 B 恢复 → 结果确认重报 | 同一尝试、一次成功事件、许可正确释放 |
|
||||
| 重复 JSON 字段、服务关闭与已接管长连接 | 非法帧关闭；节点关闭无残留长连接 |
|
||||
|
||||
WSS 测试客户端直接发送协议帧，并显式重连到另一个测试服务器。当前没有把这项测试说成 SDK 自动重连或真实主机宕机演练。两个 ExecutionStore/Hub 使用不同节点 ID、同一个真实数据库；它们位于同一测试进程。
|
||||
|
||||
## 事务与恢复约束
|
||||
|
||||
每条命令重新校验当前凭据。执行路径先锁进程头/会话，再锁队列、可选门控、任务、尝试；回收从队列开始，不反向锁 Worker。拿到相关锁后重新读取数据库时间。状态、回执、额度和事件在短事务内提交，处理器不会在数据库事务中运行。
|
||||
|
||||
会话秘密、执行秘密仅存摘要，普通任务查询与日志不输出原文。保存迟到证据不构成执行授权。粗粒度队列锁是当前正确性基线，需要后续执行计划、压力与公平性验证；没有声称支持某个生产吞吐数字。
|
||||
|
||||
## 剩余工作
|
||||
|
||||
- Worker SDK 的处理器注册与运行循环、保守租约截止、自动续租、重连、结果重报、优雅退出。
|
||||
- API/SDK 执行历史与证据查询，WS JSON Schema 和兼容性验证。
|
||||
- 运行中取消、客户端慢消费及更多网络故障、结果未知/不协作处理器的端到端用例。
|
||||
- P3 的策略化自动重试、速率和其他配额、调度及结果订阅。
|
||||
- P4–P7 的流程、运维、性能和最终交付验证。
|
||||
|
||||
当前已实现的是可以经 WSS 操作的服务端执行内核。尚未完成独立 Worker 接入交付，也未完成生产高可用和容量验收。
|
||||
5
go.mod
5
go.mod
@@ -2,6 +2,9 @@ module git.yuntuo.hk/k2safe/orbit
|
||||
|
||||
go 1.27.1
|
||||
|
||||
require github.com/go-sql-driver/mysql v1.10.1
|
||||
require (
|
||||
github.com/coder/websocket v1.8.15
|
||||
github.com/go-sql-driver/mysql v1.10.1
|
||||
)
|
||||
|
||||
require filippo.io/edwards25519 v1.2.0 // indirect
|
||||
|
||||
2
go.sum
2
go.sum
@@ -1,4 +1,6 @@
|
||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
|
||||
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
|
||||
github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME=
|
||||
github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
|
||||
@@ -14,10 +14,11 @@ import (
|
||||
|
||||
// BootstrapInput 仅供持有数据库管理权限的初始化命令使用，不暴露为公共 HTTP。
|
||||
type BootstrapInput struct {
|
||||
Application string // Application 应用名称，每次初始化创建独立应用。
|
||||
Namespace string // Namespace 命名空间名称。
|
||||
Environment string // Environment 环境名称，必须显式指定。
|
||||
Queue string // Queue 初始队列名称。
|
||||
Application string // Application 应用名称，每次初始化创建独立应用。
|
||||
Namespace string // Namespace 命名空间名称。
|
||||
Environment string // Environment 环境名称，必须显式指定。
|
||||
Queue string // Queue 初始队列名称。
|
||||
WorkerAccess bool // WorkerAccess 显式增加开发用 Worker 接入及消费权限。
|
||||
}
|
||||
|
||||
// BootstrapResult 包含一次性交付的原始凭据，禁止记录到日志或提交到版本库。
|
||||
@@ -56,6 +57,9 @@ func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func(
|
||||
}
|
||||
out.Token = secret
|
||||
permissions := auth.Permissions{Actions: []string{"tasks:submit", "tasks:read", "tasks:cancel", "queues:read", "queues:manage"}, Queues: []string{"*"}}
|
||||
if in.WorkerAccess {
|
||||
permissions.Actions = append(permissions.Actions, "workers:connect", "workers:consume")
|
||||
}
|
||||
raw, err := json.Marshal(permissions)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -191,9 +191,11 @@ func checkModelColumns(t *testing.T, ctx context.Context, db *sql.DB) {
|
||||
pulse.Queue{},
|
||||
pulse.ExecutionGate{},
|
||||
pulse.WorkerSession{},
|
||||
pulse.WorkerInstance{},
|
||||
pulse.Task{},
|
||||
pulse.Attempt{},
|
||||
pulse.AttemptReceipt{},
|
||||
pulse.AttemptEvidence{},
|
||||
pulse.ConcurrencyLimit{},
|
||||
pulse.Permit{},
|
||||
pulse.Schedule{},
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
-- Worker 会话隔离和迟到结果证据。P2 之前没有受支持的 Worker 注册接口。
|
||||
ALTER TABLE pulse_worker_sessions ADD UNIQUE KEY uq_session_instance_id(namespace_id, worker_instance_id, id);
|
||||
|
||||
CREATE TABLE pulse_worker_instances (
|
||||
namespace_id BINARY(16) NOT NULL COMMENT '进程所属命名空间，参与授权和所有关联',
|
||||
id BINARY(16) NOT NULL COMMENT 'Worker 进程实例标识，进程重启必须换新值',
|
||||
revision BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '持久化变更版本，从一开始递增',
|
||||
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '首次注册时间，UTC 微秒精度',
|
||||
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '最近变更时间，不替代会话代次',
|
||||
credential_id BINARY(16) NOT NULL COMMENT '初始注册凭据，进程生命周期内固定',
|
||||
queue_id BINARY(16) NOT NULL COMMENT '绑定队列，切换队列需创建新进程实例',
|
||||
secret_digest BINARY(32) NOT NULL COMMENT '随机进程恢复秘密的 SHA-256 摘要，不保存原文',
|
||||
registration_digest BINARY(32) NOT NULL COMMENT '队列及处理器版本集合的语义摘要，重连必须匹配',
|
||||
session_epoch BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '最大已分配连接代次，注册前为零',
|
||||
current_session_id BINARY(16) NULL COMMENT '当前会话，注册事务建立会话前为空',
|
||||
draining BOOLEAN NOT NULL DEFAULT FALSE COMMENT '持久化排空标记，重连不能清除',
|
||||
PRIMARY KEY(namespace_id,id),
|
||||
UNIQUE KEY uq_worker_credential(namespace_id,id,credential_id),
|
||||
CONSTRAINT ck_worker_instance_revision CHECK (revision>0),
|
||||
CONSTRAINT ck_worker_instance_session CHECK ((session_epoch=0 AND current_session_id IS NULL) OR (session_epoch>0 AND current_session_id IS NOT NULL)),
|
||||
CONSTRAINT fk_worker_instance_credential FOREIGN KEY(namespace_id,credential_id) REFERENCES orbit_credentials(namespace_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT fk_worker_instance_queue FOREIGN KEY(namespace_id,queue_id) REFERENCES pulse_queues(namespace_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT fk_worker_instance_session FOREIGN KEY(namespace_id,id,current_session_id) REFERENCES pulse_worker_sessions(namespace_id,worker_instance_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='Worker 进程恢复身份和当前会话头，用于跨节点隔离旧连接';
|
||||
|
||||
ALTER TABLE pulse_worker_sessions ADD CONSTRAINT fk_worker_session_instance FOREIGN KEY(namespace_id,worker_instance_id,credential_id) REFERENCES pulse_worker_instances(namespace_id,id,credential_id) ON DELETE RESTRICT ON UPDATE RESTRICT;
|
||||
|
||||
ALTER TABLE pulse_worker_sessions ADD CONSTRAINT ck_worker_session_capacity CHECK (capacity<=256);
|
||||
|
||||
ALTER TABLE pulse_attempts ADD KEY ix_attempt_session_state(namespace_id,session_id,state);
|
||||
|
||||
ALTER TABLE pulse_attempts ADD CONSTRAINT ck_attempt_lifecycle CHECK ((state IN ('LEASED','RUNNING') AND ended_at IS NULL) OR (state NOT IN ('LEASED','RUNNING') AND ended_at IS NOT NULL));
|
||||
|
||||
ALTER TABLE pulse_attempts ADD CONSTRAINT ck_attempt_start CHECK ((state='LEASED' AND authorized_at IS NULL) OR (state='RUNNING' AND authorized_at IS NOT NULL) OR state NOT IN ('LEASED','RUNNING'));
|
||||
|
||||
ALTER TABLE pulse_tasks ADD KEY ix_task_deadline(state,deadline_at,namespace_id,id);
|
||||
|
||||
ALTER TABLE pulse_tasks ADD KEY ix_task_start_deadline(state,start_deadline_at,namespace_id,id);
|
||||
|
||||
CREATE TABLE pulse_attempt_evidence (
|
||||
namespace_id BINARY(16) NOT NULL COMMENT '证据所属命名空间，与尝试一致',
|
||||
id BINARY(16) NOT NULL COMMENT '证据记录标识',
|
||||
revision BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '记录版本；证据追加保存，不覆盖',
|
||||
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '首次收到证据的数据库时间，UTC',
|
||||
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '写入时间，追加记录与创建时间相同',
|
||||
attempt_id BINARY(16) NOT NULL COMMENT '证据所属执行尝试',
|
||||
message_id BINARY(16) NOT NULL COMMENT '结果消息身份，重报保持不变',
|
||||
outcome VARCHAR(16) NOT NULL COMMENT 'Worker 声明的结果：SUCCEEDED、FAILED 或 UNKNOWN',
|
||||
error_code VARCHAR(64) NULL COMMENT '稳定错误分类；无错误时为空，不保存原始异常正文',
|
||||
payload MEDIUMTEXT NOT NULL COMMENT '结果或证据引用的 JSON 文本，保留精确数字',
|
||||
request_digest BINARY(32) NOT NULL COMMENT '结果语义摘要，排除会话与秘密',
|
||||
reason VARCHAR(32) NOT NULL COMMENT '证据未改写权威结果的原因：LEASE_LOST、NOT_STARTED 或 RESULT_CONFLICT',
|
||||
PRIMARY KEY(namespace_id,id),
|
||||
UNIQUE KEY uq_attempt_evidence_message(namespace_id,attempt_id,message_id),
|
||||
CONSTRAINT ck_attempt_evidence_revision CHECK(revision>0),
|
||||
CONSTRAINT ck_attempt_evidence_json CHECK(JSON_VALID(payload)),
|
||||
CONSTRAINT ck_attempt_evidence_outcome CHECK(outcome IN ('SUCCEEDED','FAILED','UNKNOWN')),
|
||||
CONSTRAINT ck_attempt_evidence_reason CHECK(reason IN ('LEASE_LOST','NOT_STARTED','RESULT_CONFLICT')),
|
||||
CONSTRAINT fk_attempt_evidence_attempt FOREIGN KEY(namespace_id,attempt_id) REFERENCES pulse_attempts(namespace_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='迟到与冲突结果的追加证据，不自动改变任务终态';
|
||||
|
||||
CREATE TRIGGER immutable_attempt_evidence BEFORE UPDATE ON pulse_attempt_evidence FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='attempt evidence is immutable';
|
||||
521
internal/storage/mysql/worker_execution.go
Normal file
521
internal/storage/mysql/worker_execution.go
Normal file
@@ -0,0 +1,521 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
type executionTask struct {
|
||||
view service.TaskView
|
||||
current *core.ID
|
||||
epoch uint64
|
||||
lease *time.Time
|
||||
gate *core.ID
|
||||
maxAttempts uint32
|
||||
attemptTimeout uint64
|
||||
}
|
||||
type executionRows struct {
|
||||
task executionTask
|
||||
attempt pulse.Attempt
|
||||
paused bool
|
||||
session sessionState
|
||||
}
|
||||
|
||||
func readExecutionTask(ctx context.Context, tx *sql.Tx, ns, id core.ID) (executionTask, error) {
|
||||
var row executionTask
|
||||
view, err := readTask(ctx, tx, ns, id, true)
|
||||
if err != nil {
|
||||
return row, err
|
||||
}
|
||||
row.view = view
|
||||
err = tx.QueryRowContext(ctx, `SELECT current_attempt_id,lease_epoch,lease_until,gate_id,max_attempts,attempt_timeout_ms FROM pulse_tasks WHERE namespace_id=? AND id=?`, ns, id).Scan(&row.current, &row.epoch, &row.lease, &row.gate, &row.maxAttempts, &row.attemptTimeout)
|
||||
return row, err
|
||||
}
|
||||
func lockQueue(ctx context.Context, tx *sql.Tx, ns, id core.ID) (paused bool, max uint32, err error) {
|
||||
err = tx.QueryRowContext(ctx, "SELECT paused,max_concurrency FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE", ns, id).Scan(&paused, &max)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
err = fault.NotFound
|
||||
}
|
||||
return
|
||||
}
|
||||
func lockGate(ctx context.Context, tx *sql.Tx, ns core.ID, id *core.ID) (bool, error) {
|
||||
if id == nil {
|
||||
return false, nil
|
||||
}
|
||||
var paused bool
|
||||
err := tx.QueryRowContext(ctx, "SELECT paused FROM pulse_execution_gates WHERE namespace_id=? AND id=? FOR SHARE", ns, *id).Scan(&paused)
|
||||
return paused, err
|
||||
}
|
||||
func minimumTime(first time.Time, others ...time.Time) time.Time {
|
||||
for _, other := range others {
|
||||
if other.Before(first) {
|
||||
first = other
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
func queueLimit(ctx context.Context, tx *sql.Tx, ns, queue core.ID, max uint32) (core.ID, uint32, error) {
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
digest := core.Digest(sha256.Sum256([]byte(queue)))
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_concurrency_limits(namespace_id,id,scope_key,key_digest,max_permits) VALUES (?,?,'queue',?,?) ON DUPLICATE KEY UPDATE id=pulse_concurrency_limits.id`, ns, id, digest, max)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
var used uint32
|
||||
var configured uint32
|
||||
err = tx.QueryRowContext(ctx, `SELECT id,used_permits,max_permits FROM pulse_concurrency_limits WHERE namespace_id=? AND scope_key='queue' AND key_digest=? FOR UPDATE`, ns, digest).Scan(&id, &used, &configured)
|
||||
if err == nil && configured != max {
|
||||
err = errors.New("queue limit configuration requires coordinated update")
|
||||
}
|
||||
return id, used, err
|
||||
}
|
||||
func outstanding(ctx context.Context, tx *sql.Tx, ns, instance core.ID) (uint32, error) {
|
||||
var count uint32
|
||||
err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_attempts a JOIN pulse_worker_sessions s ON s.namespace_id=a.namespace_id AND s.id=a.session_id WHERE a.namespace_id=? AND s.worker_instance_id=? AND a.state IN ('LEASED','RUNNING')`, ns, instance).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.Binding) (*worker.Assignment, error) {
|
||||
var assignment *worker.Assignment
|
||||
err := s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
state, err := s.lockSession(ctx, tx, token, b, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if state.draining || state.capacity == 0 {
|
||||
return nil
|
||||
}
|
||||
count, err := outstanding(ctx, tx, token.NamespaceID, b.InstanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if count >= state.capacity {
|
||||
return nil
|
||||
}
|
||||
paused, max, err := lockQueue(ctx, tx, token.NamespaceID, state.queue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if paused {
|
||||
return nil
|
||||
}
|
||||
limit, used, err := queueLimit(ctx, tx, token.NamespaceID, state.queue, max)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if used >= max {
|
||||
return nil
|
||||
}
|
||||
args := []any{token.NamespaceID, state.queue}
|
||||
matches := make([]string, 0, len(state.capabilities))
|
||||
for _, c := range state.capabilities {
|
||||
matches = append(matches, "(handler=? AND handler_version=? AND payload_version=?)")
|
||||
args = append(args, c.Handler, c.HandlerVersion, c.PayloadVersion)
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, `SELECT id,gate_id FROM pulse_tasks WHERE namespace_id=? AND queue_id=? AND state='WAITING' AND available_at<=UTC_TIMESTAMP(6) AND deadline_at>UTC_TIMESTAMP(6) AND (start_deadline_at IS NULL OR start_deadline_at>UTC_TIMESTAMP(6)) AND attempt_count<max_attempts AND (gate_id IS NULL OR EXISTS (SELECT 1 FROM pulse_execution_gates g WHERE g.namespace_id=pulse_tasks.namespace_id AND g.id=pulse_tasks.gate_id AND g.paused=FALSE)) AND (`+strings.Join(matches, " OR ")+`) ORDER BY available_at,priority DESC,id LIMIT 16`, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type candidate struct {
|
||||
id core.ID
|
||||
gate *core.ID
|
||||
}
|
||||
candidates := []candidate{}
|
||||
for rows.Next() {
|
||||
var c candidate
|
||||
if err := rows.Scan(&c.id, &c.gate); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
candidates = append(candidates, c)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
gated, err := lockGate(ctx, tx, token.NamespaceID, candidate.gate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if gated {
|
||||
continue
|
||||
}
|
||||
var locked core.ID
|
||||
err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_tasks WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, token.NamespaceID, candidate.id).Scan(&locked)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task, err := readExecutionTask(ctx, tx, token.NamespaceID, locked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !state.expires.After(now) {
|
||||
return worker.StaleSession
|
||||
}
|
||||
if task.view.State != pulse.TaskWaiting || task.view.AvailableAt.After(now) || !task.view.DeadlineAt.After(now) || task.view.AttemptCount >= task.maxAttempts || (task.view.StartDeadlineAt != nil && !task.view.StartDeadlineAt.After(now)) {
|
||||
continue
|
||||
}
|
||||
if task.epoch == ^uint64(0) {
|
||||
return fault.Conflict
|
||||
}
|
||||
attempt, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
secret, digest, err := worker.NewSecret()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
until := minimumTime(now.Add(s.options.LeaseDuration), task.view.DeadlineAt, state.expires)
|
||||
if task.view.StartDeadlineAt != nil {
|
||||
until = minimumTime(until, *task.view.StartDeadlineAt)
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempts(namespace_id,id,task_id,epoch,session_id,token_digest,lease_until) VALUES (?,?,?,?,?,?,?)`, token.NamespaceID, attempt, locked, task.epoch+1, b.SessionID, digest, until)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
permit, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_permits(namespace_id,id,limit_id,attempt_id,lease_until) VALUES (?,?,?,?,?)`, token.NamespaceID, permit, limit, attempt, until)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err := tx.ExecContext(ctx, `UPDATE pulse_concurrency_limits SET used_permits=used_permits+1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND used_permits<max_permits`, now, token.NamespaceID, limit)
|
||||
if err := exactlyOne(changed, err); err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='LEASED',current_attempt_id=?,lease_epoch=lease_epoch+1,lease_until=?,attempt_count=attempt_count+1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND state='WAITING' AND revision=?`, attempt, until, now, token.NamespaceID, locked, task.view.Revision)
|
||||
if err := exactlyOne(changed, err); err != nil {
|
||||
return err
|
||||
}
|
||||
view, err := readTask(ctx, tx, token.NamespaceID, locked, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, token.NamespaceID, locked, view.Revision, "task", "task.leased", map[string]any{"attempt_id": attempt, "epoch": fmt.Sprint(task.epoch + 1), "session_id": b.SessionID, "lease_until": until}); err != nil {
|
||||
return err
|
||||
}
|
||||
assignment = &worker.Assignment{Task: view, AttemptID: attempt, Epoch: task.epoch + 1, LeaseSecret: secret, LeaseUntil: until, ServerTime: now}
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return assignment, nil
|
||||
}
|
||||
func exactlyOne(result sql.Result, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != 1 {
|
||||
return errors.New("execution invariant affected unexpected rows")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// lockExecution 的跨资源顺序为进程/会话 -> 队列 -> 可选门控 -> 任务 -> 尝试。
|
||||
// 恢复扫描也从队列锁开始，不反向锁定 Worker 进程。
|
||||
func (s *ExecutionStore) lockExecution(ctx context.Context, tx *sql.Tx, token auth.Token, b worker.Binding, p auth.Permissions, ref worker.AttemptRef, checkGate bool) (executionRows, error) {
|
||||
var out executionRows
|
||||
state, err := s.lockSession(ctx, tx, token, b, p)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.session = state
|
||||
var queue core.ID
|
||||
var gate *core.ID
|
||||
err = tx.QueryRowContext(ctx, `SELECT queue_id,gate_id FROM pulse_tasks WHERE namespace_id=? AND id=?`, token.NamespaceID, ref.TaskID).Scan(&queue, &gate)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return out, fault.NotFound
|
||||
}
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if queue != state.queue || !p.AllowsQueue(queue) {
|
||||
return out, fault.NotFound
|
||||
}
|
||||
out.paused, _, err = lockQueue(ctx, tx, token.NamespaceID, queue)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if checkGate {
|
||||
paused, err := lockGate(ctx, tx, token.NamespaceID, gate)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.paused = out.paused || paused
|
||||
}
|
||||
out.task, err = readExecutionTask(ctx, tx, token.NamespaceID, ref.TaskID)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.attempt, err = readAttempt(ctx, tx, token.NamespaceID, ref.AttemptID)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
proof, err := worker.SecretDigest(ref.LeaseSecret)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if out.attempt.TaskID != ref.TaskID || out.attempt.Epoch != ref.Epoch || subtle.ConstantTimeCompare(proof[:], out.attempt.TokenDigest[:]) != 1 {
|
||||
return out, fault.Forbidden
|
||||
}
|
||||
var instance core.ID
|
||||
if err := tx.QueryRowContext(ctx, `SELECT worker_instance_id FROM pulse_worker_sessions WHERE namespace_id=? AND id=?`, token.NamespaceID, out.attempt.SessionID).Scan(&instance); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if instance != b.InstanceID {
|
||||
return out, fault.Forbidden
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func readAttempt(ctx context.Context, tx *sql.Tx, ns, id core.ID) (pulse.Attempt, error) {
|
||||
var a pulse.Attempt
|
||||
a.NamespaceID = ns
|
||||
err := tx.QueryRowContext(ctx, `SELECT id,revision,task_id,epoch,session_id,token_digest,state,lease_until,authorized_at,ended_at,error_code,result,result_digest FROM pulse_attempts WHERE namespace_id=? AND id=? FOR UPDATE`, ns, id).Scan(&a.ID, &a.Revision, &a.TaskID, &a.Epoch, &a.SessionID, &a.TokenDigest, &a.State, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Result, &a.ResultDigest)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
err = fault.NotFound
|
||||
}
|
||||
return a, err
|
||||
}
|
||||
func leaseStatus(rows executionRows, b worker.Binding, now time.Time) worker.LeaseStatus {
|
||||
t, a := rows.task, rows.attempt
|
||||
until := minimumTime(a.LeaseUntil, rows.session.expires, t.view.DeadlineAt)
|
||||
if t.lease != nil {
|
||||
until = minimumTime(until, *t.lease)
|
||||
}
|
||||
active := rows.session.expires.After(now) && t.current != nil && *t.current == a.ID && t.epoch == a.Epoch && a.SessionID == b.SessionID && t.lease != nil && t.lease.After(now) && a.LeaseUntil.After(now) && t.view.DeadlineAt.After(now) && ((t.view.State == pulse.TaskLeased && a.State == pulse.AttemptLeased) || (t.view.State == pulse.TaskRunning && a.State == pulse.AttemptRunning))
|
||||
if a.AuthorizedAt != nil {
|
||||
deadline := a.AuthorizedAt.Add(time.Duration(t.attemptTimeout) * time.Millisecond)
|
||||
active = active && deadline.After(now)
|
||||
until = minimumTime(until, deadline)
|
||||
}
|
||||
return worker.LeaseStatus{Active: active, State: a.State, LeaseUntil: until, ServerTime: now}
|
||||
}
|
||||
func attemptIdentity(ref worker.AttemptRef) any {
|
||||
return struct {
|
||||
Task core.ID
|
||||
Attempt core.ID
|
||||
Epoch string
|
||||
}{ref.TaskID, ref.AttemptID, fmt.Sprint(ref.Epoch)}
|
||||
}
|
||||
func readReceipt(ctx context.Context, tx *sql.Tx, ns, attempt, message core.ID, operation string, digest core.Digest) (*worker.Receipt, error) {
|
||||
var saved core.Digest
|
||||
var raw []byte
|
||||
err := tx.QueryRowContext(ctx, `SELECT request_digest,response FROM pulse_attempt_receipts WHERE namespace_id=? AND attempt_id=? AND operation=? AND message_id=? FOR UPDATE`, ns, attempt, operation, message).Scan(&saved, &raw)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if saved != digest {
|
||||
return nil, fault.IdempotencyConflict
|
||||
}
|
||||
var receipt worker.Receipt
|
||||
if err := json.Unmarshal(raw, &receipt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &receipt, nil
|
||||
}
|
||||
func saveReceipt(ctx context.Context, tx *sql.Tx, ns, attempt, message core.ID, operation string, digest core.Digest, receipt worker.Receipt) error {
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := json.Marshal(receipt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempt_receipts(namespace_id,id,attempt_id,operation,message_id,request_digest,response,expires_at) VALUES (?,?,?,?,?,?,?,DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 90 DAY))`, ns, id, attempt, operation, message, digest, string(raw))
|
||||
return err
|
||||
}
|
||||
func renewPermits(ctx context.Context, tx *sql.Tx, ns, attempt core.ID, until, now time.Time) error {
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_permits SET lease_until=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND attempt_id=? AND released_at IS NULL`, until, now, ns, attempt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n < 1 {
|
||||
return errors.New("active attempt has no concurrency permit")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func releasePermits(ctx context.Context, tx *sql.Tx, ns, attempt core.ID, now time.Time) error {
|
||||
rows, err := tx.QueryContext(ctx, `SELECT id,limit_id FROM pulse_permits WHERE namespace_id=? AND attempt_id=? AND released_at IS NULL ORDER BY limit_id FOR UPDATE`, ns, attempt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type permit struct{ id, limit core.ID }
|
||||
permits := []permit{}
|
||||
for rows.Next() {
|
||||
var p permit
|
||||
if err := rows.Scan(&p.id, &p.limit); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
permits = append(permits, p)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(permits) == 0 {
|
||||
return errors.New("active attempt has no permit to release")
|
||||
}
|
||||
for _, p := range permits {
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_concurrency_limits SET used_permits=used_permits-1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND used_permits>0`, now, ns, p.limit)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err = tx.ExecContext(ctx, `UPDATE pulse_permits SET released_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND released_at IS NULL`, now, now, ns, p.id)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *ExecutionStore) Start(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, ref worker.AttemptRef) (worker.CommandResult, error) {
|
||||
return s.executionCommand(ctx, token, b, message, ref, "start")
|
||||
}
|
||||
func (s *ExecutionStore) Renew(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, ref worker.AttemptRef) (worker.CommandResult, error) {
|
||||
return s.executionCommand(ctx, token, b, message, ref, "renew")
|
||||
}
|
||||
func (s *ExecutionStore) executionCommand(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, ref worker.AttemptRef, operation string) (worker.CommandResult, error) {
|
||||
var out worker.CommandResult
|
||||
digest, err := requestDigest(attemptIdentity(ref))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
err = s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
rows, err := s.lockExecution(ctx, tx, token, b, p, ref, operation == "start")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
receipt, err := readReceipt(ctx, tx, token.NamespaceID, ref.AttemptID, message, operation, digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status := leaseStatus(rows, b, now)
|
||||
if receipt != nil {
|
||||
out = worker.CommandResult{Receipt: *receipt, Lease: status, Deduplicated: true}
|
||||
return nil
|
||||
}
|
||||
if !status.Active {
|
||||
return worker.LeaseLost
|
||||
}
|
||||
task, a := rows.task, rows.attempt
|
||||
authorized := a.AuthorizedAt
|
||||
if operation == "start" {
|
||||
if a.State != pulse.AttemptLeased || task.view.State != pulse.TaskLeased {
|
||||
return fault.Conflict
|
||||
}
|
||||
if rows.paused {
|
||||
return worker.GateClosed
|
||||
}
|
||||
if task.view.StartDeadlineAt != nil && !task.view.StartDeadlineAt.After(now) {
|
||||
return worker.LeaseLost
|
||||
}
|
||||
authorized = &now
|
||||
} else if a.State != pulse.AttemptRunning || authorized == nil {
|
||||
return fault.Conflict
|
||||
}
|
||||
until := minimumTime(now.Add(s.options.LeaseDuration), authorized.Add(time.Duration(task.attemptTimeout)*time.Millisecond), task.view.DeadlineAt, rows.session.expires)
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state='RUNNING',authorized_at=?,lease_until=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, authorized, until, now, token.NamespaceID, a.ID, a.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='RUNNING',lease_until=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, until, now, token.NamespaceID, ref.TaskID, task.view.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := renewPermits(ctx, tx, token.NamespaceID, a.ID, until, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if operation == "start" {
|
||||
if err := emitEvent(ctx, tx, token.NamespaceID, ref.TaskID, task.view.Revision+1, "task", "task.started", map[string]any{"attempt_id": a.ID, "epoch": fmt.Sprint(a.Epoch), "session_id": b.SessionID}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
receipt = &worker.Receipt{Operation: operation, TaskState: pulse.TaskRunning, AttemptState: pulse.AttemptRunning, TaskRevision: task.view.Revision + 1, LeaseUntil: until, Disposition: "APPLIED"}
|
||||
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, operation, digest, *receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
out = worker.CommandResult{Receipt: *receipt, Lease: worker.LeaseStatus{Active: true, State: pulse.AttemptRunning, LeaseUntil: until, ServerTime: now}}
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (s *ExecutionStore) Resume(ctx context.Context, token auth.Token, b worker.Binding, ref worker.AttemptRef) (worker.LeaseStatus, error) {
|
||||
var out worker.LeaseStatus
|
||||
err := s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
rows, err := s.lockExecution(ctx, tx, token, b, p, ref, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 只暂按新会话计算有效性；通过所有期限和当前尝试检查后才迁移归属，绝不延长期限。
|
||||
previous := rows.attempt.SessionID
|
||||
rows.attempt.SessionID = b.SessionID
|
||||
out = leaseStatus(rows, b, now)
|
||||
if !out.Active || previous == b.SessionID {
|
||||
return nil
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET session_id=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, b.SessionID, now, token.NamespaceID, ref.AttemptID, rows.attempt.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
return emitEvent(ctx, tx, token.NamespaceID, ref.AttemptID, rows.attempt.Revision+1, "attempt", "attempt.resumed", map[string]any{"from_session_id": previous, "to_session_id": b.SessionID, "node_id": s.node})
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
638
internal/storage/mysql/worker_integration_test.go
Normal file
638
internal/storage/mysql/worker_integration_test.go
Normal file
@@ -0,0 +1,638 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
type executionFixture struct {
|
||||
t *testing.T
|
||||
ctx context.Context
|
||||
db *sql.DB
|
||||
resources BootstrapResult
|
||||
token auth.Token
|
||||
tasks *service.Service
|
||||
storeA, storeB *ExecutionStore
|
||||
a, b *worker.Service
|
||||
hello worker.ConnectInput
|
||||
binding worker.Binding
|
||||
}
|
||||
|
||||
func executionID(t *testing.T) core.ID {
|
||||
t.Helper()
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
func newExecutionFixture(t *testing.T, ctx context.Context, db *sql.DB) *executionFixture {
|
||||
t.Helper()
|
||||
f := &executionFixture{t: t, ctx: ctx, db: db, tasks: service.New(NewTaskStore(db))}
|
||||
err := Bootstrap(ctx, db, BootstrapInput{Application: "worker-" + string(executionID(t)), Namespace: "default", Environment: "test", Queue: "work"}, func(v BootstrapResult) error { f.resources = v; return nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(auth.Permissions{Actions: []string{"tasks:submit", "tasks:read", "tasks:cancel", "queues:read", "queues:manage", "workers:connect", "workers:consume"}, Queues: []string{"*"}})
|
||||
f.exec("UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?", string(raw), f.resources.NamespaceID, f.resources.CredentialID)
|
||||
f.token, err = auth.Parse(f.resources.Token)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.storeA, err = NewExecutionStore(db, executionID(t), ExecutionOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.storeB, err = NewExecutionStore(db, executionID(t), ExecutionOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.a, f.b = worker.New(f.storeA), worker.New(f.storeB)
|
||||
secret, _, err := worker.NewSecret()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.hello = worker.ConnectInput{InstanceID: executionID(t), InstanceSecret: secret, QueueID: f.resources.QueueID, Capacity: 4, Capabilities: []worker.Capability{{Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1"}}}
|
||||
f.binding, _, err = f.a.Connect(ctx, f.token, f.hello)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
func (f *executionFixture) exec(q string, args ...any) {
|
||||
f.t.Helper()
|
||||
if _, err := f.db.ExecContext(f.ctx, q, args...); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
}
|
||||
func (f *executionFixture) submit(max uint32) service.TaskView {
|
||||
f.t.Helper()
|
||||
out, err := f.tasks.Submit(f.ctx, f.token, string(executionID(f.t)), service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`), Retry: pulse.RetryPolicy{MaxAttempts: max, AttemptTimeoutMS: 120000}})
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return out.Task
|
||||
}
|
||||
func (f *executionFixture) claim() (*worker.Assignment, worker.AttemptRef) {
|
||||
f.t.Helper()
|
||||
out, err := f.a.Claim(f.ctx, f.token, f.binding)
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
if out == nil {
|
||||
f.t.Fatal("expected assignment")
|
||||
}
|
||||
return out, worker.AttemptRef{TaskID: out.Task.ID, AttemptID: out.AttemptID, Epoch: out.Epoch, LeaseSecret: out.LeaseSecret}
|
||||
}
|
||||
func (f *executionFixture) get(id core.ID) service.TaskView {
|
||||
f.t.Helper()
|
||||
out, err := f.tasks.GetTask(f.ctx, f.token, id)
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func (f *executionFixture) expire(ref worker.AttemptRef) {
|
||||
f.t.Helper()
|
||||
f.exec("UPDATE pulse_tasks SET lease_until=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, ref.TaskID)
|
||||
f.exec("UPDATE pulse_attempts SET lease_until=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, ref.AttemptID)
|
||||
}
|
||||
func (f *executionFixture) count(table, where string, args ...any) int {
|
||||
f.t.Helper()
|
||||
var n int
|
||||
if err := f.db.QueryRowContext(f.ctx, "SELECT COUNT(*) FROM "+table+" WHERE namespace_id=? AND "+where, append([]any{f.resources.NamespaceID}, args...)...).Scan(&n); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
func (f *executionFixture) used() uint32 {
|
||||
f.t.Helper()
|
||||
var n uint32
|
||||
if err := f.db.QueryRowContext(f.ctx, "SELECT COALESCE(SUM(used_permits),0) FROM pulse_concurrency_limits WHERE namespace_id=?", f.resources.NamespaceID).Scan(&n); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
func expectWorkerCode(t *testing.T, err error, code string) {
|
||||
t.Helper()
|
||||
if fault.Code(err) != code {
|
||||
t.Fatalf("wanted %s got %v", code, err)
|
||||
}
|
||||
}
|
||||
func TestIntegrationWorkerExecution(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
db, err := Open(ctx, dsn, 32, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Run("durable_start_renew_and_exact_result_confirmation", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := f.submit(3)
|
||||
assignment, ref := f.claim()
|
||||
if assignment.Task.ID != task.ID || assignment.Task.State != pulse.TaskLeased || assignment.Task.AttemptCount != 1 || f.used() != 1 {
|
||||
t.Fatal("claim state/permit invariant")
|
||||
}
|
||||
startID := executionID(t)
|
||||
started, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
|
||||
if err != nil || !started.Lease.Active || started.Receipt.TaskState != pulse.TaskRunning {
|
||||
t.Fatalf("start: %v", err)
|
||||
}
|
||||
duplicate, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
|
||||
if err != nil || !duplicate.Deduplicated || duplicate.Receipt != started.Receipt {
|
||||
t.Fatalf("start replay: %v", err)
|
||||
}
|
||||
_, err = f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
renewID := executionID(t)
|
||||
renewed, err := f.a.Renew(ctx, f.token, f.binding, renewID, ref)
|
||||
if err != nil || !renewed.Lease.Active || renewed.Lease.LeaseUntil.Before(started.Lease.LeaseUntil) {
|
||||
t.Fatalf("renew: %v", err)
|
||||
}
|
||||
repeated, err := f.a.Renew(ctx, f.token, f.binding, renewID, ref)
|
||||
if err != nil || !repeated.Deduplicated || repeated.Receipt.LeaseUntil != renewed.Receipt.LeaseUntil {
|
||||
t.Fatalf("renew replay: %v", err)
|
||||
}
|
||||
result := worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"amount":12345678901234567890.1234567890123456789,"version":9007199254740993}`)}
|
||||
message := executionID(t)
|
||||
finished, err := f.a.Finish(ctx, f.token, f.binding, message, result)
|
||||
if err != nil || finished.Lease.Active || finished.Receipt.Disposition != "APPLIED" {
|
||||
t.Fatalf("finish: %v", err)
|
||||
}
|
||||
duplicate, err = f.a.Finish(ctx, f.token, f.binding, message, result)
|
||||
if err != nil || !duplicate.Deduplicated {
|
||||
t.Fatalf("finish replay %v", err)
|
||||
}
|
||||
duplicate, err = f.a.Finish(ctx, f.token, f.binding, executionID(t), result)
|
||||
if err != nil || !duplicate.Deduplicated {
|
||||
t.Fatalf("new message equivalent result %v", err)
|
||||
}
|
||||
current := f.get(task.ID)
|
||||
if current.State != pulse.TaskSucceeded || current.CompletedAt == nil || !strings.Contains(string(current.Result), "12345678901234567890.1234567890123456789") || f.used() != 0 {
|
||||
t.Fatal("finish lost state, precision or permit")
|
||||
}
|
||||
if f.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 1 || f.count("orbit_audit_events", "resource_id=? AND action='task.result'", task.ID) != 1 {
|
||||
t.Fatal("duplicated completion history")
|
||||
}
|
||||
_, err = f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
||||
expectWorkerCode(t, err, "LEASE_LOST")
|
||||
oldAck, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
|
||||
if err != nil || !oldAck.Deduplicated || oldAck.Lease.Active {
|
||||
t.Fatalf("old start acknowledgement resurrected execution %v", err)
|
||||
}
|
||||
changed := result
|
||||
changed.Result = json.RawMessage(`{"changed":true}`)
|
||||
_, err = f.a.Finish(ctx, f.token, f.binding, message, changed)
|
||||
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
|
||||
conflict, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), changed)
|
||||
if err != nil || conflict.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(task.ID).State != pulse.TaskSucceeded {
|
||||
t.Fatalf("conflict rewrote outcome %v", err)
|
||||
}
|
||||
if f.count("pulse_attempt_evidence", "attempt_id=? AND reason='RESULT_CONFLICT'", ref.AttemptID) != 1 {
|
||||
t.Fatal("conflict evidence missing")
|
||||
}
|
||||
})
|
||||
t.Run("cross_node_resume_fences_old_session_and_requires_secrets", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
f.submit(3)
|
||||
_, ref := f.claim()
|
||||
start := executionID(t)
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, start, ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := f.get(ref.TaskID)
|
||||
newBinding, welcome, err := f.b.Connect(ctx, f.token, f.hello)
|
||||
if err != nil || welcome.Epoch != 2 {
|
||||
t.Fatalf("reconnect %v", err)
|
||||
}
|
||||
_, err = f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
||||
expectWorkerCode(t, err, "STALE_SESSION")
|
||||
_, err = f.b.Renew(ctx, f.token, newBinding, executionID(t), ref)
|
||||
expectWorkerCode(t, err, "LEASE_LOST")
|
||||
wrong := ref
|
||||
wrong.LeaseSecret = strings.Repeat("A", 43)
|
||||
_, err = f.b.Resume(ctx, f.token, newBinding, wrong)
|
||||
expectWorkerCode(t, err, "PERMISSION_DENIED")
|
||||
resumed, err := f.b.Resume(ctx, f.token, newBinding, ref)
|
||||
if err != nil || !resumed.Active || resumed.State != pulse.AttemptRunning {
|
||||
t.Fatalf("resume %v", err)
|
||||
}
|
||||
duplicate, err := f.b.Start(ctx, f.token, newBinding, start, ref)
|
||||
if err != nil || !duplicate.Deduplicated || !duplicate.Lease.Active {
|
||||
t.Fatalf("start ack recovery %v", err)
|
||||
}
|
||||
if f.get(ref.TaskID).AttemptCount != before.AttemptCount || f.used() != 1 {
|
||||
t.Fatal("resume created attempt or duplicated permit")
|
||||
}
|
||||
if _, err := f.b.Resume(ctx, f.token, newBinding, ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.count("orbit_events", "aggregate_id=? AND type='attempt.resumed'", ref.AttemptID) != 1 {
|
||||
t.Fatal("resume event duplicated")
|
||||
}
|
||||
badHello := f.hello
|
||||
badHello.InstanceSecret = strings.Repeat("A", 43)
|
||||
_, _, err = f.b.Connect(ctx, f.token, badHello)
|
||||
expectWorkerCode(t, err, "PERMISSION_DENIED")
|
||||
badHello = f.hello
|
||||
badHello.Capabilities = []worker.Capability{{Handler: "another", HandlerVersion: "v1", PayloadVersion: "v1"}}
|
||||
_, _, err = f.b.Connect(ctx, f.token, badHello)
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
f.expire(ref)
|
||||
expired, err := f.b.Resume(ctx, f.token, newBinding, ref)
|
||||
if err != nil || expired.Active {
|
||||
t.Fatalf("expired resume %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("multi_node_claim_obeys_queue_and_process_capacity", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
f.exec("UPDATE pulse_queues SET max_concurrency=2 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.QueueID)
|
||||
for i := 0; i < 8; i++ {
|
||||
f.submit(3)
|
||||
}
|
||||
secret, _, err := worker.NewSecret()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hello := f.hello
|
||||
hello.InstanceID = executionID(t)
|
||||
hello.InstanceSecret = secret
|
||||
hello.Capacity = 1
|
||||
bindingB, _, err := f.b.Connect(ctx, f.token, hello)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.a.Capacity(ctx, f.token, f.binding, worker.CapacityInput{Sequence: 1, Capacity: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assignments := make(chan *worker.Assignment, 16)
|
||||
errs := make(chan error, 16)
|
||||
var wg sync.WaitGroup
|
||||
start := make(chan struct{})
|
||||
for i := 0; i < 16; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
var a *worker.Assignment
|
||||
var err error
|
||||
if i%2 == 0 {
|
||||
a, err = f.a.Claim(ctx, f.token, f.binding)
|
||||
} else {
|
||||
a, err = f.b.Claim(ctx, f.token, bindingB)
|
||||
}
|
||||
assignments <- a
|
||||
errs <- err
|
||||
}(i)
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
close(assignments)
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ids := map[core.ID]bool{}
|
||||
for a := range assignments {
|
||||
if a == nil {
|
||||
continue
|
||||
}
|
||||
if ids[a.Task.ID] {
|
||||
t.Fatal("task dispatched twice")
|
||||
}
|
||||
ids[a.Task.ID] = true
|
||||
}
|
||||
if len(ids) != 2 || f.used() != 2 {
|
||||
t.Fatalf("concurrency overrun: assignments=%d used=%d", len(ids), f.used())
|
||||
}
|
||||
b2, _, err := f.b.Connect(ctx, f.token, hello)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, err := f.b.Claim(ctx, f.token, b2)
|
||||
if err != nil || a != nil {
|
||||
t.Fatalf("reconnect reset capacity %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("drain_is_durable_and_capacity_sequence_is_not_additive", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
f.submit(3)
|
||||
_, ref := f.claim()
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input := worker.CapacityInput{Sequence: 1, Capacity: 0, Drain: true}
|
||||
if _, err := f.a.Capacity(ctx, f.token, f.binding, input); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.a.Capacity(ctx, f.token, f.binding, input); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input.Capacity = 2
|
||||
_, err := f.a.Capacity(ctx, f.token, f.binding, input)
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
input.Sequence = 2
|
||||
input.Drain = false
|
||||
_, err = f.a.Capacity(ctx, f.token, f.binding, input)
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
f.submit(3)
|
||||
a, err := f.a.Claim(ctx, f.token, f.binding)
|
||||
if err != nil || a != nil {
|
||||
t.Fatalf("draining worker received work %v", err)
|
||||
}
|
||||
if f.used() != 1 {
|
||||
t.Fatal("capacity zero released in-flight permit")
|
||||
}
|
||||
newBinding, welcome, err := f.b.Connect(ctx, f.token, f.hello)
|
||||
if err != nil || !welcome.Draining {
|
||||
t.Fatalf("drain lost on reconnect %v", err)
|
||||
}
|
||||
if _, err := f.b.Resume(ctx, f.token, newBinding, ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.b.Finish(ctx, f.token, newBinding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.used() != 0 {
|
||||
t.Fatal("drained execution did not release capacity")
|
||||
}
|
||||
})
|
||||
t.Run("paused_gate_blocks_start_but_not_saved_result", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
gate := executionID(t)
|
||||
f.exec("INSERT INTO pulse_execution_gates(namespace_id,id,paused) VALUES (?,?,TRUE)", f.resources.NamespaceID, gate)
|
||||
// 超过一次候选页的暂停任务不能饿死后面的普通任务。
|
||||
for i := 0; i < 18; i++ {
|
||||
task := f.submit(3)
|
||||
f.exec("UPDATE pulse_tasks SET gate_id=? WHERE namespace_id=? AND id=?", gate, f.resources.NamespaceID, task.ID)
|
||||
}
|
||||
ready := f.submit(3)
|
||||
a, ref := f.claim()
|
||||
if a.Task.ID != ready.ID {
|
||||
t.Fatal("paused candidate starvation")
|
||||
}
|
||||
f.exec("UPDATE pulse_tasks SET gate_id=? WHERE namespace_id=? AND id=?", gate, f.resources.NamespaceID, ready.ID)
|
||||
start := executionID(t)
|
||||
_, err := f.a.Start(ctx, f.token, f.binding, start, ref)
|
||||
expectWorkerCode(t, err, "EXECUTION_PAUSED")
|
||||
f.exec("UPDATE pulse_execution_gates SET paused=FALSE WHERE namespace_id=? AND id=?", f.resources.NamespaceID, gate)
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, start, ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.exec("UPDATE pulse_execution_gates SET paused=TRUE WHERE namespace_id=? AND id=?", f.resources.NamespaceID, gate)
|
||||
duplicate, err := f.a.Start(ctx, f.token, f.binding, start, ref)
|
||||
if err != nil || !duplicate.Deduplicated || !duplicate.Lease.Active {
|
||||
t.Fatalf("pause blocked original start receipt %v", err)
|
||||
}
|
||||
if _, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
t.Run("recovery_is_bounded_and_unknown_effects_are_not_replayed", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := f.submit(2)
|
||||
_, ref := f.claim()
|
||||
f.expire(ref)
|
||||
stats, err := f.storeB.Recover(ctx, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current := f.get(task.ID)
|
||||
if current.State != pulse.TaskWaiting || current.AttemptCount != 1 || f.used() != 0 || stats.Requeued < 1 {
|
||||
t.Fatal("pre-start lease recovery failed")
|
||||
}
|
||||
f.exec("UPDATE pulse_tasks SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID)
|
||||
_, newRef := f.claim()
|
||||
if newRef.Epoch != 2 || newRef.AttemptID == ref.AttemptID {
|
||||
t.Fatal("attempt epoch not advanced")
|
||||
}
|
||||
_, err = f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
||||
expectWorkerCode(t, err, "LEASE_LOST")
|
||||
start := executionID(t)
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, start, newRef); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.expire(newRef)
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 2)
|
||||
for _, store := range []*ExecutionStore{f.storeA, f.storeB} {
|
||||
wg.Add(1)
|
||||
go func(store *ExecutionStore) { defer wg.Done(); _, err := store.Recover(ctx, 100); errs <- err }(store)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
current = f.get(task.ID)
|
||||
if current.State != pulse.TaskNeedsReview || current.AttemptCount != 2 || current.CompletedAt != nil || f.used() != 0 {
|
||||
t.Fatal("running expiry must become review without replay")
|
||||
}
|
||||
if f.count("orbit_events", "aggregate_id=? AND type='task.needs_review'", task.ID) != 1 {
|
||||
t.Fatal("recovery event duplicated")
|
||||
}
|
||||
late := worker.FinishInput{Ref: newRef, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"external_ref":"late-result"}`)}
|
||||
message := executionID(t)
|
||||
evidence, err := f.a.Finish(ctx, f.token, f.binding, message, late)
|
||||
if err != nil || evidence.Receipt.Disposition != "EVIDENCE_ONLY" || evidence.Lease.Active {
|
||||
t.Fatalf("late evidence %v", err)
|
||||
}
|
||||
evidence, err = f.a.Finish(ctx, f.token, f.binding, message, late)
|
||||
if err != nil || !evidence.Deduplicated {
|
||||
t.Fatalf("late replay %v", err)
|
||||
}
|
||||
if f.get(task.ID).State != pulse.TaskNeedsReview || f.count("pulse_attempt_evidence", "attempt_id=?", newRef.AttemptID) != 1 {
|
||||
t.Fatal("late result rewrote authority or duplicated evidence")
|
||||
}
|
||||
duplicate, err := f.a.Start(ctx, f.token, f.binding, start, newRef)
|
||||
if err != nil || !duplicate.Deduplicated || duplicate.Lease.Active {
|
||||
t.Fatalf("old acknowledgement restored lease %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("attempt_timeout_and_session_expiry_cap_renewal", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := f.submit(3)
|
||||
_, ref := f.claim()
|
||||
f.exec("UPDATE pulse_tasks SET attempt_timeout_ms=5000 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID)
|
||||
begun, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
renewed, err := f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
||||
if err != nil || renewed.Lease.LeaseUntil.After(begun.Lease.LeaseUntil) {
|
||||
t.Fatalf("renew bypassed absolute attempt deadline %v", err)
|
||||
}
|
||||
f.exec("UPDATE pulse_worker_sessions SET expires_at=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.binding.SessionID)
|
||||
renewed, err = f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
||||
if err != nil || renewed.Lease.LeaseUntil.Sub(renewed.Lease.ServerTime) > time.Second {
|
||||
t.Fatalf("renew exceeded session expiration %v", err)
|
||||
}
|
||||
f.exec("UPDATE pulse_worker_sessions SET expires_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.binding.SessionID)
|
||||
_, err = f.a.Capacity(ctx, f.token, f.binding, worker.CapacityInput{Sequence: 1, Capacity: 4})
|
||||
expectWorkerCode(t, err, "STALE_SESSION")
|
||||
})
|
||||
t.Run("result_event_failure_rolls_back_task_receipt_and_permit", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := f.submit(3)
|
||||
_, ref := f.claim()
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.exec("CREATE TRIGGER orbit_test_reject_worker_audit BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='worker audit failure'")
|
||||
message := executionID(t)
|
||||
in := worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}
|
||||
_, err := f.a.Finish(ctx, f.token, f.binding, message, in)
|
||||
f.exec("DROP TRIGGER orbit_test_reject_worker_audit")
|
||||
if err == nil {
|
||||
t.Fatal("expected injected failure")
|
||||
}
|
||||
if f.get(task.ID).State != pulse.TaskRunning || f.used() != 1 || f.count("pulse_attempt_receipts", "attempt_id=? AND operation='finish'", ref.AttemptID) != 0 || f.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 0 {
|
||||
t.Fatal("partial result transaction committed")
|
||||
}
|
||||
if _, err := f.a.Finish(ctx, f.token, f.binding, message, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
t.Run("finish_without_start_is_only_evidence_and_cannot_free_capacity", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := f.submit(1)
|
||||
_, ref := f.claim()
|
||||
result, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)})
|
||||
if err != nil || result.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(task.ID).State != pulse.TaskLeased || f.used() != 1 {
|
||||
t.Fatalf("unstarted result accepted %v", err)
|
||||
}
|
||||
f.expire(ref)
|
||||
if _, err := f.storeA.Recover(ctx, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.get(task.ID).State != pulse.TaskFailed || f.used() != 0 {
|
||||
t.Fatal("exhausted pre-start attempt did not terminate")
|
||||
}
|
||||
})
|
||||
t.Run("known_failure_unknown_and_waiting_deadlines", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
for _, outcome := range []string{"FAILED", "UNKNOWN"} {
|
||||
task := f.submit(3)
|
||||
_, ref := f.claim()
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: outcome, ErrorCode: "EXTERNAL_RESULT", Result: json.RawMessage(`{"evidence_ref":"object/result"}`)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := pulse.TaskFailed
|
||||
if outcome == "UNKNOWN" {
|
||||
want = pulse.TaskNeedsReview
|
||||
}
|
||||
if out.Receipt.TaskState != want || f.get(task.ID).State != want {
|
||||
t.Fatal("outcome classification lost")
|
||||
}
|
||||
}
|
||||
task := f.submit(3)
|
||||
f.exec("UPDATE pulse_tasks SET available_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 2 HOUR),deadline_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 HOUR) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID)
|
||||
if _, err := f.storeA.Recover(ctx, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.get(task.ID).State != pulse.TaskFailed {
|
||||
t.Fatal("waiting deadline not enforced")
|
||||
}
|
||||
})
|
||||
t.Run("authority_is_checked_on_every_execution_command", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
f.submit(3)
|
||||
_, ref := f.claim()
|
||||
f.exec("UPDATE orbit_credentials SET revoked_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.CredentialID)
|
||||
_, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
||||
expectWorkerCode(t, err, "UNAUTHENTICATED")
|
||||
f.exec("UPDATE orbit_credentials SET revoked_at=NULL WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.CredentialID)
|
||||
other := newExecutionFixture(t, ctx, db)
|
||||
_, err = other.a.Resume(ctx, other.token, other.binding, ref)
|
||||
expectWorkerCode(t, err, "NOT_FOUND")
|
||||
copied := f.binding
|
||||
copied.Proof = core.Digest{}
|
||||
_, err = f.a.Claim(ctx, f.token, copied)
|
||||
expectWorkerCode(t, err, "STALE_SESSION")
|
||||
cancelled, stop := context.WithCancel(ctx)
|
||||
stop()
|
||||
_, err = f.a.Start(cancelled, f.token, f.binding, executionID(t), ref)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(fmt.Sprint(err))
|
||||
}
|
||||
})
|
||||
t.Run("claim_event_failure_does_not_leave_an_invisible_lease", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
task := f.submit(3)
|
||||
f.exec("CREATE TRIGGER orbit_test_reject_claim_event BEFORE INSERT ON orbit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='claim event failure'")
|
||||
assignment, err := f.a.Claim(ctx, f.token, f.binding)
|
||||
f.exec("DROP TRIGGER orbit_test_reject_claim_event")
|
||||
if err == nil || assignment != nil {
|
||||
t.Fatal("expected failed claim without usable assignment")
|
||||
}
|
||||
if f.get(task.ID).State != pulse.TaskWaiting || f.used() != 0 || f.count("pulse_attempts", "task_id=?", task.ID) != 0 || f.count("pulse_permits", "TRUE") != 0 {
|
||||
t.Fatal("claim partially committed")
|
||||
}
|
||||
if _, err := f.a.Claim(ctx, f.token, f.binding); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
t.Run("lock_wait_does_not_reuse_an_earlier_clock_sample", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
f.submit(3)
|
||||
_, ref := f.claim()
|
||||
holder, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer holder.Rollback()
|
||||
if _, err := holder.ExecContext(ctx, "UPDATE pulse_queues SET revision=revision+1 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.QueueID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, table := range []string{"pulse_tasks", "pulse_attempts"} {
|
||||
id := ref.TaskID
|
||||
if table == "pulse_attempts" {
|
||||
id = ref.AttemptID
|
||||
}
|
||||
if _, err := holder.ExecContext(ctx, "UPDATE "+table+" SET lease_until=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 150000 MICROSECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
result := make(chan error, 1)
|
||||
go func() { _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); result <- err }()
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
if err := holder.Commit(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expectWorkerCode(t, <-result, "LEASE_LOST")
|
||||
if f.get(ref.TaskID).State != pulse.TaskLeased {
|
||||
t.Fatal("stale clock authorized start")
|
||||
}
|
||||
})
|
||||
}
|
||||
166
internal/storage/mysql/worker_recovery.go
Normal file
166
internal/storage/mysql/worker_recovery.go
Normal file
@@ -0,0 +1,166 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
// Recover 是受控后台扫描入口，每批最多 100 项、每项一个短事务，可由多个节点竞争。
|
||||
// 当前已授权执行的失联一律转核查，自动安全重试在 P3 接入冻结策略后实现。
|
||||
func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.RecoveryStats, error) {
|
||||
var out worker.RecoveryStats
|
||||
if limit < 1 || limit > 100 {
|
||||
return out, fault.Invalid("recovery batch limit must be 1 to 100")
|
||||
}
|
||||
type candidate struct{ ns, id, queue core.ID }
|
||||
candidates := []candidate{}
|
||||
seen := map[string]bool{}
|
||||
// 分别使用租约、总截止及开始截止索引，避免对整个 WAITING 队列做 COALESCE 扫描。
|
||||
for _, q := range []string{
|
||||
`SELECT namespace_id,id,queue_id FROM pulse_tasks WHERE state IN ('LEASED','RUNNING') AND lease_until<=UTC_TIMESTAMP(6) ORDER BY lease_until,namespace_id,id LIMIT ?`,
|
||||
`SELECT namespace_id,id,queue_id FROM pulse_tasks WHERE state='WAITING' AND deadline_at<=UTC_TIMESTAMP(6) ORDER BY deadline_at,namespace_id,id LIMIT ?`,
|
||||
`SELECT namespace_id,id,queue_id FROM pulse_tasks WHERE state='WAITING' AND start_deadline_at<=UTC_TIMESTAMP(6) ORDER BY start_deadline_at,namespace_id,id LIMIT ?`,
|
||||
} {
|
||||
if len(candidates) >= limit {
|
||||
break
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, q, limit-len(candidates))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var c candidate
|
||||
if err := rows.Scan(&c.ns, &c.id, &c.queue); err != nil {
|
||||
rows.Close()
|
||||
return out, err
|
||||
}
|
||||
key := string(c.ns) + string(c.id)
|
||||
if !seen[key] {
|
||||
candidates = append(candidates, c)
|
||||
seen[key] = true
|
||||
}
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
for _, c := range candidates {
|
||||
out.Examined++
|
||||
var changed pulse.TaskState
|
||||
err := WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
||||
// 与 Worker 命令一致先锁队列；有其他节点正在处理时跳过，本批不占用等待线程。
|
||||
var queue core.ID
|
||||
err := tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, c.ns, c.queue).Scan(&queue)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var taskID core.ID
|
||||
err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_tasks WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, c.ns, c.id).Scan(&taskID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task, err := readExecutionTask(ctx, tx, c.ns, c.id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if task.view.State == pulse.TaskWaiting {
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if task.view.DeadlineAt.After(now) && (task.view.StartDeadlineAt == nil || task.view.StartDeadlineAt.After(now)) {
|
||||
return nil
|
||||
}
|
||||
changed = pulse.TaskFailed
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='FAILED',completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, now, now, c.ns, c.id, task.view.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", "task.failed", map[string]any{"reason": "START_DEADLINE_EXCEEDED", "node_id": s.node})
|
||||
}
|
||||
if task.view.State != pulse.TaskLeased && task.view.State != pulse.TaskRunning {
|
||||
return nil
|
||||
}
|
||||
if task.current == nil || task.lease == nil {
|
||||
return errors.New("active task lacks attempt or lease")
|
||||
}
|
||||
attempt, err := readAttempt(ctx, tx, c.ns, *task.current)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if task.lease.After(now) || attempt.LeaseUntil.After(now) {
|
||||
return nil
|
||||
}
|
||||
if attempt.Epoch != task.epoch || attempt.TaskID != c.id {
|
||||
return errors.New("task attempt identity mismatch")
|
||||
}
|
||||
if attempt.State != pulse.AttemptLeased && attempt.State != pulse.AttemptRunning {
|
||||
return errors.New("active task references ended attempt")
|
||||
}
|
||||
attemptState := pulse.AttemptExpired
|
||||
next := pulse.TaskFailed
|
||||
reason := "LEASE_EXPIRED_BEFORE_START"
|
||||
var completed *time.Time = &now
|
||||
available := task.view.AvailableAt
|
||||
if attempt.AuthorizedAt != nil {
|
||||
attemptState = pulse.AttemptUnknown
|
||||
next = pulse.TaskNeedsReview
|
||||
reason = "EXECUTION_OUTCOME_UNKNOWN"
|
||||
completed = nil
|
||||
} else if task.view.AttemptCount < task.maxAttempts {
|
||||
// 未批准开始的尝试可以有界重派，次数不重置；开始截止与总截止仍有效。
|
||||
nextAt := now.Add(time.Second)
|
||||
if nextAt.Before(task.view.DeadlineAt) && (task.view.StartDeadlineAt == nil || nextAt.Before(*task.view.StartDeadlineAt)) {
|
||||
next = pulse.TaskWaiting
|
||||
available = nextAt
|
||||
completed = nil
|
||||
}
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, reason, now, c.ns, attempt.ID, attempt.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,lease_until=NULL,available_at=?,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, next, available, completed, now, c.ns, c.id, task.view.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := releasePermits(ctx, tx, c.ns, attempt.ID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
changed = next
|
||||
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", resultEvent(next), map[string]any{"attempt_id": attempt.ID, "epoch": fmt.Sprint(attempt.Epoch), "reason": reason, "node_id": s.node})
|
||||
})
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
switch changed {
|
||||
case pulse.TaskWaiting:
|
||||
out.Requeued++
|
||||
case pulse.TaskFailed:
|
||||
out.Failed++
|
||||
case pulse.TaskNeedsReview:
|
||||
out.NeedsReview++
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
132
internal/storage/mysql/worker_results.go
Normal file
132
internal/storage/mysql/worker_results.go
Normal file
@@ -0,0 +1,132 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
func (s *ExecutionStore) Finish(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, in worker.FinishInput) (worker.CommandResult, error) {
|
||||
var out worker.CommandResult
|
||||
digest, err := requestDigest(struct {
|
||||
Identity any
|
||||
Outcome string
|
||||
Result core.JSON
|
||||
ErrorCode string
|
||||
}{attemptIdentity(in.Ref), in.Outcome, core.JSON(in.Result), in.ErrorCode})
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
err = s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
rows, err := s.lockExecution(ctx, tx, token, b, p, in.Ref, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cached, err := readReceipt(ctx, tx, token.NamespaceID, in.Ref.AttemptID, message, "finish", digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status := leaseStatus(rows, b, now)
|
||||
if cached != nil {
|
||||
out = worker.CommandResult{Receipt: *cached, Lease: status, Deduplicated: true}
|
||||
return nil
|
||||
}
|
||||
task, a := rows.task, rows.attempt
|
||||
receipt := worker.Receipt{Operation: "finish", TaskState: task.view.State, AttemptState: a.State, TaskRevision: task.view.Revision, LeaseUntil: a.LeaseUntil, Disposition: "APPLIED"}
|
||||
if a.ResultDigest != nil && *a.ResultDigest == digest {
|
||||
// 新消息 ID 重报已保存的同一结果，也不能重复事件、许可释放或状态推进。
|
||||
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
out = worker.CommandResult{Receipt: receipt, Lease: status, Deduplicated: true}
|
||||
return nil
|
||||
}
|
||||
reason := ""
|
||||
if a.ResultDigest != nil {
|
||||
reason = "RESULT_CONFLICT"
|
||||
} else if a.AuthorizedAt == nil {
|
||||
reason = "NOT_STARTED"
|
||||
} else if !status.Active || a.State != pulse.AttemptRunning {
|
||||
reason = "LEASE_LOST"
|
||||
}
|
||||
if reason != "" {
|
||||
if err := saveEvidence(ctx, tx, token.NamespaceID, message, in, digest, reason); err != nil {
|
||||
return err
|
||||
}
|
||||
receipt.Disposition = "EVIDENCE_ONLY"
|
||||
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
out = worker.CommandResult{Receipt: receipt, Lease: status}
|
||||
return nil
|
||||
}
|
||||
attemptState, taskState := pulse.AttemptState(in.Outcome), pulse.TaskState(in.Outcome)
|
||||
var completed *time.Time = &now
|
||||
if in.Outcome == "UNKNOWN" {
|
||||
taskState = pulse.TaskNeedsReview
|
||||
completed = nil
|
||||
}
|
||||
changed, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=NULLIF(?,''),result=?,result_digest=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, in.ErrorCode, string(in.Result), digest, now, token.NamespaceID, a.ID, a.Revision)
|
||||
if err := exactlyOne(changed, err); err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,result=?,lease_until=NULL,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, taskState, string(in.Result), completed, now, token.NamespaceID, in.Ref.TaskID, task.view.Revision)
|
||||
if err := exactlyOne(changed, err); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := releasePermits(ctx, tx, token.NamespaceID, a.ID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
event := map[string]any{"attempt_id": a.ID, "epoch": fmt.Sprint(a.Epoch), "state": taskState, "outcome": in.Outcome, "error_code": in.ErrorCode}
|
||||
if err := emitEvent(ctx, tx, token.NamespaceID, in.Ref.TaskID, task.view.Revision+1, "task", resultEvent(taskState), event); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := appendAudit(ctx, tx, token, message, in.Ref.TaskID, "task", "task.result", "worker result", map[string]any{"state": task.view.State}, event); err != nil {
|
||||
return err
|
||||
}
|
||||
receipt.TaskState = taskState
|
||||
receipt.AttemptState = attemptState
|
||||
receipt.TaskRevision++
|
||||
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
out = worker.CommandResult{Receipt: receipt, Lease: worker.LeaseStatus{State: attemptState, LeaseUntil: a.LeaseUntil, ServerTime: now}}
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
func resultEvent(state pulse.TaskState) string {
|
||||
switch state {
|
||||
case pulse.TaskSucceeded:
|
||||
return "task.succeeded"
|
||||
case pulse.TaskFailed:
|
||||
return "task.failed"
|
||||
case pulse.TaskNeedsReview:
|
||||
return "task.needs_review"
|
||||
case pulse.TaskWaiting:
|
||||
return "task.requeued"
|
||||
default:
|
||||
return "task.cancelled"
|
||||
}
|
||||
}
|
||||
func saveEvidence(ctx context.Context, tx *sql.Tx, ns, message core.ID, in worker.FinishInput, digest core.Digest, reason string) error {
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempt_evidence(namespace_id,id,attempt_id,message_id,outcome,error_code,payload,request_digest,reason) VALUES (?,?,?,?,?,NULLIF(?,''),?,?,?)`, ns, id, in.Ref.AttemptID, message, in.Outcome, in.ErrorCode, string(in.Result), digest, reason)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return emitEvent(ctx, tx, ns, id, 1, "attempt_evidence", "attempt.evidence_received", map[string]any{"attempt_id": in.Ref.AttemptID, "task_id": in.Ref.TaskID, "reason": reason, "outcome": in.Outcome})
|
||||
}
|
||||
211
internal/storage/mysql/worker_sessions.go
Normal file
211
internal/storage/mysql/worker_sessions.go
Normal file
@@ -0,0 +1,211 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
// ExecutionOptions 是服务端执行窗口，不由 Worker 任意指定。
|
||||
type ExecutionOptions struct {
|
||||
LeaseDuration time.Duration // LeaseDuration 每次领取/续租的最大窗口。
|
||||
SessionTTL time.Duration // SessionTTL 会话心跳有效期，不代替执行租约。
|
||||
}
|
||||
|
||||
// ExecutionStore 所有执行权以 MySQL 为准，节点内存不持有可绕过数据库的授权。
|
||||
type ExecutionStore struct {
|
||||
db *sql.DB
|
||||
node core.ID
|
||||
options ExecutionOptions
|
||||
}
|
||||
|
||||
var _ worker.Store = (*ExecutionStore)(nil)
|
||||
|
||||
func (s *ExecutionStore) Authorize(ctx context.Context, token auth.Token) error {
|
||||
return s.transaction(ctx, token, "workers:connect", func(*sql.Tx, auth.Permissions) error { return nil })
|
||||
}
|
||||
|
||||
func NewExecutionStore(db *sql.DB, node core.ID, options ExecutionOptions) (*ExecutionStore, error) {
|
||||
if options.LeaseDuration == 0 {
|
||||
options.LeaseDuration = 30 * time.Second
|
||||
}
|
||||
if options.SessionTTL == 0 {
|
||||
options.SessionTTL = 45 * time.Second
|
||||
}
|
||||
if node.Validate() != nil || options.LeaseDuration < time.Second || options.LeaseDuration > 2*time.Minute || options.SessionTTL < options.LeaseDuration || options.SessionTTL > 5*time.Minute {
|
||||
return nil, fault.Invalid("invalid execution windows or node identity")
|
||||
}
|
||||
return &ExecutionStore{db: db, node: node, options: options}, nil
|
||||
}
|
||||
func (s *ExecutionStore) transaction(ctx context.Context, token auth.Token, action string, fn func(*sql.Tx, auth.Permissions) error) error {
|
||||
return (&TaskStore{db: s.db}).transaction(ctx, token, action, fn)
|
||||
}
|
||||
|
||||
type sessionState struct {
|
||||
queue core.ID
|
||||
capacity uint32
|
||||
sequence uint64
|
||||
draining bool
|
||||
expires time.Time
|
||||
capabilities []worker.Capability
|
||||
}
|
||||
|
||||
func (s *ExecutionStore) welcome(b worker.Binding, state sessionState, now time.Time) worker.Welcome {
|
||||
return worker.Welcome{SessionID: b.SessionID, Epoch: b.Epoch, ServerTime: now, ExpiresAt: state.expires, LeaseDurationMS: uint64(s.options.LeaseDuration / time.Millisecond), Draining: state.draining}
|
||||
}
|
||||
func (s *ExecutionStore) Connect(ctx context.Context, token auth.Token, in worker.ConnectInput) (worker.Binding, worker.Welcome, error) {
|
||||
var binding worker.Binding
|
||||
var out worker.Welcome
|
||||
proof, err := worker.SecretDigest(in.InstanceSecret)
|
||||
if err != nil {
|
||||
return binding, out, err
|
||||
}
|
||||
registration, err := requestDigest(struct {
|
||||
Queue core.ID
|
||||
Capabilities []worker.Capability
|
||||
}{in.QueueID, in.Capabilities})
|
||||
if err != nil {
|
||||
return binding, out, err
|
||||
}
|
||||
err = s.transaction(ctx, token, "workers:connect", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
if !p.AllowsQueue(in.QueueID) {
|
||||
return fault.Forbidden
|
||||
}
|
||||
_, err := tx.ExecContext(ctx, `INSERT INTO pulse_worker_instances(namespace_id,id,credential_id,queue_id,secret_digest,registration_digest) VALUES (?,?,?,?,?,?) ON DUPLICATE KEY UPDATE id=pulse_worker_instances.id`, token.NamespaceID, in.InstanceID, token.CredentialID, in.QueueID, proof, registration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var credential core.ID
|
||||
var storedProof, storedRegistration core.Digest
|
||||
var epoch uint64
|
||||
var old *core.ID
|
||||
var draining bool
|
||||
err = tx.QueryRowContext(ctx, `SELECT credential_id,secret_digest,registration_digest,session_epoch,current_session_id,draining FROM pulse_worker_instances WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, in.InstanceID).Scan(&credential, &storedProof, &storedRegistration, &epoch, &old, &draining)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if credential != token.CredentialID || subtle.ConstantTimeCompare(proof[:], storedProof[:]) != 1 {
|
||||
return fault.Forbidden
|
||||
}
|
||||
if registration != storedRegistration || epoch == ^uint64(0) {
|
||||
return fault.Conflict
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
session, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
binding = worker.Binding{InstanceID: in.InstanceID, SessionID: session, Epoch: epoch + 1, Proof: proof}
|
||||
state := sessionState{queue: in.QueueID, capacity: in.Capacity, draining: draining, expires: now.Add(s.options.SessionTTL), capabilities: in.Capabilities}
|
||||
caps, err := json.Marshal(in.Capabilities)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_worker_sessions(namespace_id,id,worker_instance_id,credential_id,node_id,session_epoch,capabilities,capacity,draining,expires_at) VALUES (?,?,?,?,?,?,?,?,?,?)`, token.NamespaceID, session, in.InstanceID, token.CredentialID, s.node, binding.Epoch, string(caps), in.Capacity, draining, state.expires)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if old != nil {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE pulse_worker_sessions SET expires_at=?,draining=TRUE,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, now, now, token.NamespaceID, *old); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE pulse_worker_instances SET current_session_id=?,session_epoch=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, session, binding.Epoch, now, token.NamespaceID, in.InstanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, token.NamespaceID, session, 1, "worker_session", "worker.connected", map[string]any{"instance_id": in.InstanceID, "node_id": s.node, "epoch": binding.Epoch}); err != nil {
|
||||
return err
|
||||
}
|
||||
out = s.welcome(binding, state, now)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return worker.Binding{}, worker.Welcome{}, err
|
||||
}
|
||||
return binding, out, nil
|
||||
}
|
||||
|
||||
// lockSession 的顺序为凭据 -> 进程头 -> 当前会话；新连接和旧连接命令在同一行上串行化。
|
||||
func (s *ExecutionStore) lockSession(ctx context.Context, tx *sql.Tx, token auth.Token, b worker.Binding, p auth.Permissions) (sessionState, error) {
|
||||
var state sessionState
|
||||
var credential core.ID
|
||||
var proof core.Digest
|
||||
var current *core.ID
|
||||
var epoch uint64
|
||||
if b.InstanceID.Validate() != nil || b.SessionID.Validate() != nil || b.Epoch == 0 {
|
||||
return state, worker.StaleSession
|
||||
}
|
||||
err := tx.QueryRowContext(ctx, `SELECT credential_id,queue_id,secret_digest,current_session_id,session_epoch,draining FROM pulse_worker_instances WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, b.InstanceID).Scan(&credential, &state.queue, &proof, ¤t, &epoch, &state.draining)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return state, worker.StaleSession
|
||||
}
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
if credential != token.CredentialID || current == nil || *current != b.SessionID || epoch != b.Epoch || subtle.ConstantTimeCompare(proof[:], b.Proof[:]) != 1 {
|
||||
return state, worker.StaleSession
|
||||
}
|
||||
if !p.AllowsQueue(state.queue) {
|
||||
return state, fault.Forbidden
|
||||
}
|
||||
var node core.ID
|
||||
var raw []byte
|
||||
err = tx.QueryRowContext(ctx, `SELECT node_id,capacity,capacity_sequence,capabilities,expires_at FROM pulse_worker_sessions WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, b.SessionID).Scan(&node, &state.capacity, &state.sequence, &raw, &state.expires)
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
if node != s.node || !state.expires.After(now) {
|
||||
return state, worker.StaleSession
|
||||
}
|
||||
if err := json.Unmarshal(raw, &state.capabilities); err != nil {
|
||||
return state, err
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
func (s *ExecutionStore) Capacity(ctx context.Context, token auth.Token, b worker.Binding, in worker.CapacityInput) (worker.Welcome, error) {
|
||||
var out worker.Welcome
|
||||
err := s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
state, err := s.lockSession(ctx, tx, token, b, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if in.Sequence < state.sequence || (in.Sequence == state.sequence && (in.Capacity != state.capacity || in.Drain != state.draining)) || (state.draining && !in.Drain) {
|
||||
return fault.Conflict
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
state.expires = now.Add(s.options.SessionTTL)
|
||||
state.capacity = in.Capacity
|
||||
state.draining = state.draining || in.Drain
|
||||
_, err = tx.ExecContext(ctx, `UPDATE pulse_worker_sessions SET capacity=?,capacity_sequence=?,draining=?,expires_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, state.capacity, in.Sequence, state.draining, state.expires, now, token.NamespaceID, b.SessionID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if in.Drain {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE pulse_worker_instances SET draining=TRUE,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND draining=FALSE`, now, token.NamespaceID, b.InstanceID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
out = s.welcome(b, state, now)
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
244
internal/storage/mysql/worker_websocket_integration_test.go
Normal file
244
internal/storage/mysql/worker_websocket_integration_test.go
Normal file
@@ -0,0 +1,244 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/transport/httpserver"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
type testWorkerFrame struct {
|
||||
Version string `json:"protocol_version"`
|
||||
Type string `json:"type"`
|
||||
MessageID core.ID `json:"message_id"`
|
||||
ReplyTo core.ID `json:"reply_to,omitempty"`
|
||||
SessionID core.ID `json:"session_id,omitempty"`
|
||||
Body json.RawMessage `json:"body,omitempty"`
|
||||
Error *struct {
|
||||
Code string `json:"code"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
type testWorkerSocket struct {
|
||||
t *testing.T
|
||||
ctx context.Context
|
||||
conn *websocket.Conn
|
||||
session core.ID
|
||||
welcome worker.Welcome
|
||||
}
|
||||
|
||||
func connectTestWorker(t *testing.T, ctx context.Context, server *httptest.Server, f *executionFixture) *testWorkerSocket {
|
||||
t.Helper()
|
||||
header := http.Header{}
|
||||
header.Set("Authorization", "Bearer "+f.resources.Token)
|
||||
conn, response, err := websocket.Dial(ctx, strings.Replace(server.URL, "https://", "wss://", 1)+"/v1/worker/connect", &websocket.DialOptions{HTTPClient: server.Client(), HTTPHeader: header, Subprotocols: []string{httpserver.WorkerSubprotocol}})
|
||||
if err != nil {
|
||||
status := 0
|
||||
if response != nil {
|
||||
status = response.StatusCode
|
||||
}
|
||||
t.Fatalf("Worker dial failed HTTP %d: %v", status, err)
|
||||
}
|
||||
t.Cleanup(func() { conn.CloseNow() })
|
||||
client := &testWorkerSocket{t: t, ctx: ctx, conn: conn}
|
||||
conn.SetReadLimit(512 << 10)
|
||||
hello := client.send("worker.hello", f.hello, "")
|
||||
reply := client.read()
|
||||
if reply.Type != "worker.welcome" || reply.ReplyTo != hello || reply.Error != nil {
|
||||
t.Fatal("missing welcome")
|
||||
}
|
||||
if err := json.Unmarshal(reply.Body, &client.welcome); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client.session = client.welcome.SessionID
|
||||
return client
|
||||
}
|
||||
func (c *testWorkerSocket) send(kind string, body any, id core.ID) core.ID {
|
||||
c.t.Helper()
|
||||
if id == "" {
|
||||
id = executionID(c.t)
|
||||
}
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
c.t.Fatal(err)
|
||||
}
|
||||
frame, err := json.Marshal(testWorkerFrame{Version: "1", Type: kind, MessageID: id, SessionID: c.session, Body: raw})
|
||||
if err != nil {
|
||||
c.t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(c.ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
if err := c.conn.Write(ctx, websocket.MessageText, frame); err != nil {
|
||||
c.t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
func (c *testWorkerSocket) read() testWorkerFrame {
|
||||
c.t.Helper()
|
||||
ctx, cancel := context.WithTimeout(c.ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
kind, raw, err := c.conn.Read(ctx)
|
||||
if err != nil {
|
||||
c.t.Fatal(err)
|
||||
}
|
||||
var frame testWorkerFrame
|
||||
if kind != websocket.MessageText || json.Unmarshal(raw, &frame) != nil || frame.Version != "1" || frame.MessageID.Validate() != nil {
|
||||
c.t.Fatal("invalid server frame")
|
||||
}
|
||||
return frame
|
||||
}
|
||||
func (c *testWorkerSocket) ack(id core.ID) json.RawMessage {
|
||||
c.t.Helper()
|
||||
frame := c.read()
|
||||
if frame.Type != "ack" || frame.ReplyTo != id || frame.SessionID != c.session || frame.Error != nil {
|
||||
code := ""
|
||||
if frame.Error != nil {
|
||||
code = frame.Error.Code
|
||||
}
|
||||
c.t.Fatalf("expected ack, got %s/%s", frame.Type, code)
|
||||
}
|
||||
return frame.Body
|
||||
}
|
||||
func TestIntegrationWorkerWebSocket(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
db, err := Open(ctx, dsn, 16, 4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fixture := newExecutionFixture(t, ctx, db)
|
||||
start := func(svc *worker.Service) (*httptest.Server, *httpserver.WorkerHub) {
|
||||
t.Helper()
|
||||
hub := httpserver.NewWorkerHub(ctx, svc)
|
||||
server := httpserver.New("", func(context.Context) error { return nil })
|
||||
httpserver.RegisterWorkers(server, hub)
|
||||
tls := httptest.NewTLSServer(server.Handler)
|
||||
t.Cleanup(func() {
|
||||
bounded, stop := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer stop()
|
||||
if err := hub.Shutdown(bounded); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tls.Close()
|
||||
})
|
||||
return tls, hub
|
||||
}
|
||||
serverA, hubA := start(fixture.a)
|
||||
serverB, _ := start(fixture.b)
|
||||
t.Run("handshake_auth_protocol_origin_and_URL", func(t *testing.T) {
|
||||
cases := []struct {
|
||||
token, protocol, path, origin string
|
||||
status int
|
||||
}{
|
||||
{"", httpserver.WorkerSubprotocol, "/v1/worker/connect", "", 401},
|
||||
{fixture.resources.Token, "unknown.v1", "/v1/worker/connect", "", 426},
|
||||
{fixture.resources.Token, httpserver.WorkerSubprotocol, "/v1/worker/connect?token=forbidden", "", 400},
|
||||
{fixture.resources.Token, httpserver.WorkerSubprotocol, "/v1/worker/connect", "https://foreign.example", 403},
|
||||
}
|
||||
for _, test := range cases {
|
||||
headers := http.Header{}
|
||||
if test.token != "" {
|
||||
headers.Set("Authorization", "Bearer "+test.token)
|
||||
}
|
||||
if test.origin != "" {
|
||||
headers.Set("Origin", test.origin)
|
||||
}
|
||||
conn, response, err := websocket.Dial(ctx, strings.Replace(serverA.URL, "https://", "wss://", 1)+test.path, &websocket.DialOptions{HTTPClient: serverA.Client(), HTTPHeader: headers, Subprotocols: []string{test.protocol}})
|
||||
if conn != nil {
|
||||
conn.CloseNow()
|
||||
}
|
||||
if err == nil || response == nil || response.StatusCode != test.status {
|
||||
t.Fatalf("handshake expected %d", test.status)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("WSS_assignment_start_and_cross_node_ack_recovery", func(t *testing.T) {
|
||||
task := fixture.submit(3)
|
||||
first := connectTestWorker(t, ctx, serverA, fixture)
|
||||
// 只有显式容量确认后才启用新派发，留出恢复核对阶段。
|
||||
first.ack(first.send("worker.capacity", worker.CapacityInput{Sequence: 1, Capacity: 1}, ""))
|
||||
frame := first.read()
|
||||
if frame.Type != "task.assign" {
|
||||
t.Fatalf("wanted assignment got %s", frame.Type)
|
||||
}
|
||||
var assignment worker.Assignment
|
||||
if err := json.Unmarshal(frame.Body, &assignment); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if assignment.Task.ID != task.ID || assignment.LeaseSecret == "" {
|
||||
t.Fatal("wrong assignment")
|
||||
}
|
||||
ref := worker.AttemptRef{TaskID: task.ID, AttemptID: assignment.AttemptID, Epoch: assignment.Epoch, LeaseSecret: assignment.LeaseSecret}
|
||||
startMessage := first.send("task.start", ref, "")
|
||||
var begun worker.CommandResult
|
||||
if err := json.Unmarshal(first.ack(startMessage), &begun); err != nil || !begun.Lease.Active || begun.Lease.State != pulse.AttemptRunning {
|
||||
t.Fatal("missing durable start")
|
||||
}
|
||||
// 模拟旧节点连接断开；新的 WSS 连接进入不同节点并原子迁移有效尝试。
|
||||
_ = first.conn.CloseNow()
|
||||
second := connectTestWorker(t, ctx, serverB, fixture)
|
||||
var resumed worker.LeaseStatus
|
||||
if err := json.Unmarshal(second.ack(second.send("worker.resume", ref, "")), &resumed); err != nil || !resumed.Active {
|
||||
t.Fatal("WSS resume failed")
|
||||
}
|
||||
var original worker.CommandResult
|
||||
if err := json.Unmarshal(second.ack(second.send("task.start", ref, startMessage)), &original); err != nil || !original.Deduplicated || !original.Lease.Active {
|
||||
t.Fatal("original start receipt not recovered")
|
||||
}
|
||||
finishMessage := second.send("task.finish", worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"output_ref":"objects/result","n":9007199254740993}`)}, "")
|
||||
var finish worker.CommandResult
|
||||
if err := json.Unmarshal(second.ack(finishMessage), &finish); err != nil || finish.Receipt.Disposition != "APPLIED" || finish.Lease.Active {
|
||||
t.Fatal("WSS finish failed")
|
||||
}
|
||||
// 丢弃该连接的操作确认上下文，再用稳定身份取回同一结果。
|
||||
_ = second.conn.CloseNow()
|
||||
third := connectTestWorker(t, ctx, serverA, fixture)
|
||||
var duplicate worker.CommandResult
|
||||
if err := json.Unmarshal(third.ack(third.send("task.finish", worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"output_ref":"objects/result","n":9007199254740993}`)}, finishMessage)), &duplicate); err != nil || !duplicate.Deduplicated {
|
||||
t.Fatal("finish receipt not recovered")
|
||||
}
|
||||
if fixture.get(task.ID).State != pulse.TaskSucceeded || fixture.used() != 0 || fixture.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 1 {
|
||||
t.Fatal("WSS result effects duplicated")
|
||||
}
|
||||
})
|
||||
t.Run("strict_worker_frame_and_hub_shutdown", func(t *testing.T) {
|
||||
conn := connectTestWorker(t, ctx, serverB, fixture)
|
||||
raw := []byte(`{"protocol_version":"1","protocol_version":"1","type":"worker.capacity"}`)
|
||||
if err := conn.conn.Write(ctx, websocket.MessageText, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
readCtx, stop := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer stop()
|
||||
_, _, err := conn.conn.Read(readCtx)
|
||||
if websocket.CloseStatus(err) != websocket.StatusPolicyViolation {
|
||||
t.Fatalf("duplicate frame not rejected: %v", err)
|
||||
}
|
||||
active := connectTestWorker(t, ctx, serverA, fixture)
|
||||
shutdown, stopShutdown := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer stopShutdown()
|
||||
if err := hubA.Shutdown(shutdown); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err = active.conn.Read(readCtx)
|
||||
if err == nil {
|
||||
t.Fatal("shutdown left a hijacked connection alive")
|
||||
}
|
||||
})
|
||||
}
|
||||
401
internal/transport/httpserver/worker.go
Normal file
401
internal/transport/httpserver/worker.go
Normal file
@@ -0,0 +1,401 @@
|
||||
package httpserver
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/jsonvalue"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
const WorkerSubprotocol = "pulse.worker.v1"
|
||||
|
||||
// WorkerHub 管理节点内连接生命周期；任务归属和容量仍由 MySQL 决定。
|
||||
type WorkerHub struct {
|
||||
service *worker.Service
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
slots chan struct{}
|
||||
mu sync.Mutex
|
||||
closing bool
|
||||
wg sync.WaitGroup
|
||||
poll time.Duration
|
||||
}
|
||||
|
||||
func NewWorkerHub(parent context.Context, service *worker.Service) *WorkerHub {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
return &WorkerHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, 128), poll: time.Second}
|
||||
}
|
||||
func RegisterWorkers(server *http.Server, hub *WorkerHub) {
|
||||
server.Handler.(*http.ServeMux).Handle("GET /v1/worker/connect", hub)
|
||||
}
|
||||
|
||||
// Shutdown 停止派发并关闭本节点连接；不清除持久化租约，由重連核对或到期恢复处理。
|
||||
func (h *WorkerHub) Shutdown(ctx context.Context) error {
|
||||
h.mu.Lock()
|
||||
h.closing = true
|
||||
h.cancel()
|
||||
h.mu.Unlock()
|
||||
done := make(chan struct{})
|
||||
go func() { h.wg.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
type workerFrame struct {
|
||||
Version string `json:"protocol_version"`
|
||||
Type string `json:"type"`
|
||||
MessageID core.ID `json:"message_id"`
|
||||
ReplyTo core.ID `json:"reply_to,omitempty"`
|
||||
SessionID core.ID `json:"session_id,omitempty"`
|
||||
Body json.RawMessage `json:"body,omitempty"`
|
||||
Error *fault.Error `json:"error,omitempty"`
|
||||
}
|
||||
type outboundFrame struct {
|
||||
data []byte
|
||||
done chan error
|
||||
}
|
||||
|
||||
func (h *WorkerHub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
requestID, _ := core.NewID()
|
||||
w.Header().Set("X-Request-ID", string(requestID))
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
if r.URL.RawQuery != "" {
|
||||
writeError(w, string(requestID), fault.Invalid("Worker credentials and options must not be placed in URL"))
|
||||
return
|
||||
}
|
||||
values := r.Header.Values("Authorization")
|
||||
if len(values) != 1 {
|
||||
writeError(w, string(requestID), fault.Unauthorized)
|
||||
return
|
||||
}
|
||||
scheme, value, ok := strings.Cut(values[0], " ")
|
||||
if !ok || !strings.EqualFold(scheme, "Bearer") {
|
||||
writeError(w, string(requestID), fault.Unauthorized)
|
||||
return
|
||||
}
|
||||
token, err := auth.Parse(value)
|
||||
if err != nil {
|
||||
writeError(w, string(requestID), err)
|
||||
return
|
||||
}
|
||||
supported := false
|
||||
for _, header := range r.Header.Values("Sec-WebSocket-Protocol") {
|
||||
for _, protocol := range strings.Split(header, ",") {
|
||||
supported = supported || strings.TrimSpace(protocol) == WorkerSubprotocol
|
||||
}
|
||||
}
|
||||
if !supported {
|
||||
writeJSON(w, http.StatusUpgradeRequired, map[string]any{"request_id": requestID, "error": fault.New("PROTOCOL_REQUIRED", "pulse.worker.v1 subprotocol is required")})
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
if h.closing || h.ctx.Err() != nil {
|
||||
h.mu.Unlock()
|
||||
writeError(w, string(requestID), fault.Unavailable)
|
||||
return
|
||||
}
|
||||
select {
|
||||
case h.slots <- struct{}{}:
|
||||
h.wg.Add(1)
|
||||
default:
|
||||
h.mu.Unlock()
|
||||
writeJSON(w, 429, map[string]any{"request_id": requestID, "error": fault.New("CAPACITY_EXCEEDED", "node connection capacity exceeded")})
|
||||
return
|
||||
}
|
||||
h.mu.Unlock()
|
||||
defer func() { <-h.slots; h.wg.Done() }()
|
||||
authorize, cancel := context.WithTimeout(r.Context(), 5*time.Second)
|
||||
err = h.service.Authorize(authorize, token)
|
||||
cancel()
|
||||
if err != nil {
|
||||
writeError(w, string(requestID), err)
|
||||
return
|
||||
}
|
||||
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{WorkerSubprotocol}, CompressionMode: websocket.CompressionDisabled})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
conn.SetReadLimit(512 << 10)
|
||||
ctx, cancel := context.WithCancel(h.ctx)
|
||||
defer cancel()
|
||||
helloCtx, helloCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
kind, raw, err := conn.Read(helloCtx)
|
||||
helloCancel()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var hello workerFrame
|
||||
var input worker.ConnectInput
|
||||
if kind != websocket.MessageText || decodeWorkerJSON(raw, &hello) != nil || hello.Version != "1" || hello.Type != "worker.hello" || hello.MessageID.Validate() != nil || hello.SessionID != "" || hello.ReplyTo != "" || hello.Error != nil || decodeWorkerJSON(hello.Body, &input) != nil {
|
||||
_ = conn.Close(websocket.StatusPolicyViolation, "invalid worker hello")
|
||||
return
|
||||
}
|
||||
call, cancelCall := context.WithTimeout(ctx, 8*time.Second)
|
||||
binding, welcome, err := h.service.Connect(call, token, input)
|
||||
cancelCall()
|
||||
if err != nil {
|
||||
_ = writeWorkerFrame(ctx, conn, workerFrame{Version: "1", Type: "error", MessageID: requestID, ReplyTo: hello.MessageID, Error: publicWorkerError(err)})
|
||||
return
|
||||
}
|
||||
welcomeRaw, err := json.Marshal(welcome)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err := writeWorkerFrame(ctx, conn, workerFrame{Version: "1", Type: "worker.welcome", MessageID: requestID, ReplyTo: hello.MessageID, SessionID: binding.SessionID, Body: welcomeRaw}); err != nil {
|
||||
return
|
||||
}
|
||||
// 有界读写队列与 SQL 控制循环分开，处理器绝不运行在这里。
|
||||
incoming := make(chan workerFrame, 4)
|
||||
outgoing := make(chan outboundFrame, 8)
|
||||
var loops sync.WaitGroup
|
||||
loops.Add(2)
|
||||
go func() {
|
||||
defer loops.Done()
|
||||
defer cancel()
|
||||
for {
|
||||
readCtx, stop := context.WithTimeout(ctx, 60*time.Second)
|
||||
kind, raw, err := conn.Read(readCtx)
|
||||
stop()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var frame workerFrame
|
||||
if kind != websocket.MessageText || decodeWorkerJSON(raw, &frame) != nil || frame.Version != "1" || frame.MessageID.Validate() != nil || frame.SessionID != binding.SessionID || frame.ReplyTo != "" || frame.Error != nil {
|
||||
_ = conn.Close(websocket.StatusPolicyViolation, "invalid worker frame")
|
||||
return
|
||||
}
|
||||
select {
|
||||
case incoming <- frame:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer loops.Done()
|
||||
defer cancel()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case frame := <-outgoing:
|
||||
writeCtx, stop := context.WithTimeout(ctx, 5*time.Second)
|
||||
err := conn.Write(writeCtx, websocket.MessageText, frame.data)
|
||||
stop()
|
||||
if frame.done != nil {
|
||||
frame.done <- err
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
defer func() { cancel(); _ = conn.CloseNow(); loops.Wait() }()
|
||||
send := func(frame workerFrame, wait bool) bool {
|
||||
frame.Version = "1"
|
||||
frame.SessionID = binding.SessionID
|
||||
if frame.MessageID == "" {
|
||||
var err error
|
||||
frame.MessageID, err = core.NewID()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
raw, err := json.Marshal(frame)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
out := outboundFrame{data: raw}
|
||||
if wait {
|
||||
out.done = make(chan error, 1)
|
||||
}
|
||||
select {
|
||||
case outgoing <- out:
|
||||
default:
|
||||
return false
|
||||
}
|
||||
if !wait {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case err := <-out.done:
|
||||
return err == nil
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
capacity := input.Capacity
|
||||
draining := welcome.Draining
|
||||
eligible := false
|
||||
leases := map[core.ID]time.Time{}
|
||||
ticker := time.NewTicker(h.poll)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case frame := <-incoming:
|
||||
call, stop := context.WithTimeout(ctx, 8*time.Second)
|
||||
var body any
|
||||
var commandErr error
|
||||
switch frame.Type {
|
||||
case "worker.capacity":
|
||||
var in worker.CapacityInput
|
||||
if commandErr = decodeWorkerJSON(frame.Body, &in); commandErr == nil {
|
||||
var out worker.Welcome
|
||||
out, commandErr = h.service.Capacity(call, token, binding, in)
|
||||
body = out
|
||||
if commandErr == nil {
|
||||
capacity = in.Capacity
|
||||
draining = out.Draining
|
||||
eligible = true
|
||||
}
|
||||
}
|
||||
case "worker.resume":
|
||||
var ref worker.AttemptRef
|
||||
if commandErr = decodeWorkerJSON(frame.Body, &ref); commandErr == nil {
|
||||
var out worker.LeaseStatus
|
||||
out, commandErr = h.service.Resume(call, token, binding, ref)
|
||||
body = out
|
||||
if commandErr == nil {
|
||||
if out.Active {
|
||||
leases[ref.AttemptID] = out.LeaseUntil
|
||||
} else {
|
||||
delete(leases, ref.AttemptID)
|
||||
}
|
||||
}
|
||||
}
|
||||
case "task.start", "task.renew":
|
||||
var ref worker.AttemptRef
|
||||
if commandErr = decodeWorkerJSON(frame.Body, &ref); commandErr == nil {
|
||||
var out worker.CommandResult
|
||||
if frame.Type == "task.start" {
|
||||
out, commandErr = h.service.Start(call, token, binding, frame.MessageID, ref)
|
||||
} else {
|
||||
out, commandErr = h.service.Renew(call, token, binding, frame.MessageID, ref)
|
||||
}
|
||||
body = out
|
||||
if commandErr == nil {
|
||||
if out.Lease.Active {
|
||||
leases[ref.AttemptID] = out.Lease.LeaseUntil
|
||||
} else {
|
||||
delete(leases, ref.AttemptID)
|
||||
}
|
||||
}
|
||||
}
|
||||
case "task.finish":
|
||||
var in worker.FinishInput
|
||||
if commandErr = decodeWorkerJSON(frame.Body, &in); commandErr == nil {
|
||||
var out worker.CommandResult
|
||||
out, commandErr = h.service.Finish(call, token, binding, frame.MessageID, in)
|
||||
body = out
|
||||
if commandErr == nil && !out.Lease.Active {
|
||||
delete(leases, in.Ref.AttemptID)
|
||||
}
|
||||
}
|
||||
default:
|
||||
commandErr = fault.Invalid("unsupported worker message type")
|
||||
}
|
||||
stop()
|
||||
reply := workerFrame{Type: "ack", ReplyTo: frame.MessageID}
|
||||
if commandErr != nil {
|
||||
reply.Type = "error"
|
||||
reply.Error = publicWorkerError(commandErr)
|
||||
} else {
|
||||
reply.Body, commandErr = json.Marshal(body)
|
||||
if commandErr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
fatal := commandErr != nil && (fault.Code(commandErr) == "STALE_SESSION" || fault.Code(commandErr) == "UNAUTHENTICATED" || fault.Code(commandErr) == "PERMISSION_DENIED")
|
||||
if !send(reply, fatal) || fatal {
|
||||
return
|
||||
}
|
||||
case <-ticker.C:
|
||||
// 本地记录仅减少空轮询；SQL 仍原子验证总容量、队列额度和有效会话。
|
||||
now := time.Now()
|
||||
for attempt, until := range leases {
|
||||
if !until.After(now) {
|
||||
delete(leases, attempt)
|
||||
}
|
||||
}
|
||||
if !eligible || draining || capacity == 0 || len(leases) >= int(capacity) || len(outgoing) >= 4 {
|
||||
continue
|
||||
}
|
||||
for burst := 0; burst < 4 && len(leases) < int(capacity) && len(outgoing) < 4; burst++ {
|
||||
call, stop := context.WithTimeout(ctx, 8*time.Second)
|
||||
assignment, err := h.service.Claim(call, token, binding)
|
||||
stop()
|
||||
if err != nil {
|
||||
if fault.Code(err) == "STALE_SESSION" || fault.Code(err) == "UNAUTHENTICATED" || fault.Code(err) == "PERMISSION_DENIED" {
|
||||
send(workerFrame{Type: "error", Error: publicWorkerError(err)}, true)
|
||||
return
|
||||
}
|
||||
break
|
||||
}
|
||||
if assignment == nil {
|
||||
break
|
||||
}
|
||||
raw, err := json.Marshal(assignment)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
leases[assignment.AttemptID] = assignment.LeaseUntil
|
||||
if !send(workerFrame{Type: "task.assign", Body: raw}, false) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
func publicWorkerError(err error) *fault.Error {
|
||||
var known *fault.Error
|
||||
if errors.As(err, &known) {
|
||||
return known
|
||||
}
|
||||
return fault.Unavailable
|
||||
}
|
||||
func writeWorkerFrame(ctx context.Context, conn *websocket.Conn, frame workerFrame) error {
|
||||
raw, err := json.Marshal(frame)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bounded, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
return conn.Write(bounded, websocket.MessageText, raw)
|
||||
}
|
||||
func decodeWorkerJSON(raw []byte, out any) error {
|
||||
if len(raw) == 0 || len(raw) > 512<<10 {
|
||||
return fault.Invalid("invalid worker message size")
|
||||
}
|
||||
if _, err := jsonvalue.Canonical(raw); err != nil {
|
||||
return fault.Invalid("invalid worker JSON")
|
||||
}
|
||||
if err := exactFields(raw, reflect.TypeOf(out).Elem()); err != nil {
|
||||
return fault.Invalid("unknown or invalid worker field")
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(out); err != nil {
|
||||
return fault.Invalid("invalid worker field value")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -58,7 +58,7 @@ type Permissions struct {
|
||||
Queues []string `json:"queues"`
|
||||
}
|
||||
|
||||
var supported = map[string]bool{"tasks:submit": true, "tasks:read": true, "tasks:cancel": true, "queues:read": true, "queues:manage": true}
|
||||
var supported = map[string]bool{"tasks:submit": true, "tasks:read": true, "tasks:cancel": true, "queues:read": true, "queues:manage": true, "workers:connect": true, "workers:consume": true}
|
||||
|
||||
func (p Permissions) Validate() error {
|
||||
if len(p.Actions) == 0 || len(p.Actions) > 32 || len(p.Queues) == 0 || len(p.Queues) > 128 {
|
||||
|
||||
25
pulse/model/attempt_evidence.go
Normal file
25
pulse/model/attempt_evidence.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package model
|
||||
|
||||
import core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
|
||||
// AttemptEvidence 保留迟到或冲突结果，作为核查证据；保存证据不会改写任务结果。
|
||||
type AttemptEvidence struct {
|
||||
// Record 包含命名空间、证据标识、并发版本与首次收到时间。
|
||||
core.Record
|
||||
// AttemptID 证据所属尝试，提交者仍需证明原执行秘密与进程身份。
|
||||
AttemptID core.ID `db:"attempt_id"`
|
||||
// MessageID 稳定结果消息标识，用于同一证据重报去重。
|
||||
MessageID core.ID `db:"message_id"`
|
||||
// Outcome Worker 报告的结果分类，不代表平台认可为任务终态。
|
||||
Outcome string `db:"outcome"`
|
||||
// ErrorCode 可选稳定错误分类；无错误分类时为空。
|
||||
ErrorCode *string `db:"error_code"`
|
||||
// Payload 结果或证据引用 JSON 文本，保留数字精度。
|
||||
Payload core.JSON `db:"payload"`
|
||||
// RequestDigest 结果语义摘要，排除会话身份与原始秘密。
|
||||
RequestDigest core.Digest `db:"request_digest"`
|
||||
// Reason 未用于更新权威状态的原因，例如授权失效或与已保存结果冲突。
|
||||
Reason string `db:"reason"`
|
||||
}
|
||||
|
||||
func (AttemptEvidence) TableName() string { return "pulse_attempt_evidence" }
|
||||
25
pulse/model/worker_instance.go
Normal file
25
pulse/model/worker_instance.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package model
|
||||
|
||||
import core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
|
||||
// WorkerInstance 保存进程实例的持久化会话头；重连推进代次，不能仅凭实例 ID 接管。
|
||||
type WorkerInstance struct {
|
||||
// Record 包含命名空间、标识、并发版本与 UTC 时间。
|
||||
core.Record
|
||||
// CredentialID 初次注册的凭据，当前进程生命周期内不可替换。
|
||||
CredentialID core.ID `db:"credential_id"`
|
||||
// QueueID 进程绑定的消费队列；切换队列应创建新实例。
|
||||
QueueID core.ID `db:"queue_id"`
|
||||
// SecretDigest 进程持有的随机恢复秘密摘要；知道实例 ID 不构成恢复授权。
|
||||
SecretDigest core.Digest `db:"secret_digest"`
|
||||
// RegistrationDigest 队列与处理器版本集合的摘要，重连不得改变注册语义。
|
||||
RegistrationDigest core.Digest `db:"registration_digest"`
|
||||
// SessionEpoch 已分配的最大连接代次，初始为零，重连递增。
|
||||
SessionEpoch uint64 `db:"session_epoch"`
|
||||
// CurrentSessionID 当前连接；注册事务创建会话之前为空。
|
||||
CurrentSessionID *core.ID `db:"current_session_id"`
|
||||
// Draining 持久化排空意图，重连保留；排空后不再给该进程新工作。
|
||||
Draining bool `db:"draining"`
|
||||
}
|
||||
|
||||
func (WorkerInstance) TableName() string { return "pulse_worker_instances" }
|
||||
139
pulse/worker/service.go
Normal file
139
pulse/worker/service.go
Normal file
@@ -0,0 +1,139 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"sort"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/jsonvalue"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
)
|
||||
|
||||
var StaleSession = fault.New("STALE_SESSION", "connection is expired or superseded")
|
||||
var LeaseLost = fault.New("LEASE_LOST", "attempt no longer grants execution authority")
|
||||
var GateClosed = fault.New("EXECUTION_PAUSED", "queue or execution gate is paused")
|
||||
|
||||
type Service struct{ store Store }
|
||||
|
||||
func New(store Store) *Service { return &Service{store: store} }
|
||||
func (s *Service) Authorize(ctx context.Context, t auth.Token) error {
|
||||
return s.store.Authorize(ctx, t)
|
||||
}
|
||||
func NewSecret() (string, core.Digest, error) {
|
||||
var raw [32]byte
|
||||
if _, err := rand.Read(raw[:]); err != nil {
|
||||
return "", core.Digest{}, err
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(raw[:]), core.Digest(sha256.Sum256(raw[:])), nil
|
||||
}
|
||||
func SecretDigest(secret string) (core.Digest, error) {
|
||||
if len(secret) != 43 {
|
||||
return core.Digest{}, fault.Unauthorized
|
||||
}
|
||||
raw, err := base64.RawURLEncoding.Strict().DecodeString(secret)
|
||||
if err != nil || len(raw) != 32 {
|
||||
return core.Digest{}, fault.Unauthorized
|
||||
}
|
||||
return core.Digest(sha256.Sum256(raw)), nil
|
||||
}
|
||||
func NormalizeConnect(in ConnectInput) (ConnectInput, error) {
|
||||
if in.InstanceID.Validate() != nil || in.QueueID.Validate() != nil || in.Capacity < 1 || in.Capacity > 256 || len(in.Capabilities) < 1 || len(in.Capabilities) > 32 {
|
||||
return in, fault.Invalid("invalid worker identity, queue, capacity or capability count")
|
||||
}
|
||||
if _, err := SecretDigest(in.InstanceSecret); err != nil {
|
||||
return in, fault.Invalid("instance_secret must encode 32 random bytes")
|
||||
}
|
||||
in.Capabilities = append([]Capability(nil), in.Capabilities...)
|
||||
for _, c := range in.Capabilities {
|
||||
if !service.ValidName(c.Handler, 128) || !service.ValidName(c.HandlerVersion, 64) || !service.ValidName(c.PayloadVersion, 32) {
|
||||
return in, fault.Invalid("invalid worker capability")
|
||||
}
|
||||
}
|
||||
sort.Slice(in.Capabilities, func(i, j int) bool {
|
||||
a, b := in.Capabilities[i], in.Capabilities[j]
|
||||
if a.Handler != b.Handler {
|
||||
return a.Handler < b.Handler
|
||||
}
|
||||
if a.HandlerVersion != b.HandlerVersion {
|
||||
return a.HandlerVersion < b.HandlerVersion
|
||||
}
|
||||
return a.PayloadVersion < b.PayloadVersion
|
||||
})
|
||||
for i := 1; i < len(in.Capabilities); i++ {
|
||||
if in.Capabilities[i] == in.Capabilities[i-1] {
|
||||
return in, fault.Invalid("duplicate worker capability")
|
||||
}
|
||||
}
|
||||
return in, nil
|
||||
}
|
||||
func (s *Service) Connect(ctx context.Context, t auth.Token, in ConnectInput) (Binding, Welcome, error) {
|
||||
in, err := NormalizeConnect(in)
|
||||
if err != nil {
|
||||
return Binding{}, Welcome{}, err
|
||||
}
|
||||
return s.store.Connect(ctx, t, in)
|
||||
}
|
||||
func (s *Service) Capacity(ctx context.Context, t auth.Token, b Binding, in CapacityInput) (Welcome, error) {
|
||||
if in.Sequence == 0 || in.Capacity > 256 {
|
||||
return Welcome{}, fault.Invalid("invalid capacity sequence or value")
|
||||
}
|
||||
return s.store.Capacity(ctx, t, b, in)
|
||||
}
|
||||
func (s *Service) Claim(ctx context.Context, t auth.Token, b Binding) (*Assignment, error) {
|
||||
return s.store.Claim(ctx, t, b)
|
||||
}
|
||||
func validRef(ref AttemptRef) error {
|
||||
if ref.TaskID.Validate() != nil || ref.AttemptID.Validate() != nil || ref.Epoch == 0 {
|
||||
return fault.Invalid("invalid attempt identity")
|
||||
}
|
||||
_, err := SecretDigest(ref.LeaseSecret)
|
||||
return err
|
||||
}
|
||||
func validCommand(message core.ID, ref AttemptRef) error {
|
||||
if message.Validate() != nil {
|
||||
return fault.Invalid("invalid operation message id")
|
||||
}
|
||||
return validRef(ref)
|
||||
}
|
||||
func (s *Service) Start(ctx context.Context, t auth.Token, b Binding, message core.ID, ref AttemptRef) (CommandResult, error) {
|
||||
if err := validCommand(message, ref); err != nil {
|
||||
return CommandResult{}, err
|
||||
}
|
||||
return s.store.Start(ctx, t, b, message, ref)
|
||||
}
|
||||
func (s *Service) Renew(ctx context.Context, t auth.Token, b Binding, message core.ID, ref AttemptRef) (CommandResult, error) {
|
||||
if err := validCommand(message, ref); err != nil {
|
||||
return CommandResult{}, err
|
||||
}
|
||||
return s.store.Renew(ctx, t, b, message, ref)
|
||||
}
|
||||
func (s *Service) Finish(ctx context.Context, t auth.Token, b Binding, message core.ID, in FinishInput) (CommandResult, error) {
|
||||
if err := validCommand(message, in.Ref); err != nil {
|
||||
return CommandResult{}, err
|
||||
}
|
||||
if in.Outcome != "SUCCEEDED" && in.Outcome != "FAILED" && in.Outcome != "UNKNOWN" {
|
||||
return CommandResult{}, fault.Invalid("invalid result outcome")
|
||||
}
|
||||
if len(in.Result) == 0 || len(in.Result) > 256<<10 || bytes.Equal(bytes.TrimSpace(in.Result), []byte("null")) {
|
||||
return CommandResult{}, fault.Invalid("result must be non-null JSON up to 256 KiB")
|
||||
}
|
||||
if _, err := jsonvalue.Canonical(in.Result); err != nil {
|
||||
return CommandResult{}, fault.Invalid("invalid result JSON")
|
||||
}
|
||||
if (in.Outcome == "SUCCEEDED" && in.ErrorCode != "") || (in.Outcome != "SUCCEEDED" && !service.ValidName(in.ErrorCode, 64)) {
|
||||
return CommandResult{}, fault.Invalid("error_code must match the outcome")
|
||||
}
|
||||
return s.store.Finish(ctx, t, b, message, in)
|
||||
}
|
||||
func (s *Service) Resume(ctx context.Context, t auth.Token, b Binding, ref AttemptRef) (LeaseStatus, error) {
|
||||
if err := validRef(ref); err != nil {
|
||||
return LeaseStatus{}, err
|
||||
}
|
||||
return s.store.Resume(ctx, t, b, ref)
|
||||
}
|
||||
38
pulse/worker/service_test.go
Normal file
38
pulse/worker/service_test.go
Normal file
@@ -0,0 +1,38 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
)
|
||||
|
||||
func TestRegistrationAndSecretBoundaries(t *testing.T) {
|
||||
secret, digest, err := NewSecret()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parsed, err := SecretDigest(secret)
|
||||
if err != nil || parsed != digest {
|
||||
t.Fatal("secret round trip")
|
||||
}
|
||||
for _, bad := range []string{secret + "=", secret + "\n", strings.Repeat("!", 43), secret[:42]} {
|
||||
if _, err := SecretDigest(bad); err == nil {
|
||||
t.Fatal("malformed secret accepted")
|
||||
}
|
||||
}
|
||||
input := ConnectInput{InstanceID: core.ID(strings.Repeat("1", 32)), InstanceSecret: secret, QueueID: core.ID(strings.Repeat("2", 32)), Capacity: 1, Capabilities: []Capability{{"z", "v1", "v1"}, {"a", "v1", "v1"}}}
|
||||
normalized, err := NormalizeConnect(input)
|
||||
if err != nil || normalized.Capabilities[0].Handler != "a" || input.Capabilities[0].Handler != "z" {
|
||||
t.Fatalf("normalization changed caller input: %v", err)
|
||||
}
|
||||
input.Capabilities = append(input.Capabilities, input.Capabilities[0])
|
||||
if _, err := NormalizeConnect(input); err == nil {
|
||||
t.Fatal("duplicate capability accepted")
|
||||
}
|
||||
input = normalized
|
||||
input.Capacity = 257
|
||||
if _, err := NormalizeConnect(input); err == nil {
|
||||
t.Fatal("excess capacity accepted")
|
||||
}
|
||||
}
|
||||
124
pulse/worker/types.go
Normal file
124
pulse/worker/types.go
Normal file
@@ -0,0 +1,124 @@
|
||||
// Package worker 定义执行控制契约；处理器本身在外部 Worker 进程执行。
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
)
|
||||
|
||||
// Capability 精确声明可处理的版本，不做静默版本降级。
|
||||
type Capability struct {
|
||||
Handler string `json:"handler"` // Handler 处理器名称。
|
||||
HandlerVersion string `json:"handler_version"` // HandlerVersion 处理器版本。
|
||||
PayloadVersion string `json:"payload_version"` // PayloadVersion 参数版本。
|
||||
}
|
||||
|
||||
// ConnectInput 在每次连接时提交；同一进程恢复时保留实例和秘密。
|
||||
type ConnectInput struct {
|
||||
InstanceID core.ID `json:"instance_id"` // InstanceID 进程标识，重启换新值。
|
||||
InstanceSecret string `json:"instance_secret"` // InstanceSecret 32 字节随机秘密，只保存摘要。
|
||||
QueueID core.ID `json:"queue_id"` // QueueID 当前进程绑定的消费队列。
|
||||
Capabilities []Capability `json:"capabilities"` // Capabilities 1–32 个精确版本组合，重连固定。
|
||||
Capacity uint32 `json:"capacity"` // Capacity 总执行槽数量，1–256。
|
||||
}
|
||||
|
||||
// Binding 是接入层持有的认证结果，不作为公开请求字段接受或写入日志。
|
||||
type Binding struct {
|
||||
InstanceID core.ID // InstanceID 已证明秘密的进程标识。
|
||||
SessionID core.ID // SessionID 当前连接标识。
|
||||
Epoch uint64 // Epoch 连接代次。
|
||||
Proof core.Digest // Proof 进程秘密摘要，持久化验证旧会话隔离。
|
||||
}
|
||||
|
||||
// Welcome 给出数据库时钟与控制参数；时间用于 SDK 计算保守的单调时钟期限。
|
||||
type Welcome struct {
|
||||
SessionID core.ID `json:"session_id"` // SessionID 本次连接标识。
|
||||
Epoch uint64 `json:"epoch,string"` // Epoch 当前连接代次。
|
||||
ServerTime time.Time `json:"server_time"` // ServerTime 本次响应对应的数据库时间。
|
||||
ExpiresAt time.Time `json:"expires_at"` // ExpiresAt 会话有效期限，不能代替任务租约。
|
||||
LeaseDurationMS uint64 `json:"lease_duration_ms"` // LeaseDurationMS 单次续租窗口，毫秒。
|
||||
Draining bool `json:"draining"` // Draining 排空标记，跨重连保留。
|
||||
}
|
||||
|
||||
// CapacityInput 使用覆盖值和序号，重复消息不会重复增加执行容量。
|
||||
type CapacityInput struct {
|
||||
Sequence uint64 `json:"sequence,string"` // Sequence 本会话内从一开始递增。
|
||||
Capacity uint32 `json:"capacity"` // Capacity 总容量，零表示暂不领取；不释放已有授权。
|
||||
Drain bool `json:"drain"` // Drain 为真后持久化排空，不能通过后续消息清除。
|
||||
}
|
||||
|
||||
// Assignment 仅通过已认证执行连接交付，包含敏感尝试秘密。
|
||||
type Assignment struct {
|
||||
Task service.TaskView `json:"task"` // Task 受理快照及业务参数。
|
||||
AttemptID core.ID `json:"attempt_id"` // AttemptID 本次尝试标识。
|
||||
Epoch uint64 `json:"epoch,string"` // Epoch 任务执行代次，隔离旧尝试。
|
||||
LeaseSecret string `json:"lease_secret"` // LeaseSecret 执行秘密，禁止普通日志及查询返回。
|
||||
LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 尚未开始时的预占有效期。
|
||||
ServerTime time.Time `json:"server_time"` // ServerTime 领取时数据库时间。
|
||||
}
|
||||
|
||||
// AttemptRef 证明对一次尝试的控制权；连接代次仍由 Binding 单独验证。
|
||||
type AttemptRef struct {
|
||||
TaskID core.ID `json:"task_id"` // TaskID 所属任务。
|
||||
AttemptID core.ID `json:"attempt_id"` // AttemptID 目标尝试。
|
||||
Epoch uint64 `json:"epoch,string"` // Epoch 目标执行代次。
|
||||
LeaseSecret string `json:"lease_secret"` // LeaseSecret 原始执行秘密，只在受控连接传输。
|
||||
}
|
||||
|
||||
// FinishInput 区分明确结果与未知效果，未知不能被解释为可以盲目重跑。
|
||||
type FinishInput struct {
|
||||
Ref AttemptRef `json:"ref"` // Ref 当前尝试授权证明。
|
||||
Outcome string `json:"outcome"` // Outcome 为 SUCCEEDED、FAILED 或 UNKNOWN。
|
||||
Result json.RawMessage `json:"result"` // Result 非 null JSON 结果或证据引用，最多 256 KiB。
|
||||
ErrorCode string `json:"error_code,omitempty"` // ErrorCode FAILED/UNKNOWN 必填稳定分类，不包含原始异常正文。
|
||||
}
|
||||
|
||||
// Receipt 是不可变的操作回执，不代表当前仍有执行权。
|
||||
type Receipt struct {
|
||||
Operation string `json:"operation"` // Operation start、renew 或 finish。
|
||||
TaskState pulse.TaskState `json:"task_state"` // TaskState 提交该操作时的任务状态。
|
||||
AttemptState pulse.AttemptState `json:"attempt_state"` // AttemptState 操作提交时的尝试状态。
|
||||
TaskRevision uint64 `json:"task_revision,string"` // TaskRevision 操作提交时的任务版本。
|
||||
LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 操作提交时租约期限，历史值不能授权新执行。
|
||||
Disposition string `json:"disposition"` // Disposition 为 APPLIED 或 EVIDENCE_ONLY。
|
||||
}
|
||||
|
||||
// LeaseStatus 每次响应重新计算；处理器仅在 Active 为真、State 为 RUNNING 且期限内推进。
|
||||
// LEASED 的 Active 只代表预占仍有效，不能代替开始授权。
|
||||
type LeaseStatus struct {
|
||||
Active bool `json:"active"` // Active 当前会话仍持有未过期的有效尝试。
|
||||
State pulse.AttemptState `json:"state"` // State 当前尝试状态。
|
||||
LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 当前租约，不因重放旧回执自动延长。
|
||||
ServerTime time.Time `json:"server_time"` // ServerTime 完成锁定后重新读取的数据库时间。
|
||||
}
|
||||
|
||||
// CommandResult 将原回执与当前执行权分开，避免旧 start ACK 使过期尝试复活。
|
||||
type CommandResult struct {
|
||||
Receipt Receipt `json:"receipt"` // Receipt 对应稳定消息身份的原回执。
|
||||
Lease LeaseStatus `json:"lease"` // Lease 当前授权判断。
|
||||
Deduplicated bool `json:"deduplicated"` // Deduplicated 是否返回已保存的回执。
|
||||
}
|
||||
|
||||
// RecoveryStats 统计本批恢复，不作为持久化成功率或容量指标。
|
||||
type RecoveryStats struct {
|
||||
Examined int
|
||||
Requeued int
|
||||
Failed int
|
||||
NeedsReview int
|
||||
}
|
||||
type Store interface {
|
||||
Authorize(context.Context, auth.Token) error
|
||||
Connect(context.Context, auth.Token, ConnectInput) (Binding, Welcome, error)
|
||||
Capacity(context.Context, auth.Token, Binding, CapacityInput) (Welcome, error)
|
||||
Claim(context.Context, auth.Token, Binding) (*Assignment, error)
|
||||
Start(context.Context, auth.Token, Binding, core.ID, AttemptRef) (CommandResult, error)
|
||||
Renew(context.Context, auth.Token, Binding, core.ID, AttemptRef) (CommandResult, error)
|
||||
Finish(context.Context, auth.Token, Binding, core.ID, FinishInput) (CommandResult, error)
|
||||
Resume(context.Context, auth.Token, Binding, AttemptRef) (LeaseStatus, error)
|
||||
}
|
||||
Reference in New Issue
Block a user