feat: add durable Worker execution and WSS server
This commit is contained in:
@@ -8,9 +8,9 @@ Orbit 是面向多个业务项目的通用任务调度与流程编排平台,
|
|||||||
|
|
||||||
## 当前进度
|
## 当前进度
|
||||||
|
|
||||||
已实现 30 个数据模型、版本化迁移、凭据及范围校验、队列创建查询、任务幂等提交/查询/分页/待执行取消,以及独立 Go SDK 和可运行示例。WSS Worker、调度器和流程执行引擎按 [持续开发计划](docs/08-development-plan.md) 继续实现。
|
已实现 32 个数据模型、版本化迁移、凭据及范围校验、队列创建查询、任务幂等提交/查询/分页/待执行取消,以及对应 Go SDK 和示例。P2 服务端执行内核和 WSS 已通过阶段内回归;Worker SDK 自动运行器、调度器和流程引擎按 [持续开发计划](docs/08-development-plan.md) 继续实现。
|
||||||
|
|
||||||
readiness 只表示数据库与迁移记录可用,不表示完整平台已具备生产服务能力。验证证据见 [P0](docs/reports/phase-0.md) 与 [P1 报告](docs/reports/phase-1.md)。
|
readiness 只表示数据库与迁移记录可用,不表示完整平台已具备生产服务能力。验证证据见 [P0](docs/reports/phase-0.md)、[P1](docs/reports/phase-1.md) 与 [P2 服务端记录](docs/reports/phase-2-server.md)。
|
||||||
|
|
||||||
## 目录
|
## 目录
|
||||||
|
|
||||||
@@ -23,6 +23,7 @@ platform/model/ 应用、命名空间、幂等、事件和审计
|
|||||||
platform/auth/ 凭据格式与动作/队列授权范围
|
platform/auth/ 凭据格式与动作/队列授权范围
|
||||||
pulse/model/ 任务、尝试、队列、计划和执行许可
|
pulse/model/ 任务、尝试、队列、计划和执行许可
|
||||||
pulse/service/ 任务与队列应用契约、输入校验
|
pulse/service/ 任务与队列应用契约、输入校验
|
||||||
|
pulse/worker/ 会话、开始、续租、结果和恢复契约
|
||||||
weave/model/ 定义、实例、步骤、信号和查询轮次
|
weave/model/ 定义、实例、步骤、信号和查询轮次
|
||||||
internal/config/ 配置校验
|
internal/config/ 配置校验
|
||||||
internal/storage/mysql/ 连接、事务、迁移及集成测试
|
internal/storage/mysql/ 连接、事务、迁移及集成测试
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ func run() error {
|
|||||||
flags.StringVar(&in.Environment, "environment", "", "environment name")
|
flags.StringVar(&in.Environment, "environment", "", "environment name")
|
||||||
flags.StringVar(&in.Queue, "queue", "", "initial queue name")
|
flags.StringVar(&in.Queue, "queue", "", "initial queue name")
|
||||||
flags.StringVar(&output, "output", "", "new private credentials JSON file")
|
flags.StringVar(&output, "output", "", "new private credentials JSON file")
|
||||||
|
flags.BoolVar(&in.WorkerAccess, "worker-access", false, "also grant Worker connect and consume actions for development")
|
||||||
if err := flags.Parse(os.Args[2:]); err != nil {
|
if err := flags.Parse(os.Args[2:]); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,7 +14,9 @@ import (
|
|||||||
"git.yuntuo.hk/k2safe/orbit/internal/config"
|
"git.yuntuo.hk/k2safe/orbit/internal/config"
|
||||||
"git.yuntuo.hk/k2safe/orbit/internal/storage/mysql"
|
"git.yuntuo.hk/k2safe/orbit/internal/storage/mysql"
|
||||||
"git.yuntuo.hk/k2safe/orbit/internal/transport/httpserver"
|
"git.yuntuo.hk/k2safe/orbit/internal/transport/httpserver"
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -49,13 +51,49 @@ func run() error {
|
|||||||
}
|
}
|
||||||
return mysql.CheckSchema(ctx, db)
|
return mysql.CheckSchema(ctx, db)
|
||||||
}, service.New(mysql.NewTaskStore(db)))
|
}, service.New(mysql.NewTaskStore(db)))
|
||||||
|
node, err := core.NewID()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
executions, err := mysql.NewExecutionStore(db, node, mysql.ExecutionOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
hub := httpserver.NewWorkerHub(root, worker.New(executions))
|
||||||
|
httpserver.RegisterWorkers(server, hub)
|
||||||
|
defer func() {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_ = hub.Shutdown(ctx)
|
||||||
|
}()
|
||||||
|
recoveryDone := make(chan struct{})
|
||||||
|
recoveryCtx, stopRecovery := context.WithCancel(root)
|
||||||
|
go func() {
|
||||||
|
defer close(recoveryDone)
|
||||||
|
ticker := time.NewTicker(2 * time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-recoveryCtx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
call, cancel := context.WithTimeout(recoveryCtx, 5*time.Second)
|
||||||
|
_, err := executions.Recover(call, 100)
|
||||||
|
cancel()
|
||||||
|
if err != nil && recoveryCtx.Err() == nil {
|
||||||
|
slog.Error("execution recovery batch failed", "node_id", node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
defer func() { stopRecovery(); <-recoveryDone }()
|
||||||
listener, err := net.Listen("tcp", cfg.HTTPAddr)
|
listener, err := net.Listen("tcp", cfg.HTTPAddr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- server.Serve(listener) }()
|
go func() { done <- server.Serve(listener) }()
|
||||||
slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task and queue management")
|
slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task, queue and Worker APIs", "node_id", node)
|
||||||
select {
|
select {
|
||||||
case err := <-done:
|
case err := <-done:
|
||||||
if errors.Is(err, http.ErrServerClosed) {
|
if errors.Is(err, http.ErrServerClosed) {
|
||||||
|
|||||||
@@ -2,6 +2,8 @@
|
|||||||
|
|
||||||
> 版本 0.2 · 2026-09-17。HTTP + WS 已确认。本文的具体路径、消息名、字段及状态码是首版设计建议,业务协议尚未实现,需评审后冻结;健康接口的已实现契约见接口手册。
|
> 版本 0.2 · 2026-09-17。HTTP + WS 已确认。本文的具体路径、消息名、字段及状态码是首版设计建议,业务协议尚未实现,需评审后冻结;健康接口的已实现契约见接口手册。
|
||||||
|
|
||||||
|
实现更新:HTTP 任务/队列接口已完成 P1;Worker 服务端已进入 P2,实际帧名、字段、回执与边界见 [Worker 契约](api/worker.md)。本文保留早期完整设计,未实现项目不能据此视为已交付。
|
||||||
|
|
||||||
## 1. 通用契约
|
## 1. 通用契约
|
||||||
|
|
||||||
- 生产使用 HTTPS / WSS;HTTP 承担任务提交、流程启动与信号、查询、管理,WS 承担 Worker 双向通信与结果订阅。
|
- 生产使用 HTTPS / WSS;HTTP 承担任务提交、流程启动与信号、查询、管理,WS 承担 Worker 双向通信与结果订阅。
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
- `model/`:ID、摘要、JSON、命名空间记录等公共原语。
|
- `model/`:ID、摘要、JSON、命名空间记录等公共原语。
|
||||||
- `platform/model/`:应用、命名空间、凭据、幂等、事件、订阅交付及审计。
|
- `platform/model/`:应用、命名空间、凭据、幂等、事件、订阅交付及审计。
|
||||||
- `pulse/model/`:队列、执行门控、会话、任务、尝试、回执、并发许可和调度计划。
|
- `pulse/model/`:队列、执行门控、进程/会话、任务、尝试、回执、迟到证据、并发许可和调度计划。
|
||||||
- `weave/model/`:定义及不可变版本、实例、步骤激活、等待、信号、查询轮次与恢复记录。
|
- `weave/model/`:定义及不可变版本、实例、步骤激活、等待、信号、查询轮次与恢复记录。
|
||||||
- `internal/storage/mysql/`:连接、事务与显式迁移,不向 SDK 暴露。
|
- `internal/storage/mysql/`:连接、事务与显式迁移,不向 SDK 暴露。
|
||||||
- `sdk/go/`:独立 Go module;对外 DTO 与服务端模型分别维护。
|
- `sdk/go/`:独立 Go module;对外 DTO 与服务端模型分别维护。
|
||||||
@@ -34,7 +34,7 @@
|
|||||||
|
|
||||||
## 3. 实体清单
|
## 3. 实体清单
|
||||||
|
|
||||||
下表与初始迁移中的 30 个实体一致;迁移自身另有 `orbit_schema_migrations` 记录。
|
当前共 32 个实体:初始迁移包含 30 个,000003 增加进程会话头和迟到证据。迁移自身另有 `orbit_schema_migrations` 记录。
|
||||||
|
|
||||||
| 模块 | Go 模型 | 数据表 |
|
| 模块 | Go 模型 | 数据表 |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
@@ -49,9 +49,11 @@
|
|||||||
| pulse | `Queue` | `pulse_queues` |
|
| pulse | `Queue` | `pulse_queues` |
|
||||||
| pulse | `ExecutionGate` | `pulse_execution_gates` |
|
| pulse | `ExecutionGate` | `pulse_execution_gates` |
|
||||||
| pulse | `WorkerSession` | `pulse_worker_sessions` |
|
| pulse | `WorkerSession` | `pulse_worker_sessions` |
|
||||||
|
| pulse | `WorkerInstance` | `pulse_worker_instances` |
|
||||||
| pulse | `Task` | `pulse_tasks` |
|
| pulse | `Task` | `pulse_tasks` |
|
||||||
| pulse | `Attempt` | `pulse_attempts` |
|
| pulse | `Attempt` | `pulse_attempts` |
|
||||||
| pulse | `AttemptReceipt` | `pulse_attempt_receipts` |
|
| pulse | `AttemptReceipt` | `pulse_attempt_receipts` |
|
||||||
|
| pulse | `AttemptEvidence` | `pulse_attempt_evidence` |
|
||||||
| pulse | `ConcurrencyLimit` | `pulse_concurrency_limits` |
|
| pulse | `ConcurrencyLimit` | `pulse_concurrency_limits` |
|
||||||
| pulse | `Permit` | `pulse_permits` |
|
| pulse | `Permit` | `pulse_permits` |
|
||||||
| pulse | `Schedule` | `pulse_schedules` |
|
| pulse | `Schedule` | `pulse_schedules` |
|
||||||
@@ -86,6 +88,8 @@
|
|||||||
|
|
||||||
外键与 `CHECK` 无法代替执行引擎。P1 已完成 HTTP 动作/队列授权、幂等摘要与原回执、受理事件,以及待执行取消的 CAS/审计事务。执行相关的以下条件继续分阶段实现:
|
外键与 `CHECK` 无法代替执行引擎。P1 已完成 HTTP 动作/队列授权、幂等摘要与原回执、受理事件,以及待执行取消的 CAS/审计事务。执行相关的以下条件继续分阶段实现:
|
||||||
|
|
||||||
|
P2 服务端已新增会话隔离、两步派发、租约、并发许可、结果回执/证据和恢复扫描,见 [执行协议](api/worker.md) 与 [阶段记录](reports/phase-2-server.md)。Worker SDK 和完整 P2 验收仍在推进。
|
||||||
|
|
||||||
1. 权限验证、凭据校验、命名空间过滤和队列消费授权。
|
1. 权限验证、凭据校验、命名空间过滤和队列消费授权。
|
||||||
2. 同一次事务更新任务、尝试、租约、并发许可、事件与流程交付记录。
|
2. 同一次事务更新任务、尝试、租约、并发许可、事件与流程交付记录。
|
||||||
3. CAS 竞争、数据库时间校验、门控与开始授权串行化。
|
3. CAS 竞争、数据库时间校验、门控与开始授权串行化。
|
||||||
|
|||||||
@@ -26,7 +26,7 @@
|
|||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| P0 基础与数据模型 | Go workspace、独立 SDK module、公共/Pulse/Weave 模型、MySQL 迁移、配置、健康检查 | 编译/竞态/静态检查;真实 MySQL 建表、隔离、唯一性、预算、不可变版本、迁移失败防护;SDK 健康调用 | 已完成并通过阶段回归(见 P0 报告) |
|
| P0 基础与数据模型 | Go workspace、独立 SDK module、公共/Pulse/Weave 模型、MySQL 迁移、配置、健康检查 | 编译/竞态/静态检查;真实 MySQL 建表、隔离、唯一性、预算、不可变版本、迁移失败防护;SDK 健康调用 | 已完成并通过阶段回归(见 P0 报告) |
|
||||||
| P1 任务受理闭环 | 应用凭据与作用域权限、队列、幂等提交、查询、待执行取消、受理事件;SDK 客户端 | API/SDK 创建查询闭环;重复键与内容冲突、响应未确认重报、跨命名空间拒绝、参数边界 | 已完成并通过阶段回归(见 P1 报告;真实断路/提交 ACK 丢失注入留后续故障阶段) |
|
| P1 任务受理闭环 | 应用凭据与作用域权限、队列、幂等提交、查询、待执行取消、受理事件;SDK 客户端 | API/SDK 创建查询闭环;重复键与内容冲突、响应未确认重报、跨命名空间拒绝、参数边界 | 已完成并通过阶段回归(见 P1 报告;真实断路/提交 ACK 丢失注入留后续故障阶段) |
|
||||||
| P2 Worker 执行闭环 | WSS 接入、能力/容量、领取与开始、租约、续租、结果确认、排空与恢复;Worker SDK | 真 Worker 执行;失联/重连/重复确认/旧尝试拒绝;业务效果未知转核查;节点故障接管 | 待开发 |
|
| P2 Worker 执行闭环 | WSS 接入、能力/容量、领取与开始、租约、续租、结果确认、排空与恢复;Worker SDK | 真 Worker 执行;失联/重连/重复确认/旧尝试拒绝;业务效果未知转核查;节点故障接管 | 开发中:服务端执行事务和 WSS 已通过阶段内回归;Worker SDK 自动运行/重连/退出及完整验收待完成 |
|
||||||
| P3 Pulse 完整能力 | 安全重试、延时、Cron/固定间隔、漏跑策略、计划版本、并发许可、配额/背压、可靠结果订阅 | 多实例领取/触发竞争、重试上限、取消竞争、队列隔离、积压恢复、事件重复交付 | 待开发 |
|
| P3 Pulse 完整能力 | 安全重试、延时、Cron/固定间隔、漏跑策略、计划版本、并发许可、配额/背压、可靠结果订阅 | 多实例领取/触发竞争、重试上限、取消竞争、队列隔离、积压恢复、事件重复交付 | 待开发 |
|
||||||
| P4 Weave 主流程 | SDK 定义构建与发布校验、实例、顺序/分支/结构化并行汇合、信号/定时等待、版本绑定 | 重启接续、早到/重复信号、并行竞争、内部事件重放、发布版本兼容;端到端通用流程 | 待开发 |
|
| P4 Weave 主流程 | SDK 定义构建与发布校验、实例、顺序/分支/结构化并行汇合、信号/定时等待、版本绑定 | 重启接续、早到/重复信号、并行竞争、内部事件重放、发布版本兼容;端到端通用流程 | 待开发 |
|
||||||
| P5 Weave 恢复与查询 | 有界查询、暂停继续、取消、人工核查恢复、控制门控与审计 | 轮次与尝试独立预算;暂停/开始/回执/超时竞争;恢复保留预算;未知结果不盲目重复 | 待开发 |
|
| P5 Weave 恢复与查询 | 有界查询、暂停继续、取消、人工核查恢复、控制门控与审计 | 轮次与尝试独立预算;暂停/开始/回执/超时竞争;恢复保留预算;未知结果不盲目重复 | 待开发 |
|
||||||
@@ -73,3 +73,4 @@ SDK、文档与示例随 P1–P5 同步开发,P7 是全量核对,不是最
|
|||||||
- [生产验收总清单](05-production-acceptance.md)
|
- [生产验收总清单](05-production-acceptance.md)
|
||||||
- [P0 验证记录](reports/phase-0.md)
|
- [P0 验证记录](reports/phase-0.md)
|
||||||
- [P1 验证记录](reports/phase-1.md)
|
- [P1 验证记录](reports/phase-1.md)
|
||||||
|
- [P2 服务端阶段记录](reports/phase-2-server.md)
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
- 文档版本:0.2
|
- 文档版本:0.2
|
||||||
- 整理日期:2026-09-17
|
- 整理日期:2026-09-17
|
||||||
- 阶段:P0 模型基础与 P1 任务受理/SDK 已实现,平台完整功能按持续计划开发;尚未完成生产验收
|
- 阶段:P0/P1 已完成,P2 服务端执行/WSS 已通过阶段内回归,Worker SDK 继续开发;尚未完成生产验收
|
||||||
- 适用范围:Orbit 统一平台,包含 Pulse 任务调度模块、Weave 流程编排模块与公共能力。平台面向多个业务系统复用。
|
- 适用范围:Orbit 统一平台,包含 Pulse 任务调度模块、Weave 流程编排模块与公共能力。平台面向多个业务系统复用。
|
||||||
|
|
||||||
## 阅读顺序
|
## 阅读顺序
|
||||||
@@ -79,4 +79,4 @@ Orbit 不内置特定行业规则;Pulse 管理执行,Weave 管理步骤依
|
|||||||
|
|
||||||
2026-09-17,文档更新至 0.2:用户确认研发定义发布流程及 Weave 首版功能,明确纳入有界重复查询和暂停/继续等生产管理能力。技术细则与参数继续按建议或待定标注,未将功能确认等同于实现或验收通过。
|
2026-09-17,文档更新至 0.2:用户确认研发定义发布流程及 Weave 首版功能,明确纳入有界重复查询和暂停/继续等生产管理能力。技术细则与参数继续按建议或待定标注,未将功能确认等同于实现或验收通过。
|
||||||
|
|
||||||
持续开发目标已建立。当前实现和验收进度以 [开发计划](08-development-plan.md)、[P0 报告](reports/phase-0.md)、[P1 报告](reports/phase-1.md) 为准;早期设计稿中的“未实现”描述保留其编写阶段语境。
|
持续开发目标已建立。当前实现和验收进度以 [开发计划](08-development-plan.md)、[P0 报告](reports/phase-0.md)、[P1 报告](reports/phase-1.md)、[P2 服务端记录](reports/phase-2-server.md) 为准;早期设计稿中的“未实现”描述保留其编写阶段语境。
|
||||||
|
|||||||
@@ -11,7 +11,7 @@
|
|||||||
| `/v1/tasks` 提交、查询、分页与待执行取消 | P1 已实现 | [任务接口](tasks.md) |
|
| `/v1/tasks` 提交、查询、分页与待执行取消 | P1 已实现 | [任务接口](tasks.md) |
|
||||||
| `/v1/queues` 创建与查询 | P1 已实现 | [队列接口](queues.md) |
|
| `/v1/queues` 创建与查询 | P1 已实现 | [队列接口](queues.md) |
|
||||||
| 凭据校验与初始化 CLI | P1 已实现 | [鉴权](authentication.md) |
|
| 凭据校验与初始化 CLI | P1 已实现 | [鉴权](authentication.md) |
|
||||||
| `/v1/worker/connect` | 计划 P2,尚未实现 | [协议草案](../03-http-websocket-protocol.md) |
|
| `/v1/worker/connect` | P2 服务端已实现,Worker SDK 仍在开发 | [Worker 契约](worker.md) |
|
||||||
| `/v1/schedules` 与计划管理 | 计划 P3,尚未实现 | [协议草案](../03-http-websocket-protocol.md) |
|
| `/v1/schedules` 与计划管理 | 计划 P3,尚未实现 | [协议草案](../03-http-websocket-protocol.md) |
|
||||||
| `/v1/events/connect` | 计划 P3,尚未实现 | [协议草案](../03-http-websocket-protocol.md) |
|
| `/v1/events/connect` | 计划 P3,尚未实现 | [协议草案](../03-http-websocket-protocol.md) |
|
||||||
| `/v1/workflows` 与流程管理 | 计划 P4–P5,尚未实现 | [流程接口规划](workflows.md) |
|
| `/v1/workflows` 与流程管理 | 计划 P4–P5,尚未实现 | [流程接口规划](workflows.md) |
|
||||||
|
|||||||
@@ -51,3 +51,5 @@ actions 不支持通配符;queues 可列出队列 ID,星号仅表示当前
|
|||||||
原始秘密错误、凭据撤销/过期、应用或命名空间停用统一返回 401。合法身份缺少动作权限返回 403。读取不在范围内的单个资源返回 404,避免披露资源内容。
|
原始秘密错误、凭据撤销/过期、应用或命名空间停用统一返回 401。合法身份缺少动作权限返回 403。读取不在范围内的单个资源返回 404,避免披露资源内容。
|
||||||
|
|
||||||
SDK 使用 `orbit.Options{Token: token}`。健康查询不发送凭据。当前不启用业务权限缓存,撤销校验不依赖缓存过期。
|
SDK 使用 `orbit.Options{Token: token}`。健康查询不发送凭据。当前不启用业务权限缓存,撤销校验不依赖缓存过期。
|
||||||
|
|
||||||
|
P2 增加 workers:connect 和 workers:consume。bootstrap 只有显式提供 --worker-access 时才同时授予这两项开发权限;现有凭据不自动变化。Worker 还需独立的进程恢复秘密和尝试秘密,见 [Worker 契约](worker.md)。
|
||||||
|
|||||||
@@ -2,6 +2,8 @@
|
|||||||
|
|
||||||
状态:P1 已实现并通过真实 MySQL 与 SDK 回归。当前提供受理、查询、分页和待执行取消;Worker 执行与自动重试在 P2–P3 实现。
|
状态:P1 已实现并通过真实 MySQL 与 SDK 回归。当前提供受理、查询、分页和待执行取消;Worker 执行与自动重试在 P2–P3 实现。
|
||||||
|
|
||||||
|
P2 进展:服务端执行内核和 WSS 已可产生 LEASED、RUNNING、SUCCEEDED、FAILED、NEEDS_REVIEW 等状态,见 [Worker 契约](worker.md);独立 Worker SDK 仍在开发,自动策略重试尚未完成。
|
||||||
|
|
||||||
## 公共约定
|
## 公共约定
|
||||||
|
|
||||||
业务接口需要 `Authorization: Bearer <token>`;写入需要 `Content-Type: application/json` 和稳定的 `Idempotency-Key`。凭据决定命名空间,请求体不能覆盖它。详见 [鉴权与初始化](authentication.md)。
|
业务接口需要 `Authorization: Bearer <token>`;写入需要 `Content-Type: application/json` 和稳定的 `Idempotency-Key`。凭据决定命名空间,请求体不能覆盖它。详见 [鉴权与初始化](authentication.md)。
|
||||||
|
|||||||
146
docs/api/worker.md
Normal file
146
docs/api/worker.md
Normal file
@@ -0,0 +1,146 @@
|
|||||||
|
# Worker 通信与执行契约 v1(开发中)
|
||||||
|
|
||||||
|
已实现服务端执行事务、WebSocket 接入和真实 WSS 回归。**P2 尚未完成**:独立 Go SDK 的 Worker 运行器、自动重连/续租、处理器生命周期与优雅退出还在开发。本页记录当前可验证的服务端契约,不把测试客户端当作可交付 Worker SDK。
|
||||||
|
|
||||||
|
## 1. 接入与身份
|
||||||
|
|
||||||
|
入口 `GET /v1/worker/connect`,子协议必须为 `pulse.worker.v1`。Authorization 使用 Bearer 凭据;禁止 URL 查询参数。握手先校验 workers:connect,消费控制操作再次校验 workers:consume、队列范围和凭据有效性。
|
||||||
|
|
||||||
|
初始化开发凭据可显式增加 Worker 权限:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/orbit-admin bootstrap \
|
||||||
|
--application worker-demo --namespace default --environment development \
|
||||||
|
--queue objects --worker-access --output .local/worker.credentials.json
|
||||||
|
```
|
||||||
|
|
||||||
|
默认 bootstrap 仍只有 P1 的五项权限,现有凭据不会自动扩权。生产细粒度凭据管理在 P6 完成。
|
||||||
|
|
||||||
|
当前 orbit 命令监听 HTTP,开发可使用 ws;生产需要受控的 TLS 终止。WSS 回归使用真实 TLS 测试服务器及受信任测试证书,没有跳过证书校验。传输库锁定 github.com/coder/websocket v1.8.15,使用上下文、读写期限、子协议和默认 Origin 校验,关闭压缩。[上游版本记录](https://github.com/coder/websocket/releases/tag/v1.8.15)、[上游接口文档](https://pkg.go.dev/github.com/coder/websocket@v1.8.15)。
|
||||||
|
|
||||||
|
## 2. 帧格式
|
||||||
|
|
||||||
|
只接受 JSON 文本帧;单帧最多 512 KiB,业务结果最多 256 KiB。重复键、未知字段、无效 Unicode 和非法数值按 HTTP 相同规则拒绝。
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"protocol_version":"1",
|
||||||
|
"type":"task.start",
|
||||||
|
"message_id":"44444444444444444444444444444444",
|
||||||
|
"session_id":"33333333333333333333333333333333",
|
||||||
|
"body":{
|
||||||
|
"task_id":"11111111111111111111111111111111",
|
||||||
|
"attempt_id":"22222222222222222222222222222222",
|
||||||
|
"epoch":"1",
|
||||||
|
"lease_secret":"<执行秘密,仅通过受控连接传输>"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
所有 ID 为 32 位小写十六进制;epoch、sequence、revision 用十进制字符串。message_id 标识一次操作,重连重报保持不变;session_id 必须换为当前连接身份。服务端响应以 reply_to 关联请求:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"protocol_version":"1",
|
||||||
|
"type":"error",
|
||||||
|
"message_id":"55555555555555555555555555555555",
|
||||||
|
"reply_to":"44444444444444444444444444444444",
|
||||||
|
"session_id":"33333333333333333333333333333333",
|
||||||
|
"error":{"code":"LEASE_LOST","message":"attempt no longer grants execution authority"}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
普通成功响应 type=ack,body 为下表对应结果。服务端主动派发 type=task.assign,无 reply_to。错误仅包含稳定分类和公开消息,不包含 SQL、业务参数或秘密。
|
||||||
|
|
||||||
|
## 3. 注册、恢复与容量
|
||||||
|
|
||||||
|
连接后 5 秒内必须发送 worker.hello,hello 不携带 session_id。
|
||||||
|
|
||||||
|
| hello 字段 | 含义 |
|
||||||
|
| --- | --- |
|
||||||
|
| instance_id | Worker 进程实例 ID;进程重启必须换新值 |
|
||||||
|
| instance_secret | 32 字节随机秘密的无填充 base64url 编码,共 43 字符;同进程重连保留 |
|
||||||
|
| queue_id | 当前进程绑定队列 |
|
||||||
|
| capabilities | 1–32 个 handler / handler_version / payload_version 精确版本组合,不可重复 |
|
||||||
|
| capacity | 总执行槽数量,1–256,不是可用槽的累加值 |
|
||||||
|
|
||||||
|
重连必须保持凭据、队列、处理器集合和进程秘密,处理器集合的排列顺序不影响身份。新连接递增会话代次并使旧会话失效。知道 instance_id 或 task_id 不能接管执行。
|
||||||
|
|
||||||
|
worker.welcome 返回 session_id、epoch、server_time、expires_at、lease_duration_ms、draining。当前窗口为会话 45 秒、执行租约最多 30 秒;注册及能力声明不等于开始授权。
|
||||||
|
|
||||||
|
**派发在收到第一次 worker.capacity 确认后才启用**,便于客户端先恢复旧尝试。
|
||||||
|
|
||||||
|
| 类型 | body | 返回 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| worker.resume | AttemptRef,字段见上方 task.start 示例 | 当前 LeaseStatus;成功迁移有效尝试到新会话,不创建尝试或延长期限 |
|
||||||
|
| worker.capacity | sequence 字符串、capacity 0–256、drain bool | 更新后的 welcome 字段 |
|
||||||
|
| task.assign | 服务端主动发送 | task 完整任务、attempt_id、epoch、lease_secret、lease_until、server_time |
|
||||||
|
|
||||||
|
capacity 使用单调递增序号和覆盖值。同序号同内容可重复,同序号不同内容或旧序号冲突;capacity=0 不释放已有任务的并发许可。drain=true 是进程级持久化排空,重连保留,不能通过后续消息清除;新进程使用新实例身份。
|
||||||
|
|
||||||
|
一个 Worker 进程当前绑定一个队列。可以运行多个 Worker 进程或后续 SDK 实例消费不同队列;不能通过重连静默切换队列。
|
||||||
|
|
||||||
|
## 4. 开始、续租与结果
|
||||||
|
|
||||||
|
| 类型 | body | 语义 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| task.start | AttemptRef | 领取后再次检查当前会话、尝试、租约、队列暂停及流程门控,保存 RUNNING 后确认 |
|
||||||
|
| task.renew | AttemptRef | 只续期当前 RUNNING;新消息可续租,重复原消息只取回原回执 |
|
||||||
|
| task.finish | ref: AttemptRef、outcome、result、error_code | 保存明确结果或未知效果;返回操作回执和当前授权 |
|
||||||
|
|
||||||
|
outcome 为 SUCCEEDED / FAILED / UNKNOWN。result 为非 null JSON 或证据引用。FAILED/UNKNOWN 必须带 1–64 字符的稳定 error_code;SUCCEEDED 不带错误分类。当前 FAILED 成为任务失败终态,UNKNOWN 转 NEEDS_REVIEW;策略化自动重试在 P3 实现。
|
||||||
|
|
||||||
|
开始、续租和结束返回 CommandResult:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"receipt":{
|
||||||
|
"operation":"start",
|
||||||
|
"task_state":"RUNNING",
|
||||||
|
"attempt_state":"RUNNING",
|
||||||
|
"task_revision":"3",
|
||||||
|
"lease_until":"2026-09-17T08:00:30Z",
|
||||||
|
"disposition":"APPLIED"
|
||||||
|
},
|
||||||
|
"lease":{
|
||||||
|
"active":true,
|
||||||
|
"state":"RUNNING",
|
||||||
|
"lease_until":"2026-09-17T08:00:30Z",
|
||||||
|
"server_time":"2026-09-17T08:00:00Z"
|
||||||
|
},
|
||||||
|
"deduplicated":false
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
receipt 是原操作的不可变确认;lease 是本次重新核验的执行权,两者不能混用:
|
||||||
|
|
||||||
|
- 只有 lease.active=true、lease.state=RUNNING,且仍在保守的本地有效期内,才允许处理器推进。LEASED 的 active 只表示预占有效。
|
||||||
|
- 原 start 回执可能仍表示 RUNNING,但当前 lease.active=false;旧 ACK 不能让已过期、已结束或已被替换的尝试复活。
|
||||||
|
- 有效期不超过租约窗口、会话期限、任务总截止和从 authorized_at 起计算的单次尝试时限。
|
||||||
|
- 客户端需要扣除请求总耗时及安全余量,用本地单调时钟停止执行,不能假定本地墙上时钟与数据库一致。
|
||||||
|
- 同结果重报不重复释放许可或产生成功事件。同消息不同内容返回 IDEMPOTENCY_CONFLICT。
|
||||||
|
- 迟到、未开始或与已保存结果冲突的内容,经过身份和秘密验证后保存为 EVIDENCE_ONLY;不改变任务权威结果,不当作新的执行授权。
|
||||||
|
- 断线后原结果仍保留稳定消息 ID,可在新会话恢复/核对后重报。不会因重连创建第二次业务执行。
|
||||||
|
|
||||||
|
## 5. 领取、并发与回收
|
||||||
|
|
||||||
|
平台在同一事务中分配队列并发许可、创建尝试、递增任务执行代次和次数、保存 LEASED 与派发事件。领取本身消耗一次 max_attempts 预算,包括最终未获开始授权的尝试,预算不会因重连或过期恢复重置。
|
||||||
|
|
||||||
|
运行中的进程总容量跨其所有旧/新会话统计。声明容量减少、断开连接或重复结果,都不能重复释放额度。当前已实施队列 max_concurrency;rate_per_second、其他配额和调度策略在 P3 完成。
|
||||||
|
|
||||||
|
每个节点每 2 秒发起一批恢复扫描,每批最多 100 项:
|
||||||
|
|
||||||
|
- 未批准开始即到期:有剩余次数和截止时间时,延后 1 秒重派;否则 FAILED。
|
||||||
|
- 已批准开始后失联:NEEDS_REVIEW,尝试 UNKNOWN,不能因网络超时断定业务未生效。
|
||||||
|
- WAITING 超过总截止或最晚开始时间:FAILED。
|
||||||
|
- 回收再次锁定队列、任务、尝试并读取数据库时间,与续租/完成串行化;多个节点不能重复释放许可或重复终结事件。
|
||||||
|
|
||||||
|
此处的安全回收不是完整重试引擎;IDEMPOTENT 的自动重试和策略退避仍是 P3 工作。
|
||||||
|
|
||||||
|
## 6. 连接保护与当前边界
|
||||||
|
|
||||||
|
当前节点上限 128 条 Worker 连接;每连接入站队列 4 帧、出站队列 8 帧,派发保留 ACK 缓冲空间,每秒最多发起 4 次新派发尝试。读空闲期限 60 秒、写期限 5 秒、单个数据库控制操作期限 8 秒。慢消费者导致连接关闭时,已提交状态仍由稳定身份恢复。
|
||||||
|
|
||||||
|
这些是有界的开发基线,不代表经过容量压测的生产参数。当前同队列操作使用较粗的行锁协调,吞吐与公平性需要 P6 按负载验证。服务退出会显式关闭被 HTTP Upgrade 接管的连接;断开不会直接撤销外部已发出的请求。
|
||||||
|
|
||||||
|
运行中取消、SDK 自动重连/续租、非协作处理器处理、完整 WS JSON Schema、凭据轮换和多节点进程级故障演练尚未全部完成。实际证据见 [P2 服务端阶段记录](../reports/phase-2-server.md)。
|
||||||
@@ -13,7 +13,7 @@
|
|||||||
## 已可运行:文件转换任务的受理与取消
|
## 已可运行:文件转换任务的受理与取消
|
||||||
|
|
||||||
- 源码:[SDK 任务受理示例](../../sdk/go/examples/tasks/main.go)。
|
- 源码:[SDK 任务受理示例](../../sdk/go/examples/tasks/main.go)。
|
||||||
- 前置:完成迁移、初始化凭据、启动服务,详见 [SDK 接入](../sdk/README.md)。
|
- 前置:完成迁移、初始化凭据、启动服务,在没有 Worker 消费的专用队列运行此受理/取消示例,详见 [SDK 接入](../sdk/README.md)。
|
||||||
- 输入:对象引用、格式与精确整数;提交两次同一身份,再查询、分页并带版本取消。
|
- 输入:对象引用、格式与精确整数;提交两次同一身份,再查询、分页并带版本取消。
|
||||||
- 预期:重复请求返回同一任务,最终 CANCELLED/revision=2。当前不启动 Worker,不实际生成文件。
|
- 预期:重复请求返回同一任务,最终 CANCELLED/revision=2。当前不启动 Worker,不实际生成文件。
|
||||||
- 异常:凭据失效、权限不足、内容冲突、版本竞争和未知网络结果,见 [任务接口](../api/tasks.md)。
|
- 异常:凭据失效、权限不足、内容冲突、版本竞争和未知网络结果,见 [任务接口](../api/tasks.md)。
|
||||||
|
|||||||
57
docs/reports/phase-2-server.md
Normal file
57
docs/reports/phase-2-server.md
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
# P2 服务端执行与 WSS 阶段记录
|
||||||
|
|
||||||
|
日期:2026-09-17。**这是 P2 的阶段内进展,不是 P2 或生产验收完成。**
|
||||||
|
|
||||||
|
## 实现
|
||||||
|
|
||||||
|
新增 WorkerInstance 与 AttemptEvidence,当前共 32 个持久化实体。字段、模型职责、单位、可空/秘密/代次语义及 SQL COMMENT 均为中文。新增前向迁移 000003_worker_execution,旧迁移不改写。
|
||||||
|
|
||||||
|
服务端现有能力包括进程恢复秘密、会话头与连接代次、能力绑定、容量覆盖和排空、两步领取/开始、尝试秘密、续租、结果回执、迟到证据、跨节点恢复,以及到期扫描。队列并发许可随执行事务领取、续期和释放。
|
||||||
|
|
||||||
|
HTTP 进程已注册 Worker WebSocket 入口,并启动受控恢复扫描;使用有界网络队列、消息/连接上限和明确的关闭流程。默认 bootstrap 不扩权,新增 --worker-access 显式为开发凭据增加 Worker 权限。
|
||||||
|
|
||||||
|
## 实际回归
|
||||||
|
|
||||||
|
环境:macOS arm64,Go 1.27.1,Docker 隔离 MySQL 8.4.10。入口为 make test、make vet、make build、make integration;数据库临时创建并清理,不使用已有业务数据库。
|
||||||
|
|
||||||
|
| 场景 | 结果 |
|
||||||
|
| --- | --- |
|
||||||
|
| 新迁移与全部 Go 模型字段匹配、前序 P0/P1 回归 | 通过 |
|
||||||
|
| 开始/续租/结果确认的原消息重报 | 原回执可查,不重复执行状态变更或延长旧续租 |
|
||||||
|
| 多消息提交同一已保存结果 | 只产生一次结果事件与审计、只释放一次许可 |
|
||||||
|
| 精确整数与小数结果 | 保存与查询不损失数值内容 |
|
||||||
|
| 不同节点重连与恢复同一尝试 | 原子迁移会话;次数和许可不增加 |
|
||||||
|
| 旧会话、错误尝试秘密、错误进程证明、注册内容改变 | 拒绝 |
|
||||||
|
| 两个实例、两个节点、16 个并发领取请求 | 队列并发 2、进程容量各 1 均不被突破 |
|
||||||
|
| 重连时旧会话已有执行 | 不重置容量计数 |
|
||||||
|
| 排空、容量零、重复/冲突序号 | 不给新工作,不误释放已有许可,排空跨重连保留 |
|
||||||
|
| 前 18 个任务门控暂停,后面有普通任务 | 普通任务仍能被领取,无候选页饥饿 |
|
||||||
|
| 暂停与开始、已授权开始回执、结果保存 | 未开始被阻止;已开始的原回执和结果仍可保存 |
|
||||||
|
| 未开始到期、已开始失联、两个恢复节点竞争 | 有限重派/FAILED/NEEDS_REVIEW 正确;无重复释放 |
|
||||||
|
| 迟到成功、未开始提交结果、冲突结果 | EVIDENCE_ONLY,保留证据,不改写权威结果 |
|
||||||
|
| 原 start ACK 在结束或过期后重报 | receipt 保留,当前 lease.active=false |
|
||||||
|
| 单次绝对时限与会话过期 | 续租不能越过限制,过期会话不能靠心跳复活 |
|
||||||
|
| 获取锁之前进入请求、锁等待期间租约过期 | 不使用早先时钟样本批准开始 |
|
||||||
|
| 领取事件或完成审计 INSERT 被注入失败 | 任务、尝试、许可、回执、事件整体回滚 |
|
||||||
|
| 凭据撤销、跨命名空间、已取消 context | 执行控制拒绝 |
|
||||||
|
| WSS 真 TLS 握手,鉴权/子协议/Origin/URL 规则 | 通过;使用受信任测试证书,不跳过验证 |
|
||||||
|
| WSS 节点 A 派发开始 → 断开 → 节点 B 恢复 → 结果确认重报 | 同一尝试、一次成功事件、许可正确释放 |
|
||||||
|
| 重复 JSON 字段、服务关闭与已接管长连接 | 非法帧关闭;节点关闭无残留长连接 |
|
||||||
|
|
||||||
|
WSS 测试客户端直接发送协议帧,并显式重连到另一个测试服务器。当前没有把这项测试说成 SDK 自动重连或真实主机宕机演练。两个 ExecutionStore/Hub 使用不同节点 ID、同一个真实数据库;它们位于同一测试进程。
|
||||||
|
|
||||||
|
## 事务与恢复约束
|
||||||
|
|
||||||
|
每条命令重新校验当前凭据。执行路径先锁进程头/会话,再锁队列、可选门控、任务、尝试;回收从队列开始,不反向锁 Worker。拿到相关锁后重新读取数据库时间。状态、回执、额度和事件在短事务内提交,处理器不会在数据库事务中运行。
|
||||||
|
|
||||||
|
会话秘密、执行秘密仅存摘要,普通任务查询与日志不输出原文。保存迟到证据不构成执行授权。粗粒度队列锁是当前正确性基线,需要后续执行计划、压力与公平性验证;没有声称支持某个生产吞吐数字。
|
||||||
|
|
||||||
|
## 剩余工作
|
||||||
|
|
||||||
|
- Worker SDK 的处理器注册与运行循环、保守租约截止、自动续租、重连、结果重报、优雅退出。
|
||||||
|
- API/SDK 执行历史与证据查询,WS JSON Schema 和兼容性验证。
|
||||||
|
- 运行中取消、客户端慢消费及更多网络故障、结果未知/不协作处理器的端到端用例。
|
||||||
|
- P3 的策略化自动重试、速率和其他配额、调度及结果订阅。
|
||||||
|
- P4–P7 的流程、运维、性能和最终交付验证。
|
||||||
|
|
||||||
|
当前已实现的是可以经 WSS 操作的服务端执行内核。尚未完成独立 Worker 接入交付,也未完成生产高可用和容量验收。
|
||||||
5
go.mod
5
go.mod
@@ -2,6 +2,9 @@ module git.yuntuo.hk/k2safe/orbit
|
|||||||
|
|
||||||
go 1.27.1
|
go 1.27.1
|
||||||
|
|
||||||
require github.com/go-sql-driver/mysql v1.10.1
|
require (
|
||||||
|
github.com/coder/websocket v1.8.15
|
||||||
|
github.com/go-sql-driver/mysql v1.10.1
|
||||||
|
)
|
||||||
|
|
||||||
require filippo.io/edwards25519 v1.2.0 // indirect
|
require filippo.io/edwards25519 v1.2.0 // indirect
|
||||||
|
|||||||
2
go.sum
2
go.sum
@@ -1,4 +1,6 @@
|
|||||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||||
|
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
|
||||||
|
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
|
||||||
github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME=
|
github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME=
|
||||||
github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ type BootstrapInput struct {
|
|||||||
Namespace string // Namespace 命名空间名称。
|
Namespace string // Namespace 命名空间名称。
|
||||||
Environment string // Environment 环境名称,必须显式指定。
|
Environment string // Environment 环境名称,必须显式指定。
|
||||||
Queue string // Queue 初始队列名称。
|
Queue string // Queue 初始队列名称。
|
||||||
|
WorkerAccess bool // WorkerAccess 显式增加开发用 Worker 接入及消费权限。
|
||||||
}
|
}
|
||||||
|
|
||||||
// BootstrapResult 包含一次性交付的原始凭据,禁止记录到日志或提交到版本库。
|
// BootstrapResult 包含一次性交付的原始凭据,禁止记录到日志或提交到版本库。
|
||||||
@@ -56,6 +57,9 @@ func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func(
|
|||||||
}
|
}
|
||||||
out.Token = secret
|
out.Token = secret
|
||||||
permissions := auth.Permissions{Actions: []string{"tasks:submit", "tasks:read", "tasks:cancel", "queues:read", "queues:manage"}, Queues: []string{"*"}}
|
permissions := auth.Permissions{Actions: []string{"tasks:submit", "tasks:read", "tasks:cancel", "queues:read", "queues:manage"}, Queues: []string{"*"}}
|
||||||
|
if in.WorkerAccess {
|
||||||
|
permissions.Actions = append(permissions.Actions, "workers:connect", "workers:consume")
|
||||||
|
}
|
||||||
raw, err := json.Marshal(permissions)
|
raw, err := json.Marshal(permissions)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -191,9 +191,11 @@ func checkModelColumns(t *testing.T, ctx context.Context, db *sql.DB) {
|
|||||||
pulse.Queue{},
|
pulse.Queue{},
|
||||||
pulse.ExecutionGate{},
|
pulse.ExecutionGate{},
|
||||||
pulse.WorkerSession{},
|
pulse.WorkerSession{},
|
||||||
|
pulse.WorkerInstance{},
|
||||||
pulse.Task{},
|
pulse.Task{},
|
||||||
pulse.Attempt{},
|
pulse.Attempt{},
|
||||||
pulse.AttemptReceipt{},
|
pulse.AttemptReceipt{},
|
||||||
|
pulse.AttemptEvidence{},
|
||||||
pulse.ConcurrencyLimit{},
|
pulse.ConcurrencyLimit{},
|
||||||
pulse.Permit{},
|
pulse.Permit{},
|
||||||
pulse.Schedule{},
|
pulse.Schedule{},
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
-- Worker 会话隔离和迟到结果证据。P2 之前没有受支持的 Worker 注册接口。
|
||||||
|
ALTER TABLE pulse_worker_sessions ADD UNIQUE KEY uq_session_instance_id(namespace_id, worker_instance_id, id);
|
||||||
|
|
||||||
|
CREATE TABLE pulse_worker_instances (
|
||||||
|
namespace_id BINARY(16) NOT NULL COMMENT '进程所属命名空间,参与授权和所有关联',
|
||||||
|
id BINARY(16) NOT NULL COMMENT 'Worker 进程实例标识,进程重启必须换新值',
|
||||||
|
revision BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '持久化变更版本,从一开始递增',
|
||||||
|
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '首次注册时间,UTC 微秒精度',
|
||||||
|
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '最近变更时间,不替代会话代次',
|
||||||
|
credential_id BINARY(16) NOT NULL COMMENT '初始注册凭据,进程生命周期内固定',
|
||||||
|
queue_id BINARY(16) NOT NULL COMMENT '绑定队列,切换队列需创建新进程实例',
|
||||||
|
secret_digest BINARY(32) NOT NULL COMMENT '随机进程恢复秘密的 SHA-256 摘要,不保存原文',
|
||||||
|
registration_digest BINARY(32) NOT NULL COMMENT '队列及处理器版本集合的语义摘要,重连必须匹配',
|
||||||
|
session_epoch BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '最大已分配连接代次,注册前为零',
|
||||||
|
current_session_id BINARY(16) NULL COMMENT '当前会话,注册事务建立会话前为空',
|
||||||
|
draining BOOLEAN NOT NULL DEFAULT FALSE COMMENT '持久化排空标记,重连不能清除',
|
||||||
|
PRIMARY KEY(namespace_id,id),
|
||||||
|
UNIQUE KEY uq_worker_credential(namespace_id,id,credential_id),
|
||||||
|
CONSTRAINT ck_worker_instance_revision CHECK (revision>0),
|
||||||
|
CONSTRAINT ck_worker_instance_session CHECK ((session_epoch=0 AND current_session_id IS NULL) OR (session_epoch>0 AND current_session_id IS NOT NULL)),
|
||||||
|
CONSTRAINT fk_worker_instance_credential FOREIGN KEY(namespace_id,credential_id) REFERENCES orbit_credentials(namespace_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||||
|
CONSTRAINT fk_worker_instance_queue FOREIGN KEY(namespace_id,queue_id) REFERENCES pulse_queues(namespace_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||||
|
CONSTRAINT fk_worker_instance_session FOREIGN KEY(namespace_id,id,current_session_id) REFERENCES pulse_worker_sessions(namespace_id,worker_instance_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='Worker 进程恢复身份和当前会话头,用于跨节点隔离旧连接';
|
||||||
|
|
||||||
|
ALTER TABLE pulse_worker_sessions ADD CONSTRAINT fk_worker_session_instance FOREIGN KEY(namespace_id,worker_instance_id,credential_id) REFERENCES pulse_worker_instances(namespace_id,id,credential_id) ON DELETE RESTRICT ON UPDATE RESTRICT;
|
||||||
|
|
||||||
|
ALTER TABLE pulse_worker_sessions ADD CONSTRAINT ck_worker_session_capacity CHECK (capacity<=256);
|
||||||
|
|
||||||
|
ALTER TABLE pulse_attempts ADD KEY ix_attempt_session_state(namespace_id,session_id,state);
|
||||||
|
|
||||||
|
ALTER TABLE pulse_attempts ADD CONSTRAINT ck_attempt_lifecycle CHECK ((state IN ('LEASED','RUNNING') AND ended_at IS NULL) OR (state NOT IN ('LEASED','RUNNING') AND ended_at IS NOT NULL));
|
||||||
|
|
||||||
|
ALTER TABLE pulse_attempts ADD CONSTRAINT ck_attempt_start CHECK ((state='LEASED' AND authorized_at IS NULL) OR (state='RUNNING' AND authorized_at IS NOT NULL) OR state NOT IN ('LEASED','RUNNING'));
|
||||||
|
|
||||||
|
ALTER TABLE pulse_tasks ADD KEY ix_task_deadline(state,deadline_at,namespace_id,id);
|
||||||
|
|
||||||
|
ALTER TABLE pulse_tasks ADD KEY ix_task_start_deadline(state,start_deadline_at,namespace_id,id);
|
||||||
|
|
||||||
|
CREATE TABLE pulse_attempt_evidence (
|
||||||
|
namespace_id BINARY(16) NOT NULL COMMENT '证据所属命名空间,与尝试一致',
|
||||||
|
id BINARY(16) NOT NULL COMMENT '证据记录标识',
|
||||||
|
revision BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '记录版本;证据追加保存,不覆盖',
|
||||||
|
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '首次收到证据的数据库时间,UTC',
|
||||||
|
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '写入时间,追加记录与创建时间相同',
|
||||||
|
attempt_id BINARY(16) NOT NULL COMMENT '证据所属执行尝试',
|
||||||
|
message_id BINARY(16) NOT NULL COMMENT '结果消息身份,重报保持不变',
|
||||||
|
outcome VARCHAR(16) NOT NULL COMMENT 'Worker 声明的结果:SUCCEEDED、FAILED 或 UNKNOWN',
|
||||||
|
error_code VARCHAR(64) NULL COMMENT '稳定错误分类;无错误时为空,不保存原始异常正文',
|
||||||
|
payload MEDIUMTEXT NOT NULL COMMENT '结果或证据引用的 JSON 文本,保留精确数字',
|
||||||
|
request_digest BINARY(32) NOT NULL COMMENT '结果语义摘要,排除会话与秘密',
|
||||||
|
reason VARCHAR(32) NOT NULL COMMENT '证据未改写权威结果的原因:LEASE_LOST、NOT_STARTED 或 RESULT_CONFLICT',
|
||||||
|
PRIMARY KEY(namespace_id,id),
|
||||||
|
UNIQUE KEY uq_attempt_evidence_message(namespace_id,attempt_id,message_id),
|
||||||
|
CONSTRAINT ck_attempt_evidence_revision CHECK(revision>0),
|
||||||
|
CONSTRAINT ck_attempt_evidence_json CHECK(JSON_VALID(payload)),
|
||||||
|
CONSTRAINT ck_attempt_evidence_outcome CHECK(outcome IN ('SUCCEEDED','FAILED','UNKNOWN')),
|
||||||
|
CONSTRAINT ck_attempt_evidence_reason CHECK(reason IN ('LEASE_LOST','NOT_STARTED','RESULT_CONFLICT')),
|
||||||
|
CONSTRAINT fk_attempt_evidence_attempt FOREIGN KEY(namespace_id,attempt_id) REFERENCES pulse_attempts(namespace_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='迟到与冲突结果的追加证据,不自动改变任务终态';
|
||||||
|
|
||||||
|
CREATE TRIGGER immutable_attempt_evidence BEFORE UPDATE ON pulse_attempt_evidence FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='attempt evidence is immutable';
|
||||||
521
internal/storage/mysql/worker_execution.go
Normal file
521
internal/storage/mysql/worker_execution.go
Normal file
@@ -0,0 +1,521 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"crypto/subtle"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||||
|
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||||
|
)
|
||||||
|
|
||||||
|
type executionTask struct {
|
||||||
|
view service.TaskView
|
||||||
|
current *core.ID
|
||||||
|
epoch uint64
|
||||||
|
lease *time.Time
|
||||||
|
gate *core.ID
|
||||||
|
maxAttempts uint32
|
||||||
|
attemptTimeout uint64
|
||||||
|
}
|
||||||
|
type executionRows struct {
|
||||||
|
task executionTask
|
||||||
|
attempt pulse.Attempt
|
||||||
|
paused bool
|
||||||
|
session sessionState
|
||||||
|
}
|
||||||
|
|
||||||
|
func readExecutionTask(ctx context.Context, tx *sql.Tx, ns, id core.ID) (executionTask, error) {
|
||||||
|
var row executionTask
|
||||||
|
view, err := readTask(ctx, tx, ns, id, true)
|
||||||
|
if err != nil {
|
||||||
|
return row, err
|
||||||
|
}
|
||||||
|
row.view = view
|
||||||
|
err = tx.QueryRowContext(ctx, `SELECT current_attempt_id,lease_epoch,lease_until,gate_id,max_attempts,attempt_timeout_ms FROM pulse_tasks WHERE namespace_id=? AND id=?`, ns, id).Scan(&row.current, &row.epoch, &row.lease, &row.gate, &row.maxAttempts, &row.attemptTimeout)
|
||||||
|
return row, err
|
||||||
|
}
|
||||||
|
func lockQueue(ctx context.Context, tx *sql.Tx, ns, id core.ID) (paused bool, max uint32, err error) {
|
||||||
|
err = tx.QueryRowContext(ctx, "SELECT paused,max_concurrency FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE", ns, id).Scan(&paused, &max)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
err = fault.NotFound
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
func lockGate(ctx context.Context, tx *sql.Tx, ns core.ID, id *core.ID) (bool, error) {
|
||||||
|
if id == nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
var paused bool
|
||||||
|
err := tx.QueryRowContext(ctx, "SELECT paused FROM pulse_execution_gates WHERE namespace_id=? AND id=? FOR SHARE", ns, *id).Scan(&paused)
|
||||||
|
return paused, err
|
||||||
|
}
|
||||||
|
func minimumTime(first time.Time, others ...time.Time) time.Time {
|
||||||
|
for _, other := range others {
|
||||||
|
if other.Before(first) {
|
||||||
|
first = other
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return first
|
||||||
|
}
|
||||||
|
func queueLimit(ctx context.Context, tx *sql.Tx, ns, queue core.ID, max uint32) (core.ID, uint32, error) {
|
||||||
|
id, err := core.NewID()
|
||||||
|
if err != nil {
|
||||||
|
return "", 0, err
|
||||||
|
}
|
||||||
|
digest := core.Digest(sha256.Sum256([]byte(queue)))
|
||||||
|
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_concurrency_limits(namespace_id,id,scope_key,key_digest,max_permits) VALUES (?,?,'queue',?,?) ON DUPLICATE KEY UPDATE id=pulse_concurrency_limits.id`, ns, id, digest, max)
|
||||||
|
if err != nil {
|
||||||
|
return "", 0, err
|
||||||
|
}
|
||||||
|
var used uint32
|
||||||
|
var configured uint32
|
||||||
|
err = tx.QueryRowContext(ctx, `SELECT id,used_permits,max_permits FROM pulse_concurrency_limits WHERE namespace_id=? AND scope_key='queue' AND key_digest=? FOR UPDATE`, ns, digest).Scan(&id, &used, &configured)
|
||||||
|
if err == nil && configured != max {
|
||||||
|
err = errors.New("queue limit configuration requires coordinated update")
|
||||||
|
}
|
||||||
|
return id, used, err
|
||||||
|
}
|
||||||
|
func outstanding(ctx context.Context, tx *sql.Tx, ns, instance core.ID) (uint32, error) {
|
||||||
|
var count uint32
|
||||||
|
err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pulse_attempts a JOIN pulse_worker_sessions s ON s.namespace_id=a.namespace_id AND s.id=a.session_id WHERE a.namespace_id=? AND s.worker_instance_id=? AND a.state IN ('LEASED','RUNNING')`, ns, instance).Scan(&count)
|
||||||
|
return count, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.Binding) (*worker.Assignment, error) {
|
||||||
|
var assignment *worker.Assignment
|
||||||
|
err := s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||||
|
state, err := s.lockSession(ctx, tx, token, b, p)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if state.draining || state.capacity == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
count, err := outstanding(ctx, tx, token.NamespaceID, b.InstanceID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if count >= state.capacity {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
paused, max, err := lockQueue(ctx, tx, token.NamespaceID, state.queue)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if paused {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
limit, used, err := queueLimit(ctx, tx, token.NamespaceID, state.queue, max)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if used >= max {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
args := []any{token.NamespaceID, state.queue}
|
||||||
|
matches := make([]string, 0, len(state.capabilities))
|
||||||
|
for _, c := range state.capabilities {
|
||||||
|
matches = append(matches, "(handler=? AND handler_version=? AND payload_version=?)")
|
||||||
|
args = append(args, c.Handler, c.HandlerVersion, c.PayloadVersion)
|
||||||
|
}
|
||||||
|
if len(matches) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
rows, err := tx.QueryContext(ctx, `SELECT id,gate_id FROM pulse_tasks WHERE namespace_id=? AND queue_id=? AND state='WAITING' AND available_at<=UTC_TIMESTAMP(6) AND deadline_at>UTC_TIMESTAMP(6) AND (start_deadline_at IS NULL OR start_deadline_at>UTC_TIMESTAMP(6)) AND attempt_count<max_attempts AND (gate_id IS NULL OR EXISTS (SELECT 1 FROM pulse_execution_gates g WHERE g.namespace_id=pulse_tasks.namespace_id AND g.id=pulse_tasks.gate_id AND g.paused=FALSE)) AND (`+strings.Join(matches, " OR ")+`) ORDER BY available_at,priority DESC,id LIMIT 16`, args...)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
type candidate struct {
|
||||||
|
id core.ID
|
||||||
|
gate *core.ID
|
||||||
|
}
|
||||||
|
candidates := []candidate{}
|
||||||
|
for rows.Next() {
|
||||||
|
var c candidate
|
||||||
|
if err := rows.Scan(&c.id, &c.gate); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
candidates = append(candidates, c)
|
||||||
|
}
|
||||||
|
err = rows.Err()
|
||||||
|
rows.Close()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, candidate := range candidates {
|
||||||
|
gated, err := lockGate(ctx, tx, token.NamespaceID, candidate.gate)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if gated {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var locked core.ID
|
||||||
|
err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_tasks WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, token.NamespaceID, candidate.id).Scan(&locked)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
task, err := readExecutionTask(ctx, tx, token.NamespaceID, locked)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
now, err := dbNow(ctx, tx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !state.expires.After(now) {
|
||||||
|
return worker.StaleSession
|
||||||
|
}
|
||||||
|
if task.view.State != pulse.TaskWaiting || task.view.AvailableAt.After(now) || !task.view.DeadlineAt.After(now) || task.view.AttemptCount >= task.maxAttempts || (task.view.StartDeadlineAt != nil && !task.view.StartDeadlineAt.After(now)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if task.epoch == ^uint64(0) {
|
||||||
|
return fault.Conflict
|
||||||
|
}
|
||||||
|
attempt, err := core.NewID()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
secret, digest, err := worker.NewSecret()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
until := minimumTime(now.Add(s.options.LeaseDuration), task.view.DeadlineAt, state.expires)
|
||||||
|
if task.view.StartDeadlineAt != nil {
|
||||||
|
until = minimumTime(until, *task.view.StartDeadlineAt)
|
||||||
|
}
|
||||||
|
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempts(namespace_id,id,task_id,epoch,session_id,token_digest,lease_until) VALUES (?,?,?,?,?,?,?)`, token.NamespaceID, attempt, locked, task.epoch+1, b.SessionID, digest, until)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
permit, err := core.NewID()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_permits(namespace_id,id,limit_id,attempt_id,lease_until) VALUES (?,?,?,?,?)`, token.NamespaceID, permit, limit, attempt, until)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
changed, err := tx.ExecContext(ctx, `UPDATE pulse_concurrency_limits SET used_permits=used_permits+1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND used_permits<max_permits`, now, token.NamespaceID, limit)
|
||||||
|
if err := exactlyOne(changed, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
changed, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='LEASED',current_attempt_id=?,lease_epoch=lease_epoch+1,lease_until=?,attempt_count=attempt_count+1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND state='WAITING' AND revision=?`, attempt, until, now, token.NamespaceID, locked, task.view.Revision)
|
||||||
|
if err := exactlyOne(changed, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
view, err := readTask(ctx, tx, token.NamespaceID, locked, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := emitEvent(ctx, tx, token.NamespaceID, locked, view.Revision, "task", "task.leased", map[string]any{"attempt_id": attempt, "epoch": fmt.Sprint(task.epoch + 1), "session_id": b.SessionID, "lease_until": until}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
assignment = &worker.Assignment{Task: view, AttemptID: attempt, Epoch: task.epoch + 1, LeaseSecret: secret, LeaseUntil: until, ServerTime: now}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return assignment, nil
|
||||||
|
}
|
||||||
|
func exactlyOne(result sql.Result, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
n, err := result.RowsAffected()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if n != 1 {
|
||||||
|
return errors.New("execution invariant affected unexpected rows")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// lockExecution 的跨资源顺序为进程/会话 -> 队列 -> 可选门控 -> 任务 -> 尝试。
|
||||||
|
// 恢复扫描也从队列锁开始,不反向锁定 Worker 进程。
|
||||||
|
func (s *ExecutionStore) lockExecution(ctx context.Context, tx *sql.Tx, token auth.Token, b worker.Binding, p auth.Permissions, ref worker.AttemptRef, checkGate bool) (executionRows, error) {
|
||||||
|
var out executionRows
|
||||||
|
state, err := s.lockSession(ctx, tx, token, b, p)
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
out.session = state
|
||||||
|
var queue core.ID
|
||||||
|
var gate *core.ID
|
||||||
|
err = tx.QueryRowContext(ctx, `SELECT queue_id,gate_id FROM pulse_tasks WHERE namespace_id=? AND id=?`, token.NamespaceID, ref.TaskID).Scan(&queue, &gate)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return out, fault.NotFound
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
if queue != state.queue || !p.AllowsQueue(queue) {
|
||||||
|
return out, fault.NotFound
|
||||||
|
}
|
||||||
|
out.paused, _, err = lockQueue(ctx, tx, token.NamespaceID, queue)
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
if checkGate {
|
||||||
|
paused, err := lockGate(ctx, tx, token.NamespaceID, gate)
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
out.paused = out.paused || paused
|
||||||
|
}
|
||||||
|
out.task, err = readExecutionTask(ctx, tx, token.NamespaceID, ref.TaskID)
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
out.attempt, err = readAttempt(ctx, tx, token.NamespaceID, ref.AttemptID)
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
proof, err := worker.SecretDigest(ref.LeaseSecret)
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
if out.attempt.TaskID != ref.TaskID || out.attempt.Epoch != ref.Epoch || subtle.ConstantTimeCompare(proof[:], out.attempt.TokenDigest[:]) != 1 {
|
||||||
|
return out, fault.Forbidden
|
||||||
|
}
|
||||||
|
var instance core.ID
|
||||||
|
if err := tx.QueryRowContext(ctx, `SELECT worker_instance_id FROM pulse_worker_sessions WHERE namespace_id=? AND id=?`, token.NamespaceID, out.attempt.SessionID).Scan(&instance); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
if instance != b.InstanceID {
|
||||||
|
return out, fault.Forbidden
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
func readAttempt(ctx context.Context, tx *sql.Tx, ns, id core.ID) (pulse.Attempt, error) {
|
||||||
|
var a pulse.Attempt
|
||||||
|
a.NamespaceID = ns
|
||||||
|
err := tx.QueryRowContext(ctx, `SELECT id,revision,task_id,epoch,session_id,token_digest,state,lease_until,authorized_at,ended_at,error_code,result,result_digest FROM pulse_attempts WHERE namespace_id=? AND id=? FOR UPDATE`, ns, id).Scan(&a.ID, &a.Revision, &a.TaskID, &a.Epoch, &a.SessionID, &a.TokenDigest, &a.State, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Result, &a.ResultDigest)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
err = fault.NotFound
|
||||||
|
}
|
||||||
|
return a, err
|
||||||
|
}
|
||||||
|
func leaseStatus(rows executionRows, b worker.Binding, now time.Time) worker.LeaseStatus {
|
||||||
|
t, a := rows.task, rows.attempt
|
||||||
|
until := minimumTime(a.LeaseUntil, rows.session.expires, t.view.DeadlineAt)
|
||||||
|
if t.lease != nil {
|
||||||
|
until = minimumTime(until, *t.lease)
|
||||||
|
}
|
||||||
|
active := rows.session.expires.After(now) && t.current != nil && *t.current == a.ID && t.epoch == a.Epoch && a.SessionID == b.SessionID && t.lease != nil && t.lease.After(now) && a.LeaseUntil.After(now) && t.view.DeadlineAt.After(now) && ((t.view.State == pulse.TaskLeased && a.State == pulse.AttemptLeased) || (t.view.State == pulse.TaskRunning && a.State == pulse.AttemptRunning))
|
||||||
|
if a.AuthorizedAt != nil {
|
||||||
|
deadline := a.AuthorizedAt.Add(time.Duration(t.attemptTimeout) * time.Millisecond)
|
||||||
|
active = active && deadline.After(now)
|
||||||
|
until = minimumTime(until, deadline)
|
||||||
|
}
|
||||||
|
return worker.LeaseStatus{Active: active, State: a.State, LeaseUntil: until, ServerTime: now}
|
||||||
|
}
|
||||||
|
func attemptIdentity(ref worker.AttemptRef) any {
|
||||||
|
return struct {
|
||||||
|
Task core.ID
|
||||||
|
Attempt core.ID
|
||||||
|
Epoch string
|
||||||
|
}{ref.TaskID, ref.AttemptID, fmt.Sprint(ref.Epoch)}
|
||||||
|
}
|
||||||
|
func readReceipt(ctx context.Context, tx *sql.Tx, ns, attempt, message core.ID, operation string, digest core.Digest) (*worker.Receipt, error) {
|
||||||
|
var saved core.Digest
|
||||||
|
var raw []byte
|
||||||
|
err := tx.QueryRowContext(ctx, `SELECT request_digest,response FROM pulse_attempt_receipts WHERE namespace_id=? AND attempt_id=? AND operation=? AND message_id=? FOR UPDATE`, ns, attempt, operation, message).Scan(&saved, &raw)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if saved != digest {
|
||||||
|
return nil, fault.IdempotencyConflict
|
||||||
|
}
|
||||||
|
var receipt worker.Receipt
|
||||||
|
if err := json.Unmarshal(raw, &receipt); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &receipt, nil
|
||||||
|
}
|
||||||
|
func saveReceipt(ctx context.Context, tx *sql.Tx, ns, attempt, message core.ID, operation string, digest core.Digest, receipt worker.Receipt) error {
|
||||||
|
id, err := core.NewID()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(receipt)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempt_receipts(namespace_id,id,attempt_id,operation,message_id,request_digest,response,expires_at) VALUES (?,?,?,?,?,?,?,DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 90 DAY))`, ns, id, attempt, operation, message, digest, string(raw))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
func renewPermits(ctx context.Context, tx *sql.Tx, ns, attempt core.ID, until, now time.Time) error {
|
||||||
|
result, err := tx.ExecContext(ctx, `UPDATE pulse_permits SET lease_until=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND attempt_id=? AND released_at IS NULL`, until, now, ns, attempt)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
n, err := result.RowsAffected()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if n < 1 {
|
||||||
|
return errors.New("active attempt has no concurrency permit")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func releasePermits(ctx context.Context, tx *sql.Tx, ns, attempt core.ID, now time.Time) error {
|
||||||
|
rows, err := tx.QueryContext(ctx, `SELECT id,limit_id FROM pulse_permits WHERE namespace_id=? AND attempt_id=? AND released_at IS NULL ORDER BY limit_id FOR UPDATE`, ns, attempt)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
type permit struct{ id, limit core.ID }
|
||||||
|
permits := []permit{}
|
||||||
|
for rows.Next() {
|
||||||
|
var p permit
|
||||||
|
if err := rows.Scan(&p.id, &p.limit); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
permits = append(permits, p)
|
||||||
|
}
|
||||||
|
err = rows.Err()
|
||||||
|
rows.Close()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(permits) == 0 {
|
||||||
|
return errors.New("active attempt has no permit to release")
|
||||||
|
}
|
||||||
|
for _, p := range permits {
|
||||||
|
result, err := tx.ExecContext(ctx, `UPDATE pulse_concurrency_limits SET used_permits=used_permits-1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND used_permits>0`, now, ns, p.limit)
|
||||||
|
if err := exactlyOne(result, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
result, err = tx.ExecContext(ctx, `UPDATE pulse_permits SET released_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND released_at IS NULL`, now, now, ns, p.id)
|
||||||
|
if err := exactlyOne(result, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ExecutionStore) Start(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, ref worker.AttemptRef) (worker.CommandResult, error) {
|
||||||
|
return s.executionCommand(ctx, token, b, message, ref, "start")
|
||||||
|
}
|
||||||
|
func (s *ExecutionStore) Renew(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, ref worker.AttemptRef) (worker.CommandResult, error) {
|
||||||
|
return s.executionCommand(ctx, token, b, message, ref, "renew")
|
||||||
|
}
|
||||||
|
func (s *ExecutionStore) executionCommand(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, ref worker.AttemptRef, operation string) (worker.CommandResult, error) {
|
||||||
|
var out worker.CommandResult
|
||||||
|
digest, err := requestDigest(attemptIdentity(ref))
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
err = s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||||
|
rows, err := s.lockExecution(ctx, tx, token, b, p, ref, operation == "start")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
receipt, err := readReceipt(ctx, tx, token.NamespaceID, ref.AttemptID, message, operation, digest)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
now, err := dbNow(ctx, tx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
status := leaseStatus(rows, b, now)
|
||||||
|
if receipt != nil {
|
||||||
|
out = worker.CommandResult{Receipt: *receipt, Lease: status, Deduplicated: true}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !status.Active {
|
||||||
|
return worker.LeaseLost
|
||||||
|
}
|
||||||
|
task, a := rows.task, rows.attempt
|
||||||
|
authorized := a.AuthorizedAt
|
||||||
|
if operation == "start" {
|
||||||
|
if a.State != pulse.AttemptLeased || task.view.State != pulse.TaskLeased {
|
||||||
|
return fault.Conflict
|
||||||
|
}
|
||||||
|
if rows.paused {
|
||||||
|
return worker.GateClosed
|
||||||
|
}
|
||||||
|
if task.view.StartDeadlineAt != nil && !task.view.StartDeadlineAt.After(now) {
|
||||||
|
return worker.LeaseLost
|
||||||
|
}
|
||||||
|
authorized = &now
|
||||||
|
} else if a.State != pulse.AttemptRunning || authorized == nil {
|
||||||
|
return fault.Conflict
|
||||||
|
}
|
||||||
|
until := minimumTime(now.Add(s.options.LeaseDuration), authorized.Add(time.Duration(task.attemptTimeout)*time.Millisecond), task.view.DeadlineAt, rows.session.expires)
|
||||||
|
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state='RUNNING',authorized_at=?,lease_until=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, authorized, until, now, token.NamespaceID, a.ID, a.Revision)
|
||||||
|
if err := exactlyOne(result, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
result, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='RUNNING',lease_until=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, until, now, token.NamespaceID, ref.TaskID, task.view.Revision)
|
||||||
|
if err := exactlyOne(result, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := renewPermits(ctx, tx, token.NamespaceID, a.ID, until, now); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if operation == "start" {
|
||||||
|
if err := emitEvent(ctx, tx, token.NamespaceID, ref.TaskID, task.view.Revision+1, "task", "task.started", map[string]any{"attempt_id": a.ID, "epoch": fmt.Sprint(a.Epoch), "session_id": b.SessionID}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
receipt = &worker.Receipt{Operation: operation, TaskState: pulse.TaskRunning, AttemptState: pulse.AttemptRunning, TaskRevision: task.view.Revision + 1, LeaseUntil: until, Disposition: "APPLIED"}
|
||||||
|
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, operation, digest, *receipt); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out = worker.CommandResult{Receipt: *receipt, Lease: worker.LeaseStatus{Active: true, State: pulse.AttemptRunning, LeaseUntil: until, ServerTime: now}}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ExecutionStore) Resume(ctx context.Context, token auth.Token, b worker.Binding, ref worker.AttemptRef) (worker.LeaseStatus, error) {
|
||||||
|
var out worker.LeaseStatus
|
||||||
|
err := s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||||
|
rows, err := s.lockExecution(ctx, tx, token, b, p, ref, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
now, err := dbNow(ctx, tx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// 只暂按新会话计算有效性;通过所有期限和当前尝试检查后才迁移归属,绝不延长期限。
|
||||||
|
previous := rows.attempt.SessionID
|
||||||
|
rows.attempt.SessionID = b.SessionID
|
||||||
|
out = leaseStatus(rows, b, now)
|
||||||
|
if !out.Active || previous == b.SessionID {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET session_id=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, b.SessionID, now, token.NamespaceID, ref.AttemptID, rows.attempt.Revision)
|
||||||
|
if err := exactlyOne(result, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return emitEvent(ctx, tx, token.NamespaceID, ref.AttemptID, rows.attempt.Revision+1, "attempt", "attempt.resumed", map[string]any{"from_session_id": previous, "to_session_id": b.SessionID, "node_id": s.node})
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
638
internal/storage/mysql/worker_integration_test.go
Normal file
638
internal/storage/mysql/worker_integration_test.go
Normal file
@@ -0,0 +1,638 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||||
|
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||||
|
)
|
||||||
|
|
||||||
|
type executionFixture struct {
|
||||||
|
t *testing.T
|
||||||
|
ctx context.Context
|
||||||
|
db *sql.DB
|
||||||
|
resources BootstrapResult
|
||||||
|
token auth.Token
|
||||||
|
tasks *service.Service
|
||||||
|
storeA, storeB *ExecutionStore
|
||||||
|
a, b *worker.Service
|
||||||
|
hello worker.ConnectInput
|
||||||
|
binding worker.Binding
|
||||||
|
}
|
||||||
|
|
||||||
|
func executionID(t *testing.T) core.ID {
|
||||||
|
t.Helper()
|
||||||
|
id, err := core.NewID()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
func newExecutionFixture(t *testing.T, ctx context.Context, db *sql.DB) *executionFixture {
|
||||||
|
t.Helper()
|
||||||
|
f := &executionFixture{t: t, ctx: ctx, db: db, tasks: service.New(NewTaskStore(db))}
|
||||||
|
err := Bootstrap(ctx, db, BootstrapInput{Application: "worker-" + string(executionID(t)), Namespace: "default", Environment: "test", Queue: "work"}, func(v BootstrapResult) error { f.resources = v; return nil })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(auth.Permissions{Actions: []string{"tasks:submit", "tasks:read", "tasks:cancel", "queues:read", "queues:manage", "workers:connect", "workers:consume"}, Queues: []string{"*"}})
|
||||||
|
f.exec("UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?", string(raw), f.resources.NamespaceID, f.resources.CredentialID)
|
||||||
|
f.token, err = auth.Parse(f.resources.Token)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.storeA, err = NewExecutionStore(db, executionID(t), ExecutionOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.storeB, err = NewExecutionStore(db, executionID(t), ExecutionOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.a, f.b = worker.New(f.storeA), worker.New(f.storeB)
|
||||||
|
secret, _, err := worker.NewSecret()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.hello = worker.ConnectInput{InstanceID: executionID(t), InstanceSecret: secret, QueueID: f.resources.QueueID, Capacity: 4, Capabilities: []worker.Capability{{Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1"}}}
|
||||||
|
f.binding, _, err = f.a.Connect(ctx, f.token, f.hello)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
func (f *executionFixture) exec(q string, args ...any) {
|
||||||
|
f.t.Helper()
|
||||||
|
if _, err := f.db.ExecContext(f.ctx, q, args...); err != nil {
|
||||||
|
f.t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (f *executionFixture) submit(max uint32) service.TaskView {
|
||||||
|
f.t.Helper()
|
||||||
|
out, err := f.tasks.Submit(f.ctx, f.token, string(executionID(f.t)), service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`), Retry: pulse.RetryPolicy{MaxAttempts: max, AttemptTimeoutMS: 120000}})
|
||||||
|
if err != nil {
|
||||||
|
f.t.Fatal(err)
|
||||||
|
}
|
||||||
|
return out.Task
|
||||||
|
}
|
||||||
|
func (f *executionFixture) claim() (*worker.Assignment, worker.AttemptRef) {
|
||||||
|
f.t.Helper()
|
||||||
|
out, err := f.a.Claim(f.ctx, f.token, f.binding)
|
||||||
|
if err != nil {
|
||||||
|
f.t.Fatal(err)
|
||||||
|
}
|
||||||
|
if out == nil {
|
||||||
|
f.t.Fatal("expected assignment")
|
||||||
|
}
|
||||||
|
return out, worker.AttemptRef{TaskID: out.Task.ID, AttemptID: out.AttemptID, Epoch: out.Epoch, LeaseSecret: out.LeaseSecret}
|
||||||
|
}
|
||||||
|
func (f *executionFixture) get(id core.ID) service.TaskView {
|
||||||
|
f.t.Helper()
|
||||||
|
out, err := f.tasks.GetTask(f.ctx, f.token, id)
|
||||||
|
if err != nil {
|
||||||
|
f.t.Fatal(err)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
func (f *executionFixture) expire(ref worker.AttemptRef) {
|
||||||
|
f.t.Helper()
|
||||||
|
f.exec("UPDATE pulse_tasks SET lease_until=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, ref.TaskID)
|
||||||
|
f.exec("UPDATE pulse_attempts SET lease_until=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, ref.AttemptID)
|
||||||
|
}
|
||||||
|
func (f *executionFixture) count(table, where string, args ...any) int {
|
||||||
|
f.t.Helper()
|
||||||
|
var n int
|
||||||
|
if err := f.db.QueryRowContext(f.ctx, "SELECT COUNT(*) FROM "+table+" WHERE namespace_id=? AND "+where, append([]any{f.resources.NamespaceID}, args...)...).Scan(&n); err != nil {
|
||||||
|
f.t.Fatal(err)
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
func (f *executionFixture) used() uint32 {
|
||||||
|
f.t.Helper()
|
||||||
|
var n uint32
|
||||||
|
if err := f.db.QueryRowContext(f.ctx, "SELECT COALESCE(SUM(used_permits),0) FROM pulse_concurrency_limits WHERE namespace_id=?", f.resources.NamespaceID).Scan(&n); err != nil {
|
||||||
|
f.t.Fatal(err)
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
func expectWorkerCode(t *testing.T, err error, code string) {
|
||||||
|
t.Helper()
|
||||||
|
if fault.Code(err) != code {
|
||||||
|
t.Fatalf("wanted %s got %v", code, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestIntegrationWorkerExecution(t *testing.T) {
|
||||||
|
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||||
|
if dsn == "" {
|
||||||
|
t.Skip("requires disposable MySQL")
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
db, err := Open(ctx, dsn, 32, 8)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
if err := Migrate(ctx, db); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Run("durable_start_renew_and_exact_result_confirmation", func(t *testing.T) {
|
||||||
|
f := newExecutionFixture(t, ctx, db)
|
||||||
|
task := f.submit(3)
|
||||||
|
assignment, ref := f.claim()
|
||||||
|
if assignment.Task.ID != task.ID || assignment.Task.State != pulse.TaskLeased || assignment.Task.AttemptCount != 1 || f.used() != 1 {
|
||||||
|
t.Fatal("claim state/permit invariant")
|
||||||
|
}
|
||||||
|
startID := executionID(t)
|
||||||
|
started, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
|
||||||
|
if err != nil || !started.Lease.Active || started.Receipt.TaskState != pulse.TaskRunning {
|
||||||
|
t.Fatalf("start: %v", err)
|
||||||
|
}
|
||||||
|
duplicate, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
|
||||||
|
if err != nil || !duplicate.Deduplicated || duplicate.Receipt != started.Receipt {
|
||||||
|
t.Fatalf("start replay: %v", err)
|
||||||
|
}
|
||||||
|
_, err = f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
||||||
|
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||||
|
renewID := executionID(t)
|
||||||
|
renewed, err := f.a.Renew(ctx, f.token, f.binding, renewID, ref)
|
||||||
|
if err != nil || !renewed.Lease.Active || renewed.Lease.LeaseUntil.Before(started.Lease.LeaseUntil) {
|
||||||
|
t.Fatalf("renew: %v", err)
|
||||||
|
}
|
||||||
|
repeated, err := f.a.Renew(ctx, f.token, f.binding, renewID, ref)
|
||||||
|
if err != nil || !repeated.Deduplicated || repeated.Receipt.LeaseUntil != renewed.Receipt.LeaseUntil {
|
||||||
|
t.Fatalf("renew replay: %v", err)
|
||||||
|
}
|
||||||
|
result := worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"amount":12345678901234567890.1234567890123456789,"version":9007199254740993}`)}
|
||||||
|
message := executionID(t)
|
||||||
|
finished, err := f.a.Finish(ctx, f.token, f.binding, message, result)
|
||||||
|
if err != nil || finished.Lease.Active || finished.Receipt.Disposition != "APPLIED" {
|
||||||
|
t.Fatalf("finish: %v", err)
|
||||||
|
}
|
||||||
|
duplicate, err = f.a.Finish(ctx, f.token, f.binding, message, result)
|
||||||
|
if err != nil || !duplicate.Deduplicated {
|
||||||
|
t.Fatalf("finish replay %v", err)
|
||||||
|
}
|
||||||
|
duplicate, err = f.a.Finish(ctx, f.token, f.binding, executionID(t), result)
|
||||||
|
if err != nil || !duplicate.Deduplicated {
|
||||||
|
t.Fatalf("new message equivalent result %v", err)
|
||||||
|
}
|
||||||
|
current := f.get(task.ID)
|
||||||
|
if current.State != pulse.TaskSucceeded || current.CompletedAt == nil || !strings.Contains(string(current.Result), "12345678901234567890.1234567890123456789") || f.used() != 0 {
|
||||||
|
t.Fatal("finish lost state, precision or permit")
|
||||||
|
}
|
||||||
|
if f.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 1 || f.count("orbit_audit_events", "resource_id=? AND action='task.result'", task.ID) != 1 {
|
||||||
|
t.Fatal("duplicated completion history")
|
||||||
|
}
|
||||||
|
_, err = f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
||||||
|
expectWorkerCode(t, err, "LEASE_LOST")
|
||||||
|
oldAck, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
|
||||||
|
if err != nil || !oldAck.Deduplicated || oldAck.Lease.Active {
|
||||||
|
t.Fatalf("old start acknowledgement resurrected execution %v", err)
|
||||||
|
}
|
||||||
|
changed := result
|
||||||
|
changed.Result = json.RawMessage(`{"changed":true}`)
|
||||||
|
_, err = f.a.Finish(ctx, f.token, f.binding, message, changed)
|
||||||
|
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
|
||||||
|
conflict, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), changed)
|
||||||
|
if err != nil || conflict.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(task.ID).State != pulse.TaskSucceeded {
|
||||||
|
t.Fatalf("conflict rewrote outcome %v", err)
|
||||||
|
}
|
||||||
|
if f.count("pulse_attempt_evidence", "attempt_id=? AND reason='RESULT_CONFLICT'", ref.AttemptID) != 1 {
|
||||||
|
t.Fatal("conflict evidence missing")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("cross_node_resume_fences_old_session_and_requires_secrets", func(t *testing.T) {
|
||||||
|
f := newExecutionFixture(t, ctx, db)
|
||||||
|
f.submit(3)
|
||||||
|
_, ref := f.claim()
|
||||||
|
start := executionID(t)
|
||||||
|
if _, err := f.a.Start(ctx, f.token, f.binding, start, ref); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before := f.get(ref.TaskID)
|
||||||
|
newBinding, welcome, err := f.b.Connect(ctx, f.token, f.hello)
|
||||||
|
if err != nil || welcome.Epoch != 2 {
|
||||||
|
t.Fatalf("reconnect %v", err)
|
||||||
|
}
|
||||||
|
_, err = f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
||||||
|
expectWorkerCode(t, err, "STALE_SESSION")
|
||||||
|
_, err = f.b.Renew(ctx, f.token, newBinding, executionID(t), ref)
|
||||||
|
expectWorkerCode(t, err, "LEASE_LOST")
|
||||||
|
wrong := ref
|
||||||
|
wrong.LeaseSecret = strings.Repeat("A", 43)
|
||||||
|
_, err = f.b.Resume(ctx, f.token, newBinding, wrong)
|
||||||
|
expectWorkerCode(t, err, "PERMISSION_DENIED")
|
||||||
|
resumed, err := f.b.Resume(ctx, f.token, newBinding, ref)
|
||||||
|
if err != nil || !resumed.Active || resumed.State != pulse.AttemptRunning {
|
||||||
|
t.Fatalf("resume %v", err)
|
||||||
|
}
|
||||||
|
duplicate, err := f.b.Start(ctx, f.token, newBinding, start, ref)
|
||||||
|
if err != nil || !duplicate.Deduplicated || !duplicate.Lease.Active {
|
||||||
|
t.Fatalf("start ack recovery %v", err)
|
||||||
|
}
|
||||||
|
if f.get(ref.TaskID).AttemptCount != before.AttemptCount || f.used() != 1 {
|
||||||
|
t.Fatal("resume created attempt or duplicated permit")
|
||||||
|
}
|
||||||
|
if _, err := f.b.Resume(ctx, f.token, newBinding, ref); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if f.count("orbit_events", "aggregate_id=? AND type='attempt.resumed'", ref.AttemptID) != 1 {
|
||||||
|
t.Fatal("resume event duplicated")
|
||||||
|
}
|
||||||
|
badHello := f.hello
|
||||||
|
badHello.InstanceSecret = strings.Repeat("A", 43)
|
||||||
|
_, _, err = f.b.Connect(ctx, f.token, badHello)
|
||||||
|
expectWorkerCode(t, err, "PERMISSION_DENIED")
|
||||||
|
badHello = f.hello
|
||||||
|
badHello.Capabilities = []worker.Capability{{Handler: "another", HandlerVersion: "v1", PayloadVersion: "v1"}}
|
||||||
|
_, _, err = f.b.Connect(ctx, f.token, badHello)
|
||||||
|
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||||
|
f.expire(ref)
|
||||||
|
expired, err := f.b.Resume(ctx, f.token, newBinding, ref)
|
||||||
|
if err != nil || expired.Active {
|
||||||
|
t.Fatalf("expired resume %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("multi_node_claim_obeys_queue_and_process_capacity", func(t *testing.T) {
|
||||||
|
f := newExecutionFixture(t, ctx, db)
|
||||||
|
f.exec("UPDATE pulse_queues SET max_concurrency=2 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.QueueID)
|
||||||
|
for i := 0; i < 8; i++ {
|
||||||
|
f.submit(3)
|
||||||
|
}
|
||||||
|
secret, _, err := worker.NewSecret()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hello := f.hello
|
||||||
|
hello.InstanceID = executionID(t)
|
||||||
|
hello.InstanceSecret = secret
|
||||||
|
hello.Capacity = 1
|
||||||
|
bindingB, _, err := f.b.Connect(ctx, f.token, hello)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := f.a.Capacity(ctx, f.token, f.binding, worker.CapacityInput{Sequence: 1, Capacity: 1}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
assignments := make(chan *worker.Assignment, 16)
|
||||||
|
errs := make(chan error, 16)
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
start := make(chan struct{})
|
||||||
|
for i := 0; i < 16; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i int) {
|
||||||
|
defer wg.Done()
|
||||||
|
<-start
|
||||||
|
var a *worker.Assignment
|
||||||
|
var err error
|
||||||
|
if i%2 == 0 {
|
||||||
|
a, err = f.a.Claim(ctx, f.token, f.binding)
|
||||||
|
} else {
|
||||||
|
a, err = f.b.Claim(ctx, f.token, bindingB)
|
||||||
|
}
|
||||||
|
assignments <- a
|
||||||
|
errs <- err
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
close(start)
|
||||||
|
wg.Wait()
|
||||||
|
close(assignments)
|
||||||
|
close(errs)
|
||||||
|
for err := range errs {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ids := map[core.ID]bool{}
|
||||||
|
for a := range assignments {
|
||||||
|
if a == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ids[a.Task.ID] {
|
||||||
|
t.Fatal("task dispatched twice")
|
||||||
|
}
|
||||||
|
ids[a.Task.ID] = true
|
||||||
|
}
|
||||||
|
if len(ids) != 2 || f.used() != 2 {
|
||||||
|
t.Fatalf("concurrency overrun: assignments=%d used=%d", len(ids), f.used())
|
||||||
|
}
|
||||||
|
b2, _, err := f.b.Connect(ctx, f.token, hello)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a, err := f.b.Claim(ctx, f.token, b2)
|
||||||
|
if err != nil || a != nil {
|
||||||
|
t.Fatalf("reconnect reset capacity %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("drain_is_durable_and_capacity_sequence_is_not_additive", func(t *testing.T) {
|
||||||
|
f := newExecutionFixture(t, ctx, db)
|
||||||
|
f.submit(3)
|
||||||
|
_, ref := f.claim()
|
||||||
|
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
input := worker.CapacityInput{Sequence: 1, Capacity: 0, Drain: true}
|
||||||
|
if _, err := f.a.Capacity(ctx, f.token, f.binding, input); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := f.a.Capacity(ctx, f.token, f.binding, input); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
input.Capacity = 2
|
||||||
|
_, err := f.a.Capacity(ctx, f.token, f.binding, input)
|
||||||
|
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||||
|
input.Sequence = 2
|
||||||
|
input.Drain = false
|
||||||
|
_, err = f.a.Capacity(ctx, f.token, f.binding, input)
|
||||||
|
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||||
|
f.submit(3)
|
||||||
|
a, err := f.a.Claim(ctx, f.token, f.binding)
|
||||||
|
if err != nil || a != nil {
|
||||||
|
t.Fatalf("draining worker received work %v", err)
|
||||||
|
}
|
||||||
|
if f.used() != 1 {
|
||||||
|
t.Fatal("capacity zero released in-flight permit")
|
||||||
|
}
|
||||||
|
newBinding, welcome, err := f.b.Connect(ctx, f.token, f.hello)
|
||||||
|
if err != nil || !welcome.Draining {
|
||||||
|
t.Fatalf("drain lost on reconnect %v", err)
|
||||||
|
}
|
||||||
|
if _, err := f.b.Resume(ctx, f.token, newBinding, ref); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := f.b.Finish(ctx, f.token, newBinding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if f.used() != 0 {
|
||||||
|
t.Fatal("drained execution did not release capacity")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("paused_gate_blocks_start_but_not_saved_result", func(t *testing.T) {
|
||||||
|
f := newExecutionFixture(t, ctx, db)
|
||||||
|
gate := executionID(t)
|
||||||
|
f.exec("INSERT INTO pulse_execution_gates(namespace_id,id,paused) VALUES (?,?,TRUE)", f.resources.NamespaceID, gate)
|
||||||
|
// 超过一次候选页的暂停任务不能饿死后面的普通任务。
|
||||||
|
for i := 0; i < 18; i++ {
|
||||||
|
task := f.submit(3)
|
||||||
|
f.exec("UPDATE pulse_tasks SET gate_id=? WHERE namespace_id=? AND id=?", gate, f.resources.NamespaceID, task.ID)
|
||||||
|
}
|
||||||
|
ready := f.submit(3)
|
||||||
|
a, ref := f.claim()
|
||||||
|
if a.Task.ID != ready.ID {
|
||||||
|
t.Fatal("paused candidate starvation")
|
||||||
|
}
|
||||||
|
f.exec("UPDATE pulse_tasks SET gate_id=? WHERE namespace_id=? AND id=?", gate, f.resources.NamespaceID, ready.ID)
|
||||||
|
start := executionID(t)
|
||||||
|
_, err := f.a.Start(ctx, f.token, f.binding, start, ref)
|
||||||
|
expectWorkerCode(t, err, "EXECUTION_PAUSED")
|
||||||
|
f.exec("UPDATE pulse_execution_gates SET paused=FALSE WHERE namespace_id=? AND id=?", f.resources.NamespaceID, gate)
|
||||||
|
if _, err := f.a.Start(ctx, f.token, f.binding, start, ref); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.exec("UPDATE pulse_execution_gates SET paused=TRUE WHERE namespace_id=? AND id=?", f.resources.NamespaceID, gate)
|
||||||
|
duplicate, err := f.a.Start(ctx, f.token, f.binding, start, ref)
|
||||||
|
if err != nil || !duplicate.Deduplicated || !duplicate.Lease.Active {
|
||||||
|
t.Fatalf("pause blocked original start receipt %v", err)
|
||||||
|
}
|
||||||
|
if _, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("recovery_is_bounded_and_unknown_effects_are_not_replayed", func(t *testing.T) {
|
||||||
|
f := newExecutionFixture(t, ctx, db)
|
||||||
|
task := f.submit(2)
|
||||||
|
_, ref := f.claim()
|
||||||
|
f.expire(ref)
|
||||||
|
stats, err := f.storeB.Recover(ctx, 100)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
current := f.get(task.ID)
|
||||||
|
if current.State != pulse.TaskWaiting || current.AttemptCount != 1 || f.used() != 0 || stats.Requeued < 1 {
|
||||||
|
t.Fatal("pre-start lease recovery failed")
|
||||||
|
}
|
||||||
|
f.exec("UPDATE pulse_tasks SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID)
|
||||||
|
_, newRef := f.claim()
|
||||||
|
if newRef.Epoch != 2 || newRef.AttemptID == ref.AttemptID {
|
||||||
|
t.Fatal("attempt epoch not advanced")
|
||||||
|
}
|
||||||
|
_, err = f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
||||||
|
expectWorkerCode(t, err, "LEASE_LOST")
|
||||||
|
start := executionID(t)
|
||||||
|
if _, err := f.a.Start(ctx, f.token, f.binding, start, newRef); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.expire(newRef)
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
errs := make(chan error, 2)
|
||||||
|
for _, store := range []*ExecutionStore{f.storeA, f.storeB} {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(store *ExecutionStore) { defer wg.Done(); _, err := store.Recover(ctx, 100); errs <- err }(store)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
close(errs)
|
||||||
|
for err := range errs {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
current = f.get(task.ID)
|
||||||
|
if current.State != pulse.TaskNeedsReview || current.AttemptCount != 2 || current.CompletedAt != nil || f.used() != 0 {
|
||||||
|
t.Fatal("running expiry must become review without replay")
|
||||||
|
}
|
||||||
|
if f.count("orbit_events", "aggregate_id=? AND type='task.needs_review'", task.ID) != 1 {
|
||||||
|
t.Fatal("recovery event duplicated")
|
||||||
|
}
|
||||||
|
late := worker.FinishInput{Ref: newRef, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"external_ref":"late-result"}`)}
|
||||||
|
message := executionID(t)
|
||||||
|
evidence, err := f.a.Finish(ctx, f.token, f.binding, message, late)
|
||||||
|
if err != nil || evidence.Receipt.Disposition != "EVIDENCE_ONLY" || evidence.Lease.Active {
|
||||||
|
t.Fatalf("late evidence %v", err)
|
||||||
|
}
|
||||||
|
evidence, err = f.a.Finish(ctx, f.token, f.binding, message, late)
|
||||||
|
if err != nil || !evidence.Deduplicated {
|
||||||
|
t.Fatalf("late replay %v", err)
|
||||||
|
}
|
||||||
|
if f.get(task.ID).State != pulse.TaskNeedsReview || f.count("pulse_attempt_evidence", "attempt_id=?", newRef.AttemptID) != 1 {
|
||||||
|
t.Fatal("late result rewrote authority or duplicated evidence")
|
||||||
|
}
|
||||||
|
duplicate, err := f.a.Start(ctx, f.token, f.binding, start, newRef)
|
||||||
|
if err != nil || !duplicate.Deduplicated || duplicate.Lease.Active {
|
||||||
|
t.Fatalf("old acknowledgement restored lease %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("attempt_timeout_and_session_expiry_cap_renewal", func(t *testing.T) {
|
||||||
|
f := newExecutionFixture(t, ctx, db)
|
||||||
|
task := f.submit(3)
|
||||||
|
_, ref := f.claim()
|
||||||
|
f.exec("UPDATE pulse_tasks SET attempt_timeout_ms=5000 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID)
|
||||||
|
begun, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
renewed, err := f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
||||||
|
if err != nil || renewed.Lease.LeaseUntil.After(begun.Lease.LeaseUntil) {
|
||||||
|
t.Fatalf("renew bypassed absolute attempt deadline %v", err)
|
||||||
|
}
|
||||||
|
f.exec("UPDATE pulse_worker_sessions SET expires_at=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.binding.SessionID)
|
||||||
|
renewed, err = f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
||||||
|
if err != nil || renewed.Lease.LeaseUntil.Sub(renewed.Lease.ServerTime) > time.Second {
|
||||||
|
t.Fatalf("renew exceeded session expiration %v", err)
|
||||||
|
}
|
||||||
|
f.exec("UPDATE pulse_worker_sessions SET expires_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.binding.SessionID)
|
||||||
|
_, err = f.a.Capacity(ctx, f.token, f.binding, worker.CapacityInput{Sequence: 1, Capacity: 4})
|
||||||
|
expectWorkerCode(t, err, "STALE_SESSION")
|
||||||
|
})
|
||||||
|
t.Run("result_event_failure_rolls_back_task_receipt_and_permit", func(t *testing.T) {
|
||||||
|
f := newExecutionFixture(t, ctx, db)
|
||||||
|
task := f.submit(3)
|
||||||
|
_, ref := f.claim()
|
||||||
|
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.exec("CREATE TRIGGER orbit_test_reject_worker_audit BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='worker audit failure'")
|
||||||
|
message := executionID(t)
|
||||||
|
in := worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}
|
||||||
|
_, err := f.a.Finish(ctx, f.token, f.binding, message, in)
|
||||||
|
f.exec("DROP TRIGGER orbit_test_reject_worker_audit")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected injected failure")
|
||||||
|
}
|
||||||
|
if f.get(task.ID).State != pulse.TaskRunning || f.used() != 1 || f.count("pulse_attempt_receipts", "attempt_id=? AND operation='finish'", ref.AttemptID) != 0 || f.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 0 {
|
||||||
|
t.Fatal("partial result transaction committed")
|
||||||
|
}
|
||||||
|
if _, err := f.a.Finish(ctx, f.token, f.binding, message, in); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("finish_without_start_is_only_evidence_and_cannot_free_capacity", func(t *testing.T) {
|
||||||
|
f := newExecutionFixture(t, ctx, db)
|
||||||
|
task := f.submit(1)
|
||||||
|
_, ref := f.claim()
|
||||||
|
result, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)})
|
||||||
|
if err != nil || result.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(task.ID).State != pulse.TaskLeased || f.used() != 1 {
|
||||||
|
t.Fatalf("unstarted result accepted %v", err)
|
||||||
|
}
|
||||||
|
f.expire(ref)
|
||||||
|
if _, err := f.storeA.Recover(ctx, 100); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if f.get(task.ID).State != pulse.TaskFailed || f.used() != 0 {
|
||||||
|
t.Fatal("exhausted pre-start attempt did not terminate")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("known_failure_unknown_and_waiting_deadlines", func(t *testing.T) {
|
||||||
|
f := newExecutionFixture(t, ctx, db)
|
||||||
|
for _, outcome := range []string{"FAILED", "UNKNOWN"} {
|
||||||
|
task := f.submit(3)
|
||||||
|
_, ref := f.claim()
|
||||||
|
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: outcome, ErrorCode: "EXTERNAL_RESULT", Result: json.RawMessage(`{"evidence_ref":"object/result"}`)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := pulse.TaskFailed
|
||||||
|
if outcome == "UNKNOWN" {
|
||||||
|
want = pulse.TaskNeedsReview
|
||||||
|
}
|
||||||
|
if out.Receipt.TaskState != want || f.get(task.ID).State != want {
|
||||||
|
t.Fatal("outcome classification lost")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
task := f.submit(3)
|
||||||
|
f.exec("UPDATE pulse_tasks SET available_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 2 HOUR),deadline_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 HOUR) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID)
|
||||||
|
if _, err := f.storeA.Recover(ctx, 100); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if f.get(task.ID).State != pulse.TaskFailed {
|
||||||
|
t.Fatal("waiting deadline not enforced")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("authority_is_checked_on_every_execution_command", func(t *testing.T) {
|
||||||
|
f := newExecutionFixture(t, ctx, db)
|
||||||
|
f.submit(3)
|
||||||
|
_, ref := f.claim()
|
||||||
|
f.exec("UPDATE orbit_credentials SET revoked_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.CredentialID)
|
||||||
|
_, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
||||||
|
expectWorkerCode(t, err, "UNAUTHENTICATED")
|
||||||
|
f.exec("UPDATE orbit_credentials SET revoked_at=NULL WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.CredentialID)
|
||||||
|
other := newExecutionFixture(t, ctx, db)
|
||||||
|
_, err = other.a.Resume(ctx, other.token, other.binding, ref)
|
||||||
|
expectWorkerCode(t, err, "NOT_FOUND")
|
||||||
|
copied := f.binding
|
||||||
|
copied.Proof = core.Digest{}
|
||||||
|
_, err = f.a.Claim(ctx, f.token, copied)
|
||||||
|
expectWorkerCode(t, err, "STALE_SESSION")
|
||||||
|
cancelled, stop := context.WithCancel(ctx)
|
||||||
|
stop()
|
||||||
|
_, err = f.a.Start(cancelled, f.token, f.binding, executionID(t), ref)
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal(fmt.Sprint(err))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("claim_event_failure_does_not_leave_an_invisible_lease", func(t *testing.T) {
|
||||||
|
f := newExecutionFixture(t, ctx, db)
|
||||||
|
task := f.submit(3)
|
||||||
|
f.exec("CREATE TRIGGER orbit_test_reject_claim_event BEFORE INSERT ON orbit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='claim event failure'")
|
||||||
|
assignment, err := f.a.Claim(ctx, f.token, f.binding)
|
||||||
|
f.exec("DROP TRIGGER orbit_test_reject_claim_event")
|
||||||
|
if err == nil || assignment != nil {
|
||||||
|
t.Fatal("expected failed claim without usable assignment")
|
||||||
|
}
|
||||||
|
if f.get(task.ID).State != pulse.TaskWaiting || f.used() != 0 || f.count("pulse_attempts", "task_id=?", task.ID) != 0 || f.count("pulse_permits", "TRUE") != 0 {
|
||||||
|
t.Fatal("claim partially committed")
|
||||||
|
}
|
||||||
|
if _, err := f.a.Claim(ctx, f.token, f.binding); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("lock_wait_does_not_reuse_an_earlier_clock_sample", func(t *testing.T) {
|
||||||
|
f := newExecutionFixture(t, ctx, db)
|
||||||
|
f.submit(3)
|
||||||
|
_, ref := f.claim()
|
||||||
|
holder, err := db.BeginTx(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer holder.Rollback()
|
||||||
|
if _, err := holder.ExecContext(ctx, "UPDATE pulse_queues SET revision=revision+1 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.QueueID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, table := range []string{"pulse_tasks", "pulse_attempts"} {
|
||||||
|
id := ref.TaskID
|
||||||
|
if table == "pulse_attempts" {
|
||||||
|
id = ref.AttemptID
|
||||||
|
}
|
||||||
|
if _, err := holder.ExecContext(ctx, "UPDATE "+table+" SET lease_until=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 150000 MICROSECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, id); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result := make(chan error, 1)
|
||||||
|
go func() { _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); result <- err }()
|
||||||
|
time.Sleep(250 * time.Millisecond)
|
||||||
|
if err := holder.Commit(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
expectWorkerCode(t, <-result, "LEASE_LOST")
|
||||||
|
if f.get(ref.TaskID).State != pulse.TaskLeased {
|
||||||
|
t.Fatal("stale clock authorized start")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
166
internal/storage/mysql/worker_recovery.go
Normal file
166
internal/storage/mysql/worker_recovery.go
Normal file
@@ -0,0 +1,166 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Recover 是受控后台扫描入口,每批最多 100 项、每项一个短事务,可由多个节点竞争。
|
||||||
|
// 当前已授权执行的失联一律转核查,自动安全重试在 P3 接入冻结策略后实现。
|
||||||
|
func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.RecoveryStats, error) {
|
||||||
|
var out worker.RecoveryStats
|
||||||
|
if limit < 1 || limit > 100 {
|
||||||
|
return out, fault.Invalid("recovery batch limit must be 1 to 100")
|
||||||
|
}
|
||||||
|
type candidate struct{ ns, id, queue core.ID }
|
||||||
|
candidates := []candidate{}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
// 分别使用租约、总截止及开始截止索引,避免对整个 WAITING 队列做 COALESCE 扫描。
|
||||||
|
for _, q := range []string{
|
||||||
|
`SELECT namespace_id,id,queue_id FROM pulse_tasks WHERE state IN ('LEASED','RUNNING') AND lease_until<=UTC_TIMESTAMP(6) ORDER BY lease_until,namespace_id,id LIMIT ?`,
|
||||||
|
`SELECT namespace_id,id,queue_id FROM pulse_tasks WHERE state='WAITING' AND deadline_at<=UTC_TIMESTAMP(6) ORDER BY deadline_at,namespace_id,id LIMIT ?`,
|
||||||
|
`SELECT namespace_id,id,queue_id FROM pulse_tasks WHERE state='WAITING' AND start_deadline_at<=UTC_TIMESTAMP(6) ORDER BY start_deadline_at,namespace_id,id LIMIT ?`,
|
||||||
|
} {
|
||||||
|
if len(candidates) >= limit {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
rows, err := s.db.QueryContext(ctx, q, limit-len(candidates))
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
var c candidate
|
||||||
|
if err := rows.Scan(&c.ns, &c.id, &c.queue); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
key := string(c.ns) + string(c.id)
|
||||||
|
if !seen[key] {
|
||||||
|
candidates = append(candidates, c)
|
||||||
|
seen[key] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
err = rows.Err()
|
||||||
|
rows.Close()
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range candidates {
|
||||||
|
out.Examined++
|
||||||
|
var changed pulse.TaskState
|
||||||
|
err := WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
||||||
|
// 与 Worker 命令一致先锁队列;有其他节点正在处理时跳过,本批不占用等待线程。
|
||||||
|
var queue core.ID
|
||||||
|
err := tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, c.ns, c.queue).Scan(&queue)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var taskID core.ID
|
||||||
|
err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_tasks WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, c.ns, c.id).Scan(&taskID)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
task, err := readExecutionTask(ctx, tx, c.ns, c.id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if task.view.State == pulse.TaskWaiting {
|
||||||
|
now, err := dbNow(ctx, tx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if task.view.DeadlineAt.After(now) && (task.view.StartDeadlineAt == nil || task.view.StartDeadlineAt.After(now)) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
changed = pulse.TaskFailed
|
||||||
|
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='FAILED',completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, now, now, c.ns, c.id, task.view.Revision)
|
||||||
|
if err := exactlyOne(result, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", "task.failed", map[string]any{"reason": "START_DEADLINE_EXCEEDED", "node_id": s.node})
|
||||||
|
}
|
||||||
|
if task.view.State != pulse.TaskLeased && task.view.State != pulse.TaskRunning {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if task.current == nil || task.lease == nil {
|
||||||
|
return errors.New("active task lacks attempt or lease")
|
||||||
|
}
|
||||||
|
attempt, err := readAttempt(ctx, tx, c.ns, *task.current)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
now, err := dbNow(ctx, tx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if task.lease.After(now) || attempt.LeaseUntil.After(now) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if attempt.Epoch != task.epoch || attempt.TaskID != c.id {
|
||||||
|
return errors.New("task attempt identity mismatch")
|
||||||
|
}
|
||||||
|
if attempt.State != pulse.AttemptLeased && attempt.State != pulse.AttemptRunning {
|
||||||
|
return errors.New("active task references ended attempt")
|
||||||
|
}
|
||||||
|
attemptState := pulse.AttemptExpired
|
||||||
|
next := pulse.TaskFailed
|
||||||
|
reason := "LEASE_EXPIRED_BEFORE_START"
|
||||||
|
var completed *time.Time = &now
|
||||||
|
available := task.view.AvailableAt
|
||||||
|
if attempt.AuthorizedAt != nil {
|
||||||
|
attemptState = pulse.AttemptUnknown
|
||||||
|
next = pulse.TaskNeedsReview
|
||||||
|
reason = "EXECUTION_OUTCOME_UNKNOWN"
|
||||||
|
completed = nil
|
||||||
|
} else if task.view.AttemptCount < task.maxAttempts {
|
||||||
|
// 未批准开始的尝试可以有界重派,次数不重置;开始截止与总截止仍有效。
|
||||||
|
nextAt := now.Add(time.Second)
|
||||||
|
if nextAt.Before(task.view.DeadlineAt) && (task.view.StartDeadlineAt == nil || nextAt.Before(*task.view.StartDeadlineAt)) {
|
||||||
|
next = pulse.TaskWaiting
|
||||||
|
available = nextAt
|
||||||
|
completed = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, reason, now, c.ns, attempt.ID, attempt.Revision)
|
||||||
|
if err := exactlyOne(result, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
result, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,lease_until=NULL,available_at=?,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, next, available, completed, now, c.ns, c.id, task.view.Revision)
|
||||||
|
if err := exactlyOne(result, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := releasePermits(ctx, tx, c.ns, attempt.ID, now); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
changed = next
|
||||||
|
return emitEvent(ctx, tx, c.ns, c.id, task.view.Revision+1, "task", resultEvent(next), map[string]any{"attempt_id": attempt.ID, "epoch": fmt.Sprint(attempt.Epoch), "reason": reason, "node_id": s.node})
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
switch changed {
|
||||||
|
case pulse.TaskWaiting:
|
||||||
|
out.Requeued++
|
||||||
|
case pulse.TaskFailed:
|
||||||
|
out.Failed++
|
||||||
|
case pulse.TaskNeedsReview:
|
||||||
|
out.NeedsReview++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
132
internal/storage/mysql/worker_results.go
Normal file
132
internal/storage/mysql/worker_results.go
Normal file
@@ -0,0 +1,132 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||||
|
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s *ExecutionStore) Finish(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, in worker.FinishInput) (worker.CommandResult, error) {
|
||||||
|
var out worker.CommandResult
|
||||||
|
digest, err := requestDigest(struct {
|
||||||
|
Identity any
|
||||||
|
Outcome string
|
||||||
|
Result core.JSON
|
||||||
|
ErrorCode string
|
||||||
|
}{attemptIdentity(in.Ref), in.Outcome, core.JSON(in.Result), in.ErrorCode})
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
err = s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||||
|
rows, err := s.lockExecution(ctx, tx, token, b, p, in.Ref, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
cached, err := readReceipt(ctx, tx, token.NamespaceID, in.Ref.AttemptID, message, "finish", digest)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
now, err := dbNow(ctx, tx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
status := leaseStatus(rows, b, now)
|
||||||
|
if cached != nil {
|
||||||
|
out = worker.CommandResult{Receipt: *cached, Lease: status, Deduplicated: true}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
task, a := rows.task, rows.attempt
|
||||||
|
receipt := worker.Receipt{Operation: "finish", TaskState: task.view.State, AttemptState: a.State, TaskRevision: task.view.Revision, LeaseUntil: a.LeaseUntil, Disposition: "APPLIED"}
|
||||||
|
if a.ResultDigest != nil && *a.ResultDigest == digest {
|
||||||
|
// 新消息 ID 重报已保存的同一结果,也不能重复事件、许可释放或状态推进。
|
||||||
|
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out = worker.CommandResult{Receipt: receipt, Lease: status, Deduplicated: true}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
reason := ""
|
||||||
|
if a.ResultDigest != nil {
|
||||||
|
reason = "RESULT_CONFLICT"
|
||||||
|
} else if a.AuthorizedAt == nil {
|
||||||
|
reason = "NOT_STARTED"
|
||||||
|
} else if !status.Active || a.State != pulse.AttemptRunning {
|
||||||
|
reason = "LEASE_LOST"
|
||||||
|
}
|
||||||
|
if reason != "" {
|
||||||
|
if err := saveEvidence(ctx, tx, token.NamespaceID, message, in, digest, reason); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
receipt.Disposition = "EVIDENCE_ONLY"
|
||||||
|
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out = worker.CommandResult{Receipt: receipt, Lease: status}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
attemptState, taskState := pulse.AttemptState(in.Outcome), pulse.TaskState(in.Outcome)
|
||||||
|
var completed *time.Time = &now
|
||||||
|
if in.Outcome == "UNKNOWN" {
|
||||||
|
taskState = pulse.TaskNeedsReview
|
||||||
|
completed = nil
|
||||||
|
}
|
||||||
|
changed, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=NULLIF(?,''),result=?,result_digest=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, in.ErrorCode, string(in.Result), digest, now, token.NamespaceID, a.ID, a.Revision)
|
||||||
|
if err := exactlyOne(changed, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
changed, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,result=?,lease_until=NULL,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, taskState, string(in.Result), completed, now, token.NamespaceID, in.Ref.TaskID, task.view.Revision)
|
||||||
|
if err := exactlyOne(changed, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := releasePermits(ctx, tx, token.NamespaceID, a.ID, now); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
event := map[string]any{"attempt_id": a.ID, "epoch": fmt.Sprint(a.Epoch), "state": taskState, "outcome": in.Outcome, "error_code": in.ErrorCode}
|
||||||
|
if err := emitEvent(ctx, tx, token.NamespaceID, in.Ref.TaskID, task.view.Revision+1, "task", resultEvent(taskState), event); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := appendAudit(ctx, tx, token, message, in.Ref.TaskID, "task", "task.result", "worker result", map[string]any{"state": task.view.State}, event); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
receipt.TaskState = taskState
|
||||||
|
receipt.AttemptState = attemptState
|
||||||
|
receipt.TaskRevision++
|
||||||
|
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out = worker.CommandResult{Receipt: receipt, Lease: worker.LeaseStatus{State: attemptState, LeaseUntil: a.LeaseUntil, ServerTime: now}}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
func resultEvent(state pulse.TaskState) string {
|
||||||
|
switch state {
|
||||||
|
case pulse.TaskSucceeded:
|
||||||
|
return "task.succeeded"
|
||||||
|
case pulse.TaskFailed:
|
||||||
|
return "task.failed"
|
||||||
|
case pulse.TaskNeedsReview:
|
||||||
|
return "task.needs_review"
|
||||||
|
case pulse.TaskWaiting:
|
||||||
|
return "task.requeued"
|
||||||
|
default:
|
||||||
|
return "task.cancelled"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func saveEvidence(ctx context.Context, tx *sql.Tx, ns, message core.ID, in worker.FinishInput, digest core.Digest, reason string) error {
|
||||||
|
id, err := core.NewID()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempt_evidence(namespace_id,id,attempt_id,message_id,outcome,error_code,payload,request_digest,reason) VALUES (?,?,?,?,?,NULLIF(?,''),?,?,?)`, ns, id, in.Ref.AttemptID, message, in.Outcome, in.ErrorCode, string(in.Result), digest, reason)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return emitEvent(ctx, tx, ns, id, 1, "attempt_evidence", "attempt.evidence_received", map[string]any{"attempt_id": in.Ref.AttemptID, "task_id": in.Ref.TaskID, "reason": reason, "outcome": in.Outcome})
|
||||||
|
}
|
||||||
211
internal/storage/mysql/worker_sessions.go
Normal file
211
internal/storage/mysql/worker_sessions.go
Normal file
@@ -0,0 +1,211 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/subtle"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ExecutionOptions 是服务端执行窗口,不由 Worker 任意指定。
|
||||||
|
type ExecutionOptions struct {
|
||||||
|
LeaseDuration time.Duration // LeaseDuration 每次领取/续租的最大窗口。
|
||||||
|
SessionTTL time.Duration // SessionTTL 会话心跳有效期,不代替执行租约。
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionStore 所有执行权以 MySQL 为准,节点内存不持有可绕过数据库的授权。
|
||||||
|
type ExecutionStore struct {
|
||||||
|
db *sql.DB
|
||||||
|
node core.ID
|
||||||
|
options ExecutionOptions
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ worker.Store = (*ExecutionStore)(nil)
|
||||||
|
|
||||||
|
func (s *ExecutionStore) Authorize(ctx context.Context, token auth.Token) error {
|
||||||
|
return s.transaction(ctx, token, "workers:connect", func(*sql.Tx, auth.Permissions) error { return nil })
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewExecutionStore(db *sql.DB, node core.ID, options ExecutionOptions) (*ExecutionStore, error) {
|
||||||
|
if options.LeaseDuration == 0 {
|
||||||
|
options.LeaseDuration = 30 * time.Second
|
||||||
|
}
|
||||||
|
if options.SessionTTL == 0 {
|
||||||
|
options.SessionTTL = 45 * time.Second
|
||||||
|
}
|
||||||
|
if node.Validate() != nil || options.LeaseDuration < time.Second || options.LeaseDuration > 2*time.Minute || options.SessionTTL < options.LeaseDuration || options.SessionTTL > 5*time.Minute {
|
||||||
|
return nil, fault.Invalid("invalid execution windows or node identity")
|
||||||
|
}
|
||||||
|
return &ExecutionStore{db: db, node: node, options: options}, nil
|
||||||
|
}
|
||||||
|
func (s *ExecutionStore) transaction(ctx context.Context, token auth.Token, action string, fn func(*sql.Tx, auth.Permissions) error) error {
|
||||||
|
return (&TaskStore{db: s.db}).transaction(ctx, token, action, fn)
|
||||||
|
}
|
||||||
|
|
||||||
|
type sessionState struct {
|
||||||
|
queue core.ID
|
||||||
|
capacity uint32
|
||||||
|
sequence uint64
|
||||||
|
draining bool
|
||||||
|
expires time.Time
|
||||||
|
capabilities []worker.Capability
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ExecutionStore) welcome(b worker.Binding, state sessionState, now time.Time) worker.Welcome {
|
||||||
|
return worker.Welcome{SessionID: b.SessionID, Epoch: b.Epoch, ServerTime: now, ExpiresAt: state.expires, LeaseDurationMS: uint64(s.options.LeaseDuration / time.Millisecond), Draining: state.draining}
|
||||||
|
}
|
||||||
|
func (s *ExecutionStore) Connect(ctx context.Context, token auth.Token, in worker.ConnectInput) (worker.Binding, worker.Welcome, error) {
|
||||||
|
var binding worker.Binding
|
||||||
|
var out worker.Welcome
|
||||||
|
proof, err := worker.SecretDigest(in.InstanceSecret)
|
||||||
|
if err != nil {
|
||||||
|
return binding, out, err
|
||||||
|
}
|
||||||
|
registration, err := requestDigest(struct {
|
||||||
|
Queue core.ID
|
||||||
|
Capabilities []worker.Capability
|
||||||
|
}{in.QueueID, in.Capabilities})
|
||||||
|
if err != nil {
|
||||||
|
return binding, out, err
|
||||||
|
}
|
||||||
|
err = s.transaction(ctx, token, "workers:connect", func(tx *sql.Tx, p auth.Permissions) error {
|
||||||
|
if !p.AllowsQueue(in.QueueID) {
|
||||||
|
return fault.Forbidden
|
||||||
|
}
|
||||||
|
_, err := tx.ExecContext(ctx, `INSERT INTO pulse_worker_instances(namespace_id,id,credential_id,queue_id,secret_digest,registration_digest) VALUES (?,?,?,?,?,?) ON DUPLICATE KEY UPDATE id=pulse_worker_instances.id`, token.NamespaceID, in.InstanceID, token.CredentialID, in.QueueID, proof, registration)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var credential core.ID
|
||||||
|
var storedProof, storedRegistration core.Digest
|
||||||
|
var epoch uint64
|
||||||
|
var old *core.ID
|
||||||
|
var draining bool
|
||||||
|
err = tx.QueryRowContext(ctx, `SELECT credential_id,secret_digest,registration_digest,session_epoch,current_session_id,draining FROM pulse_worker_instances WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, in.InstanceID).Scan(&credential, &storedProof, &storedRegistration, &epoch, &old, &draining)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if credential != token.CredentialID || subtle.ConstantTimeCompare(proof[:], storedProof[:]) != 1 {
|
||||||
|
return fault.Forbidden
|
||||||
|
}
|
||||||
|
if registration != storedRegistration || epoch == ^uint64(0) {
|
||||||
|
return fault.Conflict
|
||||||
|
}
|
||||||
|
now, err := dbNow(ctx, tx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
session, err := core.NewID()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
binding = worker.Binding{InstanceID: in.InstanceID, SessionID: session, Epoch: epoch + 1, Proof: proof}
|
||||||
|
state := sessionState{queue: in.QueueID, capacity: in.Capacity, draining: draining, expires: now.Add(s.options.SessionTTL), capabilities: in.Capabilities}
|
||||||
|
caps, err := json.Marshal(in.Capabilities)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_worker_sessions(namespace_id,id,worker_instance_id,credential_id,node_id,session_epoch,capabilities,capacity,draining,expires_at) VALUES (?,?,?,?,?,?,?,?,?,?)`, token.NamespaceID, session, in.InstanceID, token.CredentialID, s.node, binding.Epoch, string(caps), in.Capacity, draining, state.expires)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if old != nil {
|
||||||
|
if _, err := tx.ExecContext(ctx, `UPDATE pulse_worker_sessions SET expires_at=?,draining=TRUE,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, now, now, token.NamespaceID, *old); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, err = tx.ExecContext(ctx, `UPDATE pulse_worker_instances SET current_session_id=?,session_epoch=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, session, binding.Epoch, now, token.NamespaceID, in.InstanceID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := emitEvent(ctx, tx, token.NamespaceID, session, 1, "worker_session", "worker.connected", map[string]any{"instance_id": in.InstanceID, "node_id": s.node, "epoch": binding.Epoch}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out = s.welcome(binding, state, now)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return worker.Binding{}, worker.Welcome{}, err
|
||||||
|
}
|
||||||
|
return binding, out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// lockSession 的顺序为凭据 -> 进程头 -> 当前会话;新连接和旧连接命令在同一行上串行化。
|
||||||
|
func (s *ExecutionStore) lockSession(ctx context.Context, tx *sql.Tx, token auth.Token, b worker.Binding, p auth.Permissions) (sessionState, error) {
|
||||||
|
var state sessionState
|
||||||
|
var credential core.ID
|
||||||
|
var proof core.Digest
|
||||||
|
var current *core.ID
|
||||||
|
var epoch uint64
|
||||||
|
if b.InstanceID.Validate() != nil || b.SessionID.Validate() != nil || b.Epoch == 0 {
|
||||||
|
return state, worker.StaleSession
|
||||||
|
}
|
||||||
|
err := tx.QueryRowContext(ctx, `SELECT credential_id,queue_id,secret_digest,current_session_id,session_epoch,draining FROM pulse_worker_instances WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, b.InstanceID).Scan(&credential, &state.queue, &proof, ¤t, &epoch, &state.draining)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return state, worker.StaleSession
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
if credential != token.CredentialID || current == nil || *current != b.SessionID || epoch != b.Epoch || subtle.ConstantTimeCompare(proof[:], b.Proof[:]) != 1 {
|
||||||
|
return state, worker.StaleSession
|
||||||
|
}
|
||||||
|
if !p.AllowsQueue(state.queue) {
|
||||||
|
return state, fault.Forbidden
|
||||||
|
}
|
||||||
|
var node core.ID
|
||||||
|
var raw []byte
|
||||||
|
err = tx.QueryRowContext(ctx, `SELECT node_id,capacity,capacity_sequence,capabilities,expires_at FROM pulse_worker_sessions WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, b.SessionID).Scan(&node, &state.capacity, &state.sequence, &raw, &state.expires)
|
||||||
|
if err != nil {
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
now, err := dbNow(ctx, tx)
|
||||||
|
if err != nil {
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
if node != s.node || !state.expires.After(now) {
|
||||||
|
return state, worker.StaleSession
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &state.capabilities); err != nil {
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
return state, nil
|
||||||
|
}
|
||||||
|
func (s *ExecutionStore) Capacity(ctx context.Context, token auth.Token, b worker.Binding, in worker.CapacityInput) (worker.Welcome, error) {
|
||||||
|
var out worker.Welcome
|
||||||
|
err := s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||||
|
state, err := s.lockSession(ctx, tx, token, b, p)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if in.Sequence < state.sequence || (in.Sequence == state.sequence && (in.Capacity != state.capacity || in.Drain != state.draining)) || (state.draining && !in.Drain) {
|
||||||
|
return fault.Conflict
|
||||||
|
}
|
||||||
|
now, err := dbNow(ctx, tx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
state.expires = now.Add(s.options.SessionTTL)
|
||||||
|
state.capacity = in.Capacity
|
||||||
|
state.draining = state.draining || in.Drain
|
||||||
|
_, err = tx.ExecContext(ctx, `UPDATE pulse_worker_sessions SET capacity=?,capacity_sequence=?,draining=?,expires_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, state.capacity, in.Sequence, state.draining, state.expires, now, token.NamespaceID, b.SessionID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if in.Drain {
|
||||||
|
if _, err := tx.ExecContext(ctx, `UPDATE pulse_worker_instances SET draining=TRUE,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND draining=FALSE`, now, token.NamespaceID, b.InstanceID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out = s.welcome(b, state, now)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
244
internal/storage/mysql/worker_websocket_integration_test.go
Normal file
244
internal/storage/mysql/worker_websocket_integration_test.go
Normal file
@@ -0,0 +1,244 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/transport/httpserver"
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||||
|
"github.com/coder/websocket"
|
||||||
|
)
|
||||||
|
|
||||||
|
type testWorkerFrame struct {
|
||||||
|
Version string `json:"protocol_version"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
MessageID core.ID `json:"message_id"`
|
||||||
|
ReplyTo core.ID `json:"reply_to,omitempty"`
|
||||||
|
SessionID core.ID `json:"session_id,omitempty"`
|
||||||
|
Body json.RawMessage `json:"body,omitempty"`
|
||||||
|
Error *struct {
|
||||||
|
Code string `json:"code"`
|
||||||
|
} `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
type testWorkerSocket struct {
|
||||||
|
t *testing.T
|
||||||
|
ctx context.Context
|
||||||
|
conn *websocket.Conn
|
||||||
|
session core.ID
|
||||||
|
welcome worker.Welcome
|
||||||
|
}
|
||||||
|
|
||||||
|
func connectTestWorker(t *testing.T, ctx context.Context, server *httptest.Server, f *executionFixture) *testWorkerSocket {
|
||||||
|
t.Helper()
|
||||||
|
header := http.Header{}
|
||||||
|
header.Set("Authorization", "Bearer "+f.resources.Token)
|
||||||
|
conn, response, err := websocket.Dial(ctx, strings.Replace(server.URL, "https://", "wss://", 1)+"/v1/worker/connect", &websocket.DialOptions{HTTPClient: server.Client(), HTTPHeader: header, Subprotocols: []string{httpserver.WorkerSubprotocol}})
|
||||||
|
if err != nil {
|
||||||
|
status := 0
|
||||||
|
if response != nil {
|
||||||
|
status = response.StatusCode
|
||||||
|
}
|
||||||
|
t.Fatalf("Worker dial failed HTTP %d: %v", status, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { conn.CloseNow() })
|
||||||
|
client := &testWorkerSocket{t: t, ctx: ctx, conn: conn}
|
||||||
|
conn.SetReadLimit(512 << 10)
|
||||||
|
hello := client.send("worker.hello", f.hello, "")
|
||||||
|
reply := client.read()
|
||||||
|
if reply.Type != "worker.welcome" || reply.ReplyTo != hello || reply.Error != nil {
|
||||||
|
t.Fatal("missing welcome")
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(reply.Body, &client.welcome); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client.session = client.welcome.SessionID
|
||||||
|
return client
|
||||||
|
}
|
||||||
|
func (c *testWorkerSocket) send(kind string, body any, id core.ID) core.ID {
|
||||||
|
c.t.Helper()
|
||||||
|
if id == "" {
|
||||||
|
id = executionID(c.t)
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(body)
|
||||||
|
if err != nil {
|
||||||
|
c.t.Fatal(err)
|
||||||
|
}
|
||||||
|
frame, err := json.Marshal(testWorkerFrame{Version: "1", Type: kind, MessageID: id, SessionID: c.session, Body: raw})
|
||||||
|
if err != nil {
|
||||||
|
c.t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(c.ctx, 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := c.conn.Write(ctx, websocket.MessageText, frame); err != nil {
|
||||||
|
c.t.Fatal(err)
|
||||||
|
}
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
func (c *testWorkerSocket) read() testWorkerFrame {
|
||||||
|
c.t.Helper()
|
||||||
|
ctx, cancel := context.WithTimeout(c.ctx, 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
kind, raw, err := c.conn.Read(ctx)
|
||||||
|
if err != nil {
|
||||||
|
c.t.Fatal(err)
|
||||||
|
}
|
||||||
|
var frame testWorkerFrame
|
||||||
|
if kind != websocket.MessageText || json.Unmarshal(raw, &frame) != nil || frame.Version != "1" || frame.MessageID.Validate() != nil {
|
||||||
|
c.t.Fatal("invalid server frame")
|
||||||
|
}
|
||||||
|
return frame
|
||||||
|
}
|
||||||
|
func (c *testWorkerSocket) ack(id core.ID) json.RawMessage {
|
||||||
|
c.t.Helper()
|
||||||
|
frame := c.read()
|
||||||
|
if frame.Type != "ack" || frame.ReplyTo != id || frame.SessionID != c.session || frame.Error != nil {
|
||||||
|
code := ""
|
||||||
|
if frame.Error != nil {
|
||||||
|
code = frame.Error.Code
|
||||||
|
}
|
||||||
|
c.t.Fatalf("expected ack, got %s/%s", frame.Type, code)
|
||||||
|
}
|
||||||
|
return frame.Body
|
||||||
|
}
|
||||||
|
func TestIntegrationWorkerWebSocket(t *testing.T) {
|
||||||
|
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||||
|
if dsn == "" {
|
||||||
|
t.Skip("requires disposable MySQL")
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
db, err := Open(ctx, dsn, 16, 4)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
if err := Migrate(ctx, db); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fixture := newExecutionFixture(t, ctx, db)
|
||||||
|
start := func(svc *worker.Service) (*httptest.Server, *httpserver.WorkerHub) {
|
||||||
|
t.Helper()
|
||||||
|
hub := httpserver.NewWorkerHub(ctx, svc)
|
||||||
|
server := httpserver.New("", func(context.Context) error { return nil })
|
||||||
|
httpserver.RegisterWorkers(server, hub)
|
||||||
|
tls := httptest.NewTLSServer(server.Handler)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
bounded, stop := context.WithTimeout(context.Background(), 3*time.Second)
|
||||||
|
defer stop()
|
||||||
|
if err := hub.Shutdown(bounded); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
tls.Close()
|
||||||
|
})
|
||||||
|
return tls, hub
|
||||||
|
}
|
||||||
|
serverA, hubA := start(fixture.a)
|
||||||
|
serverB, _ := start(fixture.b)
|
||||||
|
t.Run("handshake_auth_protocol_origin_and_URL", func(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
token, protocol, path, origin string
|
||||||
|
status int
|
||||||
|
}{
|
||||||
|
{"", httpserver.WorkerSubprotocol, "/v1/worker/connect", "", 401},
|
||||||
|
{fixture.resources.Token, "unknown.v1", "/v1/worker/connect", "", 426},
|
||||||
|
{fixture.resources.Token, httpserver.WorkerSubprotocol, "/v1/worker/connect?token=forbidden", "", 400},
|
||||||
|
{fixture.resources.Token, httpserver.WorkerSubprotocol, "/v1/worker/connect", "https://foreign.example", 403},
|
||||||
|
}
|
||||||
|
for _, test := range cases {
|
||||||
|
headers := http.Header{}
|
||||||
|
if test.token != "" {
|
||||||
|
headers.Set("Authorization", "Bearer "+test.token)
|
||||||
|
}
|
||||||
|
if test.origin != "" {
|
||||||
|
headers.Set("Origin", test.origin)
|
||||||
|
}
|
||||||
|
conn, response, err := websocket.Dial(ctx, strings.Replace(serverA.URL, "https://", "wss://", 1)+test.path, &websocket.DialOptions{HTTPClient: serverA.Client(), HTTPHeader: headers, Subprotocols: []string{test.protocol}})
|
||||||
|
if conn != nil {
|
||||||
|
conn.CloseNow()
|
||||||
|
}
|
||||||
|
if err == nil || response == nil || response.StatusCode != test.status {
|
||||||
|
t.Fatalf("handshake expected %d", test.status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("WSS_assignment_start_and_cross_node_ack_recovery", func(t *testing.T) {
|
||||||
|
task := fixture.submit(3)
|
||||||
|
first := connectTestWorker(t, ctx, serverA, fixture)
|
||||||
|
// 只有显式容量确认后才启用新派发,留出恢复核对阶段。
|
||||||
|
first.ack(first.send("worker.capacity", worker.CapacityInput{Sequence: 1, Capacity: 1}, ""))
|
||||||
|
frame := first.read()
|
||||||
|
if frame.Type != "task.assign" {
|
||||||
|
t.Fatalf("wanted assignment got %s", frame.Type)
|
||||||
|
}
|
||||||
|
var assignment worker.Assignment
|
||||||
|
if err := json.Unmarshal(frame.Body, &assignment); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if assignment.Task.ID != task.ID || assignment.LeaseSecret == "" {
|
||||||
|
t.Fatal("wrong assignment")
|
||||||
|
}
|
||||||
|
ref := worker.AttemptRef{TaskID: task.ID, AttemptID: assignment.AttemptID, Epoch: assignment.Epoch, LeaseSecret: assignment.LeaseSecret}
|
||||||
|
startMessage := first.send("task.start", ref, "")
|
||||||
|
var begun worker.CommandResult
|
||||||
|
if err := json.Unmarshal(first.ack(startMessage), &begun); err != nil || !begun.Lease.Active || begun.Lease.State != pulse.AttemptRunning {
|
||||||
|
t.Fatal("missing durable start")
|
||||||
|
}
|
||||||
|
// 模拟旧节点连接断开;新的 WSS 连接进入不同节点并原子迁移有效尝试。
|
||||||
|
_ = first.conn.CloseNow()
|
||||||
|
second := connectTestWorker(t, ctx, serverB, fixture)
|
||||||
|
var resumed worker.LeaseStatus
|
||||||
|
if err := json.Unmarshal(second.ack(second.send("worker.resume", ref, "")), &resumed); err != nil || !resumed.Active {
|
||||||
|
t.Fatal("WSS resume failed")
|
||||||
|
}
|
||||||
|
var original worker.CommandResult
|
||||||
|
if err := json.Unmarshal(second.ack(second.send("task.start", ref, startMessage)), &original); err != nil || !original.Deduplicated || !original.Lease.Active {
|
||||||
|
t.Fatal("original start receipt not recovered")
|
||||||
|
}
|
||||||
|
finishMessage := second.send("task.finish", worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"output_ref":"objects/result","n":9007199254740993}`)}, "")
|
||||||
|
var finish worker.CommandResult
|
||||||
|
if err := json.Unmarshal(second.ack(finishMessage), &finish); err != nil || finish.Receipt.Disposition != "APPLIED" || finish.Lease.Active {
|
||||||
|
t.Fatal("WSS finish failed")
|
||||||
|
}
|
||||||
|
// 丢弃该连接的操作确认上下文,再用稳定身份取回同一结果。
|
||||||
|
_ = second.conn.CloseNow()
|
||||||
|
third := connectTestWorker(t, ctx, serverA, fixture)
|
||||||
|
var duplicate worker.CommandResult
|
||||||
|
if err := json.Unmarshal(third.ack(third.send("task.finish", worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"output_ref":"objects/result","n":9007199254740993}`)}, finishMessage)), &duplicate); err != nil || !duplicate.Deduplicated {
|
||||||
|
t.Fatal("finish receipt not recovered")
|
||||||
|
}
|
||||||
|
if fixture.get(task.ID).State != pulse.TaskSucceeded || fixture.used() != 0 || fixture.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 1 {
|
||||||
|
t.Fatal("WSS result effects duplicated")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("strict_worker_frame_and_hub_shutdown", func(t *testing.T) {
|
||||||
|
conn := connectTestWorker(t, ctx, serverB, fixture)
|
||||||
|
raw := []byte(`{"protocol_version":"1","protocol_version":"1","type":"worker.capacity"}`)
|
||||||
|
if err := conn.conn.Write(ctx, websocket.MessageText, raw); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
readCtx, stop := context.WithTimeout(ctx, 3*time.Second)
|
||||||
|
defer stop()
|
||||||
|
_, _, err := conn.conn.Read(readCtx)
|
||||||
|
if websocket.CloseStatus(err) != websocket.StatusPolicyViolation {
|
||||||
|
t.Fatalf("duplicate frame not rejected: %v", err)
|
||||||
|
}
|
||||||
|
active := connectTestWorker(t, ctx, serverA, fixture)
|
||||||
|
shutdown, stopShutdown := context.WithTimeout(ctx, 3*time.Second)
|
||||||
|
defer stopShutdown()
|
||||||
|
if err := hubA.Shutdown(shutdown); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, _, err = active.conn.Read(readCtx)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("shutdown left a hijacked connection alive")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
401
internal/transport/httpserver/worker.go
Normal file
401
internal/transport/httpserver/worker.go
Normal file
@@ -0,0 +1,401 @@
|
|||||||
|
package httpserver
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/jsonvalue"
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||||
|
"github.com/coder/websocket"
|
||||||
|
)
|
||||||
|
|
||||||
|
const WorkerSubprotocol = "pulse.worker.v1"
|
||||||
|
|
||||||
|
// WorkerHub 管理节点内连接生命周期;任务归属和容量仍由 MySQL 决定。
|
||||||
|
type WorkerHub struct {
|
||||||
|
service *worker.Service
|
||||||
|
ctx context.Context
|
||||||
|
cancel context.CancelFunc
|
||||||
|
slots chan struct{}
|
||||||
|
mu sync.Mutex
|
||||||
|
closing bool
|
||||||
|
wg sync.WaitGroup
|
||||||
|
poll time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewWorkerHub(parent context.Context, service *worker.Service) *WorkerHub {
|
||||||
|
ctx, cancel := context.WithCancel(parent)
|
||||||
|
return &WorkerHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, 128), poll: time.Second}
|
||||||
|
}
|
||||||
|
func RegisterWorkers(server *http.Server, hub *WorkerHub) {
|
||||||
|
server.Handler.(*http.ServeMux).Handle("GET /v1/worker/connect", hub)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Shutdown 停止派发并关闭本节点连接;不清除持久化租约,由重連核对或到期恢复处理。
|
||||||
|
func (h *WorkerHub) Shutdown(ctx context.Context) error {
|
||||||
|
h.mu.Lock()
|
||||||
|
h.closing = true
|
||||||
|
h.cancel()
|
||||||
|
h.mu.Unlock()
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { h.wg.Wait(); close(done) }()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
return nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type workerFrame struct {
|
||||||
|
Version string `json:"protocol_version"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
MessageID core.ID `json:"message_id"`
|
||||||
|
ReplyTo core.ID `json:"reply_to,omitempty"`
|
||||||
|
SessionID core.ID `json:"session_id,omitempty"`
|
||||||
|
Body json.RawMessage `json:"body,omitempty"`
|
||||||
|
Error *fault.Error `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
type outboundFrame struct {
|
||||||
|
data []byte
|
||||||
|
done chan error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *WorkerHub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
requestID, _ := core.NewID()
|
||||||
|
w.Header().Set("X-Request-ID", string(requestID))
|
||||||
|
w.Header().Set("Cache-Control", "no-store")
|
||||||
|
if r.URL.RawQuery != "" {
|
||||||
|
writeError(w, string(requestID), fault.Invalid("Worker credentials and options must not be placed in URL"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
values := r.Header.Values("Authorization")
|
||||||
|
if len(values) != 1 {
|
||||||
|
writeError(w, string(requestID), fault.Unauthorized)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
scheme, value, ok := strings.Cut(values[0], " ")
|
||||||
|
if !ok || !strings.EqualFold(scheme, "Bearer") {
|
||||||
|
writeError(w, string(requestID), fault.Unauthorized)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
token, err := auth.Parse(value)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, string(requestID), err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
supported := false
|
||||||
|
for _, header := range r.Header.Values("Sec-WebSocket-Protocol") {
|
||||||
|
for _, protocol := range strings.Split(header, ",") {
|
||||||
|
supported = supported || strings.TrimSpace(protocol) == WorkerSubprotocol
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !supported {
|
||||||
|
writeJSON(w, http.StatusUpgradeRequired, map[string]any{"request_id": requestID, "error": fault.New("PROTOCOL_REQUIRED", "pulse.worker.v1 subprotocol is required")})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.mu.Lock()
|
||||||
|
if h.closing || h.ctx.Err() != nil {
|
||||||
|
h.mu.Unlock()
|
||||||
|
writeError(w, string(requestID), fault.Unavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case h.slots <- struct{}{}:
|
||||||
|
h.wg.Add(1)
|
||||||
|
default:
|
||||||
|
h.mu.Unlock()
|
||||||
|
writeJSON(w, 429, map[string]any{"request_id": requestID, "error": fault.New("CAPACITY_EXCEEDED", "node connection capacity exceeded")})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.mu.Unlock()
|
||||||
|
defer func() { <-h.slots; h.wg.Done() }()
|
||||||
|
authorize, cancel := context.WithTimeout(r.Context(), 5*time.Second)
|
||||||
|
err = h.service.Authorize(authorize, token)
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, string(requestID), err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{WorkerSubprotocol}, CompressionMode: websocket.CompressionDisabled})
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer conn.CloseNow()
|
||||||
|
conn.SetReadLimit(512 << 10)
|
||||||
|
ctx, cancel := context.WithCancel(h.ctx)
|
||||||
|
defer cancel()
|
||||||
|
helloCtx, helloCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
kind, raw, err := conn.Read(helloCtx)
|
||||||
|
helloCancel()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var hello workerFrame
|
||||||
|
var input worker.ConnectInput
|
||||||
|
if kind != websocket.MessageText || decodeWorkerJSON(raw, &hello) != nil || hello.Version != "1" || hello.Type != "worker.hello" || hello.MessageID.Validate() != nil || hello.SessionID != "" || hello.ReplyTo != "" || hello.Error != nil || decodeWorkerJSON(hello.Body, &input) != nil {
|
||||||
|
_ = conn.Close(websocket.StatusPolicyViolation, "invalid worker hello")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
call, cancelCall := context.WithTimeout(ctx, 8*time.Second)
|
||||||
|
binding, welcome, err := h.service.Connect(call, token, input)
|
||||||
|
cancelCall()
|
||||||
|
if err != nil {
|
||||||
|
_ = writeWorkerFrame(ctx, conn, workerFrame{Version: "1", Type: "error", MessageID: requestID, ReplyTo: hello.MessageID, Error: publicWorkerError(err)})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
welcomeRaw, err := json.Marshal(welcome)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := writeWorkerFrame(ctx, conn, workerFrame{Version: "1", Type: "worker.welcome", MessageID: requestID, ReplyTo: hello.MessageID, SessionID: binding.SessionID, Body: welcomeRaw}); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// 有界读写队列与 SQL 控制循环分开,处理器绝不运行在这里。
|
||||||
|
incoming := make(chan workerFrame, 4)
|
||||||
|
outgoing := make(chan outboundFrame, 8)
|
||||||
|
var loops sync.WaitGroup
|
||||||
|
loops.Add(2)
|
||||||
|
go func() {
|
||||||
|
defer loops.Done()
|
||||||
|
defer cancel()
|
||||||
|
for {
|
||||||
|
readCtx, stop := context.WithTimeout(ctx, 60*time.Second)
|
||||||
|
kind, raw, err := conn.Read(readCtx)
|
||||||
|
stop()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var frame workerFrame
|
||||||
|
if kind != websocket.MessageText || decodeWorkerJSON(raw, &frame) != nil || frame.Version != "1" || frame.MessageID.Validate() != nil || frame.SessionID != binding.SessionID || frame.ReplyTo != "" || frame.Error != nil {
|
||||||
|
_ = conn.Close(websocket.StatusPolicyViolation, "invalid worker frame")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case incoming <- frame:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
defer loops.Done()
|
||||||
|
defer cancel()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case frame := <-outgoing:
|
||||||
|
writeCtx, stop := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
err := conn.Write(writeCtx, websocket.MessageText, frame.data)
|
||||||
|
stop()
|
||||||
|
if frame.done != nil {
|
||||||
|
frame.done <- err
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
defer func() { cancel(); _ = conn.CloseNow(); loops.Wait() }()
|
||||||
|
send := func(frame workerFrame, wait bool) bool {
|
||||||
|
frame.Version = "1"
|
||||||
|
frame.SessionID = binding.SessionID
|
||||||
|
if frame.MessageID == "" {
|
||||||
|
var err error
|
||||||
|
frame.MessageID, err = core.NewID()
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(frame)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
out := outboundFrame{data: raw}
|
||||||
|
if wait {
|
||||||
|
out.done = make(chan error, 1)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case outgoing <- out:
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if !wait {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case err := <-out.done:
|
||||||
|
return err == nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
capacity := input.Capacity
|
||||||
|
draining := welcome.Draining
|
||||||
|
eligible := false
|
||||||
|
leases := map[core.ID]time.Time{}
|
||||||
|
ticker := time.NewTicker(h.poll)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case frame := <-incoming:
|
||||||
|
call, stop := context.WithTimeout(ctx, 8*time.Second)
|
||||||
|
var body any
|
||||||
|
var commandErr error
|
||||||
|
switch frame.Type {
|
||||||
|
case "worker.capacity":
|
||||||
|
var in worker.CapacityInput
|
||||||
|
if commandErr = decodeWorkerJSON(frame.Body, &in); commandErr == nil {
|
||||||
|
var out worker.Welcome
|
||||||
|
out, commandErr = h.service.Capacity(call, token, binding, in)
|
||||||
|
body = out
|
||||||
|
if commandErr == nil {
|
||||||
|
capacity = in.Capacity
|
||||||
|
draining = out.Draining
|
||||||
|
eligible = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "worker.resume":
|
||||||
|
var ref worker.AttemptRef
|
||||||
|
if commandErr = decodeWorkerJSON(frame.Body, &ref); commandErr == nil {
|
||||||
|
var out worker.LeaseStatus
|
||||||
|
out, commandErr = h.service.Resume(call, token, binding, ref)
|
||||||
|
body = out
|
||||||
|
if commandErr == nil {
|
||||||
|
if out.Active {
|
||||||
|
leases[ref.AttemptID] = out.LeaseUntil
|
||||||
|
} else {
|
||||||
|
delete(leases, ref.AttemptID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "task.start", "task.renew":
|
||||||
|
var ref worker.AttemptRef
|
||||||
|
if commandErr = decodeWorkerJSON(frame.Body, &ref); commandErr == nil {
|
||||||
|
var out worker.CommandResult
|
||||||
|
if frame.Type == "task.start" {
|
||||||
|
out, commandErr = h.service.Start(call, token, binding, frame.MessageID, ref)
|
||||||
|
} else {
|
||||||
|
out, commandErr = h.service.Renew(call, token, binding, frame.MessageID, ref)
|
||||||
|
}
|
||||||
|
body = out
|
||||||
|
if commandErr == nil {
|
||||||
|
if out.Lease.Active {
|
||||||
|
leases[ref.AttemptID] = out.Lease.LeaseUntil
|
||||||
|
} else {
|
||||||
|
delete(leases, ref.AttemptID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "task.finish":
|
||||||
|
var in worker.FinishInput
|
||||||
|
if commandErr = decodeWorkerJSON(frame.Body, &in); commandErr == nil {
|
||||||
|
var out worker.CommandResult
|
||||||
|
out, commandErr = h.service.Finish(call, token, binding, frame.MessageID, in)
|
||||||
|
body = out
|
||||||
|
if commandErr == nil && !out.Lease.Active {
|
||||||
|
delete(leases, in.Ref.AttemptID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
commandErr = fault.Invalid("unsupported worker message type")
|
||||||
|
}
|
||||||
|
stop()
|
||||||
|
reply := workerFrame{Type: "ack", ReplyTo: frame.MessageID}
|
||||||
|
if commandErr != nil {
|
||||||
|
reply.Type = "error"
|
||||||
|
reply.Error = publicWorkerError(commandErr)
|
||||||
|
} else {
|
||||||
|
reply.Body, commandErr = json.Marshal(body)
|
||||||
|
if commandErr != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fatal := commandErr != nil && (fault.Code(commandErr) == "STALE_SESSION" || fault.Code(commandErr) == "UNAUTHENTICATED" || fault.Code(commandErr) == "PERMISSION_DENIED")
|
||||||
|
if !send(reply, fatal) || fatal {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case <-ticker.C:
|
||||||
|
// 本地记录仅减少空轮询;SQL 仍原子验证总容量、队列额度和有效会话。
|
||||||
|
now := time.Now()
|
||||||
|
for attempt, until := range leases {
|
||||||
|
if !until.After(now) {
|
||||||
|
delete(leases, attempt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !eligible || draining || capacity == 0 || len(leases) >= int(capacity) || len(outgoing) >= 4 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for burst := 0; burst < 4 && len(leases) < int(capacity) && len(outgoing) < 4; burst++ {
|
||||||
|
call, stop := context.WithTimeout(ctx, 8*time.Second)
|
||||||
|
assignment, err := h.service.Claim(call, token, binding)
|
||||||
|
stop()
|
||||||
|
if err != nil {
|
||||||
|
if fault.Code(err) == "STALE_SESSION" || fault.Code(err) == "UNAUTHENTICATED" || fault.Code(err) == "PERMISSION_DENIED" {
|
||||||
|
send(workerFrame{Type: "error", Error: publicWorkerError(err)}, true)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if assignment == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(assignment)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
leases[assignment.AttemptID] = assignment.LeaseUntil
|
||||||
|
if !send(workerFrame{Type: "task.assign", Body: raw}, false) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func publicWorkerError(err error) *fault.Error {
|
||||||
|
var known *fault.Error
|
||||||
|
if errors.As(err, &known) {
|
||||||
|
return known
|
||||||
|
}
|
||||||
|
return fault.Unavailable
|
||||||
|
}
|
||||||
|
func writeWorkerFrame(ctx context.Context, conn *websocket.Conn, frame workerFrame) error {
|
||||||
|
raw, err := json.Marshal(frame)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
bounded, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
return conn.Write(bounded, websocket.MessageText, raw)
|
||||||
|
}
|
||||||
|
func decodeWorkerJSON(raw []byte, out any) error {
|
||||||
|
if len(raw) == 0 || len(raw) > 512<<10 {
|
||||||
|
return fault.Invalid("invalid worker message size")
|
||||||
|
}
|
||||||
|
if _, err := jsonvalue.Canonical(raw); err != nil {
|
||||||
|
return fault.Invalid("invalid worker JSON")
|
||||||
|
}
|
||||||
|
if err := exactFields(raw, reflect.TypeOf(out).Elem()); err != nil {
|
||||||
|
return fault.Invalid("unknown or invalid worker field")
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||||
|
decoder.DisallowUnknownFields()
|
||||||
|
if err := decoder.Decode(out); err != nil {
|
||||||
|
return fault.Invalid("invalid worker field value")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -58,7 +58,7 @@ type Permissions struct {
|
|||||||
Queues []string `json:"queues"`
|
Queues []string `json:"queues"`
|
||||||
}
|
}
|
||||||
|
|
||||||
var supported = map[string]bool{"tasks:submit": true, "tasks:read": true, "tasks:cancel": true, "queues:read": true, "queues:manage": true}
|
var supported = map[string]bool{"tasks:submit": true, "tasks:read": true, "tasks:cancel": true, "queues:read": true, "queues:manage": true, "workers:connect": true, "workers:consume": true}
|
||||||
|
|
||||||
func (p Permissions) Validate() error {
|
func (p Permissions) Validate() error {
|
||||||
if len(p.Actions) == 0 || len(p.Actions) > 32 || len(p.Queues) == 0 || len(p.Queues) > 128 {
|
if len(p.Actions) == 0 || len(p.Actions) > 32 || len(p.Queues) == 0 || len(p.Queues) > 128 {
|
||||||
|
|||||||
25
pulse/model/attempt_evidence.go
Normal file
25
pulse/model/attempt_evidence.go
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
package model
|
||||||
|
|
||||||
|
import core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
|
||||||
|
// AttemptEvidence 保留迟到或冲突结果,作为核查证据;保存证据不会改写任务结果。
|
||||||
|
type AttemptEvidence struct {
|
||||||
|
// Record 包含命名空间、证据标识、并发版本与首次收到时间。
|
||||||
|
core.Record
|
||||||
|
// AttemptID 证据所属尝试,提交者仍需证明原执行秘密与进程身份。
|
||||||
|
AttemptID core.ID `db:"attempt_id"`
|
||||||
|
// MessageID 稳定结果消息标识,用于同一证据重报去重。
|
||||||
|
MessageID core.ID `db:"message_id"`
|
||||||
|
// Outcome Worker 报告的结果分类,不代表平台认可为任务终态。
|
||||||
|
Outcome string `db:"outcome"`
|
||||||
|
// ErrorCode 可选稳定错误分类;无错误分类时为空。
|
||||||
|
ErrorCode *string `db:"error_code"`
|
||||||
|
// Payload 结果或证据引用 JSON 文本,保留数字精度。
|
||||||
|
Payload core.JSON `db:"payload"`
|
||||||
|
// RequestDigest 结果语义摘要,排除会话身份与原始秘密。
|
||||||
|
RequestDigest core.Digest `db:"request_digest"`
|
||||||
|
// Reason 未用于更新权威状态的原因,例如授权失效或与已保存结果冲突。
|
||||||
|
Reason string `db:"reason"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (AttemptEvidence) TableName() string { return "pulse_attempt_evidence" }
|
||||||
25
pulse/model/worker_instance.go
Normal file
25
pulse/model/worker_instance.go
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
package model
|
||||||
|
|
||||||
|
import core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
|
||||||
|
// WorkerInstance 保存进程实例的持久化会话头;重连推进代次,不能仅凭实例 ID 接管。
|
||||||
|
type WorkerInstance struct {
|
||||||
|
// Record 包含命名空间、标识、并发版本与 UTC 时间。
|
||||||
|
core.Record
|
||||||
|
// CredentialID 初次注册的凭据,当前进程生命周期内不可替换。
|
||||||
|
CredentialID core.ID `db:"credential_id"`
|
||||||
|
// QueueID 进程绑定的消费队列;切换队列应创建新实例。
|
||||||
|
QueueID core.ID `db:"queue_id"`
|
||||||
|
// SecretDigest 进程持有的随机恢复秘密摘要;知道实例 ID 不构成恢复授权。
|
||||||
|
SecretDigest core.Digest `db:"secret_digest"`
|
||||||
|
// RegistrationDigest 队列与处理器版本集合的摘要,重连不得改变注册语义。
|
||||||
|
RegistrationDigest core.Digest `db:"registration_digest"`
|
||||||
|
// SessionEpoch 已分配的最大连接代次,初始为零,重连递增。
|
||||||
|
SessionEpoch uint64 `db:"session_epoch"`
|
||||||
|
// CurrentSessionID 当前连接;注册事务创建会话之前为空。
|
||||||
|
CurrentSessionID *core.ID `db:"current_session_id"`
|
||||||
|
// Draining 持久化排空意图,重连保留;排空后不再给该进程新工作。
|
||||||
|
Draining bool `db:"draining"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (WorkerInstance) TableName() string { return "pulse_worker_instances" }
|
||||||
139
pulse/worker/service.go
Normal file
139
pulse/worker/service.go
Normal file
@@ -0,0 +1,139 @@
|
|||||||
|
package worker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/jsonvalue"
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||||
|
)
|
||||||
|
|
||||||
|
var StaleSession = fault.New("STALE_SESSION", "connection is expired or superseded")
|
||||||
|
var LeaseLost = fault.New("LEASE_LOST", "attempt no longer grants execution authority")
|
||||||
|
var GateClosed = fault.New("EXECUTION_PAUSED", "queue or execution gate is paused")
|
||||||
|
|
||||||
|
type Service struct{ store Store }
|
||||||
|
|
||||||
|
func New(store Store) *Service { return &Service{store: store} }
|
||||||
|
func (s *Service) Authorize(ctx context.Context, t auth.Token) error {
|
||||||
|
return s.store.Authorize(ctx, t)
|
||||||
|
}
|
||||||
|
func NewSecret() (string, core.Digest, error) {
|
||||||
|
var raw [32]byte
|
||||||
|
if _, err := rand.Read(raw[:]); err != nil {
|
||||||
|
return "", core.Digest{}, err
|
||||||
|
}
|
||||||
|
return base64.RawURLEncoding.EncodeToString(raw[:]), core.Digest(sha256.Sum256(raw[:])), nil
|
||||||
|
}
|
||||||
|
func SecretDigest(secret string) (core.Digest, error) {
|
||||||
|
if len(secret) != 43 {
|
||||||
|
return core.Digest{}, fault.Unauthorized
|
||||||
|
}
|
||||||
|
raw, err := base64.RawURLEncoding.Strict().DecodeString(secret)
|
||||||
|
if err != nil || len(raw) != 32 {
|
||||||
|
return core.Digest{}, fault.Unauthorized
|
||||||
|
}
|
||||||
|
return core.Digest(sha256.Sum256(raw)), nil
|
||||||
|
}
|
||||||
|
func NormalizeConnect(in ConnectInput) (ConnectInput, error) {
|
||||||
|
if in.InstanceID.Validate() != nil || in.QueueID.Validate() != nil || in.Capacity < 1 || in.Capacity > 256 || len(in.Capabilities) < 1 || len(in.Capabilities) > 32 {
|
||||||
|
return in, fault.Invalid("invalid worker identity, queue, capacity or capability count")
|
||||||
|
}
|
||||||
|
if _, err := SecretDigest(in.InstanceSecret); err != nil {
|
||||||
|
return in, fault.Invalid("instance_secret must encode 32 random bytes")
|
||||||
|
}
|
||||||
|
in.Capabilities = append([]Capability(nil), in.Capabilities...)
|
||||||
|
for _, c := range in.Capabilities {
|
||||||
|
if !service.ValidName(c.Handler, 128) || !service.ValidName(c.HandlerVersion, 64) || !service.ValidName(c.PayloadVersion, 32) {
|
||||||
|
return in, fault.Invalid("invalid worker capability")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(in.Capabilities, func(i, j int) bool {
|
||||||
|
a, b := in.Capabilities[i], in.Capabilities[j]
|
||||||
|
if a.Handler != b.Handler {
|
||||||
|
return a.Handler < b.Handler
|
||||||
|
}
|
||||||
|
if a.HandlerVersion != b.HandlerVersion {
|
||||||
|
return a.HandlerVersion < b.HandlerVersion
|
||||||
|
}
|
||||||
|
return a.PayloadVersion < b.PayloadVersion
|
||||||
|
})
|
||||||
|
for i := 1; i < len(in.Capabilities); i++ {
|
||||||
|
if in.Capabilities[i] == in.Capabilities[i-1] {
|
||||||
|
return in, fault.Invalid("duplicate worker capability")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return in, nil
|
||||||
|
}
|
||||||
|
func (s *Service) Connect(ctx context.Context, t auth.Token, in ConnectInput) (Binding, Welcome, error) {
|
||||||
|
in, err := NormalizeConnect(in)
|
||||||
|
if err != nil {
|
||||||
|
return Binding{}, Welcome{}, err
|
||||||
|
}
|
||||||
|
return s.store.Connect(ctx, t, in)
|
||||||
|
}
|
||||||
|
func (s *Service) Capacity(ctx context.Context, t auth.Token, b Binding, in CapacityInput) (Welcome, error) {
|
||||||
|
if in.Sequence == 0 || in.Capacity > 256 {
|
||||||
|
return Welcome{}, fault.Invalid("invalid capacity sequence or value")
|
||||||
|
}
|
||||||
|
return s.store.Capacity(ctx, t, b, in)
|
||||||
|
}
|
||||||
|
func (s *Service) Claim(ctx context.Context, t auth.Token, b Binding) (*Assignment, error) {
|
||||||
|
return s.store.Claim(ctx, t, b)
|
||||||
|
}
|
||||||
|
func validRef(ref AttemptRef) error {
|
||||||
|
if ref.TaskID.Validate() != nil || ref.AttemptID.Validate() != nil || ref.Epoch == 0 {
|
||||||
|
return fault.Invalid("invalid attempt identity")
|
||||||
|
}
|
||||||
|
_, err := SecretDigest(ref.LeaseSecret)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
func validCommand(message core.ID, ref AttemptRef) error {
|
||||||
|
if message.Validate() != nil {
|
||||||
|
return fault.Invalid("invalid operation message id")
|
||||||
|
}
|
||||||
|
return validRef(ref)
|
||||||
|
}
|
||||||
|
func (s *Service) Start(ctx context.Context, t auth.Token, b Binding, message core.ID, ref AttemptRef) (CommandResult, error) {
|
||||||
|
if err := validCommand(message, ref); err != nil {
|
||||||
|
return CommandResult{}, err
|
||||||
|
}
|
||||||
|
return s.store.Start(ctx, t, b, message, ref)
|
||||||
|
}
|
||||||
|
func (s *Service) Renew(ctx context.Context, t auth.Token, b Binding, message core.ID, ref AttemptRef) (CommandResult, error) {
|
||||||
|
if err := validCommand(message, ref); err != nil {
|
||||||
|
return CommandResult{}, err
|
||||||
|
}
|
||||||
|
return s.store.Renew(ctx, t, b, message, ref)
|
||||||
|
}
|
||||||
|
func (s *Service) Finish(ctx context.Context, t auth.Token, b Binding, message core.ID, in FinishInput) (CommandResult, error) {
|
||||||
|
if err := validCommand(message, in.Ref); err != nil {
|
||||||
|
return CommandResult{}, err
|
||||||
|
}
|
||||||
|
if in.Outcome != "SUCCEEDED" && in.Outcome != "FAILED" && in.Outcome != "UNKNOWN" {
|
||||||
|
return CommandResult{}, fault.Invalid("invalid result outcome")
|
||||||
|
}
|
||||||
|
if len(in.Result) == 0 || len(in.Result) > 256<<10 || bytes.Equal(bytes.TrimSpace(in.Result), []byte("null")) {
|
||||||
|
return CommandResult{}, fault.Invalid("result must be non-null JSON up to 256 KiB")
|
||||||
|
}
|
||||||
|
if _, err := jsonvalue.Canonical(in.Result); err != nil {
|
||||||
|
return CommandResult{}, fault.Invalid("invalid result JSON")
|
||||||
|
}
|
||||||
|
if (in.Outcome == "SUCCEEDED" && in.ErrorCode != "") || (in.Outcome != "SUCCEEDED" && !service.ValidName(in.ErrorCode, 64)) {
|
||||||
|
return CommandResult{}, fault.Invalid("error_code must match the outcome")
|
||||||
|
}
|
||||||
|
return s.store.Finish(ctx, t, b, message, in)
|
||||||
|
}
|
||||||
|
func (s *Service) Resume(ctx context.Context, t auth.Token, b Binding, ref AttemptRef) (LeaseStatus, error) {
|
||||||
|
if err := validRef(ref); err != nil {
|
||||||
|
return LeaseStatus{}, err
|
||||||
|
}
|
||||||
|
return s.store.Resume(ctx, t, b, ref)
|
||||||
|
}
|
||||||
38
pulse/worker/service_test.go
Normal file
38
pulse/worker/service_test.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package worker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRegistrationAndSecretBoundaries(t *testing.T) {
|
||||||
|
secret, digest, err := NewSecret()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
parsed, err := SecretDigest(secret)
|
||||||
|
if err != nil || parsed != digest {
|
||||||
|
t.Fatal("secret round trip")
|
||||||
|
}
|
||||||
|
for _, bad := range []string{secret + "=", secret + "\n", strings.Repeat("!", 43), secret[:42]} {
|
||||||
|
if _, err := SecretDigest(bad); err == nil {
|
||||||
|
t.Fatal("malformed secret accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
input := ConnectInput{InstanceID: core.ID(strings.Repeat("1", 32)), InstanceSecret: secret, QueueID: core.ID(strings.Repeat("2", 32)), Capacity: 1, Capabilities: []Capability{{"z", "v1", "v1"}, {"a", "v1", "v1"}}}
|
||||||
|
normalized, err := NormalizeConnect(input)
|
||||||
|
if err != nil || normalized.Capabilities[0].Handler != "a" || input.Capabilities[0].Handler != "z" {
|
||||||
|
t.Fatalf("normalization changed caller input: %v", err)
|
||||||
|
}
|
||||||
|
input.Capabilities = append(input.Capabilities, input.Capabilities[0])
|
||||||
|
if _, err := NormalizeConnect(input); err == nil {
|
||||||
|
t.Fatal("duplicate capability accepted")
|
||||||
|
}
|
||||||
|
input = normalized
|
||||||
|
input.Capacity = 257
|
||||||
|
if _, err := NormalizeConnect(input); err == nil {
|
||||||
|
t.Fatal("excess capacity accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
124
pulse/worker/types.go
Normal file
124
pulse/worker/types.go
Normal file
@@ -0,0 +1,124 @@
|
|||||||
|
// Package worker 定义执行控制契约;处理器本身在外部 Worker 进程执行。
|
||||||
|
package worker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||||
|
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Capability 精确声明可处理的版本,不做静默版本降级。
|
||||||
|
type Capability struct {
|
||||||
|
Handler string `json:"handler"` // Handler 处理器名称。
|
||||||
|
HandlerVersion string `json:"handler_version"` // HandlerVersion 处理器版本。
|
||||||
|
PayloadVersion string `json:"payload_version"` // PayloadVersion 参数版本。
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConnectInput 在每次连接时提交;同一进程恢复时保留实例和秘密。
|
||||||
|
type ConnectInput struct {
|
||||||
|
InstanceID core.ID `json:"instance_id"` // InstanceID 进程标识,重启换新值。
|
||||||
|
InstanceSecret string `json:"instance_secret"` // InstanceSecret 32 字节随机秘密,只保存摘要。
|
||||||
|
QueueID core.ID `json:"queue_id"` // QueueID 当前进程绑定的消费队列。
|
||||||
|
Capabilities []Capability `json:"capabilities"` // Capabilities 1–32 个精确版本组合,重连固定。
|
||||||
|
Capacity uint32 `json:"capacity"` // Capacity 总执行槽数量,1–256。
|
||||||
|
}
|
||||||
|
|
||||||
|
// Binding 是接入层持有的认证结果,不作为公开请求字段接受或写入日志。
|
||||||
|
type Binding struct {
|
||||||
|
InstanceID core.ID // InstanceID 已证明秘密的进程标识。
|
||||||
|
SessionID core.ID // SessionID 当前连接标识。
|
||||||
|
Epoch uint64 // Epoch 连接代次。
|
||||||
|
Proof core.Digest // Proof 进程秘密摘要,持久化验证旧会话隔离。
|
||||||
|
}
|
||||||
|
|
||||||
|
// Welcome 给出数据库时钟与控制参数;时间用于 SDK 计算保守的单调时钟期限。
|
||||||
|
type Welcome struct {
|
||||||
|
SessionID core.ID `json:"session_id"` // SessionID 本次连接标识。
|
||||||
|
Epoch uint64 `json:"epoch,string"` // Epoch 当前连接代次。
|
||||||
|
ServerTime time.Time `json:"server_time"` // ServerTime 本次响应对应的数据库时间。
|
||||||
|
ExpiresAt time.Time `json:"expires_at"` // ExpiresAt 会话有效期限,不能代替任务租约。
|
||||||
|
LeaseDurationMS uint64 `json:"lease_duration_ms"` // LeaseDurationMS 单次续租窗口,毫秒。
|
||||||
|
Draining bool `json:"draining"` // Draining 排空标记,跨重连保留。
|
||||||
|
}
|
||||||
|
|
||||||
|
// CapacityInput 使用覆盖值和序号,重复消息不会重复增加执行容量。
|
||||||
|
type CapacityInput struct {
|
||||||
|
Sequence uint64 `json:"sequence,string"` // Sequence 本会话内从一开始递增。
|
||||||
|
Capacity uint32 `json:"capacity"` // Capacity 总容量,零表示暂不领取;不释放已有授权。
|
||||||
|
Drain bool `json:"drain"` // Drain 为真后持久化排空,不能通过后续消息清除。
|
||||||
|
}
|
||||||
|
|
||||||
|
// Assignment 仅通过已认证执行连接交付,包含敏感尝试秘密。
|
||||||
|
type Assignment struct {
|
||||||
|
Task service.TaskView `json:"task"` // Task 受理快照及业务参数。
|
||||||
|
AttemptID core.ID `json:"attempt_id"` // AttemptID 本次尝试标识。
|
||||||
|
Epoch uint64 `json:"epoch,string"` // Epoch 任务执行代次,隔离旧尝试。
|
||||||
|
LeaseSecret string `json:"lease_secret"` // LeaseSecret 执行秘密,禁止普通日志及查询返回。
|
||||||
|
LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 尚未开始时的预占有效期。
|
||||||
|
ServerTime time.Time `json:"server_time"` // ServerTime 领取时数据库时间。
|
||||||
|
}
|
||||||
|
|
||||||
|
// AttemptRef 证明对一次尝试的控制权;连接代次仍由 Binding 单独验证。
|
||||||
|
type AttemptRef struct {
|
||||||
|
TaskID core.ID `json:"task_id"` // TaskID 所属任务。
|
||||||
|
AttemptID core.ID `json:"attempt_id"` // AttemptID 目标尝试。
|
||||||
|
Epoch uint64 `json:"epoch,string"` // Epoch 目标执行代次。
|
||||||
|
LeaseSecret string `json:"lease_secret"` // LeaseSecret 原始执行秘密,只在受控连接传输。
|
||||||
|
}
|
||||||
|
|
||||||
|
// FinishInput 区分明确结果与未知效果,未知不能被解释为可以盲目重跑。
|
||||||
|
type FinishInput struct {
|
||||||
|
Ref AttemptRef `json:"ref"` // Ref 当前尝试授权证明。
|
||||||
|
Outcome string `json:"outcome"` // Outcome 为 SUCCEEDED、FAILED 或 UNKNOWN。
|
||||||
|
Result json.RawMessage `json:"result"` // Result 非 null JSON 结果或证据引用,最多 256 KiB。
|
||||||
|
ErrorCode string `json:"error_code,omitempty"` // ErrorCode FAILED/UNKNOWN 必填稳定分类,不包含原始异常正文。
|
||||||
|
}
|
||||||
|
|
||||||
|
// Receipt 是不可变的操作回执,不代表当前仍有执行权。
|
||||||
|
type Receipt struct {
|
||||||
|
Operation string `json:"operation"` // Operation start、renew 或 finish。
|
||||||
|
TaskState pulse.TaskState `json:"task_state"` // TaskState 提交该操作时的任务状态。
|
||||||
|
AttemptState pulse.AttemptState `json:"attempt_state"` // AttemptState 操作提交时的尝试状态。
|
||||||
|
TaskRevision uint64 `json:"task_revision,string"` // TaskRevision 操作提交时的任务版本。
|
||||||
|
LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 操作提交时租约期限,历史值不能授权新执行。
|
||||||
|
Disposition string `json:"disposition"` // Disposition 为 APPLIED 或 EVIDENCE_ONLY。
|
||||||
|
}
|
||||||
|
|
||||||
|
// LeaseStatus 每次响应重新计算;处理器仅在 Active 为真、State 为 RUNNING 且期限内推进。
|
||||||
|
// LEASED 的 Active 只代表预占仍有效,不能代替开始授权。
|
||||||
|
type LeaseStatus struct {
|
||||||
|
Active bool `json:"active"` // Active 当前会话仍持有未过期的有效尝试。
|
||||||
|
State pulse.AttemptState `json:"state"` // State 当前尝试状态。
|
||||||
|
LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 当前租约,不因重放旧回执自动延长。
|
||||||
|
ServerTime time.Time `json:"server_time"` // ServerTime 完成锁定后重新读取的数据库时间。
|
||||||
|
}
|
||||||
|
|
||||||
|
// CommandResult 将原回执与当前执行权分开,避免旧 start ACK 使过期尝试复活。
|
||||||
|
type CommandResult struct {
|
||||||
|
Receipt Receipt `json:"receipt"` // Receipt 对应稳定消息身份的原回执。
|
||||||
|
Lease LeaseStatus `json:"lease"` // Lease 当前授权判断。
|
||||||
|
Deduplicated bool `json:"deduplicated"` // Deduplicated 是否返回已保存的回执。
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecoveryStats 统计本批恢复,不作为持久化成功率或容量指标。
|
||||||
|
type RecoveryStats struct {
|
||||||
|
Examined int
|
||||||
|
Requeued int
|
||||||
|
Failed int
|
||||||
|
NeedsReview int
|
||||||
|
}
|
||||||
|
type Store interface {
|
||||||
|
Authorize(context.Context, auth.Token) error
|
||||||
|
Connect(context.Context, auth.Token, ConnectInput) (Binding, Welcome, error)
|
||||||
|
Capacity(context.Context, auth.Token, Binding, CapacityInput) (Welcome, error)
|
||||||
|
Claim(context.Context, auth.Token, Binding) (*Assignment, error)
|
||||||
|
Start(context.Context, auth.Token, Binding, core.ID, AttemptRef) (CommandResult, error)
|
||||||
|
Renew(context.Context, auth.Token, Binding, core.ID, AttemptRef) (CommandResult, error)
|
||||||
|
Finish(context.Context, auth.Token, Binding, core.ID, FinishInput) (CommandResult, error)
|
||||||
|
Resume(context.Context, auth.Token, Binding, AttemptRef) (LeaseStatus, error)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user