feat: add durable schedule management and multi-node crash recovery
This commit is contained in:
@@ -8,9 +8,9 @@ Orbit 是面向多个业务项目的通用任务调度与流程编排平台,
|
|||||||
|
|
||||||
## 当前进度
|
## 当前进度
|
||||||
|
|
||||||
已完成 P0–P2 阶段验收:32 个数据模型、版本化迁移、凭据及范围校验、队列与任务 API、WSS 执行、独立 SDK、自动 Worker 和 [执行尝试/证据查询](docs/api/execution-history.md)。[Worker Schema](docs/api/worker-schema.md) 与真实进程强杀恢复回归已通过;[安全重试](docs/api/retries.md) 已接入服务端和 SDK,调度器和流程引擎按 [持续开发计划](docs/08-development-plan.md) 继续实现。
|
已完成 P0–P2 阶段验收:32 个数据模型、版本化迁移、凭据及范围校验、队列与任务 API、WSS 执行、独立 SDK、自动 Worker 和 [执行尝试/证据查询](docs/api/execution-history.md)。[Worker Schema](docs/api/worker-schema.md) 与真实进程强杀恢复回归已通过;[安全重试](docs/api/retries.md) 与 [定时计划](docs/api/schedules.md) 已接入服务端和 SDK,剩余 Pulse 能力与流程引擎按 [持续开发计划](docs/08-development-plan.md) 继续实现。
|
||||||
|
|
||||||
readiness 只表示数据库与迁移记录可用,不表示完整平台已具备生产服务能力。验证证据见 [P0](docs/reports/phase-0.md)、[P1](docs/reports/phase-1.md) 与 [P2 阶段验收](docs/reports/phase-2.md),后者汇总服务端、SDK、执行历史及故障验证的详细记录。P3 的当前进展见 [安全重试报告](docs/reports/phase-3-safe-retry.md)。
|
readiness 只表示数据库与迁移记录可用,不表示完整平台已具备生产服务能力。验证证据见 [P0](docs/reports/phase-0.md)、[P1](docs/reports/phase-1.md) 与 [P2 阶段验收](docs/reports/phase-2.md),后者汇总服务端、SDK、执行历史及故障验证的详细记录。P3 的当前进展见 [安全重试报告](docs/reports/phase-3-safe-retry.md) 和 [计划调度报告](docs/reports/phase-3-schedules.md)。
|
||||||
|
|
||||||
## 目录
|
## 目录
|
||||||
|
|
||||||
@@ -25,6 +25,8 @@ pulse/model/ 任务、尝试、队列、计划和执行许可
|
|||||||
pulse/service/ 任务与队列应用契约、输入校验
|
pulse/service/ 任务与队列应用契约、输入校验
|
||||||
pulse/worker/ 会话、开始、续租、结果和恢复契约
|
pulse/worker/ 会话、开始、续租、结果和恢复契约
|
||||||
pulse/retry/ 冻结策略、有限退避与未知效果决策
|
pulse/retry/ 冻结策略、有限退避与未知效果决策
|
||||||
|
pulse/schedule/ Cron/间隔、漏跑、版本及计划接口
|
||||||
|
internal/timezone/ 随计划冻结的内置时区数据快照
|
||||||
weave/model/ 定义、实例、步骤、信号和查询轮次
|
weave/model/ 定义、实例、步骤、信号和查询轮次
|
||||||
internal/config/ 配置校验
|
internal/config/ 配置校验
|
||||||
internal/storage/mysql/ 连接、事务、迁移及集成测试
|
internal/storage/mysql/ 连接、事务、迁移及集成测试
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ func run() error {
|
|||||||
flags.StringVar(&in.Queue, "queue", "", "initial queue name")
|
flags.StringVar(&in.Queue, "queue", "", "initial queue name")
|
||||||
flags.StringVar(&output, "output", "", "new private credentials JSON file")
|
flags.StringVar(&output, "output", "", "new private credentials JSON file")
|
||||||
flags.BoolVar(&in.WorkerAccess, "worker-access", false, "also grant Worker connect and consume actions for development")
|
flags.BoolVar(&in.WorkerAccess, "worker-access", false, "also grant Worker connect and consume actions for development")
|
||||||
|
flags.BoolVar(&in.ScheduleAccess, "schedule-access", false, "also grant schedule read and manage actions for development")
|
||||||
if err := flags.Parse(os.Args[2:]); err != nil {
|
if err := flags.Parse(os.Args[2:]); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"git.yuntuo.hk/k2safe/orbit/internal/storage/mysql"
|
"git.yuntuo.hk/k2safe/orbit/internal/storage/mysql"
|
||||||
"git.yuntuo.hk/k2safe/orbit/internal/transport/httpserver"
|
"git.yuntuo.hk/k2safe/orbit/internal/transport/httpserver"
|
||||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
|
||||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||||
)
|
)
|
||||||
@@ -61,6 +62,32 @@ func run() error {
|
|||||||
}
|
}
|
||||||
hub := httpserver.NewWorkerHub(root, worker.New(executions))
|
hub := httpserver.NewWorkerHub(root, worker.New(executions))
|
||||||
httpserver.RegisterWorkers(server, hub)
|
httpserver.RegisterWorkers(server, hub)
|
||||||
|
schedules, err := mysql.NewScheduleStore(db, node)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
httpserver.RegisterSchedules(server, schedule.New(schedules))
|
||||||
|
scheduleCtx, stopSchedules := context.WithCancel(root)
|
||||||
|
scheduleDone := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(scheduleDone)
|
||||||
|
ticker := time.NewTicker(time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-scheduleCtx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
call, cancel := context.WithTimeout(scheduleCtx, 5*time.Second)
|
||||||
|
_, err := schedules.Tick(call, 100)
|
||||||
|
cancel()
|
||||||
|
if err != nil && scheduleCtx.Err() == nil {
|
||||||
|
slog.Error("schedule batch failed", "node_id", node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
defer func() { stopSchedules(); <-scheduleDone }()
|
||||||
defer func() {
|
defer func() {
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
@@ -93,7 +120,7 @@ func run() error {
|
|||||||
}
|
}
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- server.Serve(listener) }()
|
go func() { done <- server.Serve(listener) }()
|
||||||
slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task, queue and Worker APIs", "node_id", node)
|
slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task, queue, schedule and Worker APIs", "node_id", node)
|
||||||
select {
|
select {
|
||||||
case err := <-done:
|
case err := <-done:
|
||||||
if errors.Is(err, http.ErrServerClosed) {
|
if errors.Is(err, http.ErrServerClosed) {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Orbit 生产验收与发布门槛
|
# Orbit 生产验收与发布门槛
|
||||||
|
|
||||||
> 版本 0.2 · 2026-09-17。本文是全平台生产发布门槛。P0–P2 已积累阶段证据,见 [开发计划](08-development-plan.md) 与 [P2 验收](reports/phase-2.md);未覆盖的场景继续实施,尚无全平台生产验收或容量结论。
|
> 版本 0.2 · 2026-09-17。本文是全平台生产发布门槛。P0–P2 已积累阶段证据,见 [开发计划](08-development-plan.md) 与 [P2 验收](reports/phase-2.md);P3 新增 [计划调度证据](reports/phase-3-schedules.md)。未覆盖的场景继续实施,尚无全平台生产验收或容量结论。
|
||||||
|
|
||||||
## 1. 验收对象与证据
|
## 1. 验收对象与证据
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# 数据模型与实现基线
|
# 数据模型与实现基线
|
||||||
|
|
||||||
更新:2026-09-17。当前为基础实现阶段,完整调度与编排引擎仍在开发计划中。
|
更新:2026-09-17。当前已实现任务执行、安全重试与计划调度;剩余资源控制、订阅及流程编排继续开发。
|
||||||
|
|
||||||
## 1. 模块与依赖
|
## 1. 模块与依赖
|
||||||
|
|
||||||
@@ -113,4 +113,6 @@ P2 服务端已新增会话隔离、两步派发、租约、并发许可、结
|
|||||||
|
|
||||||
000005_safe_retry 增加任务 EffectUncertain、尝试与证据 Retryable 字段,Go 与 SQL 均附中文说明;实体数仍为 32。跨尝试的不确定效果必须保留,历史 NEEDS_REVIEW 保守回填 true,旧状态、版本和冻结策略不变。回执增加 `(namespace_id, attempt_id, operation, request_digest)` 索引,换消息身份重报结果也返回原已应用回执。字段语义、升级策略启用与限制见 [安全重试](api/retries.md),升级回归见 [阶段报告](reports/phase-3-safe-retry.md)。
|
000005_safe_retry 增加任务 EffectUncertain、尝试与证据 Retryable 字段,Go 与 SQL 均附中文说明;实体数仍为 32。跨尝试的不确定效果必须保留,历史 NEEDS_REVIEW 保守回填 true,旧状态、版本和冻结策略不变。回执增加 `(namespace_id, attempt_id, operation, request_digest)` 索引,换消息身份重报结果也返回原已应用回执。字段语义、升级策略启用与限制见 [安全重试](api/retries.md),升级回归见 [阶段报告](reports/phase-3-safe-retry.md)。
|
||||||
|
|
||||||
|
000006_schedule_runtime 增加 Schedule.QueueID / ProcessingVersion / CheckAt 以及 ScheduleVersion.CatchupWindowMS / GraceMS。字段、单位、可空语义均有中文注释;CoveredUntil 明确为合并或跳过范围的排除边界。实体数不变,增加扫描、列表和任务来源索引。时间规则 JSON 冻结 CalendarVersion,服务内置相应时区快照。调度记录、生成任务、事件和进度同事务写入,旧版本积压不会被新发布覆盖;详见 [计划契约](api/schedules.md)。
|
||||||
|
|
||||||
命令与验证记录见 [开发计划](08-development-plan.md)、[基础阶段验证报告](reports/phase-0.md) 与 [执行历史回归](reports/phase-2-execution-history.md)。
|
命令与验证记录见 [开发计划](08-development-plan.md)、[基础阶段验证报告](reports/phase-0.md) 与 [执行历史回归](reports/phase-2-execution-history.md)。
|
||||||
|
|||||||
@@ -27,7 +27,7 @@
|
|||||||
| P0 基础与数据模型 | Go workspace、独立 SDK module、公共/Pulse/Weave 模型、MySQL 迁移、配置、健康检查 | 编译/竞态/静态检查;真实 MySQL 建表、隔离、唯一性、预算、不可变版本、迁移失败防护;SDK 健康调用 | 已完成并通过阶段回归(见 P0 报告) |
|
| P0 基础与数据模型 | Go workspace、独立 SDK module、公共/Pulse/Weave 模型、MySQL 迁移、配置、健康检查 | 编译/竞态/静态检查;真实 MySQL 建表、隔离、唯一性、预算、不可变版本、迁移失败防护;SDK 健康调用 | 已完成并通过阶段回归(见 P0 报告) |
|
||||||
| P1 任务受理闭环 | 应用凭据与作用域权限、队列、幂等提交、查询、待执行取消、受理事件;SDK 客户端 | API/SDK 创建查询闭环;重复键与内容冲突、响应未确认重报、跨命名空间拒绝、参数边界 | 已完成并通过阶段回归(见 P1 报告;真实断路/提交 ACK 丢失注入留后续故障阶段) |
|
| P1 任务受理闭环 | 应用凭据与作用域权限、队列、幂等提交、查询、待执行取消、受理事件;SDK 客户端 | API/SDK 创建查询闭环;重复键与内容冲突、响应未确认重报、跨命名空间拒绝、参数边界 | 已完成并通过阶段回归(见 P1 报告;真实断路/提交 ACK 丢失注入留后续故障阶段) |
|
||||||
| P2 Worker 执行闭环 | WSS 接入、能力/容量、领取与开始、租约、续租、结果确认、排空与恢复;Worker SDK | 真 Worker 执行;失联/重连/重复确认/旧尝试拒绝;业务效果未知转核查;节点故障接管 | 已完成并通过阶段回归:服务端、自动 SDK、历史查询、完整 Schema 与真实进程强杀恢复;测试拓扑和剩余生产故障矩阵见 P2 验收报告 |
|
| P2 Worker 执行闭环 | WSS 接入、能力/容量、领取与开始、租约、续租、结果确认、排空与恢复;Worker SDK | 真 Worker 执行;失联/重连/重复确认/旧尝试拒绝;业务效果未知转核查;节点故障接管 | 已完成并通过阶段回归:服务端、自动 SDK、历史查询、完整 Schema 与真实进程强杀恢复;测试拓扑和剩余生产故障矩阵见 P2 验收报告 |
|
||||||
| P3 Pulse 完整能力 | 安全重试、延时、Cron/固定间隔、漏跑策略、计划版本、并发许可、配额/背压、可靠结果订阅 | 多实例领取/触发竞争、重试上限、取消竞争、队列隔离、积压恢复、事件重复交付 | 进行中:安全重试、退避、未知效果保留、升级回填及 SDK 场景已通过回归;Cron/固定间隔、计划管理、运行中取消、配额/背压和订阅待开发 |
|
| P3 Pulse 完整能力 | 安全重试、延时、Cron/固定间隔、漏跑策略、计划版本、并发许可、配额/背压、可靠结果订阅 | 多实例领取/触发竞争、重试上限、取消竞争、队列隔离、积压恢复、事件重复交付 | 进行中:安全重试、Cron/固定间隔、版本、暂停恢复、有限漏跑/重叠策略和 SDK 场景已实现并回归;运行中取消、配额/背压和可靠订阅待开发 |
|
||||||
| P4 Weave 主流程 | SDK 定义构建与发布校验、实例、顺序/分支/结构化并行汇合、信号/定时等待、版本绑定 | 重启接续、早到/重复信号、并行竞争、内部事件重放、发布版本兼容;端到端通用流程 | 待开发 |
|
| P4 Weave 主流程 | SDK 定义构建与发布校验、实例、顺序/分支/结构化并行汇合、信号/定时等待、版本绑定 | 重启接续、早到/重复信号、并行竞争、内部事件重放、发布版本兼容;端到端通用流程 | 待开发 |
|
||||||
| P5 Weave 恢复与查询 | 有界查询、暂停继续、取消、人工核查恢复、控制门控与审计 | 轮次与尝试独立预算;暂停/开始/回执/超时竞争;恢复保留预算;未知结果不盲目重复 | 待开发 |
|
| P5 Weave 恢复与查询 | 有界查询、暂停继续、取消、人工核查恢复、控制门控与审计 | 轮次与尝试独立预算;暂停/开始/回执/超时竞争;恢复保留预算;未知结果不盲目重复 | 待开发 |
|
||||||
| P6 生产验证与运维 | 完整监控、凭据轮换、备份恢复、滚动升级、运行手册、可复现负载和故障工具 | 对照生产验收清单执行,报告环境/指标/遗留项;容量、RPO/RTO 按真实拓扑验证 | 待开发 |
|
| P6 生产验证与运维 | 完整监控、凭据轮换、备份恢复、滚动升级、运行手册、可复现负载和故障工具 | 对照生产验收清单执行,报告环境/指标/遗留项;容量、RPO/RTO 按真实拓扑验证 | 待开发 |
|
||||||
@@ -42,7 +42,7 @@ SDK、文档与示例随 P1–P5 同步开发,P7 是全量核对,不是最
|
|||||||
| 健康检查 | 创建客户端 → 存活/就绪查询 | P0 |
|
| 健康检查 | 创建客户端 → 存活/就绪查询 | P0 |
|
||||||
| 文件转换任务 | 提交幂等任务 → Worker 处理 → 查询结果 | P1–P2 |
|
| 文件转换任务 | 提交幂等任务 → Worker 处理 → 查询结果 | P1–P2 |
|
||||||
| 有限安全重试 | 明确失败 → 持久化退避 → 新尝试 → 查询完整历史 | P3,已实现完整示例 |
|
| 有限安全重试 | 明确失败 → 持久化退避 → 新尝试 → 查询完整历史 | P3,已实现完整示例 |
|
||||||
| 定期数据同步 | 发布计划 → 触发任务 → 暂停/恢复 → 查看触发记录 | P3 |
|
| 定期数据同步 | 发布计划 → 触发任务 → 暂停/恢复 → 查看触发记录 | P3,已实现通用快照示例 |
|
||||||
| 并行生成结果 | 发布流程 → 并行执行 → 汇合 → 完成 | P4 |
|
| 并行生成结果 | 发布流程 → 并行执行 → 汇合 → 完成 | P4 |
|
||||||
| 审核与外部回执 | 提交流程 → 等待信号 → 授权信号注入 → 继续 | P4 |
|
| 审核与外部回执 | 提交流程 → 等待信号 → 授权信号注入 → 继续 | P4 |
|
||||||
| 外部作业轮询 | 提交一次 → 有界查询处理中状态 → 回执或最终结果 → 超限核查 | P5 |
|
| 外部作业轮询 | 提交一次 → 有界查询处理中状态 → 回执或最终结果 → 超限核查 | P5 |
|
||||||
@@ -77,6 +77,7 @@ SDK、文档与示例随 P1–P5 同步开发,P7 是全量核对,不是最
|
|||||||
- [P1 验证记录](reports/phase-1.md)
|
- [P1 验证记录](reports/phase-1.md)
|
||||||
- [P2 阶段验收与进程故障记录](reports/phase-2.md)
|
- [P2 阶段验收与进程故障记录](reports/phase-2.md)
|
||||||
- [P3 安全重试阶段记录](reports/phase-3-safe-retry.md)
|
- [P3 安全重试阶段记录](reports/phase-3-safe-retry.md)
|
||||||
|
- [P3 计划调度阶段记录](reports/phase-3-schedules.md)
|
||||||
- [P2 服务端阶段记录](reports/phase-2-server.md)
|
- [P2 服务端阶段记录](reports/phase-2-server.md)
|
||||||
- [P2 SDK 协议阶段记录](reports/phase-2-sdk-protocol.md)
|
- [P2 SDK 协议阶段记录](reports/phase-2-sdk-protocol.md)
|
||||||
- [P2 自动 Worker 运行器记录](reports/phase-2-worker-runtime.md)
|
- [P2 自动 Worker 运行器记录](reports/phase-2-worker-runtime.md)
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
- 文档版本:0.2
|
- 文档版本:0.2
|
||||||
- 整理日期:2026-09-17
|
- 整理日期:2026-09-17
|
||||||
- 阶段:P0–P2 已通过阶段验收,包括协议 Schema 与真实进程故障恢复;P3 安全重试已通过回归,P3–P7 继续推进,尚未完成全平台生产验收
|
- 阶段:P0–P2 已通过阶段验收,包括协议 Schema 与真实进程故障恢复;P3 安全重试与计划调度已通过回归,P3–P7 继续推进,尚未完成全平台生产验收
|
||||||
- 适用范围:Orbit 统一平台,包含 Pulse 任务调度模块、Weave 流程编排模块与公共能力。平台面向多个业务系统复用。
|
- 适用范围:Orbit 统一平台,包含 Pulse 任务调度模块、Weave 流程编排模块与公共能力。平台面向多个业务系统复用。
|
||||||
|
|
||||||
## 阅读顺序
|
## 阅读顺序
|
||||||
@@ -58,7 +58,7 @@ Orbit 不内置特定行业规则;Pulse 管理执行,Weave 管理步骤依
|
|||||||
| Weave 技术细则 | 功能范围已确认;SDK/定义 Schema、暂停门控与计时、查询预算、并行失败策略、人工恢复身份仍需评审 | [流程](06-workflow-spec.md) |
|
| Weave 技术细则 | 功能范围已确认;SDK/定义 Schema、暂停门控与计时、查询预算、并行失败策略、人工恢复身份仍需评审 | [流程](06-workflow-spec.md) |
|
||||||
| WS 库及后续依赖版本 | 当前基础使用 Go 1.27.1、MySQL 8.4+ 与显式 SQL;WS 已锁定 coder/websocket v1.8.15,后续依赖需单独验证 | [模型基线](07-data-model.md) |
|
| WS 库及后续依赖版本 | 当前基础使用 Go 1.27.1、MySQL 8.4+ 与显式 SQL;WS 已锁定 coder/websocket v1.8.15,后续依赖需单独验证 | [模型基线](07-data-model.md) |
|
||||||
| 人工重试的任务身份 | 建议创建关联的新任务,自动重试保持原任务 ID | [功能](01-functional-spec.md) |
|
| 人工重试的任务身份 | 建议创建关联的新任务,自动重试保持原任务 ID | [功能](01-functional-spec.md) |
|
||||||
| Cron、固定间隔与夏令时细节 | 建议五段 Cron、固定间隔首版采用 fixed-rate;夏令时行为需明确 | [功能](01-functional-spec.md) |
|
| Cron、固定间隔与夏令时细节 | 已实现五段 Cron、fixed-rate;缺失时刻跳过、重复时刻按 UTC 分开;时区快照随版本冻结 | [计划接口](api/schedules.md) |
|
||||||
| 批量大小、消息上限、心跳、租约、扫描周期 | 均需设置有限上限;数值通过原型与压测确定 | [验收](05-production-acceptance.md) |
|
| 批量大小、消息上限、心跳、租约、扫描周期 | 均需设置有限上限;数值通过原型与压测确定 | [验收](05-production-acceptance.md) |
|
||||||
| 幂等、事件、审计的保留周期 | 按恢复与业务追溯要求确定,不能仅随任务清理 | [数据](04-reliability-and-data.md) |
|
| 幂等、事件、审计的保留周期 | 按恢复与业务追溯要求确定,不能仅随任务清理 | [数据](04-reliability-and-data.md) |
|
||||||
| 数据库高可用与恢复目标 | 明确故障模型、复制确认、RPO / RTO 及运维责任 | [验收](05-production-acceptance.md) |
|
| 数据库高可用与恢复目标 | 明确故障模型、复制确认、RPO / RTO 及运维责任 | [验收](05-production-acceptance.md) |
|
||||||
@@ -79,4 +79,4 @@ Orbit 不内置特定行业规则;Pulse 管理执行,Weave 管理步骤依
|
|||||||
|
|
||||||
2026-09-17,文档更新至 0.2:用户确认研发定义发布流程及 Weave 首版功能,明确纳入有界重复查询和暂停/继续等生产管理能力。技术细则与参数继续按建议或待定标注,未将功能确认等同于实现或验收通过。
|
2026-09-17,文档更新至 0.2:用户确认研发定义发布流程及 Weave 首版功能,明确纳入有界重复查询和暂停/继续等生产管理能力。技术细则与参数继续按建议或待定标注,未将功能确认等同于实现或验收通过。
|
||||||
|
|
||||||
持续开发目标已建立。当前实现和验收进度以 [开发计划](08-development-plan.md)、[P0 报告](reports/phase-0.md)、[P1 报告](reports/phase-1.md) 与 [P2 阶段验收](reports/phase-2.md) 和 [P3 安全重试报告](reports/phase-3-safe-retry.md) 为准;早期设计稿中的“未实现”描述保留其编写阶段语境。
|
持续开发目标已建立。当前实现和验收进度以 [开发计划](08-development-plan.md)、[P0 报告](reports/phase-0.md)、[P1 报告](reports/phase-1.md) 与 [P2 阶段验收](reports/phase-2.md) 、[P3 安全重试报告](reports/phase-3-safe-retry.md) 和 [P3 计划调度报告](reports/phase-3-schedules.md) 为准;早期设计稿中的“未实现”描述保留其编写阶段语境。
|
||||||
|
|||||||
@@ -15,7 +15,7 @@
|
|||||||
| 凭据校验与初始化 CLI | P1 已实现 | [鉴权](authentication.md) |
|
| 凭据校验与初始化 CLI | P1 已实现 | [鉴权](authentication.md) |
|
||||||
| `/v1/worker/connect` | P2 已实现并通过阶段验收 | [Worker 契约](worker.md)、[Schema](worker-schema.md) |
|
| `/v1/worker/connect` | P2 已实现并通过阶段验收 | [Worker 契约](worker.md)、[Schema](worker-schema.md) |
|
||||||
| 任务冻结策略与 Worker `retryable` | P3 安全重试已实现 | [安全重试](retries.md) |
|
| 任务冻结策略与 Worker `retryable` | P3 安全重试已实现 | [安全重试](retries.md) |
|
||||||
| `/v1/schedules` 与计划管理 | 计划 P3,尚未实现 | [协议草案](../03-http-websocket-protocol.md) |
|
| `/v1/schedules`、不可变版本、暂停/恢复及触发查询 | P3 已实现 | [计划接口与 SDK 场景](schedules.md) |
|
||||||
| `/v1/events/connect` | 计划 P3,尚未实现 | [协议草案](../03-http-websocket-protocol.md) |
|
| `/v1/events/connect` | 计划 P3,尚未实现 | [协议草案](../03-http-websocket-protocol.md) |
|
||||||
| `/v1/workflows` 与流程管理 | 计划 P4–P5,尚未实现 | [流程接口规划](workflows.md) |
|
| `/v1/workflows` 与流程管理 | 计划 P4–P5,尚未实现 | [流程接口规划](workflows.md) |
|
||||||
|
|
||||||
|
|||||||
@@ -52,4 +52,6 @@ actions 不支持通配符;queues 可列出队列 ID,星号仅表示当前
|
|||||||
|
|
||||||
SDK 使用 `orbit.Options{Token: token}`。健康查询不发送凭据。当前不启用业务权限缓存,撤销校验不依赖缓存过期。
|
SDK 使用 `orbit.Options{Token: token}`。健康查询不发送凭据。当前不启用业务权限缓存,撤销校验不依赖缓存过期。
|
||||||
|
|
||||||
|
P3 增加 schedules:read 和 schedules:manage。bootstrap 显式传 --schedule-access 才授予;创建/发布计划还需要 tasks:submit 与队列范围。已有计划属于命名空间,创建凭据到期或撤销不自动暂停计划;停止新触发应暂停计划,或禁用应用/命名空间。详见 [计划接口](schedules.md)。
|
||||||
|
|
||||||
P2 增加 workers:connect 和 workers:consume。bootstrap 只有显式提供 --worker-access 时才同时授予这两项开发权限;现有凭据不自动变化。Worker 还需独立的进程恢复秘密和尝试秘密,见 [Worker 契约](worker.md)。
|
P2 增加 workers:connect 和 workers:consume。bootstrap 只有显式提供 --worker-access 时才同时授予这两项开发权限;现有凭据不自动变化。Worker 还需独立的进程恢复秘密和尝试秘密,见 [Worker 契约](worker.md)。
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# 安全重试 v1
|
# 安全重试 v1
|
||||||
|
|
||||||
状态:P3 的安全重试已实现。使用现有任务提交与 Worker 结果接口,无独立的自动重试路由;Cron、运行中取消、人工恢复与可靠结果订阅继续按 [计划](../08-development-plan.md) 开发。验证证据见 [本阶段报告](../reports/phase-3-safe-retry.md)。
|
状态:P3 的安全重试已实现。使用现有任务提交与 Worker 结果接口,无独立的自动重试路由;[Cron/固定间隔计划](schedules.md) 已接入;运行中取消、人工恢复与可靠结果订阅继续按 [计划](../08-development-plan.md) 开发。验证证据见 [本阶段报告](../reports/phase-3-safe-retry.md)。
|
||||||
|
|
||||||
## 1. 三种不同的重报
|
## 1. 三种不同的重报
|
||||||
|
|
||||||
|
|||||||
219
docs/api/schedules.md
Normal file
219
docs/api/schedules.md
Normal file
@@ -0,0 +1,219 @@
|
|||||||
|
# 调度计划接口 v1
|
||||||
|
|
||||||
|
状态:已实现五段 Cron、fixed-rate、不可变版本、暂停/恢复、三种漏跑与重叠策略、触发历史及独立 SDK。调度记录、任务创建、事件与进度同事务提交。当前实际验证与边界见 [阶段报告](../reports/phase-3-schedules.md),整体仍按 [开发计划](../08-development-plan.md) 推进。
|
||||||
|
|
||||||
|
## 1. 权限与生命周期
|
||||||
|
|
||||||
|
| 接口 | 权限 | SDK |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| POST /v1/schedules | schedules:manage + tasks:submit + 模板队列范围 | CreateSchedule |
|
||||||
|
| GET /v1/schedules?queue_id=… | schedules:read + 队列范围 | ListSchedules |
|
||||||
|
| GET /v1/schedules/{id} | schedules:read + 计划队列范围 | GetSchedule |
|
||||||
|
| PATCH /v1/schedules/{id} | schedules:manage + tasks:submit + 计划队列范围 | PublishSchedule |
|
||||||
|
| POST /v1/schedules/{id}:pause | schedules:manage + 计划队列范围 | PauseSchedule |
|
||||||
|
| POST /v1/schedules/{id}:resume | schedules:manage + 计划队列范围 | ResumeSchedule |
|
||||||
|
| GET /v1/schedules/{id}/versions/{version} | schedules:read + 计划队列范围 | GetScheduleVersion |
|
||||||
|
| GET /v1/schedules/{id}/occurrences | schedules:read + 计划队列范围 | ListScheduleOccurrences |
|
||||||
|
|
||||||
|
计划终身绑定一个队列。发布完整新定义时可以改变时间、参数和策略,不能更换队列。计划属于命名空间;创建凭据后续到期/撤销不自动撤销已创建的计划。停止计划使用暂停接口,禁用应用或命名空间会阻止后台继续创建任务。每次管理与查询仍重新校验当前凭据。
|
||||||
|
|
||||||
|
计划暂停只停止产生新的任务,已经受理的任务继续执行。队列暂停只停止任务派发,不停止计划受理,可能形成积压。SERIAL 策略将 WAITING、LEASED、RUNNING、NEEDS_REVIEW 都视为未完成;未知效果未处理时不能开始下一轮。
|
||||||
|
|
||||||
|
请求使用 Bearer、application/json 和稳定的 Idempotency-Key,沿用 [任务接口](tasks.md) 的 512 KiB 请求、严格 JSON、命名空间隔离和精确数字规则。ID 为 32 位小写十六进制,revision/version 为十进制字符串,时间为 UTC RFC 3339 微秒精度。
|
||||||
|
|
||||||
|
## 2. 创建与定义字段
|
||||||
|
|
||||||
|
CreateScheduleRequest:name、definition 必填;paused 默认 false。name 为命名空间内唯一的 1–128 位 ASCII 名称,字符规则同任务处理器。definition 包含 rule、policy、task,三者均需明确提供。
|
||||||
|
|
||||||
|
| rule 字段 | 含义与边界 |
|
||||||
|
| --- | --- |
|
||||||
|
| kind | CRON 或 FIXED_RATE |
|
||||||
|
| timezone | 必填 IANA 名称,如 UTC、Asia/Shanghai;不接受 Local 或文件路径 |
|
||||||
|
| calendar_version | 可省略,v1 固定默认 `go1.27.1-b2d18a7c8fa8`;规范化后写入版本快照,其他未支持值拒绝 |
|
||||||
|
| start_at | 必填基准,包含边界;年份 2000–9998,最多微秒 |
|
||||||
|
| end_at | 可选排除边界,必须晚于 start_at;达到后不再产生新触发点 |
|
||||||
|
| cron | CRON 必填五段表达式,最多 256 字符;不接受秒段、@every 等描述符或内嵌 TZ= |
|
||||||
|
| interval_ms | FIXED_RATE 必填,1000–2592000000 毫秒;此时不能设置 cron |
|
||||||
|
|
||||||
|
新计划的第一个点不早于数据库创建时刻;start_at 在过去时只用作规则基准,不追溯生成计划存在前的任务。创建时必须还有未来触发点。新发布版本可以设置已经结束的规则,此时先处理旧积压,再结束后续调度。
|
||||||
|
|
||||||
|
Cron 段顺序为分钟、小时、月日、月、星期;支持 `*`、列表、范围、步长、月份/星期名称,星期为 0–6(周日为 0)。月日与星期均受限时按 OR 匹配,有通配段时按标准组合规则匹配。采用 [robfig/cron v3.0.1 的解析器](https://github.com/robfig/cron/tree/v3.0.1),Orbit 自行按时区偏移区间转换 UTC 触发点,不启动该库的内存调度器。
|
||||||
|
|
||||||
|
时区快照内置于服务二进制,不读取宿主机时区文件或 ZONEINFO。快照来自 Go 1.27.1 的 zoneinfo.zip,完整 SHA-256 为 `b2d18a7c8fa8142097a48c99609fb3c92db5ee98bc740294e57eab8ae9f94779`。升级时必须保留旧快照可读能力和 v1 默认语义,不能把旧计划静默换成新的时区规则。
|
||||||
|
|
||||||
|
- 夏令时缺失的本地时刻跳过;重复的本地时刻按两个不同 UTC 时间分别触发。
|
||||||
|
- 半小时跳变、整天跳过、非整点时区和世纪闰日都有测试。
|
||||||
|
- fixed-rate 始终按 `start_at + n × interval_ms` 的实际经过时间计算,不跟随上一任务的完成时间漂移。
|
||||||
|
- 无解日期规则、混合字段和无效时区在发布前拒绝。无后续触发点与计算错误分开处理,错误不能静默当作完成。
|
||||||
|
|
||||||
|
| policy 字段 | 要求 |
|
||||||
|
| --- | --- |
|
||||||
|
| misfire | 必填:SKIP、COALESCE、CATCH_UP |
|
||||||
|
| overlap | 必填:PARALLEL、SERIAL、SKIP |
|
||||||
|
| catchup_limit | 必填 1–100,每批最多生成任务数;大参数批次还受约 1 MiB 参数复制量限制 |
|
||||||
|
| catchup_window_ms | 必填 1000–2592000000,未处理触发点最多保留的时间窗口 |
|
||||||
|
| grace_ms | 必填 1000–60000,正常派发允许延迟;不能超过 catchup_window_ms |
|
||||||
|
|
||||||
|
所有任务策略在 task 内冻结,字段同 SubmitTaskRequest;payload 最大 256 KiB,数字文本保持精确。task 中禁止绝对 available_at / start_deadline_at,每次生成时立即可受理,task_timeout_ms 从本次实际创建时间计算。计划结束不取消已经生成的任务,也不缩短其任务时限。
|
||||||
|
|
||||||
|
创建请求示例,ID 和未来时间替换为实际值:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"name":"object-snapshot",
|
||||||
|
"paused":true,
|
||||||
|
"definition":{
|
||||||
|
"rule":{"kind":"FIXED_RATE","timezone":"UTC","start_at":"2026-10-01T00:00:00Z","interval_ms":60000},
|
||||||
|
"policy":{"misfire":"COALESCE","overlap":"SERIAL","catchup_limit":10,"catchup_window_ms":86400000,"grace_ms":5000},
|
||||||
|
"task":{"queue_id":"11111111111111111111111111111111","handler":"object.snapshot","handler_version":"v1","payload_version":"v1","payload":{"source_ref":"objects/sample","sequence":9007199254740993},"task_timeout_ms":30000,"retry":{"max_attempts":1,"attempt_timeout_ms":5000}}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
```sh
|
||||||
|
curl "$ORBIT_URL/v1/schedules" \
|
||||||
|
-H "Authorization: Bearer $ORBIT_TOKEN" \
|
||||||
|
-H 'Content-Type: application/json' -H 'Idempotency-Key: snapshot-plan-001' \
|
||||||
|
--data-binary @schedule.json
|
||||||
|
```
|
||||||
|
|
||||||
|
将上方 JSON 保存为 schedule.json。首次返回 201,同键同内容返回 200;均不表示已有任务完成。响应示例:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"request_id":"44444444444444444444444444444444",
|
||||||
|
"data":{
|
||||||
|
"deduplicated":false,
|
||||||
|
"schedule":{
|
||||||
|
"id":"22222222222222222222222222222222","namespace_id":"33333333333333333333333333333333",
|
||||||
|
"queue_id":"11111111111111111111111111111111","name":"object-snapshot",
|
||||||
|
"revision":"1","current_version":"1","processing_version":"1","paused":true,
|
||||||
|
"next_fire_at":"2026-10-01T00:00:00Z","check_at":"2026-10-01T00:00:00Z",
|
||||||
|
"created_at":"2026-09-17T08:00:00Z","updated_at":"2026-09-17T08:00:00Z",
|
||||||
|
"current":{
|
||||||
|
"version":"1","effective_at":"2026-09-17T08:00:00Z",
|
||||||
|
"definition":{
|
||||||
|
"rule":{"calendar_version":"go1.27.1-b2d18a7c8fa8","kind":"FIXED_RATE","timezone":"UTC","start_at":"2026-10-01T00:00:00Z","interval_ms":60000},
|
||||||
|
"policy":{"misfire":"COALESCE","overlap":"SERIAL","catchup_limit":10,"catchup_window_ms":86400000,"grace_ms":5000},
|
||||||
|
"task":{"queue_id":"11111111111111111111111111111111","handler":"object.snapshot","handler_version":"v1","payload_version":"v1","payload":{"source_ref":"objects/sample","sequence":9007199254740993},"task_timeout_ms":30000,"recovery_safety":"RECONCILE_ONLY","retry":{"max_attempts":1,"initial_delay_ms":0,"max_delay_ms":0,"attempt_timeout_ms":5000}}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
GetSchedule 的 data 直接为 schedule 对象。列表 Items 省略 current/大参数,其他字段相同。next_fire_at 是下一原计划时间,积压时可能在过去;check_at 是平台下次允许扫描时间,并不保证届时执行。next_fire_at 为空时,可能尚待初始化某个版本的首个触发点,也可能已经结束;即使 processing_version 等于 current_version,也必须结合 check_at 判断。check_at 非空表示仍有扫描工作,为空且处理到最新版本表示已无后续触发。
|
||||||
|
|
||||||
|
## 3. 漏跑、重叠与有限补偿
|
||||||
|
|
||||||
|
先处理超过 catchup_window_ms 的范围并记录 CATCHUP_WINDOW_EXCEEDED,再应用漏跑规则:
|
||||||
|
|
||||||
|
| 策略 | 行为 |
|
||||||
|
| --- | --- |
|
||||||
|
| SKIP | 早于 now − grace_ms 的点合为跳过范围;容差内的点仍可受理 |
|
||||||
|
| COALESCE | 若最早积压点超过 grace_ms,将当前版本内本轮所有已到期点合为一个任务 |
|
||||||
|
| CATCH_UP | 按原时间逐点创建,单批不超过 catchup_limit,剩余进度保留到下一批 |
|
||||||
|
| PARALLEL | 可以产生多条未完成任务,实际执行仍受队列与 Worker 并发许可限制 |
|
||||||
|
| SERIAL | 有任一先前计划任务未终结时保留游标等待;没有未完成任务时本批最多创建一条 |
|
||||||
|
| SKIP 重叠策略 | 有先前未终结任务时跳过本轮已到期范围;否则本批最多创建一条 |
|
||||||
|
|
||||||
|
SERIAL 等待也受时间窗口约束,过旧范围会被记录为跳过,避免无界补偿。NEEDS_REVIEW 不属于明确终态,会持续阻止串行后续任务;取消或失败不能假装外部效果已消失。
|
||||||
|
|
||||||
|
每节点每秒扫描最多 100 个候选计划,每个计划单独事务。同一计划相邻批次的数据库决策时间至少间隔一秒,check_at 持久化,多节点不会各自补跑一份。每批最多创建一个历史版本的任务,跨版本也不绕过批量上限;空版本每批最多推进 16 个。当前是有界运行基线,不是已验证的生产吞吐承诺。
|
||||||
|
|
||||||
|
每个触发身份由命名空间、计划 ID、原 UTC scheduled_at 唯一确定。暂停、重连和发布新版本都不会改变旧身份。跳过/合并记录的覆盖区间为 `[scheduled_at, covered_until)`,结束边界不包含;范围不会跨越版本生效时间,也不会为范围内每个点额外创建任务。
|
||||||
|
|
||||||
|
## 4. 发布、暂停与恢复
|
||||||
|
|
||||||
|
PATCH 是完整发布,必须包含 expected_revision(非零十进制字符串)、reason(非空,最多 1024 字符)、definition(完整新定义)。发布后 current_version 增加一,新版本以事务读取的数据库时间立即生效。
|
||||||
|
|
||||||
|
尚未处理且早于新版本生效边界的点仍按旧规则、旧模板和旧策略决定。后续原时间由新版本计算,已生成任务保持原快照。数据库触发器禁止覆盖已发布版本。GET versions/{version} 返回 version、effective_at、definition;version 必须是无前导零的非零十进制数。
|
||||||
|
|
||||||
|
暂停/恢复请求体相同:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"expected_revision":"1","reason":"start regular snapshots"}
|
||||||
|
```
|
||||||
|
|
||||||
|
成功返回 200,data 与创建回执结构相同。暂停不清空游标,恢复按冻结漏跑规则处理暂停期间的点。对已经处于目标暂停状态的计划换键操作返回 STATE_CONFLICT;原键原内容重报仍返回原回执。
|
||||||
|
|
||||||
|
- 管理操作、不可变版本、事件、审计和幂等回执同事务保存。后台触发持久化 occurrence、task.accepted、schedule.decided、schedule.advanced,记录执行节点。
|
||||||
|
- 每次扫描也更新 revision。遇到明确 STATE_CONFLICT,先 GET 新状态,再用新的命令身份和新 expected_revision 操作;网络结果不明时必须复用原键和原版本核对。
|
||||||
|
- 同键更改定义、reason 或 expected_revision 返回 IDEMPOTENCY_CONFLICT。原回执不表示计划当前仍是该状态。
|
||||||
|
- 计划名称不能修改,暂无删除或单次手动触发接口。WAITING 的已生成计划任务可用任务取消接口取消,计划本身继续运行;流程归属任务仍由后续流程控制管理。
|
||||||
|
|
||||||
|
## 5. 列表与触发记录
|
||||||
|
|
||||||
|
ListSchedules 需要 queue_id;计划列表和 occurrences 均可用 limit(默认 20,1–100)、cursor(最多 1024 字符)。未知、重复查询参数或错误游标返回 400。详情接口不接受查询参数。
|
||||||
|
|
||||||
|
计划按 `(created_at DESC,id DESC)`,触发按 `(scheduled_at DESC,id DESC)` 分页,不使用 offset。游标绑定命名空间、队列或计划及查询类型。读取不承诺跨页快照;空列表为 items: [],有下一页才返回 next_cursor。
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"request_id":"44444444444444444444444444444444",
|
||||||
|
"data":{"items":[{
|
||||||
|
"id":"55555555555555555555555555555555","schedule_id":"22222222222222222222222222222222",
|
||||||
|
"schedule_version":"1","scheduled_at":"2026-10-01T00:00:00Z","decision":"COALESCED",
|
||||||
|
"task_id":"66666666666666666666666666666666","covered_until":"2026-10-01T00:03:00.000001Z",
|
||||||
|
"reason":"MISFIRE_COALESCED","created_at":"2026-10-01T00:03:00Z"
|
||||||
|
}]}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
FIRED 表示已创建单点任务;COALESCED 表示范围已合并到 task_id;SKIPPED 没有 task_id。它们都不是业务执行结果,继续 GetTask 查询,再通过 [尝试与证据接口](execution-history.md) 核对执行。
|
||||||
|
|
||||||
|
## 6. SDK 与完整示例
|
||||||
|
|
||||||
|
所有方法首参数为 context.Context,写操作需要稳定 key。ListSchedules 的参数为 `(ctx, queueID, SchedulePageRequest)`;ListScheduleOccurrences 为 `(ctx, scheduleID, SchedulePageRequest)`。
|
||||||
|
|
||||||
|
在已有服务和当前迁移基础上,创建专用开发凭据:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/orbit-admin bootstrap \
|
||||||
|
--application schedule-demo --namespace default --environment development \
|
||||||
|
--queue snapshots --worker-access --schedule-access \
|
||||||
|
--output .local/schedule.credentials.json
|
||||||
|
cd sdk/go
|
||||||
|
ORBIT_URL=http://127.0.0.1:8080 \
|
||||||
|
ORBIT_CREDENTIALS_FILE=../../.local/schedule.credentials.json \
|
||||||
|
go run ./examples/schedules
|
||||||
|
```
|
||||||
|
|
||||||
|
前置数据库与目录步骤见 [SDK 接入](../sdk/README.md)。源码:[完整调度示例](../../sdk/go/examples/schedules/main.go)。示例创建暂停的每秒计划,恢复后让自动 Worker 返回对象输入快照,查询两次完成,再暂停并排空。计划本身有十秒结束边界,任务与历史保留;重复运行生成独立身份,不调用外部业务系统。
|
||||||
|
|
||||||
|
```text
|
||||||
|
schedule=<计划ID> occurrence=<触发ID> task=<任务ID> version=1 state=SUCCEEDED
|
||||||
|
schedule=<计划ID> occurrence=<触发ID> task=<任务ID> version=1 state=SUCCEEDED
|
||||||
|
paused=true completed_tasks=2 current_version=1
|
||||||
|
worker_drained=true running=0
|
||||||
|
```
|
||||||
|
|
||||||
|
调度或查询有延迟时可能观察到更多已产生的任务。异常时非零退出,Worker 尝试有界收尾;暂停不回滚已产生任务。接入函数可直接用以下片段发布 Cron 版本:
|
||||||
|
|
||||||
|
```go
|
||||||
|
current, err := client.GetSchedule(ctx, scheduleID)
|
||||||
|
if err != nil { return err }
|
||||||
|
definition := current.Current.Definition
|
||||||
|
definition.Rule.Kind = "CRON"
|
||||||
|
definition.Rule.Cron = "0 9 * * MON-FRI"
|
||||||
|
definition.Rule.IntervalMS = 0
|
||||||
|
definition.Rule.Timezone = "Asia/Shanghai"
|
||||||
|
updated, err := client.PublishSchedule(ctx, scheduleID, operationKey, orbit.PublishScheduleRequest{
|
||||||
|
ExpectedRevision: current.Revision,
|
||||||
|
Reason: "use weekday snapshot cadence",
|
||||||
|
Definition: definition,
|
||||||
|
})
|
||||||
|
if err != nil { return err }
|
||||||
|
fmt.Println(updated.Schedule.CurrentVersion)
|
||||||
|
```
|
||||||
|
|
||||||
|
保留原 calendar_version,并核对 start_at/end_at 是否仍覆盖期望时间。发布时不会取消旧版本已创建任务。
|
||||||
|
|
||||||
|
## 7. 错误与升级
|
||||||
|
|
||||||
|
错误信封同 [任务接口](tasks.md)。400 表示字段、时间、时区快照、游标或队列变更非法;401 表示凭据/应用/命名空间不可用;403 表示缺少动作或列表/创建队列范围;404 隐藏不存在或单资源越权;409 表示名称、状态、版本或幂等内容冲突;503 时写操作可能已经提交,应保留原身份核对。
|
||||||
|
|
||||||
|
迁移 000006 增加绑定队列、历史处理版本、扫描时间、窗口与容差,以及调度/任务来源索引。此前只有结构、没有创建计划接口;原人工行的 queue_id/check_at 为 NULL,不自动启用。迁移不会猜测任务模板对应的授权队列;此类行需由管理员核查后通过正式接口建立受控计划。
|
||||||
|
|
||||||
|
服务仍只依赖 MySQL,不需要外部 Cron、消息队列或协调服务。生产跨主机、数据库切换、在线迁移、海量积压与公平性测试尚待 P6;当前计划能力不代表完整生产验收已通过。
|
||||||
@@ -54,12 +54,21 @@
|
|||||||
- 异常:次数/时限耗尽停止重试;未知效果默认转核查。`Retryable=true` 需要明确的安全依据,不能用于普通网络超时。
|
- 异常:次数/时限耗尽停止重试;未知效果默认转核查。`Retryable=true` 需要明确的安全依据,不能用于普通网络超时。
|
||||||
- 清理:示例不调用外部业务系统,每次创建独立任务并保留历史;失败时非零退出并有界收尾。
|
- 清理:示例不调用外部业务系统,每次创建独立任务并保留历史;失败时非零退出并有界收尾。
|
||||||
|
|
||||||
|
## 已可运行:周期对象快照
|
||||||
|
|
||||||
|
- 源码:[调度示例](../../sdk/go/examples/schedules/main.go),[初始化、调用与输出](../api/schedules.md#6-sdk-与完整示例)。
|
||||||
|
- 前置:当前迁移、服务,显式 --schedule-access 与 --worker-access 的专用开发凭据。
|
||||||
|
- 步骤:创建暂停的有限每秒计划 → 恢复 → 自动 Worker 返回输入快照 → 查到至少两次任务成功 → 暂停并排空。
|
||||||
|
- 预期:不同 occurrence/task 身份、相同计划版本、paused=true、running=0;精确整数保留。
|
||||||
|
- 异常:串行前项未知时不新建后续任务;恢复按版本与漏跑策略处理,超过窗口记录跳过。暂停不撤销已生成任务。
|
||||||
|
- 清理:计划最终暂停,自带十秒结束边界;失败非零退出,历史保留,无外部业务副作用。
|
||||||
|
|
||||||
## 计划场景与验收标准
|
## 计划场景与验收标准
|
||||||
|
|
||||||
| 场景 | 输入与步骤 | 预期结果 | 必测异常 | 状态 |
|
| 场景 | 输入与步骤 | 预期结果 | 必测异常 | 状态 |
|
||||||
| --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- |
|
||||||
| 文件转换 | 对象引用 → 提交任务 → Worker 转换 → 结果引用 | 同一操作获得一次业务结果,历史可查 | 重复提交、Worker 在结果上报前中断 | 受理与执行框架已实现,实际转换示例待补齐 |
|
| 文件转换 | 对象引用 → 提交任务 → Worker 转换 → 结果引用 | 同一操作获得一次业务结果,历史可查 | 重复提交、Worker 在结果上报前中断 | 受理与执行框架已实现,实际转换示例待补齐 |
|
||||||
| 定期同步 | 发布计划 → 周期触发 → 暂停 → 恢复 | 触发身份唯一,漏跑策略可追溯 | 多节点竞争、停机后补跑上限 | P3 待开发 |
|
| 定期同步 | 发布计划 → 周期触发 → 暂停 → 恢复 | 触发身份唯一,漏跑策略可追溯 | 多节点竞争、停机后补跑上限 | P3 通用快照示例已完成,业务同步由接入方处理器实现 |
|
||||||
| 并行生成 | 输入对象 → 并行计算两份结果 → 汇合 | 两支满足条件后只创建一份后继工作 | 一支失败、重复结果、汇合竞争 | P4 待开发 |
|
| 并行生成 | 输入对象 → 并行计算两份结果 → 汇合 | 两支满足条件后只创建一份后继工作 | 一支失败、重复结果、汇合竞争 | P4 待开发 |
|
||||||
| 等待审核 | 校验 → 等待信号 → 发布结果 | 只接受授权、匹配当前操作的信号 | 早到、重复、过期与越权信号 | P4 待开发 |
|
| 等待审核 | 校验 → 等待信号 → 发布结果 | 只接受授权、匹配当前操作的信号 | 早到、重复、过期与越权信号 | P4 待开发 |
|
||||||
| 外部作业查询 | 提交一次 → 间隔查询 → 最终结果 | 处理中消耗轮次,请求失败消耗尝试 | 回执与查询竞争、预算耗尽、响应未知 | P5 待开发 |
|
| 外部作业查询 | 提交一次 → 间隔查询 → 最终结果 | 处理中消耗轮次,请求失败消耗尝试 | 回执与查询竞争、预算耗尽、响应未知 | P5 待开发 |
|
||||||
|
|||||||
96
docs/reports/phase-3-schedules.md
Normal file
96
docs/reports/phase-3-schedules.md
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
# P3 计划调度阶段记录
|
||||||
|
|
||||||
|
验证日期:2026-09-17。已交付计划管理与持久化调度闭环,P3 仍有运行中取消、配额/背压、可靠订阅待完成;本报告不替代全平台生产验收。
|
||||||
|
|
||||||
|
## 1. 已实现行为
|
||||||
|
|
||||||
|
- 五段 Cron 与 fixed-rate;显式时区、开始和结束边界,固定间隔保持基准,不按任务完成时间漂移。
|
||||||
|
- 时区快照随定义冻结,服务从内置数据加载,不依赖宿主机或 ZONEINFO;兼容已有快照是后续升级约束。
|
||||||
|
- 创建、队列分页、详情、完整版本发布、历史版本、暂停/恢复、触发历史。每个命令有当前权限校验、幂等回执与管理审计。
|
||||||
|
- SKIP / COALESCE / CATCH_UP 漏跑策略与 PARALLEL / SERIAL / SKIP 重叠策略;明确补偿窗口、延迟容差、批量和扫描间隔。
|
||||||
|
- 发布新版本保留旧生效边界前的积压;跳过和合并范围不越过版本边界。已产生任务保留当时模板。
|
||||||
|
- 多节点用数据库锁与唯一触发身份竞争;触发决定、对应任务、事件及进度一起提交。
|
||||||
|
- 实际 orbit 命令持续扫描,独立 SDK 与 [完整周期快照示例](../../sdk/go/examples/schedules/main.go) 同步交付。
|
||||||
|
- 迁移 000006 增加运行字段与索引,实体仍为 32 个;新模型字段及 SQL COMMENT 均为中文;源码核对 32 个持久化模型、253 个直接声明的持久化字段均有中文说明。
|
||||||
|
|
||||||
|
完整字段、权限、状态与示例见 [接口手册](../api/schedules.md)。
|
||||||
|
|
||||||
|
## 2. 环境、命令及结果
|
||||||
|
|
||||||
|
macOS 本机,Go 1.27.1,Docker 单实例 MySQL 8.4.10。临时随机凭据、回环随机端口,运行结束清理容器;不使用业务数据库。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make fmt
|
||||||
|
make test vet build
|
||||||
|
make integration
|
||||||
|
cd sdk/go
|
||||||
|
GOWORK=off go test -race ./...
|
||||||
|
GOWORK=off go vet ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
上述检查均已通过。未配置集成环境时,普通 go test 会跳过真实数据库/进程用例;实际证据来自 make integration。
|
||||||
|
|
||||||
|
最终完整集成输出:
|
||||||
|
|
||||||
|
| 测试组 | 本次结果 |
|
||||||
|
| --- | --- |
|
||||||
|
| MySQL 全部集成 | PASS,44.712 秒 |
|
||||||
|
| 计划事务与管理子集 | PASS,31.62 秒 |
|
||||||
|
| SDK Worker 连接/运行器及安全重试 | PASS,62.912 秒 |
|
||||||
|
| SDK 计划生命周期 | PASS,4.43 秒,含包启动/竞态开销 5.675 秒 |
|
||||||
|
| 真实进程故障组 | PASS,21.487 秒 |
|
||||||
|
| 新增调度进程强杀恢复 | PASS,10.23 秒 |
|
||||||
|
| 七个可运行示例 | health / tasks / worker-protocol / worker / history / retry / schedules 全部通过 |
|
||||||
|
|
||||||
|
时间为一次测试运行耗时,不能用作吞吐、延迟或容量指标。
|
||||||
|
|
||||||
|
## 3. 时钟与规则证据
|
||||||
|
|
||||||
|
[规则测试](../../pulse/schedule/rule_test.go) 验证:五段/分钟边界、亚洲时区、纽约缺失/重复小时、Lord Howe 半小时夏令时变更、Apia 整天跳过、2096→2104 世纪闰日、月日/星期 OR、固定间隔微秒和跨 292 年计算、结束排除边界、规则副本隔离及无解日期拒绝。
|
||||||
|
|
||||||
|
[时区测试](../../internal/timezone/zone_test.go) 核对内置快照完整 SHA-256、禁用外部 ZONEINFO 后仍能加载、精确名称及缓存,以及未知快照拒绝。快照来自 Go 1.27.1,附 Go 许可。Cron 库版本固定为 v3.0.1,只用于解析和 UTC 日历计算,运行不依赖外部调度服务;SDK 未新增该依赖。
|
||||||
|
|
||||||
|
[有限决策测试](../../pulse/schedule/planner_test.go) 覆盖补偿窗口截断、跳过、合并、有限逐点补跑、串行阻塞、重叠跳过、未来点不处理,以及合并不越过版本生效边界。计算器不自行读取系统当前时间,执行使用资源锁定后的数据库时间。
|
||||||
|
|
||||||
|
## 4. 实际数据库与 SDK 证据
|
||||||
|
|
||||||
|
[数据库用例](../../internal/storage/mysql/schedule_integration_test.go):
|
||||||
|
|
||||||
|
- 两个不同节点身份并发扫描同一到期计划,只创建本批三个唯一任务;再次扫描不能绕过持久化冷却时间。
|
||||||
|
- 禁用命名空间后,即使计划已到期也不再产生任务。
|
||||||
|
- 创建原键重报、同键不同内容冲突、暂停禁止触发、恢复原回执、跨命名空间/队列范围拒绝。
|
||||||
|
- 暂停期间发布新模板,旧积压仍用旧版本;历史版本不可被新模板覆盖,旧点处理后游标推进到新版本。
|
||||||
|
- 连续发布三个有限版本:第二版本尚待初始化时再次发布,仍保存其积压;空历史版本之后的末版全部处理完时,不重新扫描已完成触发点。新增用例先复现了漏掉中间版本和重复写入冲突,再验证修复。
|
||||||
|
- 超过时间窗口的跳过范围、漏跑合并范围与任务关联实际落库,范围结束边界大于起点。
|
||||||
|
- SERIAL 在待执行任务存在时等待;明确取消前项后继续;前项进入 UNKNOWN / NEEDS_REVIEW 后再次停止推进。
|
||||||
|
- 通过测试触发器让 occurrence 写入失败,任务、触发记录及游标整体回滚;故障解除后正常提交。
|
||||||
|
- 触发游标分页无重复,不能跨计划复用游标;任务模板中的大整数文本保留。
|
||||||
|
|
||||||
|
上述用例以真实时间等待形成积压,不把原触发时间改到计划创建前。少量用例将受控行 check_at 设为当前时间,以加速核对连续历史版本或阻塞状态;冷却限制另有未改写 check_at 的竞争测试。
|
||||||
|
|
||||||
|
[SDK 用例](../../sdk/go/schedule_integration_test.go) 连接实际服务,通过独立 SDK 创建暂停计划、幂等核对、恢复、自动 Worker 成功、暂停、发布 Cron 新版本、读取两个版本、原发布回执重报、列表及暂停期间不新增触发。业务结果中的 `9007199254740993` 保持精确。
|
||||||
|
|
||||||
|
## 5. 实际进程强杀恢复
|
||||||
|
|
||||||
|
新增 [TestIntegrationWorkerProcessScheduleRecovery](../../sdk/go/worker_process_integration_test.go) 使用构建后的 orbit 可执行文件和临时 MySQL:
|
||||||
|
|
||||||
|
1. 节点 A 创建一个每秒触发、共八个原触发点的有限计划,CATCH_UP / PARALLEL,每批上限二。
|
||||||
|
2. 观察到首个触发已经落库后,向 A 发送真实强杀并 Wait 确认退出。
|
||||||
|
3. 保留三秒无 Orbit 服务的间隙形成漏跑,不修改数据库时钟或计划游标。
|
||||||
|
4. 启动身份不同的 B、C 两个真实服务进程,从持久化进度竞争接续。
|
||||||
|
5. 查询至计划结束,严格核对恰好八条 FIRED、八个不同 TaskID、原 scheduled_at 序列及相同版本;每条都有完整 WAITING 任务与原模板。
|
||||||
|
|
||||||
|
这证明了本地单 MySQL 下服务进程消失后的持久化接续与多进程触发去重。本用例不启动业务 Worker,不宣称验证外部副作用 exactly-once。原开始/完成 ACK 丢失后强杀接管、Worker 进程中断转核查等 P2 用例也同步通过。
|
||||||
|
|
||||||
|
## 6. 可运行场景与当前边界
|
||||||
|
|
||||||
|
周期快照示例输出两条不同的 occurrence/task 身份,均完成 SUCCEEDED,随后 `paused=true completed_tasks=2 current_version=1`、`worker_drained=true running=0`。示例不访问外部业务资源;计划有十秒结束边界,正常完成后暂停,历史保留。
|
||||||
|
|
||||||
|
仍需后续验证或开发:
|
||||||
|
|
||||||
|
- 运行中任务取消、细粒度限流与队列配额、可靠结果订阅;流程引擎及人工核查控制。
|
||||||
|
- 调度期间数据库故障切换、提交 ACK 丢失的专门故障矩阵、跨主机网络分区、备份恢复和混合版本升级。
|
||||||
|
- 大表在线 DDL、历史人工计划行核查;000006 不自动启用旧的未绑定队列行。新建数据库与既有 v4 数据升级到当前结构的通用测试通过,但这不是旧人工计划内容的完整迁移验收。
|
||||||
|
- 大规模公平性、行锁竞争、重试/补跑风暴、历史保留与清理、租户配额。当前批量、扫描间隔和参数复制限制只建立有界基线,没有百万租户容量结论。
|
||||||
|
|
||||||
|
后续继续 [持续计划](../08-development-plan.md),不将本阶段闭环标为整个 P3 或最终目标完成。
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
# Go SDK 接入说明
|
# Go SDK 接入说明
|
||||||
|
|
||||||
已实现健康查询、任务提交/查询/分页/待执行取消、队列创建/查询、[自动 Worker](worker-runtime.md)、[Worker 底层协议 SDK](worker.md) 和 [执行历史查询](../api/execution-history.md)。[安全重试](../api/retries.md) 已提供有限策略、WorkerFailure.Retryable、历史查询及完整例子。流程构建等后续接口按 [开发计划](../08-development-plan.md) 继续实现。
|
已实现健康查询、任务提交/查询/分页/待执行取消、队列创建/查询、[自动 Worker](worker-runtime.md)、[Worker 底层协议 SDK](worker.md) 和 [执行历史查询](../api/execution-history.md)。[安全重试](../api/retries.md) 已提供有限策略、WorkerFailure.Retryable、历史查询及完整例子。[计划调度](../api/schedules.md) 已支持创建、查询、版本发布、暂停/恢复、触发历史与完整示例。流程构建等后续接口按 [开发计划](../08-development-plan.md) 继续实现。
|
||||||
|
|
||||||
module:`git.yuntuo.hk/k2safe/orbit/sdk/go`。SDK 使用独立 go.mod,依赖 Go 标准库与 `github.com/coder/websocket v1.8.15`,不引用服务端模型或数据库实现。
|
module:`git.yuntuo.hk/k2safe/orbit/sdk/go`。SDK 使用独立 go.mod,依赖 Go 标准库与 `github.com/coder/websocket v1.8.15`,不引用服务端模型或数据库实现。
|
||||||
|
|
||||||
@@ -105,6 +105,9 @@ fmt.Println(cancelled.Task.State)
|
|||||||
|
|
||||||
| 方法 | 请求 / 返回 | 文档 |
|
| 方法 | 请求 / 返回 | 文档 |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
|
| CreateSchedule / GetSchedule / ListSchedules | 计划受理、详情与队列分页 | [计划](../api/schedules.md) |
|
||||||
|
| PublishSchedule / GetScheduleVersion | 不可变发布与历史版本 | [计划](../api/schedules.md) |
|
||||||
|
| PauseSchedule / ResumeSchedule / ListScheduleOccurrences | 控制与触发记录 | [计划](../api/schedules.md) |
|
||||||
| Liveness / Readiness | Health | [健康](../api/health.md) |
|
| Liveness / Readiness | Health | [健康](../api/health.md) |
|
||||||
| SubmitTask | SubmitTaskRequest / SubmitTaskResult | [任务](../api/tasks.md) |
|
| SubmitTask | SubmitTaskRequest / SubmitTaskResult | [任务](../api/tasks.md) |
|
||||||
| GetTask | taskID / Task | [任务](../api/tasks.md) |
|
| GetTask | taskID / Task | [任务](../api/tasks.md) |
|
||||||
@@ -117,7 +120,7 @@ fmt.Println(cancelled.Task.State)
|
|||||||
| ListAttemptEvidence | taskID、attemptID、HistoryPageRequest / AttemptEvidencePage | [执行历史](../api/execution-history.md) |
|
| ListAttemptEvidence | taskID、attemptID、HistoryPageRequest / AttemptEvidencePage | [执行历史](../api/execution-history.md) |
|
||||||
| GetAttemptEvidence | taskID、attemptID、evidenceID / AttemptEvidence | [执行历史](../api/execution-history.md) |
|
| GetAttemptEvidence | taskID、attemptID、evidenceID / AttemptEvidence | [执行历史](../api/execution-history.md) |
|
||||||
|
|
||||||
所有方法第一个参数为 context.Context,返回值之外都返回 error。命令方法要求调用方提供稳定 key,不自动生成或重试。ListTasksRequest 接受 QueueID、State、Limit、Cursor。
|
所有方法第一个参数为 context.Context,返回值之外都返回 error。计划方法的具体参数见对应手册。命令方法要求调用方提供稳定 key,不自动生成或重试。ListTasksRequest 接受 QueueID、State、Limit、Cursor。
|
||||||
|
|
||||||
Options.HTTPClient 可注入专用 HTTP 客户端。SDK 复制配置,不修改调用方实例;默认超时 10 秒,可用 context 缩短。禁止自动跟随重定向,避免凭据被转交其他地址。生产配置 HTTPS;SDK 不承担部署 TLS 终止。
|
Options.HTTPClient 可注入专用 HTTP 客户端。SDK 复制配置,不修改调用方实例;默认超时 10 秒,可用 context 缩短。禁止自动跟随重定向,避免凭据被转交其他地址。生产配置 HTTPS;SDK 不承担部署 TLS 终止。
|
||||||
|
|
||||||
@@ -176,4 +179,4 @@ task=<任务ID> state=SUCCEEDED attempts=1 evidence=1 reason=RESULT_CONFLICT dis
|
|||||||
|
|
||||||
## 7. 后续交付
|
## 7. 后续交付
|
||||||
|
|
||||||
P2 已提供底层协议、[自动 Worker 运行器及示例](worker-runtime.md) 与历史查询,并通过 [Schema 及进程故障阶段验收](../reports/phase-2.md);P3 已实现 [安全重试与示例](../api/retries.md),计划和结果订阅继续开发;P4–P5 提供流程构建发布、信号、等待、轮询与人工恢复。每项随实现同步交付可运行示例及异常回归。
|
P2 已提供底层协议、[自动 Worker 运行器及示例](worker-runtime.md) 与历史查询,并通过 [Schema 及进程故障阶段验收](../reports/phase-2.md);P3 已实现 [安全重试与示例](../api/retries.md) 和 [计划调度与示例](../api/schedules.md),运行中取消、资源配额和结果订阅继续开发;P4–P5 提供流程构建发布、信号、等待、轮询与人工恢复。每项随实现同步交付可运行示例及异常回归。
|
||||||
|
|||||||
1
go.mod
1
go.mod
@@ -5,6 +5,7 @@ go 1.27.1
|
|||||||
require (
|
require (
|
||||||
github.com/coder/websocket v1.8.15
|
github.com/coder/websocket v1.8.15
|
||||||
github.com/go-sql-driver/mysql v1.10.1
|
github.com/go-sql-driver/mysql v1.10.1
|
||||||
|
github.com/robfig/cron/v3 v3.0.1
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
2
go.sum
2
go.sum
@@ -6,6 +6,8 @@ github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxK
|
|||||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||||
github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME=
|
github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME=
|
||||||
github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||||
|
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||||
|
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||||
|
|||||||
@@ -14,11 +14,12 @@ import (
|
|||||||
|
|
||||||
// BootstrapInput 仅供持有数据库管理权限的初始化命令使用,不暴露为公共 HTTP。
|
// BootstrapInput 仅供持有数据库管理权限的初始化命令使用,不暴露为公共 HTTP。
|
||||||
type BootstrapInput struct {
|
type BootstrapInput struct {
|
||||||
Application string // Application 应用名称,每次初始化创建独立应用。
|
Application string // Application 应用名称,每次初始化创建独立应用。
|
||||||
Namespace string // Namespace 命名空间名称。
|
Namespace string // Namespace 命名空间名称。
|
||||||
Environment string // Environment 环境名称,必须显式指定。
|
Environment string // Environment 环境名称,必须显式指定。
|
||||||
Queue string // Queue 初始队列名称。
|
Queue string // Queue 初始队列名称。
|
||||||
WorkerAccess bool // WorkerAccess 显式增加开发用 Worker 接入及消费权限。
|
WorkerAccess bool // WorkerAccess 显式增加开发用 Worker 接入及消费权限。
|
||||||
|
ScheduleAccess bool // ScheduleAccess 显式增加开发用计划查询及管理权限。
|
||||||
}
|
}
|
||||||
|
|
||||||
// BootstrapResult 包含一次性交付的原始凭据,禁止记录到日志或提交到版本库。
|
// BootstrapResult 包含一次性交付的原始凭据,禁止记录到日志或提交到版本库。
|
||||||
@@ -60,6 +61,9 @@ func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func(
|
|||||||
if in.WorkerAccess {
|
if in.WorkerAccess {
|
||||||
permissions.Actions = append(permissions.Actions, "workers:connect", "workers:consume")
|
permissions.Actions = append(permissions.Actions, "workers:connect", "workers:consume")
|
||||||
}
|
}
|
||||||
|
if in.ScheduleAccess {
|
||||||
|
permissions.Actions = append(permissions.Actions, "schedules:read", "schedules:manage")
|
||||||
|
}
|
||||||
raw, err := json.Marshal(permissions)
|
raw, err := json.Marshal(permissions)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
-- 旧版本只定义了结构,没有公开创建计划的入口。未绑定队列的历史行不启用扫描。
|
||||||
|
ALTER TABLE pulse_schedules
|
||||||
|
ADD COLUMN queue_id BINARY(16) NULL COMMENT '计划终身绑定队列;旧结构人工行为空,不允许执行',
|
||||||
|
ADD COLUMN processing_version BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '当前处理积压的不可变版本,不因发布新版本丢弃旧进度',
|
||||||
|
ADD COLUMN check_at DATETIME(6) NULL COMMENT '下一可扫描时间,UTC;补跑批次之间持久化至少一秒间隔,为空表示没有扫描工作',
|
||||||
|
ADD CONSTRAINT ck_schedule_cursor CHECK (processing_version>0 AND processing_version<=current_version),
|
||||||
|
ADD CONSTRAINT ck_schedule_enabled_scan CHECK (queue_id IS NOT NULL OR check_at IS NULL),
|
||||||
|
ADD CONSTRAINT fk_schedule_queue FOREIGN KEY(namespace_id,queue_id) REFERENCES pulse_queues(namespace_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||||
|
ADD KEY ix_schedule_scan(paused,check_at,namespace_id,id),
|
||||||
|
ADD KEY ix_schedule_list(namespace_id,queue_id,created_at,id);
|
||||||
|
ALTER TABLE pulse_schedule_versions
|
||||||
|
ADD COLUMN catchup_window_ms BIGINT UNSIGNED NOT NULL DEFAULT 86400000 COMMENT '未处理触发点最大保留时长,单位毫秒,过旧范围记录跳过',
|
||||||
|
ADD COLUMN grace_ms BIGINT UNSIGNED NOT NULL DEFAULT 5000 COMMENT '正常派发允许延迟的毫秒数,超过后按漏跑策略处理',
|
||||||
|
ADD CONSTRAINT ck_schedule_window CHECK(catchup_window_ms BETWEEN 1000 AND 2592000000 AND grace_ms BETWEEN 1000 AND 60000 AND grace_ms<=catchup_window_ms);
|
||||||
|
ALTER TABLE pulse_schedule_occurrences
|
||||||
|
MODIFY COLUMN covered_until DATETIME(6) NULL COMMENT '合并或跳过范围的排除边界,范围为原计划时间至该边界,不包含边界',
|
||||||
|
ADD CONSTRAINT ck_occurrence_range CHECK(covered_until IS NULL OR covered_until>scheduled_at);
|
||||||
|
ALTER TABLE pulse_tasks ADD KEY ix_task_owner_state(namespace_id,owner_kind,owner_id,state,id);
|
||||||
139
internal/storage/mysql/schedule_history.go
Normal file
139
internal/storage/mysql/schedule_history.go
Normal file
@@ -0,0 +1,139 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/jsonvalue"
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
|
||||||
|
)
|
||||||
|
|
||||||
|
type scheduleCursor struct {
|
||||||
|
Version int `json:"v"`
|
||||||
|
Kind string `json:"k"`
|
||||||
|
Namespace core.ID `json:"n"`
|
||||||
|
Parent core.ID `json:"p"`
|
||||||
|
At time.Time `json:"t"`
|
||||||
|
ID core.ID `json:"i"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseScheduleCursor(raw string, ns, parent core.ID, kind string) (scheduleCursor, error) {
|
||||||
|
var c scheduleCursor
|
||||||
|
b, err := base64.RawURLEncoding.Strict().DecodeString(raw)
|
||||||
|
if err != nil || len(b) > 768 {
|
||||||
|
return c, fault.Invalid("invalid schedule cursor")
|
||||||
|
}
|
||||||
|
if _, err := jsonvalue.Canonical(b); err != nil {
|
||||||
|
return c, fault.Invalid("invalid schedule cursor")
|
||||||
|
}
|
||||||
|
var fields map[string]json.RawMessage
|
||||||
|
if json.Unmarshal(b, &fields) != nil || len(fields) != 6 {
|
||||||
|
return c, fault.Invalid("invalid schedule cursor")
|
||||||
|
}
|
||||||
|
for _, key := range []string{"v", "k", "n", "p", "t", "i"} {
|
||||||
|
if _, ok := fields[key]; !ok {
|
||||||
|
return c, fault.Invalid("invalid schedule cursor")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
d := json.NewDecoder(bytes.NewReader(b))
|
||||||
|
d.DisallowUnknownFields()
|
||||||
|
if d.Decode(&c) != nil || c.Version != 1 || c.Namespace != ns || c.Parent != parent || c.Kind != kind || c.ID.Validate() != nil || c.At.Year() < 2000 || c.At.Year() > 9998 || c.At.Nanosecond()%1000 != 0 {
|
||||||
|
return c, fault.Invalid("cursor does not match schedule query")
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
func makeScheduleCursor(ns, parent core.ID, kind string, at time.Time, id core.ID) (string, error) {
|
||||||
|
b, err := json.Marshal(scheduleCursor{1, kind, ns, parent, at, id})
|
||||||
|
return base64.RawURLEncoding.EncodeToString(b), err
|
||||||
|
}
|
||||||
|
func (s *ScheduleStore) List(ctx context.Context, t auth.Token, queue core.ID, in schedule.PageInput) (schedule.Page, error) {
|
||||||
|
out := schedule.Page{Items: []schedule.Summary{}}
|
||||||
|
err := s.transaction(ctx, t, "schedules:read", func(tx *sql.Tx, p auth.Permissions) error {
|
||||||
|
if !p.AllowsQueue(queue) {
|
||||||
|
return fault.Forbidden
|
||||||
|
}
|
||||||
|
q := `SELECT ` + scheduleColumns + ` FROM pulse_schedules WHERE namespace_id=? AND queue_id=?`
|
||||||
|
args := []any{t.NamespaceID, queue}
|
||||||
|
if in.Cursor != "" {
|
||||||
|
c, err := parseScheduleCursor(in.Cursor, t.NamespaceID, queue, "schedules")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
q += ` AND (created_at<? OR (created_at=? AND id<?))`
|
||||||
|
args = append(args, c.At, c.At, c.ID)
|
||||||
|
}
|
||||||
|
q += ` ORDER BY created_at DESC,id DESC LIMIT ?`
|
||||||
|
args = append(args, in.Limit+1)
|
||||||
|
rows, err := tx.QueryContext(ctx, q, args...)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
v, err := scanSchedule(rows)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out.Items = append(out.Items, v)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(out.Items) > in.Limit {
|
||||||
|
out.Items = out.Items[:in.Limit]
|
||||||
|
last := out.Items[len(out.Items)-1]
|
||||||
|
out.NextCursor, err = makeScheduleCursor(t.NamespaceID, queue, "schedules", last.CreatedAt, last.ID)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
func (s *ScheduleStore) Occurrences(ctx context.Context, t auth.Token, id core.ID, in schedule.PageInput) (schedule.OccurrencePage, error) {
|
||||||
|
out := schedule.OccurrencePage{Items: []schedule.OccurrenceView{}}
|
||||||
|
err := s.transaction(ctx, t, "schedules:read", func(tx *sql.Tx, p auth.Permissions) error {
|
||||||
|
if _, err := authorizeSchedule(ctx, tx, t.NamespaceID, id, p); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
q := `SELECT id,schedule_id,schedule_version,scheduled_at,decision,task_id,covered_until,COALESCE(reason,''),created_at FROM pulse_schedule_occurrences WHERE namespace_id=? AND schedule_id=?`
|
||||||
|
args := []any{t.NamespaceID, id}
|
||||||
|
if in.Cursor != "" {
|
||||||
|
c, err := parseScheduleCursor(in.Cursor, t.NamespaceID, id, "occurrences")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
q += ` AND (scheduled_at<? OR (scheduled_at=? AND id<?))`
|
||||||
|
args = append(args, c.At, c.At, c.ID)
|
||||||
|
}
|
||||||
|
q += ` ORDER BY scheduled_at DESC,id DESC LIMIT ?`
|
||||||
|
args = append(args, in.Limit+1)
|
||||||
|
rows, err := tx.QueryContext(ctx, q, args...)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
var v schedule.OccurrenceView
|
||||||
|
if err := rows.Scan(&v.ID, &v.ScheduleID, &v.ScheduleVersion, &v.ScheduledAt, &v.Decision, &v.TaskID, &v.CoveredUntil, &v.Reason, &v.CreatedAt); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out.Items = append(out.Items, v)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(out.Items) > in.Limit {
|
||||||
|
out.Items = out.Items[:in.Limit]
|
||||||
|
last := out.Items[len(out.Items)-1]
|
||||||
|
out.NextCursor, err = makeScheduleCursor(t.NamespaceID, id, "occurrences", last.ScheduledAt, last.ID)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
400
internal/storage/mysql/schedule_integration_test.go
Normal file
400
internal/storage/mysql/schedule_integration_test.go
Normal file
@@ -0,0 +1,400 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||||
|
)
|
||||||
|
|
||||||
|
type scheduleFixture struct {
|
||||||
|
*executionFixture
|
||||||
|
store *ScheduleStore
|
||||||
|
api *schedule.Service
|
||||||
|
}
|
||||||
|
|
||||||
|
func newScheduleFixture(t *testing.T, f *executionFixture) *scheduleFixture {
|
||||||
|
t.Helper()
|
||||||
|
raw, _ := json.Marshal(auth.Permissions{Actions: []string{"tasks:submit", "tasks:read", "tasks:cancel", "queues:read", "schedules:read", "schedules:manage", "workers:connect", "workers:consume"}, Queues: []string{"*"}})
|
||||||
|
f.exec(`UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?`, string(raw), f.resources.NamespaceID, f.resources.CredentialID)
|
||||||
|
store, err := NewScheduleStore(f.db, executionID(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return &scheduleFixture{f, store, schedule.New(store)}
|
||||||
|
}
|
||||||
|
func (f *scheduleFixture) definition() schedule.Definition {
|
||||||
|
return schedule.Definition{Rule: schedule.Rule{Kind: "FIXED_RATE", Timezone: "UTC", StartAt: time.Now().UTC().Truncate(time.Microsecond).Add(-time.Hour), IntervalMS: 1000}, Policy: schedule.Policy{Misfire: "CATCH_UP", Overlap: "PARALLEL", CatchupLimit: 3, CatchupWindowMS: 30000, GraceMS: 1000}, Task: service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"number":9007199254740993}`)}}
|
||||||
|
}
|
||||||
|
func (f *scheduleFixture) create(in schedule.Definition) schedule.View {
|
||||||
|
f.t.Helper()
|
||||||
|
out, err := f.api.Create(f.ctx, f.token, string(executionID(f.t)), schedule.CreateInput{Name: "plan-" + string(executionID(f.t)), Definition: in})
|
||||||
|
if err != nil {
|
||||||
|
f.t.Fatal(err)
|
||||||
|
}
|
||||||
|
return out.Schedule
|
||||||
|
}
|
||||||
|
func (f *scheduleFixture) current(id core.ID) schedule.View {
|
||||||
|
f.t.Helper()
|
||||||
|
out, err := f.api.Get(f.ctx, f.token, id)
|
||||||
|
if err != nil {
|
||||||
|
f.t.Fatal(err)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
func (f *scheduleFixture) occurrences(id core.ID) []schedule.OccurrenceView {
|
||||||
|
f.t.Helper()
|
||||||
|
out, err := f.api.Occurrences(f.ctx, f.token, id, schedule.PageInput{Limit: 100})
|
||||||
|
if err != nil {
|
||||||
|
f.t.Fatal(err)
|
||||||
|
}
|
||||||
|
return out.Items
|
||||||
|
}
|
||||||
|
|
||||||
|
// 等待真实时间形成积压,不将游标改到计划实际生效之前。
|
||||||
|
func (f *scheduleFixture) due(v schedule.View, age time.Duration) {
|
||||||
|
f.t.Helper()
|
||||||
|
if delay := time.Until(v.NextFireAt.Add(age)); delay > 0 {
|
||||||
|
timer := time.NewTimer(delay)
|
||||||
|
defer timer.Stop()
|
||||||
|
select {
|
||||||
|
case <-timer.C:
|
||||||
|
case <-f.ctx.Done():
|
||||||
|
f.t.Fatal(f.ctx.Err())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||||
|
}
|
||||||
|
func (f *scheduleFixture) tick() ScheduleStats {
|
||||||
|
f.t.Helper()
|
||||||
|
out, err := f.store.Tick(f.ctx, 100)
|
||||||
|
if err != nil {
|
||||||
|
f.t.Fatal(err)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
func TestIntegrationSchedules(t *testing.T) {
|
||||||
|
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||||
|
if dsn == "" {
|
||||||
|
t.Skip("requires disposable MySQL")
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
db, err := Open(ctx, dsn, 32, 8)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
if err := Migrate(ctx, db); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fixture := func(t *testing.T) *scheduleFixture { return newScheduleFixture(t, newExecutionFixture(t, ctx, db)) }
|
||||||
|
// 连续发布经过“下一版本尚待初始化”和空版本时,不能丢失积压或重新开始已结束版本。
|
||||||
|
for _, emptyMiddle := range []bool{false, true} {
|
||||||
|
name := "publish_preserves_uninitialized_version"
|
||||||
|
if emptyMiddle {
|
||||||
|
name = "empty_version_then_finished_latest_stays_finished"
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
f := fixture(t)
|
||||||
|
finite := func(delay time.Duration) schedule.Definition {
|
||||||
|
in := f.definition()
|
||||||
|
in.Rule.StartAt = time.Now().UTC().Truncate(time.Microsecond).Add(delay)
|
||||||
|
end := in.Rule.StartAt.Add(time.Millisecond)
|
||||||
|
in.Rule.EndAt = &end
|
||||||
|
return in
|
||||||
|
}
|
||||||
|
publish := func(id core.ID, in schedule.Definition) schedule.View {
|
||||||
|
t.Helper()
|
||||||
|
current := f.current(id)
|
||||||
|
out, err := f.api.Publish(ctx, f.token, id, string(executionID(t)), schedule.PublishInput{ExpectedRevision: current.Revision, Reason: "version boundary regression", Definition: in})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return out.Schedule
|
||||||
|
}
|
||||||
|
waitUntil := func(at time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
timer := time.NewTimer(time.Until(at.Add(10 * time.Millisecond)))
|
||||||
|
defer timer.Stop()
|
||||||
|
select {
|
||||||
|
case <-timer.C:
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal(ctx.Err())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
v := f.create(finite(300 * time.Millisecond))
|
||||||
|
waitUntil(*v.NextFireAt)
|
||||||
|
delay := 300 * time.Millisecond
|
||||||
|
if emptyMiddle {
|
||||||
|
delay = 30 * time.Second
|
||||||
|
}
|
||||||
|
middle := finite(delay)
|
||||||
|
publish(v.ID, middle)
|
||||||
|
var latest schedule.Definition
|
||||||
|
if emptyMiddle {
|
||||||
|
latest = finite(300 * time.Millisecond)
|
||||||
|
publish(v.ID, latest)
|
||||||
|
}
|
||||||
|
f.tick() // 第一版本结束,第二版本尚未计算第一个触发点。
|
||||||
|
pending := f.current(v.ID)
|
||||||
|
if pending.ProcessingVersion != 2 || pending.NextFireAt != nil || pending.CheckAt == nil {
|
||||||
|
t.Fatal("expected pending initialization of version 2")
|
||||||
|
}
|
||||||
|
if !emptyMiddle {
|
||||||
|
waitUntil(middle.Rule.StartAt)
|
||||||
|
latest = finite(300 * time.Millisecond)
|
||||||
|
published := publish(v.ID, latest)
|
||||||
|
if published.ProcessingVersion != 2 {
|
||||||
|
t.Fatal("publishing discarded an uninitialized version")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
waitUntil(latest.Rule.StartAt)
|
||||||
|
for i := 0; i < 3 && f.current(v.ID).CheckAt != nil; i++ {
|
||||||
|
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||||
|
f.tick()
|
||||||
|
}
|
||||||
|
finished := f.current(v.ID)
|
||||||
|
if finished.ProcessingVersion != 3 || finished.CheckAt != nil || finished.NextFireAt != nil {
|
||||||
|
t.Fatal("finished latest version retained scan work")
|
||||||
|
}
|
||||||
|
got := f.occurrences(v.ID)
|
||||||
|
want := map[uint64]time.Time{1: *v.NextFireAt, 3: latest.Rule.StartAt}
|
||||||
|
if !emptyMiddle {
|
||||||
|
want[2] = middle.Rule.StartAt
|
||||||
|
}
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Fatalf("got %d occurrences, want %d", len(got), len(want))
|
||||||
|
}
|
||||||
|
for _, o := range got {
|
||||||
|
at, ok := want[o.ScheduleVersion]
|
||||||
|
if !ok || !o.ScheduledAt.Equal(at) || o.Decision != "FIRED" || o.TaskID == nil {
|
||||||
|
t.Fatal("version occurrence lost or duplicated")
|
||||||
|
}
|
||||||
|
delete(want, o.ScheduleVersion)
|
||||||
|
}
|
||||||
|
f.tick()
|
||||||
|
if f.current(v.ID).Revision != finished.Revision {
|
||||||
|
t.Fatal("finished schedule advanced again")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
t.Run("multi_node_atomic_trigger_and_rate", func(t *testing.T) {
|
||||||
|
f := fixture(t)
|
||||||
|
v := f.create(f.definition())
|
||||||
|
f.due(v, 5*time.Second)
|
||||||
|
other, err := NewScheduleStore(db, executionID(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
failures := make(chan error, 2)
|
||||||
|
for _, s := range []*ScheduleStore{f.store, other} {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(s *ScheduleStore) { defer wg.Done(); _, err := s.Tick(ctx, 100); failures <- err }(s)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
close(failures)
|
||||||
|
for err := range failures {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got := f.occurrences(v.ID)
|
||||||
|
if len(got) != 3 || f.count("pulse_tasks", "owner_kind='schedule' AND owner_id=?", v.ID) != 3 {
|
||||||
|
t.Fatal("duplicate or partial trigger")
|
||||||
|
}
|
||||||
|
current := f.current(v.ID)
|
||||||
|
if current.CheckAt == nil || !current.CheckAt.After(current.UpdatedAt) {
|
||||||
|
t.Fatal("catchup rate not durable")
|
||||||
|
}
|
||||||
|
f.tick()
|
||||||
|
if len(f.occurrences(v.ID)) != 3 {
|
||||||
|
t.Fatal("repeat scan bypassed cooldown")
|
||||||
|
}
|
||||||
|
for _, o := range got {
|
||||||
|
if o.TaskID == nil || o.ScheduleVersion != 1 {
|
||||||
|
t.Fatal("missing ownership")
|
||||||
|
}
|
||||||
|
task := f.get(*o.TaskID)
|
||||||
|
if string(task.Payload) != `{"number":9007199254740993}` || task.AttemptCount != 0 {
|
||||||
|
t.Fatal("template snapshot damaged")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f.exec(`UPDATE orbit_namespaces SET enabled=FALSE WHERE id=?`, f.resources.NamespaceID)
|
||||||
|
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||||
|
f.tick()
|
||||||
|
if f.count("pulse_tasks", "owner_kind='schedule' AND owner_id=?", v.ID) != 3 {
|
||||||
|
t.Fatal("disabled namespace generated tasks")
|
||||||
|
}
|
||||||
|
f.exec(`UPDATE orbit_namespaces SET enabled=TRUE WHERE id=?`, f.resources.NamespaceID)
|
||||||
|
})
|
||||||
|
t.Run("idempotency_pause_scope_and_version", func(t *testing.T) {
|
||||||
|
f := fixture(t)
|
||||||
|
in := schedule.CreateInput{Name: "stable", Definition: f.definition(), Paused: true}
|
||||||
|
key := string(executionID(t))
|
||||||
|
out, err := f.api.Create(ctx, f.token, key, in)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
again, err := f.api.Create(ctx, f.token, key, in)
|
||||||
|
if err != nil || !again.Deduplicated || again.Schedule.ID != out.Schedule.ID {
|
||||||
|
t.Fatal("creation replay")
|
||||||
|
}
|
||||||
|
in.Paused = false
|
||||||
|
_, err = f.api.Create(ctx, f.token, key, in)
|
||||||
|
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
|
||||||
|
v := out.Schedule
|
||||||
|
f.due(v, 4*time.Second)
|
||||||
|
f.tick()
|
||||||
|
if len(f.occurrences(v.ID)) != 0 {
|
||||||
|
t.Fatal("paused schedule fired")
|
||||||
|
}
|
||||||
|
next := f.definition()
|
||||||
|
next.Task.Payload = json.RawMessage(`{"version":2}`)
|
||||||
|
published, err := f.api.Publish(ctx, f.token, v.ID, string(executionID(t)), schedule.PublishInput{ExpectedRevision: v.Revision, Reason: "new template", Definition: next})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if published.Schedule.ProcessingVersion != 1 || published.Schedule.CurrentVersion != 2 {
|
||||||
|
t.Fatal("publish erased old backlog")
|
||||||
|
}
|
||||||
|
old, err := f.api.Version(ctx, f.token, v.ID, 1)
|
||||||
|
if err != nil || string(old.Definition.Task.Payload) != `{"number":9007199254740993}` {
|
||||||
|
t.Fatal("old version changed")
|
||||||
|
}
|
||||||
|
resumeKey := string(executionID(t))
|
||||||
|
resume := schedule.ControlInput{ExpectedRevision: published.Schedule.Revision, Reason: "resume work"}
|
||||||
|
resumed, err := f.api.Control(ctx, f.token, v.ID, resumeKey, resume, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
repeated, err := f.api.Control(ctx, f.token, v.ID, resumeKey, resume, false)
|
||||||
|
if err != nil || !repeated.Deduplicated || repeated.Schedule.Revision != resumed.Schedule.Revision {
|
||||||
|
t.Fatal("control replay")
|
||||||
|
}
|
||||||
|
f.tick()
|
||||||
|
for _, o := range f.occurrences(v.ID) {
|
||||||
|
if o.ScheduleVersion != 1 || !o.ScheduledAt.Before(published.Schedule.Current.EffectiveAt) {
|
||||||
|
t.Fatal("historical version boundary lost")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i := 0; i < 3 && f.current(v.ID).ProcessingVersion == 1; i++ {
|
||||||
|
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||||
|
f.tick()
|
||||||
|
}
|
||||||
|
if f.current(v.ID).ProcessingVersion != 2 {
|
||||||
|
t.Fatal("old backlog never advanced to published version")
|
||||||
|
}
|
||||||
|
for _, o := range f.occurrences(v.ID) {
|
||||||
|
if o.ScheduleVersion == 1 && !o.ScheduledAt.Before(published.Schedule.Current.EffectiveAt) {
|
||||||
|
t.Fatal("old version crossed effective boundary")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 另一命名空间知道 ID 也不可读或管理。
|
||||||
|
stranger := fixture(t)
|
||||||
|
_, err = stranger.api.Get(ctx, stranger.token, v.ID)
|
||||||
|
expectWorkerCode(t, err, "NOT_FOUND")
|
||||||
|
raw, _ := json.Marshal(auth.Permissions{Actions: []string{"schedules:read", "schedules:manage"}, Queues: []string{string(executionID(t))}})
|
||||||
|
f.exec(`UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?`, string(raw), f.resources.NamespaceID, f.resources.CredentialID)
|
||||||
|
_, err = f.api.Get(ctx, f.token, v.ID)
|
||||||
|
expectWorkerCode(t, err, "NOT_FOUND")
|
||||||
|
})
|
||||||
|
t.Run("persisted_skip_and_coalesce_ranges", func(t *testing.T) {
|
||||||
|
f := fixture(t)
|
||||||
|
in := f.definition()
|
||||||
|
in.Policy.CatchupWindowMS = 2000
|
||||||
|
in.Policy.Misfire = "COALESCE"
|
||||||
|
coalesce := f.create(in)
|
||||||
|
in.Policy.Misfire = "SKIP"
|
||||||
|
skip := f.create(in)
|
||||||
|
f.due(coalesce, 4*time.Second)
|
||||||
|
f.tick()
|
||||||
|
for _, v := range []schedule.View{coalesce, skip} {
|
||||||
|
items := f.occurrences(v.ID)
|
||||||
|
skipped, merged := false, false
|
||||||
|
for _, o := range items {
|
||||||
|
if o.Decision == "SKIPPED" {
|
||||||
|
skipped = true
|
||||||
|
if o.TaskID != nil || o.CoveredUntil == nil || !o.CoveredUntil.After(o.ScheduledAt) {
|
||||||
|
t.Fatal("invalid skipped range")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if o.Decision == "COALESCED" {
|
||||||
|
merged = true
|
||||||
|
if o.TaskID == nil || o.CoveredUntil == nil {
|
||||||
|
t.Fatal("coalesced range lost task")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !skipped || (v.ID == coalesce.ID && !merged) {
|
||||||
|
t.Fatal("misfire decisions not persisted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("overlap_serial_unknown_and_waiting_cancel", func(t *testing.T) {
|
||||||
|
f := fixture(t)
|
||||||
|
in := f.definition()
|
||||||
|
in.Policy.Overlap = "SERIAL"
|
||||||
|
v := f.create(in)
|
||||||
|
f.due(v, 5*time.Second)
|
||||||
|
f.tick()
|
||||||
|
items := f.occurrences(v.ID)
|
||||||
|
if len(items) != 1 || items[0].TaskID == nil {
|
||||||
|
t.Fatal("serial did not start one")
|
||||||
|
}
|
||||||
|
first := f.current(v.ID)
|
||||||
|
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||||
|
f.tick()
|
||||||
|
if len(f.occurrences(v.ID)) != 1 || !f.current(v.ID).NextFireAt.Equal(*first.NextFireAt) {
|
||||||
|
t.Fatal("serial skipped waiting task")
|
||||||
|
}
|
||||||
|
task := f.get(*items[0].TaskID)
|
||||||
|
if _, err := f.tasks.CancelTask(ctx, f.token, task.ID, string(executionID(t)), service.CancelInput{ExpectedRevision: task.Revision, Reason: "cancel pending scheduled task"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||||
|
f.tick()
|
||||||
|
if len(f.occurrences(v.ID)) != 2 {
|
||||||
|
t.Fatal("serial did not continue after terminal task")
|
||||||
|
}
|
||||||
|
retryFinish(t, f.executionFixture, retryStart(t, f.executionFixture), "UNKNOWN", false)
|
||||||
|
waiting := f.current(v.ID)
|
||||||
|
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||||
|
f.tick()
|
||||||
|
if len(f.occurrences(v.ID)) != 2 || !f.current(v.ID).NextFireAt.Equal(*waiting.NextFireAt) {
|
||||||
|
t.Fatal("unknown outcome did not block serial schedule")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("transaction_rollback_and_read_pagination", func(t *testing.T) {
|
||||||
|
f := fixture(t)
|
||||||
|
v := f.create(f.definition())
|
||||||
|
f.due(v, 6*time.Second)
|
||||||
|
prior := f.current(v.ID)
|
||||||
|
f.exec(`CREATE TRIGGER orbit_test_schedule_event BEFORE INSERT ON pulse_schedule_occurrences FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='controlled trigger failure'`)
|
||||||
|
_, err := f.store.Tick(ctx, 100)
|
||||||
|
f.exec(`DROP TRIGGER orbit_test_schedule_event`)
|
||||||
|
if err == nil || len(f.occurrences(v.ID)) != 0 || f.count("pulse_tasks", "owner_id=?", v.ID) != 0 || !f.current(v.ID).NextFireAt.Equal(*prior.NextFireAt) {
|
||||||
|
t.Fatal("partial schedule transaction")
|
||||||
|
}
|
||||||
|
f.tick()
|
||||||
|
page, err := f.api.Occurrences(ctx, f.token, v.ID, schedule.PageInput{Limit: 1})
|
||||||
|
if err != nil || len(page.Items) != 1 || page.NextCursor == "" {
|
||||||
|
t.Fatal("missing page")
|
||||||
|
}
|
||||||
|
second, err := f.api.Occurrences(ctx, f.token, v.ID, schedule.PageInput{Limit: 1, Cursor: page.NextCursor})
|
||||||
|
if err != nil || len(second.Items) != 1 || second.Items[0].ID == page.Items[0].ID {
|
||||||
|
t.Fatal("duplicate page")
|
||||||
|
}
|
||||||
|
another := f.create(f.definition())
|
||||||
|
_, err = f.api.Occurrences(ctx, f.token, another.ID, schedule.PageInput{Cursor: page.NextCursor})
|
||||||
|
expectWorkerCode(t, err, "INVALID_ARGUMENT")
|
||||||
|
})
|
||||||
|
}
|
||||||
294
internal/storage/mysql/schedule_store.go
Normal file
294
internal/storage/mysql/schedule_store.go
Normal file
@@ -0,0 +1,294 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/jsonwire"
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
|
||||||
|
driver "github.com/go-sql-driver/mysql"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ScheduleStore 管理计划控制与扫描;共享任务存储的事务认证,不在事务外推进游标。
|
||||||
|
type ScheduleStore struct {
|
||||||
|
*TaskStore
|
||||||
|
node core.ID
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewScheduleStore(db *sql.DB, node core.ID) (*ScheduleStore, error) {
|
||||||
|
if node.Validate() != nil {
|
||||||
|
return nil, errors.New("scheduler node id required")
|
||||||
|
}
|
||||||
|
return &ScheduleStore{TaskStore: NewTaskStore(db), node: node}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ schedule.Store = (*ScheduleStore)(nil)
|
||||||
|
|
||||||
|
const scheduleColumns = `id,namespace_id,queue_id,name,revision,current_version,processing_version,paused,next_fire_at,check_at,created_at,updated_at`
|
||||||
|
|
||||||
|
func scanSchedule(row scanner) (schedule.Summary, error) {
|
||||||
|
var v schedule.Summary
|
||||||
|
err := row.Scan(&v.ID, &v.NamespaceID, &v.QueueID, &v.Name, &v.Revision, &v.CurrentVersion, &v.ProcessingVersion, &v.Paused, &v.NextFireAt, &v.CheckAt, &v.CreatedAt, &v.UpdatedAt)
|
||||||
|
return v, err
|
||||||
|
}
|
||||||
|
func readSchedule(ctx context.Context, tx *sql.Tx, ns, id core.ID, lock bool) (schedule.Summary, error) {
|
||||||
|
q := `SELECT ` + scheduleColumns + ` FROM pulse_schedules WHERE namespace_id=? AND id=? AND queue_id IS NOT NULL`
|
||||||
|
if lock {
|
||||||
|
q += ` FOR UPDATE`
|
||||||
|
}
|
||||||
|
v, err := scanSchedule(tx.QueryRowContext(ctx, q, ns, id))
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
err = fault.NotFound
|
||||||
|
}
|
||||||
|
return v, err
|
||||||
|
}
|
||||||
|
func readScheduleVersion(ctx context.Context, tx *sql.Tx, ns, id core.ID, version uint64) (schedule.VersionView, error) {
|
||||||
|
var v schedule.VersionView
|
||||||
|
var rule, task []byte
|
||||||
|
err := tx.QueryRowContext(ctx, `SELECT version,effective_at,specification,task_template,misfire_policy,overlap_policy,catchup_limit,catchup_window_ms,grace_ms FROM pulse_schedule_versions WHERE namespace_id=? AND schedule_id=? AND version=?`, ns, id, version).Scan(&v.Version, &v.EffectiveAt, &rule, &task, &v.Definition.Policy.Misfire, &v.Definition.Policy.Overlap, &v.Definition.Policy.CatchupLimit, &v.Definition.Policy.CatchupWindowMS, &v.Definition.Policy.GraceMS)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return v, fault.NotFound
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return v, err
|
||||||
|
}
|
||||||
|
if err = json.Unmarshal(rule, &v.Definition.Rule); err != nil {
|
||||||
|
return v, err
|
||||||
|
}
|
||||||
|
err = json.Unmarshal(task, &v.Definition.Task)
|
||||||
|
return v, err
|
||||||
|
}
|
||||||
|
func insertScheduleVersion(ctx context.Context, tx *sql.Tx, ns, id core.ID, v schedule.VersionView) error {
|
||||||
|
identity, err := core.NewID()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rule, err := json.Marshal(v.Definition.Rule)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
task, err := jsonwire.Marshal(v.Definition.Task)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p := v.Definition.Policy
|
||||||
|
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_schedule_versions(namespace_id,id,schedule_id,version,effective_at,timezone,specification,task_template,misfire_policy,overlap_policy,catchup_limit,catchup_window_ms,grace_ms) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)`, ns, identity, id, v.Version, v.EffectiveAt, v.Definition.Rule.Timezone, string(rule), string(task), p.Misfire, p.Overlap, p.CatchupLimit, p.CatchupWindowMS, p.GraceMS)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
func scheduleView(ctx context.Context, tx *sql.Tx, ns, id core.ID) (schedule.View, error) {
|
||||||
|
summary, err := readSchedule(ctx, tx, ns, id, false)
|
||||||
|
if err != nil {
|
||||||
|
return schedule.View{}, err
|
||||||
|
}
|
||||||
|
current, err := readScheduleVersion(ctx, tx, ns, id, summary.CurrentVersion)
|
||||||
|
return schedule.View{Summary: summary, Current: current}, err
|
||||||
|
}
|
||||||
|
func authorizeSchedule(ctx context.Context, tx *sql.Tx, ns, id core.ID, p auth.Permissions) (schedule.Summary, error) {
|
||||||
|
v, err := readSchedule(ctx, tx, ns, id, false)
|
||||||
|
if err != nil {
|
||||||
|
return v, err
|
||||||
|
}
|
||||||
|
if !p.AllowsQueue(v.QueueID) {
|
||||||
|
return v, fault.NotFound
|
||||||
|
}
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
func (s *ScheduleStore) Create(ctx context.Context, t auth.Token, key string, in schedule.CreateInput) (schedule.Result, error) {
|
||||||
|
var out schedule.Result
|
||||||
|
err := s.transaction(ctx, t, "schedules:manage", func(tx *sql.Tx, p auth.Permissions) error {
|
||||||
|
if !p.Allows("tasks:submit") || !p.AllowsQueue(in.Definition.Task.QueueID) {
|
||||||
|
return fault.Forbidden
|
||||||
|
}
|
||||||
|
id, err := core.NewID()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "schedule.create", "", key, "schedule", id, in)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if cached != nil {
|
||||||
|
if err := json.Unmarshal(cached, &out); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out.Deduplicated = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var queue core.ID
|
||||||
|
err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR SHARE`, t.NamespaceID, in.Definition.Task.QueueID).Scan(&queue)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return fault.NotFound
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
now, err := dbNow(ctx, tx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c, err := schedule.Compile(in.Definition.Rule)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
next, err := c.AtOrAfter(now)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if next == nil {
|
||||||
|
return fault.Invalid("schedule has no future occurrence")
|
||||||
|
}
|
||||||
|
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_schedules(namespace_id,id,queue_id,name,current_version,processing_version,paused,next_fire_at,check_at,created_at,updated_at) VALUES (?,?,?,?,1,1,?,?,?,?,?)`, t.NamespaceID, id, queue, in.Name, in.Paused, next, next, now, now)
|
||||||
|
var me *driver.MySQLError
|
||||||
|
if errors.As(err, &me) && me.Number == 1062 {
|
||||||
|
return fault.Conflict
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := insertScheduleVersion(ctx, tx, t.NamespaceID, id, schedule.VersionView{Version: 1, EffectiveAt: now, Definition: in.Definition}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out.Schedule, err = scheduleView(ctx, tx, t.NamespaceID, id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := emitEvent(ctx, tx, t.NamespaceID, id, 1, "schedule", "schedule.created", map[string]any{"queue_id": queue, "version": "1", "paused": in.Paused}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := appendAudit(ctx, tx, t, identity, id, "schedule", "schedule.create", "create schedule", nil, out.Schedule.Summary); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
func (s *ScheduleStore) Get(ctx context.Context, t auth.Token, id core.ID) (schedule.View, error) {
|
||||||
|
var out schedule.View
|
||||||
|
err := s.transaction(ctx, t, "schedules:read", func(tx *sql.Tx, p auth.Permissions) error {
|
||||||
|
if _, err := authorizeSchedule(ctx, tx, t.NamespaceID, id, p); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
out, err = scheduleView(ctx, tx, t.NamespaceID, id)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
func (s *ScheduleStore) Publish(ctx context.Context, t auth.Token, id core.ID, key string, in schedule.PublishInput) (schedule.Result, error) {
|
||||||
|
return s.mutate(ctx, t, id, key, "schedule.publish", in, in.ExpectedRevision, in.Reason, func(tx *sql.Tx, v schedule.Summary, now time.Time) error {
|
||||||
|
if in.Definition.Task.QueueID != v.QueueID {
|
||||||
|
return fault.Invalid("schedule queue cannot change")
|
||||||
|
}
|
||||||
|
last, err := readScheduleVersion(ctx, tx, t.NamespaceID, id, v.CurrentVersion)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !now.After(last.EffectiveAt) {
|
||||||
|
return fault.Unavailable
|
||||||
|
}
|
||||||
|
if err := insertScheduleVersion(ctx, tx, t.NamespaceID, id, schedule.VersionView{Version: v.CurrentVersion + 1, EffectiveAt: now, Definition: in.Definition}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
cursor, next := v.ProcessingVersion, v.NextFireAt
|
||||||
|
// nil 游标也可能表示刚进入最新版本、尚未初始化;只有没有扫描工作才表示结束。
|
||||||
|
if (next == nil && cursor == v.CurrentVersion && v.CheckAt == nil) || (next != nil && !next.Before(now)) {
|
||||||
|
c, err := schedule.Compile(in.Definition.Rule)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
next, err = c.AtOrAfter(now)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
cursor = v.CurrentVersion + 1
|
||||||
|
}
|
||||||
|
check := &now
|
||||||
|
if next != nil && next.After(now) {
|
||||||
|
check = next
|
||||||
|
} else if next == nil && cursor == v.CurrentVersion+1 {
|
||||||
|
check = nil
|
||||||
|
}
|
||||||
|
_, err = tx.ExecContext(ctx, `UPDATE pulse_schedules SET current_version=current_version+1,processing_version=?,next_fire_at=?,check_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, cursor, next, check, now, t.NamespaceID, id)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
func (s *ScheduleStore) Control(ctx context.Context, t auth.Token, id core.ID, key string, in schedule.ControlInput, paused bool) (schedule.Result, error) {
|
||||||
|
operation := "schedule.resume"
|
||||||
|
if paused {
|
||||||
|
operation = "schedule.pause"
|
||||||
|
}
|
||||||
|
return s.mutate(ctx, t, id, key, operation, in, in.ExpectedRevision, in.Reason, func(tx *sql.Tx, v schedule.Summary, now time.Time) error {
|
||||||
|
if v.Paused == paused {
|
||||||
|
return fault.Conflict
|
||||||
|
}
|
||||||
|
_, err := tx.ExecContext(ctx, `UPDATE pulse_schedules SET paused=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, paused, now, t.NamespaceID, id)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
func (s *ScheduleStore) mutate(ctx context.Context, t auth.Token, id core.ID, key, operation string, input any, revision uint64, reason string, change func(*sql.Tx, schedule.Summary, time.Time) error) (schedule.Result, error) {
|
||||||
|
var out schedule.Result
|
||||||
|
err := s.transaction(ctx, t, "schedules:manage", func(tx *sql.Tx, p auth.Permissions) error {
|
||||||
|
if operation == "schedule.publish" && !p.Allows("tasks:submit") {
|
||||||
|
return fault.Forbidden
|
||||||
|
}
|
||||||
|
if _, err := authorizeSchedule(ctx, tx, t.NamespaceID, id, p); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, operation, string(id), key, "schedule", id, input)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if cached != nil {
|
||||||
|
if err := json.Unmarshal(cached, &out); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out.Deduplicated = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
v, err := readSchedule(ctx, tx, t.NamespaceID, id, true)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if v.Revision != revision {
|
||||||
|
return fault.Conflict
|
||||||
|
}
|
||||||
|
now, err := dbNow(ctx, tx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := change(tx, v, now); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out.Schedule, err = scheduleView(ctx, tx, t.NamespaceID, id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := appendAudit(ctx, tx, t, identity, id, "schedule", operation, reason, v, out.Schedule.Summary); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := emitEvent(ctx, tx, t.NamespaceID, id, out.Schedule.Revision, "schedule", operation, map[string]any{"version": fmt.Sprint(out.Schedule.CurrentVersion), "paused": out.Schedule.Paused}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
func (s *ScheduleStore) Version(ctx context.Context, t auth.Token, id core.ID, version uint64) (schedule.VersionView, error) {
|
||||||
|
var out schedule.VersionView
|
||||||
|
err := s.transaction(ctx, t, "schedules:read", func(tx *sql.Tx, p auth.Permissions) error {
|
||||||
|
if _, err := authorizeSchedule(ctx, tx, t.NamespaceID, id, p); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
out, err = readScheduleVersion(ctx, tx, t.NamespaceID, id, version)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
225
internal/storage/mysql/schedule_tick.go
Normal file
225
internal/storage/mysql/schedule_tick.go
Normal file
@@ -0,0 +1,225 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ScheduleStats 是本次已确认提交的批次计数,事务失败不计作成功。
|
||||||
|
type ScheduleStats struct {
|
||||||
|
Examined int // Examined 已检查的候选计划数。
|
||||||
|
Fired int // Fired 本轮创建的任务数,合并范围计一个。
|
||||||
|
Skipped int // Skipped 本轮保存的跳过范围数,不是被覆盖触发点总数。
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tick 按持久化 check_at 扫描,一次最多 100 个计划,每个计划独立短事务。
|
||||||
|
func (s *ScheduleStore) Tick(ctx context.Context, limit int) (ScheduleStats, error) {
|
||||||
|
var out ScheduleStats
|
||||||
|
if limit < 1 || limit > 100 {
|
||||||
|
return out, fault.Invalid("schedule batch limit must be 1 to 100")
|
||||||
|
}
|
||||||
|
type candidate struct{ ns, id, queue core.ID }
|
||||||
|
candidates := []candidate{}
|
||||||
|
rows, err := s.db.QueryContext(ctx, `SELECT s.namespace_id,s.id,s.queue_id FROM pulse_schedules s JOIN orbit_namespaces n ON n.id=s.namespace_id JOIN orbit_applications a ON a.id=n.application_id WHERE s.paused=FALSE AND s.check_at<=UTC_TIMESTAMP(6) AND n.enabled=TRUE AND a.enabled=TRUE ORDER BY s.check_at,s.namespace_id,s.id LIMIT ?`, limit)
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
var c candidate
|
||||||
|
if err := rows.Scan(&c.ns, &c.id, &c.queue); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
candidates = append(candidates, c)
|
||||||
|
}
|
||||||
|
err = rows.Err()
|
||||||
|
rows.Close()
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
var failures []error
|
||||||
|
for _, c := range candidates {
|
||||||
|
out.Examined++
|
||||||
|
fired, skipped, err := s.tickOne(ctx, c.ns, c.id, c.queue)
|
||||||
|
if err != nil {
|
||||||
|
failures = append(failures, err)
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out.Fired += fired
|
||||||
|
out.Skipped += skipped
|
||||||
|
}
|
||||||
|
return out, errors.Join(failures...)
|
||||||
|
}
|
||||||
|
func (s *ScheduleStore) tickOne(ctx context.Context, ns, id, queue core.ID) (int, int, error) {
|
||||||
|
fired, skipped := 0, 0
|
||||||
|
err := WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
||||||
|
var allowed core.ID
|
||||||
|
err := tx.QueryRowContext(ctx, `SELECT n.id FROM orbit_namespaces n JOIN orbit_applications a ON a.id=n.application_id WHERE n.id=? AND n.enabled=TRUE AND a.enabled=TRUE FOR SHARE`, ns).Scan(&allowed)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// 和任务执行一致先锁队列,再锁计划;其他扫描节点不等待同一计划所在队列。
|
||||||
|
err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, ns, queue).Scan(&allowed)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
v, err := readSchedule(ctx, tx, ns, id, true)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
now, err := dbNow(ctx, tx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if v.Paused || v.CheckAt == nil || v.CheckAt.After(now) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if v.QueueID != queue {
|
||||||
|
return errors.New("schedule queue changed")
|
||||||
|
}
|
||||||
|
cursor, next := v.ProcessingVersion, v.NextFireAt
|
||||||
|
needsInitialization := false
|
||||||
|
// 单批最多跨过 16 个空历史版本,避免长事务;下次从已保存版本继续。
|
||||||
|
for n := 0; n < 16; n++ {
|
||||||
|
version, err := readScheduleVersion(ctx, tx, ns, id, cursor)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c, err := schedule.Compile(version.Definition.Rule)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var before *time.Time
|
||||||
|
if cursor < v.CurrentVersion {
|
||||||
|
newer, err := readScheduleVersion(ctx, tx, ns, id, cursor+1)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
before = &newer.EffectiveAt
|
||||||
|
}
|
||||||
|
needsInitialization = false
|
||||||
|
if next == nil {
|
||||||
|
next, err = c.AtOrAfter(version.EffectiveAt)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if next != nil && next.Before(version.EffectiveAt) {
|
||||||
|
return errors.New("schedule cursor predates its version")
|
||||||
|
}
|
||||||
|
if next == nil || (before != nil && !next.Before(*before)) {
|
||||||
|
next = nil
|
||||||
|
if cursor < v.CurrentVersion {
|
||||||
|
cursor++
|
||||||
|
needsInitialization = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if next.After(now) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
var active bool
|
||||||
|
if version.Definition.Policy.Overlap != "PARALLEL" {
|
||||||
|
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pulse_tasks WHERE namespace_id=? AND owner_kind='schedule' AND owner_id=? AND state IN ('WAITING','LEASED','RUNNING','NEEDS_REVIEW'))`, ns, id).Scan(&active); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
policy := version.Definition.Policy
|
||||||
|
// 大参数也要保持事务有界;每批复制的业务参数最多约 1 MiB。
|
||||||
|
if size := len(version.Definition.Task.Payload); size > 0 {
|
||||||
|
if max := uint32((1 << 20) / size); max < policy.CatchupLimit {
|
||||||
|
policy.CatchupLimit = max
|
||||||
|
}
|
||||||
|
}
|
||||||
|
batch, err := schedule.Plan(c, policy, *next, now, before, active)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, decision := range batch.Decisions {
|
||||||
|
if err := s.saveOccurrence(ctx, tx, v, version, decision, now); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if decision.Kind == "SKIPPED" {
|
||||||
|
skipped++
|
||||||
|
} else {
|
||||||
|
fired++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
next = batch.Next
|
||||||
|
if next == nil && cursor < v.CurrentVersion {
|
||||||
|
cursor++
|
||||||
|
needsInitialization = true
|
||||||
|
}
|
||||||
|
break // 每计划每批只创建一个版本的有限任务,跨版本也不绕过速率上限。
|
||||||
|
}
|
||||||
|
var check *time.Time
|
||||||
|
if next != nil || cursor < v.CurrentVersion || needsInitialization {
|
||||||
|
t := now.Add(time.Second)
|
||||||
|
check = &t
|
||||||
|
if next != nil && next.After(t) {
|
||||||
|
check = next
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result, err := tx.ExecContext(ctx, `UPDATE pulse_schedules SET processing_version=?,next_fire_at=?,check_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, cursor, next, check, now, ns, id, v.Revision)
|
||||||
|
if err := exactlyOne(result, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return emitEvent(ctx, tx, ns, id, v.Revision+1, "schedule", "schedule.advanced", map[string]any{"node_id": s.node, "processing_version": fmt.Sprint(cursor), "next_fire_at": next, "fired": fired, "skipped": skipped})
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
return fired, skipped, nil
|
||||||
|
}
|
||||||
|
func (s *ScheduleStore) saveOccurrence(ctx context.Context, tx *sql.Tx, v schedule.Summary, version schedule.VersionView, d schedule.Decision, now time.Time) error {
|
||||||
|
id, err := core.NewID()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var task *core.ID
|
||||||
|
if d.Kind != "SKIPPED" {
|
||||||
|
value, err := core.NewID()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
task = &value
|
||||||
|
in := version.Definition.Task
|
||||||
|
if in.QueueID != v.QueueID {
|
||||||
|
return errors.New("schedule template queue mismatch")
|
||||||
|
}
|
||||||
|
policy, err := json.Marshal(in.Retry)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_tasks(namespace_id,id,queue_id,handler,handler_version,payload_version,payload,policy_snapshot,recovery_safety,business_key,owner_kind,owner_id,available_at,deadline_at,attempt_timeout_ms,max_attempts,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,NULLIF(?,''),'schedule',?,?,?,?,?,?,?)`, v.NamespaceID, value, v.QueueID, in.Handler, in.HandlerVersion, in.PayloadVersion, string(in.Payload), string(policy), in.RecoverySafety, in.BusinessKey, v.ID, now, now.Add(time.Duration(in.TaskTimeoutMS)*time.Millisecond), in.Retry.AttemptTimeoutMS, in.Retry.MaxAttempts, now, now)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := emitEvent(ctx, tx, v.NamespaceID, value, 1, "task", "task.accepted", map[string]any{"schedule_id": v.ID, "occurrence_id": id, "schedule_version": fmt.Sprint(version.Version), "scheduled_at": d.ScheduledAt, "node_id": s.node}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_schedule_occurrences(namespace_id,id,schedule_id,schedule_version,scheduled_at,decision,task_id,covered_until,reason,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)`, v.NamespaceID, id, v.ID, version.Version, d.ScheduledAt, d.Kind, task, d.CoveredUntil, d.Reason, now, now)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return emitEvent(ctx, tx, v.NamespaceID, id, 1, "schedule_occurrence", "schedule.decided", map[string]any{"schedule_id": v.ID, "version": fmt.Sprint(version.Version), "scheduled_at": d.ScheduledAt, "decision": d.Kind, "covered_until": d.CoveredUntil, "reason": d.Reason, "task_id": task, "node_id": s.node})
|
||||||
|
}
|
||||||
@@ -354,7 +354,7 @@ func (s *TaskStore) CancelTask(ctx context.Context, token auth.Token, id core.ID
|
|||||||
if err := tx.QueryRowContext(ctx, "SELECT owner_kind FROM pulse_tasks WHERE namespace_id=? AND id=?", token.NamespaceID, id).Scan(&owner); err != nil {
|
if err := tx.QueryRowContext(ctx, "SELECT owner_kind FROM pulse_tasks WHERE namespace_id=? AND id=?", token.NamespaceID, id).Scan(&owner); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if owner.Valid {
|
if owner.Valid && owner.String != "schedule" {
|
||||||
return fault.Conflict
|
return fault.Conflict
|
||||||
}
|
}
|
||||||
now, err := dbNow(ctx, tx)
|
now, err := dbNow(ctx, tx)
|
||||||
|
|||||||
27
internal/timezone/LICENSE
Normal file
27
internal/timezone/LICENSE
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
Copyright 2009 The Go Authors.
|
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without
|
||||||
|
modification, are permitted provided that the following conditions are
|
||||||
|
met:
|
||||||
|
|
||||||
|
* Redistributions of source code must retain the above copyright
|
||||||
|
notice, this list of conditions and the following disclaimer.
|
||||||
|
* Redistributions in binary form must reproduce the above
|
||||||
|
copyright notice, this list of conditions and the following disclaimer
|
||||||
|
in the documentation and/or other materials provided with the
|
||||||
|
distribution.
|
||||||
|
* Neither the name of Google LLC nor the names of its
|
||||||
|
contributors may be used to endorse or promote products derived from
|
||||||
|
this software without specific prior written permission.
|
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
61
internal/timezone/zone.go
Normal file
61
internal/timezone/zone.go
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
// Package timezone 提供随计划冻结的时区快照,禁止读取宿主机或 ZONEINFO 环境配置。
|
||||||
|
package timezone
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"bytes"
|
||||||
|
_ "embed"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Version 标识 Go 1.27.1 附带的 IANA 快照及文件摘要;升级时须保留旧版本可读能力。
|
||||||
|
const Version = "go1.27.1-b2d18a7c8fa8"
|
||||||
|
|
||||||
|
//go:embed zoneinfo.zip
|
||||||
|
var data []byte
|
||||||
|
var zones sync.Map
|
||||||
|
var archive = sync.OnceValues(func() (*zip.Reader, error) { return zip.NewReader(bytes.NewReader(data), int64(len(data))) })
|
||||||
|
|
||||||
|
// Load 仅接受精确的内置名称,返回只读 Location;不接受 Local 或文件系统路径。
|
||||||
|
func Load(name, version string) (*time.Location, error) {
|
||||||
|
if version != Version || name == "" || name == "Local" || len(name) > 64 {
|
||||||
|
return nil, errors.New("unsupported frozen timezone version or name")
|
||||||
|
}
|
||||||
|
if name == "UTC" {
|
||||||
|
return time.UTC, nil
|
||||||
|
}
|
||||||
|
if v, ok := zones.Load(name); ok {
|
||||||
|
return v.(*time.Location), nil
|
||||||
|
}
|
||||||
|
a, err := archive()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, entry := range a.File {
|
||||||
|
if entry.Name != name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
r, err := entry.Open()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
b, err := io.ReadAll(io.LimitReader(r, 1<<20))
|
||||||
|
closeErr := r.Close()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if closeErr != nil {
|
||||||
|
return nil, closeErr
|
||||||
|
}
|
||||||
|
loc, err := time.LoadLocationFromTZData(name, b)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
actual, _ := zones.LoadOrStore(name, loc)
|
||||||
|
return actual.(*time.Location), nil
|
||||||
|
}
|
||||||
|
return nil, errors.New("unknown IANA timezone")
|
||||||
|
}
|
||||||
38
internal/timezone/zone_test.go
Normal file
38
internal/timezone/zone_test.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package timezone
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFrozenSnapshot(t *testing.T) {
|
||||||
|
if got := fmt.Sprintf("%x", sha256.Sum256(data)); got != "b2d18a7c8fa8142097a48c99609fb3c92db5ee98bc740294e57eab8ae9f94779" {
|
||||||
|
t.Fatal("snapshot changed without a new version")
|
||||||
|
}
|
||||||
|
t.Setenv("ZONEINFO", t.TempDir())
|
||||||
|
for _, name := range []string{"UTC", "Asia/Shanghai", "America/New_York", "Australia/Lord_Howe", "Pacific/Apia"} {
|
||||||
|
zone, err := Load(name, Version)
|
||||||
|
if err != nil || zone == nil {
|
||||||
|
t.Fatalf("load %s: %v", name, err)
|
||||||
|
}
|
||||||
|
again, err := Load(name, Version)
|
||||||
|
if err != nil || again != zone {
|
||||||
|
t.Fatal("cache changed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
zone, _ := Load("Asia/Shanghai", Version)
|
||||||
|
_, offset := time.Date(2026, 1, 1, 0, 0, 0, 0, zone).Zone()
|
||||||
|
if offset != 8*3600 {
|
||||||
|
t.Fatal("wrong frozen zone")
|
||||||
|
}
|
||||||
|
for _, name := range []string{"", "Local", "../UTC", "/etc/localtime"} {
|
||||||
|
if _, err := Load(name, Version); err == nil {
|
||||||
|
t.Fatal("accepted external timezone")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := Load("UTC", "unavailable-version"); err == nil {
|
||||||
|
t.Fatal("silently upgraded frozen version")
|
||||||
|
}
|
||||||
|
}
|
||||||
BIN
internal/timezone/zoneinfo.zip
Normal file
BIN
internal/timezone/zoneinfo.zip
Normal file
Binary file not shown.
106
internal/transport/httpserver/schedules.go
Normal file
106
internal/transport/httpserver/schedules.go
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
package httpserver
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RegisterSchedules 接入同一 HTTP 信封、认证、大小和严格 JSON 边界。
|
||||||
|
func RegisterSchedules(server *http.Server, s *schedule.Service) {
|
||||||
|
mux := server.Handler.(*http.ServeMux)
|
||||||
|
mux.HandleFunc("POST /v1/schedules", wrap(func(r *http.Request, t auth.Token) (any, int, error) {
|
||||||
|
var in schedule.CreateInput
|
||||||
|
if err := decodeBody(r, &in); err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
key, err := idempotencyKey(r)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
out, err := s.Create(r.Context(), t, key, in)
|
||||||
|
return out, createdStatus(out.Deduplicated), err
|
||||||
|
}))
|
||||||
|
mux.HandleFunc("GET /v1/schedules", wrap(func(r *http.Request, t auth.Token) (any, int, error) {
|
||||||
|
q, err := url.ParseQuery(r.URL.RawQuery)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, fault.Invalid("invalid query encoding")
|
||||||
|
}
|
||||||
|
for k, v := range q {
|
||||||
|
if len(v) != 1 || (k != "queue_id" && k != "limit" && k != "cursor") {
|
||||||
|
return nil, 0, fault.Invalid("invalid schedule query")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var in schedule.PageInput
|
||||||
|
in.Cursor = q.Get("cursor")
|
||||||
|
if q.Has("limit") {
|
||||||
|
in.Limit, err = strconv.Atoi(q.Get("limit"))
|
||||||
|
if err != nil || in.Limit < 1 {
|
||||||
|
return nil, 0, fault.Invalid("invalid limit")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out, err := s.List(r.Context(), t, core.ID(q.Get("queue_id")), in)
|
||||||
|
return out, 200, err
|
||||||
|
}))
|
||||||
|
mux.HandleFunc("GET /v1/schedules/{id}", wrap(func(r *http.Request, t auth.Token) (any, int, error) {
|
||||||
|
if _, err := historyQuery(r, false); err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
out, err := s.Get(r.Context(), t, core.ID(r.PathValue("id")))
|
||||||
|
return out, 200, err
|
||||||
|
}))
|
||||||
|
mux.HandleFunc("PATCH /v1/schedules/{id}", wrap(func(r *http.Request, t auth.Token) (any, int, error) {
|
||||||
|
var in schedule.PublishInput
|
||||||
|
if err := decodeBody(r, &in); err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
key, err := idempotencyKey(r)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
out, err := s.Publish(r.Context(), t, core.ID(r.PathValue("id")), key, in)
|
||||||
|
return out, 200, err
|
||||||
|
}))
|
||||||
|
mux.HandleFunc("POST /v1/schedules/{command}", wrap(func(r *http.Request, t auth.Token) (any, int, error) {
|
||||||
|
id, command, ok := strings.Cut(r.PathValue("command"), ":")
|
||||||
|
if !ok || (command != "pause" && command != "resume") {
|
||||||
|
return nil, 0, fault.NotFound
|
||||||
|
}
|
||||||
|
var in schedule.ControlInput
|
||||||
|
if err := decodeBody(r, &in); err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
key, err := idempotencyKey(r)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
out, err := s.Control(r.Context(), t, core.ID(id), key, in, command == "pause")
|
||||||
|
return out, 200, err
|
||||||
|
}))
|
||||||
|
mux.HandleFunc("GET /v1/schedules/{id}/versions/{version}", wrap(func(r *http.Request, t auth.Token) (any, int, error) {
|
||||||
|
if _, err := historyQuery(r, false); err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
raw := r.PathValue("version")
|
||||||
|
version, err := strconv.ParseUint(raw, 10, 64)
|
||||||
|
if err != nil || version == 0 || strconv.FormatUint(version, 10) != raw {
|
||||||
|
return nil, 0, fault.Invalid("invalid version")
|
||||||
|
}
|
||||||
|
out, err := s.Version(r.Context(), t, core.ID(r.PathValue("id")), version)
|
||||||
|
return out, 200, err
|
||||||
|
}))
|
||||||
|
mux.HandleFunc("GET /v1/schedules/{id}/occurrences", wrap(func(r *http.Request, t auth.Token) (any, int, error) {
|
||||||
|
in, err := historyQuery(r, true)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
out, err := s.Occurrences(r.Context(), t, core.ID(r.PathValue("id")), schedule.PageInput{Limit: in.Limit, Cursor: in.Cursor})
|
||||||
|
return out, 200, err
|
||||||
|
}))
|
||||||
|
}
|
||||||
@@ -58,7 +58,7 @@ type Permissions struct {
|
|||||||
Queues []string `json:"queues"`
|
Queues []string `json:"queues"`
|
||||||
}
|
}
|
||||||
|
|
||||||
var supported = map[string]bool{"tasks:submit": true, "tasks:read": true, "tasks:cancel": true, "queues:read": true, "queues:manage": true, "workers:connect": true, "workers:consume": true}
|
var supported = map[string]bool{"tasks:submit": true, "tasks:read": true, "tasks:cancel": true, "queues:read": true, "queues:manage": true, "workers:connect": true, "workers:consume": true, "schedules:read": true, "schedules:manage": true}
|
||||||
|
|
||||||
func (p Permissions) Validate() error {
|
func (p Permissions) Validate() error {
|
||||||
if len(p.Actions) == 0 || len(p.Actions) > 32 || len(p.Queues) == 0 || len(p.Queues) > 128 {
|
if len(p.Actions) == 0 || len(p.Actions) > 32 || len(p.Queues) == 0 || len(p.Queues) > 128 {
|
||||||
|
|||||||
@@ -19,11 +19,11 @@ type Occurrence struct {
|
|||||||
ScheduledAt time.Time `db:"scheduled_at"`
|
ScheduledAt time.Time `db:"scheduled_at"`
|
||||||
// Decision 触发处理决定:已触发、已跳过或已合并。
|
// Decision 触发处理决定:已触发、已跳过或已合并。
|
||||||
Decision string `db:"decision"`
|
Decision string `db:"decision"`
|
||||||
// TaskID 关联的 Pulse 任务标识;可为空,表示尚未设置该关联或值。
|
// TaskID 已触发或合并生成的 Pulse 任务标识;跳过时为空。
|
||||||
TaskID *core.ID `db:"task_id"`
|
TaskID *core.ID `db:"task_id"`
|
||||||
// CoveredUntil 合并触发覆盖的结束边界;没有合并范围时为空。
|
// CoveredUntil 合并或跳过范围的排除边界;单点处理时为空。
|
||||||
CoveredUntil *time.Time `db:"covered_until"`
|
CoveredUntil *time.Time `db:"covered_until"`
|
||||||
// Reason 操作原因或处理说明,需要保留审计信息;可为空,表示尚未设置该关联或值。
|
// Reason 稳定处理分类;新调度器始终写入,旧人工记录可为空。
|
||||||
Reason *string `db:"reason"`
|
Reason *string `db:"reason"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -13,11 +13,17 @@ type Schedule struct {
|
|||||||
core.Record
|
core.Record
|
||||||
// Name 可读名称,唯一性范围由对应数据表约束。
|
// Name 可读名称,唯一性范围由对应数据表约束。
|
||||||
Name string `db:"name"`
|
Name string `db:"name"`
|
||||||
|
// QueueID 计划终身绑定的队列;旧结构人工行可为空,不允许参与扫描。
|
||||||
|
QueueID *core.ID `db:"queue_id"`
|
||||||
|
// ProcessingVersion 当前积压处理所用版本;发布新版本不覆盖旧进度。
|
||||||
|
ProcessingVersion uint64 `db:"processing_version"`
|
||||||
|
// CheckAt 下次允许扫描的 UTC 时间;为空表示当前没有扫描工作。
|
||||||
|
CheckAt *time.Time `db:"check_at"`
|
||||||
// CurrentVersion 当前生效的计划版本号,发布时与版本记录原子更新。
|
// CurrentVersion 当前生效的计划版本号,发布时与版本记录原子更新。
|
||||||
CurrentVersion uint64 `db:"current_version"`
|
CurrentVersion uint64 `db:"current_version"`
|
||||||
// Paused 暂停标记;停止新的受控工作,不表示在途请求已停止。
|
// Paused 暂停标记;停止新的受控工作,不表示在途请求已停止。
|
||||||
Paused bool `db:"paused"`
|
Paused bool `db:"paused"`
|
||||||
// NextFireAt 下一计划触发时间;没有后续触发时为空。
|
// NextFireAt 下一原计划触发时间;版本刚切换尚待计算或已经结束时为空,需结合 CheckAt 判断。
|
||||||
NextFireAt *time.Time `db:"next_fire_at"`
|
NextFireAt *time.Time `db:"next_fire_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -27,8 +27,12 @@ type ScheduleVersion struct {
|
|||||||
MisfirePolicy string `db:"misfire_policy"`
|
MisfirePolicy string `db:"misfire_policy"`
|
||||||
// OverlapPolicy 上次未完成时的策略:并行、串行或跳过。
|
// OverlapPolicy 上次未完成时的策略:并行、串行或跳过。
|
||||||
OverlapPolicy string `db:"overlap_policy"`
|
OverlapPolicy string `db:"overlap_policy"`
|
||||||
// CatchupLimit 单次漏跑补偿上限;补跑策略必须配置正数。
|
// CatchupLimit 每批生成任务数量上限,所有策略均要求 1–100。
|
||||||
CatchupLimit uint32 `db:"catchup_limit"`
|
CatchupLimit uint32 `db:"catchup_limit"`
|
||||||
|
// CatchupWindowMS 未处理触发点最多保留的毫秒数,过旧范围记录跳过。
|
||||||
|
CatchupWindowMS uint64 `db:"catchup_window_ms"`
|
||||||
|
// GraceMS 正常派发允许延迟的毫秒数,超出后按漏跑策略处理。
|
||||||
|
GraceMS uint64 `db:"grace_ms"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ScheduleVersion) TableName() string { return "pulse_schedule_versions" }
|
func (ScheduleVersion) TableName() string { return "pulse_schedule_versions" }
|
||||||
|
|||||||
135
pulse/schedule/planner.go
Normal file
135
pulse/schedule/planner.go
Normal file
@@ -0,0 +1,135 @@
|
|||||||
|
package schedule
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Policy 显式规定漏跑与重叠;窗口、批量和持久化扫描间隔共同限制补跑。
|
||||||
|
type Policy struct {
|
||||||
|
Misfire string `json:"misfire"` // Misfire SKIP、COALESCE 或 CATCH_UP,必须明确提供。
|
||||||
|
Overlap string `json:"overlap"` // Overlap PARALLEL、SERIAL 或 SKIP,必须明确提供。
|
||||||
|
CatchupLimit uint32 `json:"catchup_limit"` // CatchupLimit 每批最多生成的任务数,1–100;串行最多一个。
|
||||||
|
CatchupWindowMS uint64 `json:"catchup_window_ms"` // CatchupWindowMS 最多保留多久的未处理触发点,1000 毫秒–30 天。
|
||||||
|
GraceMS uint64 `json:"grace_ms"` // GraceMS 允许正常派发延迟的毫秒数,1000–60000,不超过窗口。
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p Policy) Validate() error {
|
||||||
|
if p.Misfire != "SKIP" && p.Misfire != "COALESCE" && p.Misfire != "CATCH_UP" {
|
||||||
|
return errors.New("explicit misfire policy required")
|
||||||
|
}
|
||||||
|
if p.Overlap != "PARALLEL" && p.Overlap != "SERIAL" && p.Overlap != "SKIP" {
|
||||||
|
return errors.New("explicit overlap policy required")
|
||||||
|
}
|
||||||
|
if p.CatchupLimit < 1 || p.CatchupLimit > 100 || p.CatchupWindowMS < 1000 || p.CatchupWindowMS > 2592000000 || p.GraceMS < 1000 || p.GraceMS > 60000 || p.GraceMS > p.CatchupWindowMS {
|
||||||
|
return errors.New("invalid bounded catchup policy")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decision 描述单点或连续范围的处理;范围为 [ScheduledAt,CoveredUntil),结束不包含。
|
||||||
|
type Decision struct {
|
||||||
|
ScheduledAt time.Time // ScheduledAt 范围中的首个原触发点,作为唯一去重身份。
|
||||||
|
CoveredUntil *time.Time // CoveredUntil 合并/跳过范围的排除边界;单点为空。
|
||||||
|
Kind string // Kind FIRED、COALESCED 或 SKIPPED。
|
||||||
|
Reason string // Reason 稳定处理分类,不包含业务参数。
|
||||||
|
}
|
||||||
|
|
||||||
|
// Batch 是同一个不可变版本的一批决定,必须与任务及进度原子持久化。
|
||||||
|
type Batch struct {
|
||||||
|
Decisions []Decision // Decisions 有界决策,最多 CatchupLimit+2 条。
|
||||||
|
Next *time.Time // Next 当前版本下一触发点;nil 表示版本区间结束。
|
||||||
|
Blocked bool // Blocked SERIAL 正在等待已有非终态任务,不能跨越该点。
|
||||||
|
}
|
||||||
|
|
||||||
|
// Plan 不读取墙上时钟。now/active 由资源锁定后的数据库事实提供。
|
||||||
|
// before 是下一版本的生效排除边界,旧积压仍按旧版本计算。
|
||||||
|
func Plan(c *Calendar, p Policy, due, now time.Time, before *time.Time, active bool) (Batch, error) {
|
||||||
|
out := Batch{Decisions: []Decision{}}
|
||||||
|
if err := p.Validate(); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
if due.IsZero() || now.IsZero() {
|
||||||
|
return out, errors.New("schedule facts required")
|
||||||
|
}
|
||||||
|
actual, err := c.AtOrAfter(due)
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
if actual == nil || !actual.Equal(due) {
|
||||||
|
return out, errors.New("cursor is not an occurrence")
|
||||||
|
}
|
||||||
|
if before != nil && !due.Before(*before) {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
out.Next = &due
|
||||||
|
upper := now.Add(time.Microsecond)
|
||||||
|
if before != nil && before.Before(upper) {
|
||||||
|
upper = *before
|
||||||
|
}
|
||||||
|
advance := func(at time.Time) error {
|
||||||
|
next, e := c.AtOrAfter(at)
|
||||||
|
if e != nil {
|
||||||
|
return e
|
||||||
|
}
|
||||||
|
if next != nil && before != nil && !next.Before(*before) {
|
||||||
|
next = nil
|
||||||
|
}
|
||||||
|
out.Next = next
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
skip := func(until time.Time, reason string) error {
|
||||||
|
out.Decisions = append(out.Decisions, Decision{ScheduledAt: *out.Next, CoveredUntil: &until, Kind: "SKIPPED", Reason: reason})
|
||||||
|
return advance(until)
|
||||||
|
}
|
||||||
|
cutoff := now.Add(-time.Duration(p.CatchupWindowMS) * time.Millisecond)
|
||||||
|
if upper.Before(cutoff) {
|
||||||
|
cutoff = upper
|
||||||
|
}
|
||||||
|
if out.Next.Before(cutoff) {
|
||||||
|
if err := skip(cutoff, "CATCHUP_WINDOW_EXCEEDED"); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if out.Next == nil || !out.Next.Before(upper) {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
if p.Misfire == "SKIP" {
|
||||||
|
grace := now.Add(-time.Duration(p.GraceMS) * time.Millisecond)
|
||||||
|
if upper.Before(grace) {
|
||||||
|
grace = upper
|
||||||
|
}
|
||||||
|
if out.Next.Before(grace) {
|
||||||
|
if err := skip(grace, "MISFIRE_SKIPPED"); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if out.Next == nil || !out.Next.Before(upper) {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
if active && p.Overlap == "SERIAL" {
|
||||||
|
out.Blocked = true
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
if active && p.Overlap == "SKIP" {
|
||||||
|
err := skip(upper, "OVERLAP_SKIPPED")
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
if p.Misfire == "COALESCE" && out.Next.Before(now.Add(-time.Duration(p.GraceMS)*time.Millisecond)) {
|
||||||
|
out.Decisions = append(out.Decisions, Decision{ScheduledAt: *out.Next, CoveredUntil: &upper, Kind: "COALESCED", Reason: "MISFIRE_COALESCED"})
|
||||||
|
err := advance(upper)
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
for n := uint32(0); n < p.CatchupLimit && out.Next != nil && out.Next.Before(upper); n++ {
|
||||||
|
at := *out.Next
|
||||||
|
out.Decisions = append(out.Decisions, Decision{ScheduledAt: at, Kind: "FIRED", Reason: "SCHEDULED"})
|
||||||
|
if err := advance(at.Add(time.Microsecond)); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
if p.Overlap == "SERIAL" || p.Overlap == "SKIP" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
56
pulse/schedule/planner_test.go
Normal file
56
pulse/schedule/planner_test.go
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
package schedule
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMisfireOverlapAndBoundedCatchup(t *testing.T) {
|
||||||
|
start := stamp("2026-09-17T00:00:00Z")
|
||||||
|
c, err := Compile(Rule{Kind: "FIXED_RATE", Timezone: "UTC", StartAt: start, IntervalMS: 1000})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
base := Policy{Misfire: "CATCH_UP", Overlap: "PARALLEL", CatchupLimit: 3, CatchupWindowMS: 10000, GraceMS: 1000}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, misfire, overlap string
|
||||||
|
active bool
|
||||||
|
kinds []string
|
||||||
|
next int
|
||||||
|
blocked bool
|
||||||
|
}{
|
||||||
|
{"catchup", "CATCH_UP", "PARALLEL", false, []string{"SKIPPED", "FIRED", "FIRED", "FIRED"}, 13, false},
|
||||||
|
{"coalesce", "COALESCE", "PARALLEL", false, []string{"SKIPPED", "COALESCED"}, 21, false},
|
||||||
|
{"skip", "SKIP", "PARALLEL", false, []string{"SKIPPED", "SKIPPED", "FIRED", "FIRED"}, 21, false},
|
||||||
|
{"serial_wait", "CATCH_UP", "SERIAL", true, []string{"SKIPPED"}, 10, true},
|
||||||
|
{"serial_one", "CATCH_UP", "SERIAL", false, []string{"SKIPPED", "FIRED"}, 11, false},
|
||||||
|
{"overlap_skip", "CATCH_UP", "SKIP", true, []string{"SKIPPED", "SKIPPED"}, 21, false},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
p := base
|
||||||
|
p.Misfire = tc.misfire
|
||||||
|
p.Overlap = tc.overlap
|
||||||
|
out, err := Plan(c, p, start, start.Add(20*time.Second), nil, tc.active)
|
||||||
|
if err != nil || out.Next == nil || !out.Next.Equal(start.Add(time.Duration(tc.next)*time.Second)) || out.Blocked != tc.blocked || len(out.Decisions) != len(tc.kinds) {
|
||||||
|
t.Fatalf("unexpected batch %+v %v", out, err)
|
||||||
|
}
|
||||||
|
for i, kind := range tc.kinds {
|
||||||
|
if out.Decisions[i].Kind != kind {
|
||||||
|
t.Fatal("wrong decision", out.Decisions)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// 旧版本只覆盖生效边界之前,不能把新版本的时间合并进去。
|
||||||
|
before := start.Add(15 * time.Second)
|
||||||
|
base.Misfire = "COALESCE"
|
||||||
|
out, err := Plan(c, base, start, start.Add(20*time.Second), &before, false)
|
||||||
|
if err != nil || out.Next != nil || len(out.Decisions) != 2 || !out.Decisions[1].CoveredUntil.Equal(before) {
|
||||||
|
t.Fatalf("version boundary lost: %+v %v", out, err)
|
||||||
|
}
|
||||||
|
// 未到期不产生记录;同一快照重算得到同一原触发点。
|
||||||
|
out, err = Plan(c, base, start, start.Add(-time.Second), nil, false)
|
||||||
|
if err != nil || len(out.Decisions) != 0 || out.Next == nil || !out.Next.Equal(start) {
|
||||||
|
t.Fatal("future point processed")
|
||||||
|
}
|
||||||
|
}
|
||||||
173
pulse/schedule/rule.go
Normal file
173
pulse/schedule/rule.go
Normal file
@@ -0,0 +1,173 @@
|
|||||||
|
// Package schedule 计算计划触发点和有限的漏跑决策;不运行内存 Cron 或业务处理器。
|
||||||
|
package schedule
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/timezone"
|
||||||
|
|
||||||
|
"github.com/robfig/cron/v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Rule 是不可变计划版本中的时间规则。所有边界精确到 UTC 微秒。
|
||||||
|
type Rule struct {
|
||||||
|
CalendarVersion string `json:"calendar_version"` // CalendarVersion 冻结的时区快照版本;提交为空时采用 v1 固定默认快照,升级不能静默改变。
|
||||||
|
Kind string `json:"kind"` // Kind CRON 或 FIXED_RATE;不支持以完成时间推进的 fixed-delay。
|
||||||
|
Timezone string `json:"timezone"` // Timezone 显式 IANA 名称,禁止依赖节点 Local 设置。
|
||||||
|
StartAt time.Time `json:"start_at"` // StartAt 包含此边界;固定间隔以此为唯一基准。
|
||||||
|
EndAt *time.Time `json:"end_at,omitempty"` // EndAt 排除此边界;为空时运行至支持的日期上限。
|
||||||
|
Cron string `json:"cron,omitempty"` // Cron 五段表达式:分钟、小时、月日、月、星期。
|
||||||
|
IntervalMS uint64 `json:"interval_ms,omitempty"` // IntervalMS 固定间隔毫秒数,范围 1000–2592000000。
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calendar 持有已验证且不可变的计算规则,可并发读取,不包含当前进度。
|
||||||
|
type Calendar struct {
|
||||||
|
rule Rule
|
||||||
|
zone *time.Location
|
||||||
|
cron *cron.SpecSchedule
|
||||||
|
}
|
||||||
|
|
||||||
|
func validTime(t time.Time) bool {
|
||||||
|
return t.Year() >= 2000 && t.Year() <= 9998 && t.Nanosecond()%1000 == 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compile 规范化规则并拒绝无解、混合字段和隐式本地时区。
|
||||||
|
func Compile(in Rule) (*Calendar, error) {
|
||||||
|
in.StartAt = in.StartAt.UTC()
|
||||||
|
if !validTime(in.StartAt) {
|
||||||
|
return nil, errors.New("start_at must use UTC-compatible microseconds and years 2000 to 9998")
|
||||||
|
}
|
||||||
|
if in.EndAt != nil {
|
||||||
|
end := in.EndAt.UTC()
|
||||||
|
if !validTime(end) || !end.After(in.StartAt) {
|
||||||
|
return nil, errors.New("end_at must be after start_at")
|
||||||
|
}
|
||||||
|
in.EndAt = &end
|
||||||
|
}
|
||||||
|
if in.Timezone == "" || in.Timezone == "Local" || len(in.Timezone) > 64 {
|
||||||
|
return nil, errors.New("explicit IANA timezone required")
|
||||||
|
}
|
||||||
|
if in.CalendarVersion == "" {
|
||||||
|
in.CalendarVersion = timezone.Version
|
||||||
|
}
|
||||||
|
zone, err := timezone.Load(in.Timezone, in.CalendarVersion)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New("invalid IANA timezone")
|
||||||
|
}
|
||||||
|
c := &Calendar{rule: in, zone: zone}
|
||||||
|
switch in.Kind {
|
||||||
|
case "FIXED_RATE":
|
||||||
|
if in.Cron != "" || in.IntervalMS < 1000 || in.IntervalMS > 2592000000 {
|
||||||
|
return nil, errors.New("fixed rate requires interval_ms 1000 to 2592000000 and no cron")
|
||||||
|
}
|
||||||
|
case "CRON":
|
||||||
|
fields := strings.Fields(in.Cron)
|
||||||
|
if in.IntervalMS != 0 || len(in.Cron) > 256 || len(fields) != 5 || strings.HasPrefix(fields[0], "@") || strings.Contains(fields[0], "=") {
|
||||||
|
return nil, errors.New("cron requires exactly five fields and no interval or embedded timezone")
|
||||||
|
}
|
||||||
|
c.rule.Cron = strings.Join(fields, " ")
|
||||||
|
parsed, err := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow).Parse(c.rule.Cron)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New("invalid five-field cron")
|
||||||
|
}
|
||||||
|
c.cron = parsed.(*cron.SpecSchedule)
|
||||||
|
// 仅用上游计算 UTC 日历;实际时区转换由恒定偏移区间处理,避免非整小时夏令时跳变遗漏。
|
||||||
|
c.cron.Location = time.UTC
|
||||||
|
default:
|
||||||
|
return nil, errors.New("rule kind must be CRON or FIXED_RATE")
|
||||||
|
}
|
||||||
|
first, err := c.AtOrAfter(in.StartAt)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if first == nil {
|
||||||
|
return nil, errors.New("rule has no occurrence in its time bounds")
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rule 返回独立规则副本,不暴露内部可变时间指针。
|
||||||
|
func (c *Calendar) Rule() Rule {
|
||||||
|
out := c.rule
|
||||||
|
if out.EndAt != nil {
|
||||||
|
v := *out.EndAt
|
||||||
|
out.EndAt = &v
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// AtOrAfter 返回不早于指定时刻的第一个原计划触发点;无后续返回 nil。
|
||||||
|
// DST 缺失时刻跳过,重复时刻分别按两个 UTC 身份触发;固定间隔按真实经过时间推进。
|
||||||
|
func (c *Calendar) AtOrAfter(at time.Time) (*time.Time, error) {
|
||||||
|
at = at.UTC()
|
||||||
|
if at.Before(c.rule.StartAt) {
|
||||||
|
at = c.rule.StartAt
|
||||||
|
}
|
||||||
|
end := time.Date(9999, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||||
|
if c.rule.EndAt != nil {
|
||||||
|
end = *c.rule.EndAt
|
||||||
|
}
|
||||||
|
if !at.Before(end) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if c.cron == nil {
|
||||||
|
// 使用微秒整数而非 time.Sub,避免跨越 292 年时 Duration 饱和。
|
||||||
|
start := c.rule.StartAt.UnixMicro()
|
||||||
|
step := int64(c.rule.IntervalMS) * 1000
|
||||||
|
delta := at.UnixMicro() - start
|
||||||
|
n := delta / step
|
||||||
|
if delta%step != 0 || at.Nanosecond()%1000 != 0 {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
next := time.UnixMicro(start + n*step).UTC()
|
||||||
|
if !next.Before(end) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return &next, nil
|
||||||
|
}
|
||||||
|
// 五段 Gregorian 规则的最大日期间隔为世纪闰年边界的八年;十年是计算保护界。
|
||||||
|
limit := at.AddDate(10, 0, 0)
|
||||||
|
if end.Before(limit) {
|
||||||
|
limit = end
|
||||||
|
}
|
||||||
|
for segment := 0; segment < 256 && at.Before(limit); segment++ {
|
||||||
|
local := at.In(c.zone)
|
||||||
|
_, offset := local.Zone()
|
||||||
|
_, until := local.ZoneBounds()
|
||||||
|
wall := at.Add(time.Duration(offset) * time.Second)
|
||||||
|
candidate := c.cron.Next(wall.Add(-time.Nanosecond))
|
||||||
|
if candidate.IsZero() {
|
||||||
|
// 上游单次只查五年;2096→2104 等合法闰日规则需要第二个区间。
|
||||||
|
candidate = c.cron.Next(wall.AddDate(5, 0, 0).Add(-time.Nanosecond))
|
||||||
|
}
|
||||||
|
if candidate.IsZero() {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
candidate = candidate.Add(-time.Duration(offset) * time.Second).UTC()
|
||||||
|
if !until.IsZero() && !candidate.Before(until) {
|
||||||
|
if !until.After(at) {
|
||||||
|
return nil, errors.New("timezone interval did not advance")
|
||||||
|
}
|
||||||
|
at = until.UTC()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if candidate.Before(at) {
|
||||||
|
return nil, errors.New("calendar moved backwards")
|
||||||
|
}
|
||||||
|
if !candidate.Before(limit) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return &candidate, nil
|
||||||
|
}
|
||||||
|
if at.Before(limit) {
|
||||||
|
return nil, errors.New("timezone transition budget exceeded")
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// After 返回严格晚于该触发点的下一个 UTC 身份。
|
||||||
|
func (c *Calendar) After(at time.Time) (*time.Time, error) {
|
||||||
|
return c.AtOrAfter(at.Add(time.Microsecond))
|
||||||
|
}
|
||||||
107
pulse/schedule/rule_test.go
Normal file
107
pulse/schedule/rule_test.go
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
package schedule
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func stamp(v string) time.Time {
|
||||||
|
t, err := time.Parse(time.RFC3339Nano, v)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
func calendar(t *testing.T, expr, zone, start string) *Calendar {
|
||||||
|
t.Helper()
|
||||||
|
c, err := Compile(Rule{Kind: "CRON", Cron: expr, Timezone: zone, StartAt: stamp(start)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
func TestCronCalendarUTCAndDST(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, expr, zone, from string
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{"five_fields", "*/15 9-10 * * MON-FRI", "Asia/Shanghai", "2026-09-18T00:59:59Z", []string{"2026-09-18T01:00:00Z", "2026-09-18T01:15:00Z"}},
|
||||||
|
{"spring_missing", "30 2 * * *", "America/New_York", "2026-03-08T00:00:00Z", []string{"2026-03-09T06:30:00Z"}},
|
||||||
|
{"autumn_repeated", "30 1 * * *", "America/New_York", "2026-11-01T00:00:00Z", []string{"2026-11-01T05:30:00Z", "2026-11-01T06:30:00Z", "2026-11-02T06:30:00Z"}},
|
||||||
|
{"half_hour_spring", "45 2 * * *", "Australia/Lord_Howe", "2026-10-03T14:59:00Z", []string{"2026-10-03T15:45:00Z", "2026-10-04T15:45:00Z"}},
|
||||||
|
{"half_hour_autumn", "45 1 * * *", "Australia/Lord_Howe", "2026-04-04T13:00:00Z", []string{"2026-04-04T14:45:00Z", "2026-04-04T15:15:00Z"}},
|
||||||
|
{"skipped_date", "0 0 * * *", "Pacific/Apia", "2011-12-29T10:00:00Z", []string{"2011-12-29T10:00:00Z", "2011-12-30T10:00:00Z"}},
|
||||||
|
{"century_leap", "0 0 29 2 *", "UTC", "2096-03-01T00:00:00Z", []string{"2104-02-29T00:00:00Z", "2108-02-29T00:00:00Z"}},
|
||||||
|
{"day_or_weekday", "0 0 13 * MON", "UTC", "2026-09-12T00:00:00Z", []string{"2026-09-13T00:00:00Z", "2026-09-14T00:00:00Z"}},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
c := calendar(t, tc.expr, tc.zone, tc.from)
|
||||||
|
at := stamp(tc.from)
|
||||||
|
for _, want := range tc.want {
|
||||||
|
got, err := c.AtOrAfter(at)
|
||||||
|
if err != nil || got == nil || !got.Equal(stamp(want)) {
|
||||||
|
t.Fatalf("at %s got %v %v want %s", at, got, err, want)
|
||||||
|
}
|
||||||
|
at = got.Add(time.Microsecond)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestRuleBoundsAndFixedRate(t *testing.T) {
|
||||||
|
start := stamp("2026-09-17T00:00:00.000001Z")
|
||||||
|
end := start.Add(3 * time.Second)
|
||||||
|
c, err := Compile(Rule{Kind: "FIXED_RATE", Timezone: "UTC", StartAt: start, EndAt: &end, IntervalMS: 1000})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, at := range []time.Time{start.Add(-time.Hour), start, start.Add(time.Nanosecond), start.Add(1500 * time.Millisecond), end} {
|
||||||
|
got, err := c.AtOrAfter(at)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if at.Equal(end) {
|
||||||
|
if got != nil {
|
||||||
|
t.Fatal("end is inclusive")
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if got == nil || got.Before(at) || !got.Before(end) || (got.UnixMicro()-start.UnixMicro())%1000000 != 0 {
|
||||||
|
t.Fatal("fixed rate drift")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 规则副本不能修改已编译日历。
|
||||||
|
copy := c.Rule()
|
||||||
|
*copy.EndAt = start
|
||||||
|
if got, _ := c.AtOrAfter(start); got == nil {
|
||||||
|
t.Fatal("rule alias")
|
||||||
|
}
|
||||||
|
c, err = Compile(Rule{Kind: "FIXED_RATE", Timezone: "UTC", StartAt: stamp("2000-01-01T00:00:00Z"), IntervalMS: 1000})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, err := c.AtOrAfter(stamp("9998-12-31T23:59:59.000001Z")); err != nil || got != nil {
|
||||||
|
t.Fatal("calendar overflow")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestRuleRejectsAmbiguityAndImpossibleDates(t *testing.T) {
|
||||||
|
base := Rule{Kind: "CRON", Timezone: "UTC", StartAt: stamp("2026-01-01T00:00:00Z"), Cron: "* * * * *"}
|
||||||
|
for _, expr := range []string{"@every 1s", "0 0 0 * * *", "TZ=UTC 0 0 * * *", "0 0 31 2 *", "60 * * * *", "0 0 * * 7", "*/0 * * * *"} {
|
||||||
|
in := base
|
||||||
|
in.Cron = expr
|
||||||
|
if _, err := Compile(in); err == nil {
|
||||||
|
t.Fatalf("accepted %q", expr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, zone := range []string{"", "Local", "../UTC", "Unknown/Zone"} {
|
||||||
|
in := base
|
||||||
|
in.Timezone = zone
|
||||||
|
if _, err := Compile(in); err == nil {
|
||||||
|
t.Fatalf("accepted zone %s", zone)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
in := base
|
||||||
|
in.IntervalMS = 1000
|
||||||
|
if _, err := Compile(in); err == nil {
|
||||||
|
t.Fatal("accepted mixed rule")
|
||||||
|
}
|
||||||
|
}
|
||||||
207
pulse/schedule/service.go
Normal file
207
pulse/schedule/service.go
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
package schedule
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||||
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||||
|
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Definition 是每次发布冻结的完整定义,业务内容仍由独立 Worker 解释。
|
||||||
|
type Definition struct {
|
||||||
|
Rule Rule `json:"rule"` // Rule 时间规则;start_at 是基准,不允许追溯到版本生效前产生新任务。
|
||||||
|
Policy Policy `json:"policy"` // Policy 明确的漏跑、重叠和补跑边界。
|
||||||
|
Task service.SubmitInput `json:"task"` // Task 每次触发的模板,禁止绝对 available_at/start_deadline_at。
|
||||||
|
}
|
||||||
|
|
||||||
|
// CreateInput 创建持久化计划,暂停计划也保存完整的触发进度。
|
||||||
|
type CreateInput struct {
|
||||||
|
Name string `json:"name"` // Name 命名空间内唯一的 ASCII 名称。
|
||||||
|
Definition Definition `json:"definition"` // Definition 首个不可变版本。
|
||||||
|
Paused bool `json:"paused"` // Paused 是否从暂停状态开始,恢复时处理期间漏跑。
|
||||||
|
}
|
||||||
|
|
||||||
|
// PublishInput 发布立即生效的新版本;旧积压仍按当时版本处理。
|
||||||
|
type PublishInput struct {
|
||||||
|
ExpectedRevision uint64 `json:"expected_revision,string"` // ExpectedRevision 当前控制版本,必须非零。
|
||||||
|
Reason string `json:"reason"` // Reason 必填操作原因,写入审计。
|
||||||
|
Definition Definition `json:"definition"` // Definition 完整替换定义;队列不能更换。
|
||||||
|
}
|
||||||
|
|
||||||
|
// ControlInput 暂停/恢复计划,不能取消已产生的任务。
|
||||||
|
type ControlInput struct {
|
||||||
|
ExpectedRevision uint64 `json:"expected_revision,string"` // ExpectedRevision 乐观并发版本。
|
||||||
|
Reason string `json:"reason"` // Reason 必填操作原因,最多 1024 字符。
|
||||||
|
}
|
||||||
|
|
||||||
|
// VersionView 保留规则、模板和原生效时间,已生成任务不会采用后续版本。
|
||||||
|
type VersionView struct {
|
||||||
|
Version uint64 `json:"version,string"` // Version 发布序号,从一递增。
|
||||||
|
EffectiveAt time.Time `json:"effective_at"` // EffectiveAt 数据库事务内取得的生效边界,包含此时间。
|
||||||
|
Definition Definition `json:"definition"` // Definition 已冻结内容。
|
||||||
|
}
|
||||||
|
|
||||||
|
// Summary 用于计划列表,不包含大任务参数。
|
||||||
|
type Summary struct {
|
||||||
|
ID core.ID `json:"id"` // ID 计划稳定身份。
|
||||||
|
NamespaceID core.ID `json:"namespace_id"` // NamespaceID 隔离范围。
|
||||||
|
QueueID core.ID `json:"queue_id"` // QueueID 计划终身绑定队列。
|
||||||
|
Name string `json:"name"` // Name 同命名空间内唯一名称。
|
||||||
|
Revision uint64 `json:"revision,string"` // Revision 控制和进度变更版本。
|
||||||
|
CurrentVersion uint64 `json:"current_version,string"` // CurrentVersion 最新发布版本。
|
||||||
|
ProcessingVersion uint64 `json:"processing_version,string"` // ProcessingVersion 尚在处理积压的历史版本。
|
||||||
|
Paused bool `json:"paused"` // Paused 为真时停止新触发。
|
||||||
|
NextFireAt *time.Time `json:"next_fire_at,omitempty"` // NextFireAt 原计划时间;积压时可在过去,待初始化或结束时为空,需结合 CheckAt。
|
||||||
|
CheckAt *time.Time `json:"check_at,omitempty"` // CheckAt 下次可扫描时间;为空表示没有扫描工作,非空也可能尚待初始化触发点。
|
||||||
|
CreatedAt time.Time `json:"created_at"` // CreatedAt UTC 创建时间。
|
||||||
|
UpdatedAt time.Time `json:"updated_at"` // UpdatedAt UTC 最近变更时间。
|
||||||
|
}
|
||||||
|
|
||||||
|
// View 返回控制状态和最新版本,历史版本通过专用查询读取。
|
||||||
|
type View struct {
|
||||||
|
Summary
|
||||||
|
Current VersionView `json:"current"` // Current 最新完整发布版本,不一定是当前积压所用版本。
|
||||||
|
}
|
||||||
|
|
||||||
|
// Result 是幂等命令的原始回执,当前状态应使用 Get 查询。
|
||||||
|
type Result struct {
|
||||||
|
Schedule View `json:"schedule"` // Schedule 本次命令提交时的快照。
|
||||||
|
Deduplicated bool `json:"deduplicated"` // Deduplicated 是否取回原回执。
|
||||||
|
}
|
||||||
|
|
||||||
|
// OccurrenceView 描述一个已决定的触发点或不重叠范围。
|
||||||
|
type OccurrenceView struct {
|
||||||
|
ID core.ID `json:"id"` // ID 触发记录标识。
|
||||||
|
ScheduleID core.ID `json:"schedule_id"` // ScheduleID 所属计划。
|
||||||
|
ScheduleVersion uint64 `json:"schedule_version,string"` // ScheduleVersion 当时适用的不可变版本。
|
||||||
|
ScheduledAt time.Time `json:"scheduled_at"` // ScheduledAt 原时间,参与唯一去重。
|
||||||
|
Decision string `json:"decision"` // Decision FIRED、SKIPPED 或 COALESCED。
|
||||||
|
TaskID *core.ID `json:"task_id,omitempty"` // TaskID 已生成任务;跳过时为空。
|
||||||
|
CoveredUntil *time.Time `json:"covered_until,omitempty"` // CoveredUntil 范围排除边界,不一定是触发点。
|
||||||
|
Reason string `json:"reason"` // Reason 稳定决策分类。
|
||||||
|
CreatedAt time.Time `json:"created_at"` // CreatedAt 数据库记录时间。
|
||||||
|
}
|
||||||
|
type Page struct {
|
||||||
|
Items []Summary `json:"items"` // Items 当前页摘要,始终为数组。
|
||||||
|
NextCursor string `json:"next_cursor,omitempty"` // NextCursor 不透明后续游标。
|
||||||
|
}
|
||||||
|
type OccurrencePage struct {
|
||||||
|
Items []OccurrenceView `json:"items"` // Items 当前页触发历史。
|
||||||
|
NextCursor string `json:"next_cursor,omitempty"` // NextCursor 原样用于相同父资源查询。
|
||||||
|
}
|
||||||
|
type PageInput struct {
|
||||||
|
Limit int // Limit 默认 20,最大 100。
|
||||||
|
Cursor string // Cursor 最长 1024 字符,不能作为权限证明。
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store 必须原子执行命令授权、版本、游标、审计和幂等回执。
|
||||||
|
type Store interface {
|
||||||
|
Create(context.Context, auth.Token, string, CreateInput) (Result, error)
|
||||||
|
Get(context.Context, auth.Token, core.ID) (View, error)
|
||||||
|
Publish(context.Context, auth.Token, core.ID, string, PublishInput) (Result, error)
|
||||||
|
Control(context.Context, auth.Token, core.ID, string, ControlInput, bool) (Result, error)
|
||||||
|
List(context.Context, auth.Token, core.ID, PageInput) (Page, error)
|
||||||
|
Version(context.Context, auth.Token, core.ID, uint64) (VersionView, error)
|
||||||
|
Occurrences(context.Context, auth.Token, core.ID, PageInput) (OccurrencePage, error)
|
||||||
|
}
|
||||||
|
type Service struct{ store Store }
|
||||||
|
|
||||||
|
func New(store Store) *Service { return &Service{store: store} }
|
||||||
|
func Normalize(in Definition) (Definition, error) {
|
||||||
|
c, err := Compile(in.Rule)
|
||||||
|
if err != nil {
|
||||||
|
return in, fault.Invalid(err.Error())
|
||||||
|
}
|
||||||
|
in.Rule = c.Rule()
|
||||||
|
if err := in.Policy.Validate(); err != nil {
|
||||||
|
return in, fault.Invalid(err.Error())
|
||||||
|
}
|
||||||
|
if in.Task.AvailableAt != nil || in.Task.StartDeadlineAt != nil {
|
||||||
|
return in, fault.Invalid("schedule templates cannot contain absolute task times")
|
||||||
|
}
|
||||||
|
in.Task, err = service.NormalizeSubmit(in.Task)
|
||||||
|
return in, err
|
||||||
|
}
|
||||||
|
func control(id core.ID, key string, revision uint64, reason string) error {
|
||||||
|
if id.Validate() != nil || revision == 0 || !utf8.ValidString(reason) || strings.TrimSpace(reason) == "" || utf8.RuneCountInString(reason) > 1024 {
|
||||||
|
return fault.Invalid("valid schedule id, expected_revision and reason are required")
|
||||||
|
}
|
||||||
|
return service.ValidateKey(key)
|
||||||
|
}
|
||||||
|
func page(in PageInput) (PageInput, error) {
|
||||||
|
if in.Limit == 0 {
|
||||||
|
in.Limit = 20
|
||||||
|
}
|
||||||
|
if in.Limit < 1 || in.Limit > 100 || len(in.Cursor) > 1024 {
|
||||||
|
return in, fault.Invalid("invalid schedule page")
|
||||||
|
}
|
||||||
|
return in, nil
|
||||||
|
}
|
||||||
|
func (s *Service) Create(ctx context.Context, t auth.Token, key string, in CreateInput) (Result, error) {
|
||||||
|
if err := service.ValidateKey(key); err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
if !service.ValidName(in.Name, 128) {
|
||||||
|
return Result{}, fault.Invalid("schedule name is invalid")
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
in.Definition, err = Normalize(in.Definition)
|
||||||
|
if err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
return s.store.Create(ctx, t, key, in)
|
||||||
|
}
|
||||||
|
func (s *Service) Get(ctx context.Context, t auth.Token, id core.ID) (View, error) {
|
||||||
|
if id.Validate() != nil {
|
||||||
|
return View{}, fault.Invalid("invalid schedule id")
|
||||||
|
}
|
||||||
|
return s.store.Get(ctx, t, id)
|
||||||
|
}
|
||||||
|
func (s *Service) Publish(ctx context.Context, t auth.Token, id core.ID, key string, in PublishInput) (Result, error) {
|
||||||
|
if err := control(id, key, in.ExpectedRevision, in.Reason); err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
in.Definition, err = Normalize(in.Definition)
|
||||||
|
if err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
return s.store.Publish(ctx, t, id, key, in)
|
||||||
|
}
|
||||||
|
func (s *Service) Control(ctx context.Context, t auth.Token, id core.ID, key string, in ControlInput, paused bool) (Result, error) {
|
||||||
|
if err := control(id, key, in.ExpectedRevision, in.Reason); err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
return s.store.Control(ctx, t, id, key, in, paused)
|
||||||
|
}
|
||||||
|
func (s *Service) List(ctx context.Context, t auth.Token, queue core.ID, in PageInput) (Page, error) {
|
||||||
|
if queue.Validate() != nil {
|
||||||
|
return Page{}, fault.Invalid("queue_id is required")
|
||||||
|
}
|
||||||
|
in, err := page(in)
|
||||||
|
if err != nil {
|
||||||
|
return Page{}, err
|
||||||
|
}
|
||||||
|
return s.store.List(ctx, t, queue, in)
|
||||||
|
}
|
||||||
|
func (s *Service) Version(ctx context.Context, t auth.Token, id core.ID, version uint64) (VersionView, error) {
|
||||||
|
if id.Validate() != nil || version == 0 {
|
||||||
|
return VersionView{}, fault.Invalid("invalid schedule id or version")
|
||||||
|
}
|
||||||
|
return s.store.Version(ctx, t, id, version)
|
||||||
|
}
|
||||||
|
func (s *Service) Occurrences(ctx context.Context, t auth.Token, id core.ID, in PageInput) (OccurrencePage, error) {
|
||||||
|
if id.Validate() != nil {
|
||||||
|
return OccurrencePage{}, fault.Invalid("invalid schedule id")
|
||||||
|
}
|
||||||
|
in, err := page(in)
|
||||||
|
if err != nil {
|
||||||
|
return OccurrencePage{}, err
|
||||||
|
}
|
||||||
|
return s.store.Occurrences(ctx, t, id, in)
|
||||||
|
}
|
||||||
@@ -48,12 +48,14 @@ done
|
|||||||
if [ -z "$TASK_HTTP_PORT" ]; then printf '%s\n' 'Bootstrap HTTP server did not start.' >&2; exit 1; fi
|
if [ -z "$TASK_HTTP_PORT" ]; then printf '%s\n' 'Bootstrap HTTP server did not start.' >&2; exit 1; fi
|
||||||
(cd sdk/go && ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" "$TASK_GO" run ./examples/health)
|
(cd sdk/go && ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" "$TASK_GO" run ./examples/health)
|
||||||
(cd sdk/go && ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_TEST_DIR/sdk.credentials.json" "$TASK_GO" run ./examples/tasks)
|
(cd sdk/go && ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_TEST_DIR/sdk.credentials.json" "$TASK_GO" run ./examples/tasks)
|
||||||
ORBIT_MYSQL_DSN="$ORBIT_TEST_MYSQL_DSN" "$TASK_TEST_DIR/orbit-admin" bootstrap --application sdk-worker --namespace default --environment test --queue worker-examples --worker-access --output "$TASK_TEST_DIR/worker.credentials.json"
|
ORBIT_MYSQL_DSN="$ORBIT_TEST_MYSQL_DSN" "$TASK_TEST_DIR/orbit-admin" bootstrap --application sdk-worker --namespace default --environment test --queue worker-examples --worker-access --schedule-access --output "$TASK_TEST_DIR/worker.credentials.json"
|
||||||
(cd sdk/go && ORBIT_TEST_WORKER_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_TEST_WORKER_CREDENTIALS_FILE="$TASK_TEST_DIR/worker.credentials.json" "$TASK_GO" test -race -count=1 -run 'IntegrationWorker(Connection|Runtime)' -v .)
|
(cd sdk/go && ORBIT_TEST_WORKER_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_TEST_WORKER_CREDENTIALS_FILE="$TASK_TEST_DIR/worker.credentials.json" "$TASK_GO" test -race -count=1 -run 'IntegrationWorker(Connection|Runtime)' -v .)
|
||||||
|
(cd sdk/go && ORBIT_TEST_WORKER_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_TEST_WORKER_CREDENTIALS_FILE="$TASK_TEST_DIR/worker.credentials.json" "$TASK_GO" test -race -count=1 -run '^TestIntegrationSchedule' -v .)
|
||||||
(cd sdk/go && ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_TEST_DIR/worker.credentials.json" "$TASK_GO" run ./examples/worker-protocol)
|
(cd sdk/go && ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_TEST_DIR/worker.credentials.json" "$TASK_GO" run ./examples/worker-protocol)
|
||||||
(cd sdk/go && ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_TEST_DIR/worker.credentials.json" "$TASK_GO" run ./examples/worker)
|
(cd sdk/go && ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_TEST_DIR/worker.credentials.json" "$TASK_GO" run ./examples/worker)
|
||||||
(cd sdk/go && ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_TEST_DIR/worker.credentials.json" "$TASK_GO" run ./examples/history)
|
(cd sdk/go && ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_TEST_DIR/worker.credentials.json" "$TASK_GO" run ./examples/history)
|
||||||
(cd sdk/go && ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_TEST_DIR/worker.credentials.json" "$TASK_GO" run ./examples/retry)
|
(cd sdk/go && ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_TEST_DIR/worker.credentials.json" "$TASK_GO" run ./examples/retry)
|
||||||
|
(cd sdk/go && ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_TEST_DIR/worker.credentials.json" "$TASK_GO" run ./examples/schedules)
|
||||||
|
|
||||||
# 停止原服务后再做进程故障演练,避免其他恢复扫描器掩盖接管验证。
|
# 停止原服务后再做进程故障演练,避免其他恢复扫描器掩盖接管验证。
|
||||||
kill "$TASK_SERVER_PID"
|
kill "$TASK_SERVER_PID"
|
||||||
|
|||||||
149
sdk/go/examples/schedules/main.go
Normal file
149
sdk/go/examples/schedules/main.go
Normal file
@@ -0,0 +1,149 @@
|
|||||||
|
// schedules 演示持久化周期任务:创建、恢复、Worker 执行、触发历史和暂停。
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if err := run(); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func run() error {
|
||||||
|
token, queue := os.Getenv("ORBIT_TOKEN"), os.Getenv("ORBIT_QUEUE_ID")
|
||||||
|
if path := os.Getenv("ORBIT_CREDENTIALS_FILE"); path != "" {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read private credentials: %w", err)
|
||||||
|
}
|
||||||
|
var credentials struct {
|
||||||
|
Token string `json:"token"`
|
||||||
|
QueueID string `json:"queue_id"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(raw, &credentials) != nil {
|
||||||
|
return errors.New("invalid credentials file")
|
||||||
|
}
|
||||||
|
if token == "" {
|
||||||
|
token = credentials.Token
|
||||||
|
}
|
||||||
|
if queue == "" {
|
||||||
|
queue = credentials.QueueID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
client, err := orbit.NewClient(os.Getenv("ORBIT_URL"), orbit.Options{Token: token})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
operation, err := orbit.NewWorkerMessageID()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
handler := "object.snapshot." + operation
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
worker, err := orbit.NewWorker(client, orbit.WorkerOptions{QueueID: queue, Capacity: 1, ShutdownTimeout: 5 * time.Second, Handlers: []orbit.WorkerHandler{{Capability: orbit.WorkerCapability{Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(ctx context.Context, e orbit.WorkerExecution) (json.RawMessage, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// 实际副作用应按 Task.ID 或业务操作键去重;此示例只返回输入快照。
|
||||||
|
return e.Task.Payload, nil
|
||||||
|
}}}})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
start := time.Now().UTC().Truncate(time.Microsecond).Add(1500 * time.Millisecond)
|
||||||
|
end := start.Add(10 * time.Second)
|
||||||
|
created, err := client.CreateSchedule(ctx, "create-"+operation, orbit.CreateScheduleRequest{Name: handler, Paused: true, Definition: orbit.ScheduleDefinition{
|
||||||
|
Rule: orbit.ScheduleRule{Kind: "FIXED_RATE", Timezone: "UTC", StartAt: start, EndAt: &end, IntervalMS: 1000},
|
||||||
|
Policy: orbit.SchedulePolicy{Misfire: "COALESCE", Overlap: "SERIAL", CatchupLimit: 2, CatchupWindowMS: 30000, GraceMS: 1000},
|
||||||
|
Task: orbit.SubmitTaskRequest{QueueID: queue, Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"source_ref":"objects/sample","sequence":9007199254740993}`)},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := client.ResumeSchedule(ctx, created.Schedule.ID, "resume-"+operation, orbit.ControlScheduleRequest{ExpectedRevision: created.Schedule.Revision, Reason: "start bounded example"}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
workerCtx, stop := context.WithCancel(ctx)
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- worker.Run(workerCtx) }()
|
||||||
|
waited := false
|
||||||
|
defer func() {
|
||||||
|
stop()
|
||||||
|
if !waited {
|
||||||
|
<-done
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
completed := map[string]bool{}
|
||||||
|
for len(completed) < 2 {
|
||||||
|
page, err := client.ListScheduleOccurrences(ctx, created.Schedule.ID, orbit.SchedulePageRequest{})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, item := range page.Items {
|
||||||
|
if item.TaskID == nil || completed[*item.TaskID] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
task, err := client.GetTask(ctx, *item.TaskID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if task.State == orbit.TaskSucceeded {
|
||||||
|
completed[task.ID] = true
|
||||||
|
fmt.Printf("schedule=%s occurrence=%s task=%s version=%d state=%s\n", created.Schedule.ID, item.ID, task.ID, item.ScheduleVersion, task.State)
|
||||||
|
}
|
||||||
|
if task.State == orbit.TaskFailed || task.State == orbit.TaskNeedsReview {
|
||||||
|
return fmt.Errorf("scheduled task requires attention: %s", task.State)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(completed) >= 2 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
waited = true
|
||||||
|
return fmt.Errorf("Worker stopped before completion: %v", err)
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-time.After(100 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 扫描也改变计划版本;遇到 CAS 冲突读取新状态,再以新操作身份重试暂停。
|
||||||
|
for {
|
||||||
|
current, err := client.GetSchedule(ctx, created.Schedule.ID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
key, err := orbit.NewWorkerMessageID()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
paused, err := client.PauseSchedule(ctx, current.ID, key, orbit.ControlScheduleRequest{ExpectedRevision: current.Revision, Reason: "example finished"})
|
||||||
|
if err == nil {
|
||||||
|
fmt.Printf("paused=%t completed_tasks=%d current_version=%d\n", paused.Schedule.Paused, len(completed), paused.Schedule.CurrentVersion)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
var apiErr *orbit.APIError
|
||||||
|
if !errors.As(err, &apiErr) || apiErr.Code != "STATE_CONFLICT" {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stop()
|
||||||
|
err = <-done
|
||||||
|
waited = true
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("worker_drained=true running=%d\n", worker.Stats().Running)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
134
sdk/go/schedule_integration_test.go
Normal file
134
sdk/go/schedule_integration_test.go
Normal file
@@ -0,0 +1,134 @@
|
|||||||
|
package orbit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIntegrationScheduleLifecycle(t *testing.T) {
|
||||||
|
api, queue, _ := integrationWorkerClient(t)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
name := "scheduled." + workerTestID(t)
|
||||||
|
w, err := NewWorker(api, WorkerOptions{QueueID: queue, Capacity: 1, Handlers: []WorkerHandler{runtimeHandler(name, func(ctx context.Context, e WorkerExecution) (json.RawMessage, error) { return e.Task.Payload, nil })}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
run, stop := context.WithCancel(ctx)
|
||||||
|
defer stop()
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- w.Run(run) }()
|
||||||
|
ended := false
|
||||||
|
defer func() {
|
||||||
|
stop()
|
||||||
|
if !ended {
|
||||||
|
<-done
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
start := time.Now().UTC().Truncate(time.Microsecond).Add(1500 * time.Millisecond)
|
||||||
|
in := CreateScheduleRequest{Name: name, Paused: true, Definition: ScheduleDefinition{Rule: ScheduleRule{Kind: "FIXED_RATE", Timezone: "UTC", StartAt: start, IntervalMS: 1000}, Policy: SchedulePolicy{Misfire: "COALESCE", Overlap: "SERIAL", CatchupLimit: 2, CatchupWindowMS: 60000, GraceMS: 1000}, Task: SubmitTaskRequest{QueueID: queue, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`)}}}
|
||||||
|
key := "schedule-" + workerTestID(t)
|
||||||
|
accepted, err := api.CreateSchedule(ctx, key, in)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
duplicate, err := api.CreateSchedule(ctx, key, in)
|
||||||
|
if err != nil || !duplicate.Deduplicated || duplicate.Schedule.ID != accepted.Schedule.ID {
|
||||||
|
t.Fatal("schedule submit replay", err)
|
||||||
|
}
|
||||||
|
resumed, err := api.ResumeSchedule(ctx, accepted.Schedule.ID, workerTestID(t), ControlScheduleRequest{ExpectedRevision: accepted.Schedule.Revision, Reason: "run integration schedule"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var occurrence ScheduleOccurrence
|
||||||
|
for occurrence.TaskID == nil {
|
||||||
|
page, err := api.ListScheduleOccurrences(ctx, resumed.Schedule.ID, SchedulePageRequest{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, item := range page.Items {
|
||||||
|
if item.TaskID != nil {
|
||||||
|
occurrence = item
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if occurrence.TaskID != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal(ctx.Err())
|
||||||
|
case <-time.After(80 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
task := waitRuntimeState(t, ctx, api, *occurrence.TaskID, TaskSucceeded)
|
||||||
|
if string(task.Result) != `{"n":9007199254740993}` || task.AttemptCount != 1 {
|
||||||
|
t.Fatal("scheduled Worker result corrupted")
|
||||||
|
}
|
||||||
|
var paused ScheduleResult
|
||||||
|
for {
|
||||||
|
current, err := api.GetSchedule(ctx, resumed.Schedule.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
paused, err = api.PauseSchedule(ctx, current.ID, workerTestID(t), ControlScheduleRequest{ExpectedRevision: current.Revision, Reason: "publish next definition"})
|
||||||
|
if err == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
var apiErr *APIError
|
||||||
|
if !errors.As(err, &apiErr) || apiErr.Code != "STATE_CONFLICT" {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
in.Definition.Rule = ScheduleRule{Kind: "CRON", Timezone: "Australia/Lord_Howe", StartAt: start, Cron: "*/5 * * * *"}
|
||||||
|
publish := PublishScheduleRequest{ExpectedRevision: paused.Schedule.Revision, Reason: "test immutable cron version", Definition: in.Definition}
|
||||||
|
publishKey := workerTestID(t)
|
||||||
|
updated, err := api.PublishSchedule(ctx, paused.Schedule.ID, publishKey, publish)
|
||||||
|
if err != nil || updated.Schedule.CurrentVersion != 2 || !updated.Schedule.Paused {
|
||||||
|
t.Fatal("publish schedule", err)
|
||||||
|
}
|
||||||
|
old, err := api.GetScheduleVersion(ctx, paused.Schedule.ID, 1)
|
||||||
|
if err != nil || old.Definition.Rule.Kind != "FIXED_RATE" {
|
||||||
|
t.Fatal("old version lost", err)
|
||||||
|
}
|
||||||
|
current, err := api.GetScheduleVersion(ctx, paused.Schedule.ID, 2)
|
||||||
|
if err != nil || current.Definition.Rule.Kind != "CRON" {
|
||||||
|
t.Fatal("new cron version lost", err)
|
||||||
|
}
|
||||||
|
duplicate, err = api.PublishSchedule(ctx, paused.Schedule.ID, publishKey, publish)
|
||||||
|
if err != nil || !duplicate.Deduplicated || duplicate.Schedule.Revision != updated.Schedule.Revision {
|
||||||
|
t.Fatal("publish replay changed revision")
|
||||||
|
}
|
||||||
|
list, err := api.ListSchedules(ctx, queue, SchedulePageRequest{Limit: 100})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, v := range list.Items {
|
||||||
|
found = found || v.ID == updated.Schedule.ID
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatal("schedule missing from list")
|
||||||
|
}
|
||||||
|
before, err := api.ListScheduleOccurrences(ctx, updated.Schedule.ID, SchedulePageRequest{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal(ctx.Err())
|
||||||
|
case <-time.After(1200 * time.Millisecond):
|
||||||
|
}
|
||||||
|
after, err := api.ListScheduleOccurrences(ctx, updated.Schedule.ID, SchedulePageRequest{})
|
||||||
|
if err != nil || len(after.Items) != len(before.Items) {
|
||||||
|
t.Fatal("paused version fired")
|
||||||
|
}
|
||||||
|
stop()
|
||||||
|
if err := <-done; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ended = true
|
||||||
|
}
|
||||||
110
sdk/go/schedule_types.go
Normal file
110
sdk/go/schedule_types.go
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
package orbit
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// ScheduleRule 是不可变计划版本中的时间规则。所有边界精确到 UTC 微秒。
|
||||||
|
type ScheduleRule struct {
|
||||||
|
CalendarVersion string `json:"calendar_version,omitempty"` // CalendarVersion 留空时采用 v1 固定默认时区快照;读取后保留原值。
|
||||||
|
Kind string `json:"kind"` // Kind CRON 或 FIXED_RATE;不支持以完成时间推进的 fixed-delay。
|
||||||
|
Timezone string `json:"timezone"` // Timezone 显式 IANA 名称,禁止依赖节点 Local 设置。
|
||||||
|
StartAt time.Time `json:"start_at"` // StartAt 包含此边界;固定间隔以此为唯一基准。
|
||||||
|
EndAt *time.Time `json:"end_at,omitempty"` // EndAt 排除此边界;为空时运行至支持的日期上限。
|
||||||
|
Cron string `json:"cron,omitempty"` // Cron 五段表达式:分钟、小时、月日、月、星期。
|
||||||
|
IntervalMS uint64 `json:"interval_ms,omitempty"` // IntervalMS 固定间隔毫秒数,范围 1000–2592000000。
|
||||||
|
}
|
||||||
|
|
||||||
|
// SchedulePolicy 显式规定漏跑与重叠;窗口、批量和持久化扫描间隔共同限制补跑。
|
||||||
|
type SchedulePolicy struct {
|
||||||
|
Misfire string `json:"misfire"` // Misfire SKIP、COALESCE 或 CATCH_UP,必须明确提供。
|
||||||
|
Overlap string `json:"overlap"` // Overlap PARALLEL、SERIAL 或 SKIP,必须明确提供。
|
||||||
|
CatchupLimit uint32 `json:"catchup_limit"` // CatchupLimit 每批最多生成的任务数,1–100;串行最多一个。
|
||||||
|
CatchupWindowMS uint64 `json:"catchup_window_ms"` // CatchupWindowMS 最多保留多久的未处理触发点,1000 毫秒–30 天。
|
||||||
|
GraceMS uint64 `json:"grace_ms"` // GraceMS 允许正常派发延迟的毫秒数,1000–60000,不超过窗口。
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScheduleDefinition 是每次发布冻结的完整定义,业务内容仍由独立 Worker 解释。
|
||||||
|
type ScheduleDefinition struct {
|
||||||
|
Rule ScheduleRule `json:"rule"` // Rule 时间规则;start_at 是基准,不允许追溯到版本生效前产生新任务。
|
||||||
|
Policy SchedulePolicy `json:"policy"` // Policy 明确的漏跑、重叠和补跑边界。
|
||||||
|
Task SubmitTaskRequest `json:"task"` // Task 每次触发的模板,禁止绝对 available_at/start_deadline_at。
|
||||||
|
}
|
||||||
|
|
||||||
|
// CreateScheduleRequest 创建持久化计划,暂停计划也保存完整的触发进度。
|
||||||
|
type CreateScheduleRequest struct {
|
||||||
|
Name string `json:"name"` // Name 命名空间内唯一的 ASCII 名称。
|
||||||
|
Definition ScheduleDefinition `json:"definition"` // Definition 首个不可变版本。
|
||||||
|
Paused bool `json:"paused"` // Paused 是否从暂停状态开始,恢复时处理期间漏跑。
|
||||||
|
}
|
||||||
|
|
||||||
|
// PublishScheduleRequest 发布立即生效的新版本;旧积压仍按当时版本处理。
|
||||||
|
type PublishScheduleRequest struct {
|
||||||
|
ExpectedRevision uint64 `json:"expected_revision,string"` // ExpectedRevision 当前控制版本,必须非零。
|
||||||
|
Reason string `json:"reason"` // Reason 必填操作原因,写入审计。
|
||||||
|
Definition ScheduleDefinition `json:"definition"` // Definition 完整替换定义;队列不能更换。
|
||||||
|
}
|
||||||
|
|
||||||
|
// ControlScheduleRequest 暂停/恢复计划,不能取消已产生的任务。
|
||||||
|
type ControlScheduleRequest struct {
|
||||||
|
ExpectedRevision uint64 `json:"expected_revision,string"` // ExpectedRevision 乐观并发版本。
|
||||||
|
Reason string `json:"reason"` // Reason 必填操作原因,最多 1024 字符。
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScheduleVersion 保留规则、模板和原生效时间,已生成任务不会采用后续版本。
|
||||||
|
type ScheduleVersion struct {
|
||||||
|
Version uint64 `json:"version,string"` // Version 发布序号,从一递增。
|
||||||
|
EffectiveAt time.Time `json:"effective_at"` // EffectiveAt 数据库事务内取得的生效边界,包含此时间。
|
||||||
|
Definition ScheduleDefinition `json:"definition"` // Definition 已冻结内容。
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScheduleSummary 用于计划列表,不包含大任务参数。
|
||||||
|
type ScheduleSummary struct {
|
||||||
|
ID string `json:"id"` // ID 计划稳定身份。
|
||||||
|
NamespaceID string `json:"namespace_id"` // NamespaceID 隔离范围。
|
||||||
|
QueueID string `json:"queue_id"` // QueueID 计划终身绑定队列。
|
||||||
|
Name string `json:"name"` // Name 同命名空间内唯一名称。
|
||||||
|
Revision uint64 `json:"revision,string"` // Revision 控制和进度变更版本。
|
||||||
|
CurrentVersion uint64 `json:"current_version,string"` // CurrentVersion 最新发布版本。
|
||||||
|
ProcessingVersion uint64 `json:"processing_version,string"` // ProcessingVersion 尚在处理积压的历史版本。
|
||||||
|
Paused bool `json:"paused"` // Paused 为真时停止新触发。
|
||||||
|
NextFireAt *time.Time `json:"next_fire_at,omitempty"` // NextFireAt 原计划时间;积压时可在过去,待初始化或结束时为空,需结合 CheckAt。
|
||||||
|
CheckAt *time.Time `json:"check_at,omitempty"` // CheckAt 下次可扫描时间;为空表示没有扫描工作,非空也可能尚待初始化触发点。
|
||||||
|
CreatedAt time.Time `json:"created_at"` // CreatedAt UTC 创建时间。
|
||||||
|
UpdatedAt time.Time `json:"updated_at"` // UpdatedAt UTC 最近变更时间。
|
||||||
|
}
|
||||||
|
|
||||||
|
// Schedule 返回控制状态和最新版本,历史版本通过专用查询读取。
|
||||||
|
type Schedule struct {
|
||||||
|
ScheduleSummary
|
||||||
|
Current ScheduleVersion `json:"current"` // Current 最新完整发布版本,不一定是当前积压所用版本。
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScheduleResult 是幂等命令的原始回执,当前状态应使用 Get 查询。
|
||||||
|
type ScheduleResult struct {
|
||||||
|
Schedule Schedule `json:"schedule"` // Schedule 本次命令提交时的快照。
|
||||||
|
Deduplicated bool `json:"deduplicated"` // Deduplicated 是否取回原回执。
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScheduleOccurrence 描述一个已决定的触发点或不重叠范围。
|
||||||
|
type ScheduleOccurrence struct {
|
||||||
|
ID string `json:"id"` // ID 触发记录标识。
|
||||||
|
ScheduleID string `json:"schedule_id"` // ScheduleID 所属计划。
|
||||||
|
ScheduleVersion uint64 `json:"schedule_version,string"` // ScheduleVersion 当时适用的不可变版本。
|
||||||
|
ScheduledAt time.Time `json:"scheduled_at"` // ScheduledAt 原时间,参与唯一去重。
|
||||||
|
Decision string `json:"decision"` // Decision FIRED、SKIPPED 或 COALESCED。
|
||||||
|
TaskID *string `json:"task_id,omitempty"` // TaskID 已生成任务;跳过时为空。
|
||||||
|
CoveredUntil *time.Time `json:"covered_until,omitempty"` // CoveredUntil 范围排除边界,不一定是触发点。
|
||||||
|
Reason string `json:"reason"` // Reason 稳定决策分类。
|
||||||
|
CreatedAt time.Time `json:"created_at"` // CreatedAt 数据库记录时间。
|
||||||
|
}
|
||||||
|
type SchedulePage struct {
|
||||||
|
Items []ScheduleSummary `json:"items"` // Items 当前页摘要,始终为数组。
|
||||||
|
NextCursor string `json:"next_cursor,omitempty"` // NextCursor 不透明后续游标。
|
||||||
|
}
|
||||||
|
type ScheduleOccurrencePage struct {
|
||||||
|
Items []ScheduleOccurrence `json:"items"` // Items 当前页触发历史。
|
||||||
|
NextCursor string `json:"next_cursor,omitempty"` // NextCursor 原样用于相同父资源查询。
|
||||||
|
}
|
||||||
|
type SchedulePageRequest struct {
|
||||||
|
Limit int // Limit 默认 20,最大 100。
|
||||||
|
Cursor string // Cursor 最长 1024 字符,不能作为权限证明。
|
||||||
|
}
|
||||||
83
sdk/go/schedules.go
Normal file
83
sdk/go/schedules.go
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
package orbit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CreateSchedule 使用稳定幂等键受理一个计划;回执不代表任何一次触发已经执行。
|
||||||
|
func (c *Client) CreateSchedule(ctx context.Context, key string, in CreateScheduleRequest) (ScheduleResult, error) {
|
||||||
|
var out ScheduleResult
|
||||||
|
err := c.call(ctx, "POST", "/v1/schedules", key, in, &out)
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
func (c *Client) GetSchedule(ctx context.Context, id string) (Schedule, error) {
|
||||||
|
var out Schedule
|
||||||
|
if !validID(id) {
|
||||||
|
return out, errors.New("invalid schedule id")
|
||||||
|
}
|
||||||
|
err := c.call(ctx, "GET", "/v1/schedules/"+id, "", nil, &out)
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// PublishSchedule 发布完整新版本。保留原 key/revision 重报,不能重置原版本的积压身份。
|
||||||
|
func (c *Client) PublishSchedule(ctx context.Context, id, key string, in PublishScheduleRequest) (ScheduleResult, error) {
|
||||||
|
var out ScheduleResult
|
||||||
|
if !validID(id) {
|
||||||
|
return out, errors.New("invalid schedule id")
|
||||||
|
}
|
||||||
|
err := c.call(ctx, "PATCH", "/v1/schedules/"+id, key, in, &out)
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
func (c *Client) PauseSchedule(ctx context.Context, id, key string, in ControlScheduleRequest) (ScheduleResult, error) {
|
||||||
|
return c.controlSchedule(ctx, id, key, in, "pause")
|
||||||
|
}
|
||||||
|
func (c *Client) ResumeSchedule(ctx context.Context, id, key string, in ControlScheduleRequest) (ScheduleResult, error) {
|
||||||
|
return c.controlSchedule(ctx, id, key, in, "resume")
|
||||||
|
}
|
||||||
|
func (c *Client) controlSchedule(ctx context.Context, id, key string, in ControlScheduleRequest, action string) (ScheduleResult, error) {
|
||||||
|
var out ScheduleResult
|
||||||
|
if !validID(id) {
|
||||||
|
return out, errors.New("invalid schedule id")
|
||||||
|
}
|
||||||
|
err := c.call(ctx, "POST", "/v1/schedules/"+id+":"+action, key, in, &out)
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
func (c *Client) ListSchedules(ctx context.Context, queue string, in SchedulePageRequest) (SchedulePage, error) {
|
||||||
|
var out SchedulePage
|
||||||
|
if !validID(queue) {
|
||||||
|
return out, errors.New("invalid queue id")
|
||||||
|
}
|
||||||
|
q := scheduleQuery(in)
|
||||||
|
q.Set("queue_id", queue)
|
||||||
|
err := c.call(ctx, "GET", "/v1/schedules?"+q.Encode(), "", nil, &out)
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
func (c *Client) GetScheduleVersion(ctx context.Context, id string, version uint64) (ScheduleVersion, error) {
|
||||||
|
var out ScheduleVersion
|
||||||
|
if !validID(id) || version == 0 {
|
||||||
|
return out, errors.New("invalid schedule id or version")
|
||||||
|
}
|
||||||
|
err := c.call(ctx, "GET", "/v1/schedules/"+id+"/versions/"+strconv.FormatUint(version, 10), "", nil, &out)
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
func (c *Client) ListScheduleOccurrences(ctx context.Context, id string, in SchedulePageRequest) (ScheduleOccurrencePage, error) {
|
||||||
|
var out ScheduleOccurrencePage
|
||||||
|
if !validID(id) {
|
||||||
|
return out, errors.New("invalid schedule id")
|
||||||
|
}
|
||||||
|
err := c.call(ctx, "GET", "/v1/schedules/"+id+"/occurrences?"+scheduleQuery(in).Encode(), "", nil, &out)
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
func scheduleQuery(in SchedulePageRequest) url.Values {
|
||||||
|
q := url.Values{}
|
||||||
|
if in.Limit != 0 {
|
||||||
|
q.Set("limit", strconv.Itoa(in.Limit))
|
||||||
|
}
|
||||||
|
if in.Cursor != "" {
|
||||||
|
q.Set("cursor", in.Cursor)
|
||||||
|
}
|
||||||
|
return q
|
||||||
|
}
|
||||||
@@ -392,3 +392,83 @@ func TestIntegrationWorkerProcessCrashUnknown(t *testing.T) {
|
|||||||
t.Fatal("external work marker lost")
|
t.Fatal("external work marker lost")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 在没有其他 Orbit 节点的间隙真实强杀调度进程,再由两个新进程从数据库接续。
|
||||||
|
func TestIntegrationWorkerProcessScheduleRecovery(t *testing.T) {
|
||||||
|
a := startOrbitProcess(t)
|
||||||
|
_, queue, token := integrationWorkerClient(t)
|
||||||
|
apiA := processClient(t, a, token)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
name := "schedule-crash." + workerTestID(t)
|
||||||
|
start := time.Now().UTC().Truncate(time.Microsecond).Add(time.Second)
|
||||||
|
end := start.Add(8 * time.Second)
|
||||||
|
created, err := apiA.CreateSchedule(ctx, workerTestID(t), CreateScheduleRequest{Name: name, Definition: ScheduleDefinition{
|
||||||
|
Rule: ScheduleRule{Kind: "FIXED_RATE", Timezone: "UTC", StartAt: start, EndAt: &end, IntervalMS: 1000},
|
||||||
|
Policy: SchedulePolicy{Misfire: "CATCH_UP", Overlap: "PARALLEL", CatchupLimit: 2, CatchupWindowMS: 60000, GraceMS: 1000},
|
||||||
|
Task: SubmitTaskRequest{QueueID: queue, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"source_ref":"objects/recovery"}`)},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
page, err := apiA.ListScheduleOccurrences(ctx, created.Schedule.ID, SchedulePageRequest{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(page.Items) > 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal(ctx.Err())
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.kill(t)
|
||||||
|
// 三秒无服务形成真实漏跑,不改数据库时钟或游标。
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal(ctx.Err())
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
}
|
||||||
|
b := startOrbitProcess(t)
|
||||||
|
c := startOrbitProcess(t)
|
||||||
|
if a.nodeID == b.nodeID || a.nodeID == c.nodeID || b.nodeID == c.nodeID {
|
||||||
|
t.Fatal("scheduler nodes reused identity")
|
||||||
|
}
|
||||||
|
apiB := processClient(t, b, token)
|
||||||
|
for {
|
||||||
|
current, err := apiB.GetSchedule(ctx, created.Schedule.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if current.CheckAt == nil && current.NextFireAt == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal(ctx.Err())
|
||||||
|
case <-time.After(80 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
page, err := apiB.ListScheduleOccurrences(ctx, created.Schedule.ID, SchedulePageRequest{Limit: 100})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(page.Items) != 8 {
|
||||||
|
t.Fatalf("expected eight durable occurrences, got %d", len(page.Items))
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for i, item := range page.Items {
|
||||||
|
want := start.Add(time.Duration(7-i) * time.Second)
|
||||||
|
if item.TaskID == nil || item.Decision != "FIRED" || item.ScheduleVersion != 1 || !item.ScheduledAt.Equal(want) || seen[*item.TaskID] {
|
||||||
|
t.Fatal("recovery duplicated, skipped, or retimed occurrence")
|
||||||
|
}
|
||||||
|
seen[*item.TaskID] = true
|
||||||
|
task, err := apiB.GetTask(ctx, *item.TaskID)
|
||||||
|
if err != nil || task.State != TaskWaiting || string(task.Payload) != `{"source_ref":"objects/recovery"}` {
|
||||||
|
t.Fatal("occurrence/task transaction lost", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user