feat: tune websocket hub backpressure
This commit is contained in:
@@ -12,6 +12,12 @@ ORBIT_RECOVERY_TICK_INTERVAL=2s
|
||||
ORBIT_RUNTIME_BATCH_LIMIT=100
|
||||
# 收到 SIGTERM 后的排空与优雅关闭窗口，范围1s–5m，默认10s。
|
||||
ORBIT_SHUTDOWN_TIMEOUT=10s
|
||||
# 单节点 Worker/Event WebSocket 连接上限，范围1–4096；连接数仍受数据库和业务容量约束。
|
||||
ORBIT_WORKER_CONNECTION_LIMIT=128
|
||||
ORBIT_EVENT_CONNECTION_LIMIT=128
|
||||
# 每个 WebSocket Hub 的数据库领取并发和有界内存排队，范围分别为1–256、1–65536。
|
||||
ORBIT_DISPATCH_WORKERS=4
|
||||
ORBIT_DISPATCH_CAPACITY=128
|
||||
# Empty disables metrics. To enable a separate private listener, use e.g.
|
||||
# ORBIT_METRICS_ADDR=127.0.0.1:9090. Never expose it on the public API ingress.
|
||||
ORBIT_METRICS_ADDR=
|
||||
|
||||
@@ -70,11 +70,15 @@ func run() error {
|
||||
defer metricsListener.Close()
|
||||
metrics = observability.NewRegistry()
|
||||
metrics.RecordRuntimeConfig(observability.RuntimeConfig{
|
||||
WorkflowTickInterval: cfg.WorkflowTickInterval,
|
||||
ScheduleTickInterval: cfg.ScheduleTickInterval,
|
||||
RecoveryTickInterval: cfg.RecoveryTickInterval,
|
||||
BatchLimit: cfg.RuntimeBatchLimit,
|
||||
ShutdownTimeout: cfg.ShutdownTimeout,
|
||||
WorkflowTickInterval: cfg.WorkflowTickInterval,
|
||||
ScheduleTickInterval: cfg.ScheduleTickInterval,
|
||||
RecoveryTickInterval: cfg.RecoveryTickInterval,
|
||||
BatchLimit: cfg.RuntimeBatchLimit,
|
||||
ShutdownTimeout: cfg.ShutdownTimeout,
|
||||
WorkerConnectionLimit: cfg.WorkerConnectionLimit,
|
||||
EventConnectionLimit: cfg.EventConnectionLimit,
|
||||
DispatchWorkers: cfg.DispatchWorkers,
|
||||
DispatchCapacity: cfg.DispatchCapacity,
|
||||
})
|
||||
metrics.RecordDatabase(db.Stats())
|
||||
}
|
||||
@@ -88,10 +92,18 @@ func run() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hub := httpserver.NewWorkerHub(root, worker.New(executions))
|
||||
hub := httpserver.NewWorkerHub(root, worker.New(executions), httpserver.HubOptions{
|
||||
ConnectionLimit: cfg.WorkerConnectionLimit,
|
||||
DispatchWorkers: cfg.DispatchWorkers,
|
||||
DispatchCapacity: cfg.DispatchCapacity,
|
||||
})
|
||||
httpserver.RegisterWorkers(server, hub)
|
||||
eventService := events.New(mysql.NewEventStore(db))
|
||||
eventHub := httpserver.NewEventHub(root, eventService)
|
||||
eventHub := httpserver.NewEventHub(root, eventService, httpserver.HubOptions{
|
||||
ConnectionLimit: cfg.EventConnectionLimit,
|
||||
DispatchWorkers: cfg.DispatchWorkers,
|
||||
DispatchCapacity: cfg.DispatchCapacity,
|
||||
})
|
||||
httpserver.RegisterEventSubscriptions(server, eventService)
|
||||
httpserver.RegisterEventConsumers(server, eventHub)
|
||||
defer func() {
|
||||
@@ -250,7 +262,7 @@ func run() error {
|
||||
slog.Info("Orbit metrics server starting", "metrics_address", metricsListener.Addr().String())
|
||||
}
|
||||
go func() { done <- server.Serve(listener) }()
|
||||
slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task, queue, schedule, subscription, workflow and Worker APIs", "node_id", node, "workflow_tick_interval", cfg.WorkflowTickInterval.String(), "schedule_tick_interval", cfg.ScheduleTickInterval.String(), "recovery_tick_interval", cfg.RecoveryTickInterval.String(), "runtime_batch_limit", cfg.RuntimeBatchLimit, "shutdown_timeout", cfg.ShutdownTimeout.String())
|
||||
slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task, queue, schedule, subscription, workflow and Worker APIs", "node_id", node, "workflow_tick_interval", cfg.WorkflowTickInterval.String(), "schedule_tick_interval", cfg.ScheduleTickInterval.String(), "recovery_tick_interval", cfg.RecoveryTickInterval.String(), "runtime_batch_limit", cfg.RuntimeBatchLimit, "shutdown_timeout", cfg.ShutdownTimeout.String(), "worker_connection_limit", cfg.WorkerConnectionLimit, "event_connection_limit", cfg.EventConnectionLimit, "dispatch_workers", cfg.DispatchWorkers, "dispatch_capacity", cfg.DispatchCapacity)
|
||||
select {
|
||||
case err := <-done:
|
||||
if errors.Is(err, http.ErrServerClosed) {
|
||||
|
||||
@@ -81,7 +81,7 @@ P4/P5定时等待与后续查询轮次的实现及回归记录见[时钟报告](
|
||||
### P6 首批实施顺序
|
||||
|
||||
1. 已补[首批可复现故障工具](reports/phase-6-fault-boundaries.md)：A04“派发已提交但消息尚未送达”、W01/W13事务提交结果不明，以及W16最终回执/旧PENDING结果重放/下一轮竞争。迁移器同时提供当前版本无外键新装基线；每个切点必须有实际数据库或进程证据，不能用响应错误推断事务是否已提交，也不能用普通SIGNAL竞争替代最终查询场景。
|
||||
2. 已实现[基础运行观测](api/observability.md)及[有界业务状态与事件采样](reports/phase-6-business-observability.md)：HTTP/WS、数据库连接池、后台批次、固定任务/交付/流程/Inbox状态计数、最近事件类别、终态耗时样本、固定事件阶段耗时、最早年龄、截断/新鲜度指标和可离线验证的告警规则；独立采集监听默认关闭，固定指标目录拒绝动态标签；发布前只读结构核验由[`orbit-admin verify`](api/operations.md)提供，运维人员也可用 `orbit-admin observe snapshot` 获取同一有界单节点快照；后台循环间隔和批量上限已支持有界环境配置并在启动日志中记录。仍需端到端外部效果延迟、全局持久吞吐、Inbox从事件产生到推进完成的精确阶段延迟、全局发送队列、多节点聚合及生产采集/通知验证；SDK受控重连风暴和真实慢消费者背压已补[边界回归](reports/phase-6-reconnect-storm.md)与[慢消费者报告](reports/phase-6-slow-consumer.md)，跨节点重连仍待目标拓扑。
|
||||
2. 已实现[基础运行观测](api/observability.md)及[有界业务状态与事件采样](reports/phase-6-business-observability.md)：HTTP/WS、数据库连接池、后台批次、固定任务/交付/流程/Inbox状态计数、最近事件类别、终态耗时样本、固定事件阶段耗时、最早年龄、截断/新鲜度指标和可离线验证的告警规则；独立采集监听默认关闭，固定指标目录拒绝动态标签；发布前只读结构核验由[`orbit-admin verify`](api/operations.md)提供，运维人员也可用 `orbit-admin observe snapshot` 获取同一有界单节点快照；后台循环间隔、批量上限、Worker/Event连接上限、Hub领取协程和有界内存排队均支持有界环境配置，并在启动日志和固定指标中记录。仍需端到端外部效果延迟、全局持久吞吐、Inbox从事件产生到推进完成的精确阶段延迟、全局发送队列、多节点聚合及生产采集/通知验证；SDK受控重连风暴和真实慢消费者背压已补[边界回归](reports/phase-6-reconnect-storm.md)与[慢消费者报告](reports/phase-6-slow-consumer.md)，跨节点重连仍待目标拓扑。
|
||||
3. 已补齐受保护管理环境中的固定凭据操作文件、轮换、撤销、提交不明故障切点、幂等回执和脱敏管理审计；轮换保留授权绑定并要求新令牌重新建立会话，旧 Worker 会话被排空且立即过期，撤销允许在停用范围内执行且不改写历史记录。已用一次性 MySQL 定向验证旧令牌失效、新令牌重绑和零外键；仍需多节点安全重绑演练、凭据最小权限配置核对和生产密钥托管/审计演练。
|
||||
4. 已补[`orbit-admin retention inventory`](api/retention.md)只读盘点和[阶段报告](reports/phase-6-retention-inventory.md)，能区分过期幂等、确定终态、未知效果、未终结流程、已确认/未确认交付、已消费/未消费Inbox、事件和审计规模；它不执行归档或删除。已补[容量模型](19-capacity-model.md)，把企业数转换为任务、流程、事件、连接、写放大和保留期输入；仍需评审各类保留周期、归档格式、受保护记录核对，再执行备份恢复、兼容升级和六类容量负载。未确认交付、有效幂等身份、当前/历史执行证据和未终结流程不能被普通清理破坏。生产HA及RPO/RTO继续按实际拓扑验证，不缩减为本机多进程检查。
|
||||
5. 已补[`orbit-load`](api/load.md)有界受理负载工具及24次/1000次隔离样本，见[受理报告](reports/phase-6-load.md)。新增独立[`orbit-exec-load`](api/execution-load.md)，通过真实Worker、任务结果和尝试历史核对，提供短/混合/中长任务的受理、排队、授权执行及完成耗时分布；具体环境、原始JSON和失败分母见[执行测量报告](reports/phase-6-execution-load.md)。新增[`orbit-workflow-load`](api/workflow-load.md)，用本次发布版本的双分支并行流程，逐个核对不同任务、唯一尝试、来源、精确输出与本地调用；24/100实例及24实例截止负例已重新构建回归并绑定源码/二进制哈希，见[流程测量报告](reports/phase-6-workflow-load.md)。新增[`orbit-schedule-load`](api/schedule-load.md)，已完成24/128次定时集中触发与24次计划量的截止负例，保存实际分母、唯一任务/尝试、来源与暂停收尾证据，见[定时测量报告](reports/phase-6-schedule-load.md)。Go 1.27.1 模块级门禁、MySQL 存储集成、关键 SDK/Orbit 进程故障组、24个可运行 SDK 示例真实服务串跑和 result-inbox/task-outbox 真实业务集成已复跑通过。已补[长期积压与资源曲线样本](reports/phase-6-backlog.md)：32×8 共256个触发点，在1个Worker/容量1和180秒截止下保留173个成功、83个未核验任务，并以独立数据库读取记录等待积压曲线；该单节点样本不替代目标拓扑容量。SDK受控重连风暴及[真实慢消费者背压](reports/phase-6-slow-consumer.md)已补边界回归；下一步补指标采集对照、长期存储增长、跨节点重连与目标数据库拓扑容量。
|
||||
|
||||
@@ -45,7 +45,7 @@ curl -fsS http://127.0.0.1:8080/health/ready
|
||||
|
||||
`live` 只表示进程仍在运行；`ready` 同时核对 MySQL、迁移账本和零外键要求。收到 `503` 时停止向该节点发送新的受理、执行授权和流程推进请求，保留原幂等键并等待核对，不把网络错误当作业务失败。
|
||||
|
||||
后台调度默认每秒检查 Weave 和计划、每两秒执行一次执行恢复，每批最多处理100条持久化候选。容量验证后可以通过 `ORBIT_WORKFLOW_TICK_INTERVAL`、`ORBIT_SCHEDULE_TICK_INTERVAL`、`ORBIT_RECOVERY_TICK_INTERVAL` 和 `ORBIT_RUNTIME_BATCH_LIMIT` 调整；间隔只允许 `100ms` 到 `1m`，批量只允许 `1` 到 `100`。`ORBIT_SHUTDOWN_TIMEOUT` 控制 SIGTERM 后的排空和优雅关闭窗口，允许 `1s` 到 `5m`，默认 `10s`。调优必须和数据库锁等待、连接池、积压及恢复窗口一起记录，不能只提高批量或缩短间隔来掩盖容量不足。
|
||||
后台调度默认每秒检查 Weave 和计划、每两秒执行一次执行恢复，每批最多处理100条持久化候选。容量验证后可以通过 `ORBIT_WORKFLOW_TICK_INTERVAL`、`ORBIT_SCHEDULE_TICK_INTERVAL`、`ORBIT_RECOVERY_TICK_INTERVAL` 和 `ORBIT_RUNTIME_BATCH_LIMIT` 调整；间隔只允许 `100ms` 到 `1m`，批量只允许 `1` 到 `100`。`ORBIT_SHUTDOWN_TIMEOUT` 控制 SIGTERM 后的排空和优雅关闭窗口，允许 `1s` 到 `5m`，默认 `10s`。Worker/Event WebSocket 连接上限分别由 `ORBIT_WORKER_CONNECTION_LIMIT`、`ORBIT_EVENT_CONNECTION_LIMIT` 控制，范围 `1` 到 `4096`；每个 Hub 的数据库领取协程和有界内存排队由 `ORBIT_DISPATCH_WORKERS`、`ORBIT_DISPATCH_CAPACITY` 控制，范围分别为 `1` 到 `256`、`1` 到 `65536`。这些参数只影响单节点内存和领取并发，真实授权、租约和配额仍由 MySQL 决定。调优必须和数据库锁等待、连接池、积压及恢复窗口一起记录，不能只提高批量或缩短间隔来掩盖容量不足。
|
||||
|
||||
## 3. 滚动停止与替换
|
||||
|
||||
|
||||
@@ -54,6 +54,10 @@ docker run --rm \
|
||||
--env ORBIT_SCHEDULE_TICK_INTERVAL=1s \
|
||||
--env ORBIT_RECOVERY_TICK_INTERVAL=2s \
|
||||
--env ORBIT_SHUTDOWN_TIMEOUT=10s \
|
||||
--env ORBIT_WORKER_CONNECTION_LIMIT=128 \
|
||||
--env ORBIT_EVENT_CONNECTION_LIMIT=128 \
|
||||
--env ORBIT_DISPATCH_WORKERS=4 \
|
||||
--env ORBIT_DISPATCH_CAPACITY=128 \
|
||||
--env ORBIT_METRICS_ADDR=127.0.0.1:9090 \
|
||||
--publish 8080:8080 \
|
||||
registry.example/orbit:dev
|
||||
|
||||
@@ -27,7 +27,7 @@ curl -fsS http://127.0.0.1:9090/metrics
|
||||
|
||||
每节点同时最多2个采集请求，超出返回503及 `Retry-After: 1`，不排无限等待队列。独立HTTP服务器的读头/读/写/空闲超时分别为2/3/5/15秒，头部限制8 KiB。每次HTTP采集只访问进程内存，不查询业务表；DB池快照在启动时及之后每5秒读取 `database/sql.DBStats`，该调用不执行SQL。指标启用后同一个5秒后台采样器另外执行一次有界业务快照：查询固定状态、固定表和固定索引，单状态或事件类别最多读取10000条；固定事件类别窗口为最近5分钟；数据库调用使用900ms截止时间；失败只增加错误/超时计数并保留上次成功值，不改变业务事务。
|
||||
|
||||
所有指标名称、类型、标签名和取值在代码目录中固定。未知名称、错误类型、非法标签、非有限数值和计数器倒退会被丢弃，并增加 `orbit_observability_rejected_samples_total`。新增指标需要修改目录和回归，不允许业务方动态注册标签。HTTP标签为8种方法、12种路由分类和6种状态分类，两组HTTP族各最多576种标签组合。禁止将任务/流程/租户ID、业务键、URL、参数、原始错误或凭据写入指标。
|
||||
所有指标名称、类型、标签名和取值在代码目录中固定。未知名称、错误类型、非法标签、非有限数值和计数器倒退会被丢弃，并增加 `orbit_observability_rejected_samples_total`。新增指标需要修改目录和回归，不允许业务方动态注册标签。HTTP标签为8种方法、12种路由分类和6种状态分类，两组HTTP族各最多576种标签组合。禁止将任务/流程/租户ID、业务键、URL、参数、原始错误或凭据写入指标。Worker/Event连接上限、Hub领取协程数和内存排队槽位也以固定基数导出，便于核对节点实际生效的背压边界。
|
||||
|
||||
格式化输出前只短暂复制注册器引用；不在全局记录锁下排序、格式化或写网络。直方图的桶、count、sum在同一局部锁下获取，保持单条直方图快照一致。指标在重启后重置，不能用于账务审计、幂等判断或恢复授权。
|
||||
|
||||
@@ -47,6 +47,10 @@ curl -fsS http://127.0.0.1:9090/metrics
|
||||
| `orbit_runtime_tick_interval_seconds{component}` | gauge | 节点实际生效的后台扫描间隔，component为workflow、schedule、execution_recovery |
|
||||
| `orbit_runtime_batch_limit` | gauge | 节点实际生效的后台批量上限 |
|
||||
| `orbit_shutdown_timeout_seconds` | gauge | 节点实际生效的排空与优雅关闭窗口（秒） |
|
||||
| `orbit_worker_connection_limit` | gauge | 节点实际生效的 Worker WebSocket 连接上限 |
|
||||
| `orbit_event_connection_limit` | gauge | 节点实际生效的 Event WebSocket 连接上限 |
|
||||
| `orbit_dispatch_workers` | gauge | 每个 WebSocket Hub 同时执行数据库领取调用的协程上限 |
|
||||
| `orbit_dispatch_capacity` | gauge | 每个 WebSocket Hub 领取调用的有界内存排队槽位 |
|
||||
| `orbit_db_connections_open` / `orbit_db_connections_in_use` / `orbit_db_connections_idle` | gauge | 本节点MySQL连接池当前连接数 |
|
||||
| `orbit_db_connections_max_open` | gauge | 池配置的最大连接数 |
|
||||
| `orbit_db_wait_count_total` / `orbit_db_wait_duration_seconds_total` | counter | 连接池累计等待次数/秒数，不是数据库锁等待 |
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
| 服务端单元与竞态、vet、构建 | Docker `golang:1.27.1`，仓库声明版本；`GOWORK=off go test -race ./... && go vet ./... && go build ./...` | PASS |
|
||||
| 后台运行参数边界 | `internal/config.TestRuntimeDefaultsAndBoundedOverrides`；默认值与五个 `ORBIT_*` 覆盖项 | PASS；循环间隔限定100ms–1m，批量限定1–100，排空超时限定1s–5m，服务启动日志记录生效值 |
|
||||
| 生效运行参数指标 | `internal/observability.TestRuntimeObservationDoesNotExposeErrors`；固定指标目录 | PASS；workflow/schedule/execution_recovery 间隔、批量上限和排空超时以固定基数指标导出；排空状态同步为固定 gauge，不进入业务或租户标签 |
|
||||
| WebSocket Hub 背压参数 | 2026-09-20；`go test -race ./internal/config ./internal/observability ./internal/transport/httpserver ./cmd/orbit` | PASS；Worker/Event 连接上限、每个 Hub 的领取协程和有界内存排队可由环境配置，范围校验、应用和固定运行指标均有测试；持久化授权、租约和配额规则不由内存参数替代 |
|
||||
| SDK 与独立示例竞态、vet、构建 | Docker `golang:1.27.1`；SDK 全 module、全部示例、`orbit-load`、`orbit-exec-load`、`orbit-schedule-load`、`orbit-workflow-load`，以及 result-inbox/task-outbox | PASS；健康/版本 SDK 契约包含 `Client.Version(ctx)`，返回协议身份且不暴露敏感响应 |
|
||||
| 容器发布物构建 | `make image`；宿主平台 linux/arm64；显式 `docker build --platform linux/amd64`；带 `VERSION`、`REVISION`、`BUILD_TIME` 构建参数的版本化镜像；镜像内 `GOWORK=off` | PASS；native 与 amd64 镜像均构建成功，包含 `orbit`、`orbit-migrate`、`orbit-admin`，运行用户为非 root；版本化 amd64 二进制和 OCI 标签均包含注入的构建身份，`GET /version` 契约单测通过；无 DSN 时服务按预期拒绝启动 |
|
||||
| 当前协议门禁 | `make protocol-check` | PASS，当前 v1 契约同步 |
|
||||
|
||||
@@ -10,31 +10,39 @@ import (
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
HTTPAddr string
|
||||
MySQLDSN string
|
||||
MaxOpen int
|
||||
MaxIdle int
|
||||
MetricsAddr string
|
||||
WorkflowTickInterval time.Duration // WorkflowTickInterval Weave 持久化推进扫描间隔，范围100ms–1m。
|
||||
ScheduleTickInterval time.Duration // ScheduleTickInterval 计划扫描间隔，范围100ms–1m。
|
||||
RecoveryTickInterval time.Duration // RecoveryTickInterval 执行租约恢复扫描间隔，范围100ms–1m。
|
||||
RuntimeBatchLimit int // RuntimeBatchLimit 每个后台批次的候选上限，范围1–100。
|
||||
ShutdownTimeout time.Duration // ShutdownTimeout 优雅排空和关闭窗口，范围1s–5m。
|
||||
HTTPAddr string
|
||||
MySQLDSN string
|
||||
MaxOpen int
|
||||
MaxIdle int
|
||||
MetricsAddr string
|
||||
WorkflowTickInterval time.Duration // WorkflowTickInterval Weave 持久化推进扫描间隔，范围100ms–1m。
|
||||
ScheduleTickInterval time.Duration // ScheduleTickInterval 计划扫描间隔，范围100ms–1m。
|
||||
RecoveryTickInterval time.Duration // RecoveryTickInterval 执行租约恢复扫描间隔，范围100ms–1m。
|
||||
RuntimeBatchLimit int // RuntimeBatchLimit 每个后台批次的候选上限，范围1–100。
|
||||
ShutdownTimeout time.Duration // ShutdownTimeout 优雅排空和关闭窗口，范围1s–5m。
|
||||
WorkerConnectionLimit int // WorkerConnectionLimit 单节点 Worker WebSocket 连接上限，范围1–4096。
|
||||
EventConnectionLimit int // EventConnectionLimit 单节点 Event WebSocket 连接上限，范围1–4096。
|
||||
DispatchWorkers int // DispatchWorkers 单个 Hub 并发执行数据库领取调用的协程数，范围1–256。
|
||||
DispatchCapacity int // DispatchCapacity 单个 Hub 领取调用的有界内存排队槽位，范围1–65536。
|
||||
}
|
||||
|
||||
func Load() (Config, error) { return FromEnv(os.Getenv) }
|
||||
func FromEnv(get func(string) string) (Config, error) {
|
||||
c := Config{
|
||||
HTTPAddr: get("ORBIT_HTTP_ADDR"),
|
||||
MySQLDSN: get("ORBIT_MYSQL_DSN"),
|
||||
MaxOpen: 20,
|
||||
MaxIdle: 5,
|
||||
MetricsAddr: get("ORBIT_METRICS_ADDR"),
|
||||
WorkflowTickInterval: time.Second,
|
||||
ScheduleTickInterval: time.Second,
|
||||
RecoveryTickInterval: 2 * time.Second,
|
||||
RuntimeBatchLimit: 100,
|
||||
ShutdownTimeout: 10 * time.Second,
|
||||
HTTPAddr: get("ORBIT_HTTP_ADDR"),
|
||||
MySQLDSN: get("ORBIT_MYSQL_DSN"),
|
||||
MaxOpen: 20,
|
||||
MaxIdle: 5,
|
||||
MetricsAddr: get("ORBIT_METRICS_ADDR"),
|
||||
WorkflowTickInterval: time.Second,
|
||||
ScheduleTickInterval: time.Second,
|
||||
RecoveryTickInterval: 2 * time.Second,
|
||||
RuntimeBatchLimit: 100,
|
||||
ShutdownTimeout: 10 * time.Second,
|
||||
WorkerConnectionLimit: 128,
|
||||
EventConnectionLimit: 128,
|
||||
DispatchWorkers: 4,
|
||||
DispatchCapacity: 128,
|
||||
}
|
||||
if c.HTTPAddr == "" {
|
||||
c.HTTPAddr = "127.0.0.1:8080"
|
||||
@@ -86,6 +94,25 @@ func FromEnv(get func(string) string) (Config, error) {
|
||||
}
|
||||
c.ShutdownTimeout = d
|
||||
}
|
||||
for _, field := range []struct {
|
||||
name string
|
||||
dst *int
|
||||
min int
|
||||
max int
|
||||
}{
|
||||
{"ORBIT_WORKER_CONNECTION_LIMIT", &c.WorkerConnectionLimit, 1, 4096},
|
||||
{"ORBIT_EVENT_CONNECTION_LIMIT", &c.EventConnectionLimit, 1, 4096},
|
||||
{"ORBIT_DISPATCH_WORKERS", &c.DispatchWorkers, 1, 256},
|
||||
{"ORBIT_DISPATCH_CAPACITY", &c.DispatchCapacity, 1, 65536},
|
||||
} {
|
||||
if raw := get(field.name); raw != "" {
|
||||
n, err := strconv.Atoi(raw)
|
||||
if err != nil || n < field.min || n > field.max {
|
||||
return Config{}, fmt.Errorf("%s must be an integer from %d to %d", field.name, field.min, field.max)
|
||||
}
|
||||
*field.dst = n
|
||||
}
|
||||
}
|
||||
if c.MetricsAddr != "" {
|
||||
_, port, err := net.SplitHostPort(c.MetricsAddr)
|
||||
n, parseErr := strconv.Atoi(port)
|
||||
|
||||
@@ -9,7 +9,7 @@ func TestRuntimeDefaultsAndBoundedOverrides(t *testing.T) {
|
||||
base := map[string]string{"ORBIT_MYSQL_DSN": "private-dsn"}
|
||||
get := func(k string) string { return base[k] }
|
||||
c, err := FromEnv(get)
|
||||
if err != nil || c.WorkflowTickInterval != time.Second || c.ScheduleTickInterval != time.Second || c.RecoveryTickInterval != 2*time.Second || c.RuntimeBatchLimit != 100 || c.ShutdownTimeout != 10*time.Second {
|
||||
if err != nil || c.WorkflowTickInterval != time.Second || c.ScheduleTickInterval != time.Second || c.RecoveryTickInterval != 2*time.Second || c.RuntimeBatchLimit != 100 || c.ShutdownTimeout != 10*time.Second || c.WorkerConnectionLimit != 128 || c.EventConnectionLimit != 128 || c.DispatchWorkers != 4 || c.DispatchCapacity != 128 {
|
||||
t.Fatalf("unexpected runtime defaults: %+v err=%v", c, err)
|
||||
}
|
||||
base["ORBIT_WORKFLOW_TICK_INTERVAL"] = "250ms"
|
||||
@@ -17,16 +17,24 @@ func TestRuntimeDefaultsAndBoundedOverrides(t *testing.T) {
|
||||
base["ORBIT_RECOVERY_TICK_INTERVAL"] = "1m"
|
||||
base["ORBIT_RUNTIME_BATCH_LIMIT"] = "37"
|
||||
base["ORBIT_SHUTDOWN_TIMEOUT"] = "45s"
|
||||
base["ORBIT_WORKER_CONNECTION_LIMIT"] = "512"
|
||||
base["ORBIT_EVENT_CONNECTION_LIMIT"] = "256"
|
||||
base["ORBIT_DISPATCH_WORKERS"] = "16"
|
||||
base["ORBIT_DISPATCH_CAPACITY"] = "2048"
|
||||
c, err = FromEnv(get)
|
||||
if err != nil || c.WorkflowTickInterval != 250*time.Millisecond || c.ScheduleTickInterval != 1500*time.Millisecond || c.RecoveryTickInterval != time.Minute || c.RuntimeBatchLimit != 37 || c.ShutdownTimeout != 45*time.Second {
|
||||
if err != nil || c.WorkflowTickInterval != 250*time.Millisecond || c.ScheduleTickInterval != 1500*time.Millisecond || c.RecoveryTickInterval != time.Minute || c.RuntimeBatchLimit != 37 || c.ShutdownTimeout != 45*time.Second || c.WorkerConnectionLimit != 512 || c.EventConnectionLimit != 256 || c.DispatchWorkers != 16 || c.DispatchCapacity != 2048 {
|
||||
t.Fatalf("valid runtime override rejected: %+v err=%v", c, err)
|
||||
}
|
||||
for key, value := range map[string]string{
|
||||
"ORBIT_WORKFLOW_TICK_INTERVAL": "99ms",
|
||||
"ORBIT_SCHEDULE_TICK_INTERVAL": "2m",
|
||||
"ORBIT_RECOVERY_TICK_INTERVAL": "not-a-duration",
|
||||
"ORBIT_RUNTIME_BATCH_LIMIT": "101",
|
||||
"ORBIT_SHUTDOWN_TIMEOUT": "500ms",
|
||||
"ORBIT_WORKFLOW_TICK_INTERVAL": "99ms",
|
||||
"ORBIT_SCHEDULE_TICK_INTERVAL": "2m",
|
||||
"ORBIT_RECOVERY_TICK_INTERVAL": "not-a-duration",
|
||||
"ORBIT_RUNTIME_BATCH_LIMIT": "101",
|
||||
"ORBIT_SHUTDOWN_TIMEOUT": "500ms",
|
||||
"ORBIT_WORKER_CONNECTION_LIMIT": "0",
|
||||
"ORBIT_EVENT_CONNECTION_LIMIT": "4097",
|
||||
"ORBIT_DISPATCH_WORKERS": "257",
|
||||
"ORBIT_DISPATCH_CAPACITY": "65537",
|
||||
} {
|
||||
base[key] = value
|
||||
if _, err := FromEnv(get); err == nil {
|
||||
|
||||
@@ -66,5 +66,9 @@ func catalog() map[string]descriptor {
|
||||
c["orbit_runtime_tick_interval_seconds"] = descriptor{gauge, "Configured background controller tick interval in seconds", map[string][]string{"component": components}}
|
||||
c["orbit_runtime_batch_limit"] = descriptor{gauge, "Configured maximum candidates per background batch", nil}
|
||||
c["orbit_shutdown_timeout_seconds"] = descriptor{gauge, "Configured graceful drain and shutdown window in seconds", nil}
|
||||
c["orbit_worker_connection_limit"] = descriptor{gauge, "Configured maximum Worker WebSocket connections on this node", nil}
|
||||
c["orbit_event_connection_limit"] = descriptor{gauge, "Configured maximum Event WebSocket connections on this node", nil}
|
||||
c["orbit_dispatch_workers"] = descriptor{gauge, "Configured database claim workers per WebSocket hub", nil}
|
||||
c["orbit_dispatch_capacity"] = descriptor{gauge, "Configured bounded in-memory claim queue per WebSocket hub", nil}
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -96,7 +96,7 @@ func TestInvalidNumbersAndMonotoneCounters(t *testing.T) {
|
||||
}
|
||||
func TestRuntimeObservationDoesNotExposeErrors(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
r.RecordRuntimeConfig(RuntimeConfig{WorkflowTickInterval: time.Second, ScheduleTickInterval: 1500 * time.Millisecond, RecoveryTickInterval: 2 * time.Second, BatchLimit: 37, ShutdownTimeout: 45 * time.Second})
|
||||
r.RecordRuntimeConfig(RuntimeConfig{WorkflowTickInterval: time.Second, ScheduleTickInterval: 1500 * time.Millisecond, RecoveryTickInterval: 2 * time.Second, BatchLimit: 37, ShutdownTimeout: 45 * time.Second, WorkerConnectionLimit: 512, EventConnectionLimit: 256, DispatchWorkers: 16, DispatchCapacity: 2048})
|
||||
r.RecordDraining(true)
|
||||
r.RecordBatch("workflow", time.Now().Add(-time.Millisecond), nil)
|
||||
r.RecordBatch("workflow", time.Now(), errors.New("private-dsn"))
|
||||
@@ -108,7 +108,7 @@ func TestRuntimeObservationDoesNotExposeErrors(t *testing.T) {
|
||||
t.Fatal("missing runtime evidence", want)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{`orbit_runtime_tick_interval_seconds{component="workflow"} 1`, `orbit_runtime_tick_interval_seconds{component="schedule"} 1.5`, `orbit_runtime_tick_interval_seconds{component="execution_recovery"} 2`, "orbit_runtime_batch_limit 37\n", "orbit_shutdown_timeout_seconds 45\n", "orbit_process_draining 1\n"} {
|
||||
for _, want := range []string{`orbit_runtime_tick_interval_seconds{component="workflow"} 1`, `orbit_runtime_tick_interval_seconds{component="schedule"} 1.5`, `orbit_runtime_tick_interval_seconds{component="execution_recovery"} 2`, "orbit_runtime_batch_limit 37\n", "orbit_shutdown_timeout_seconds 45\n", "orbit_worker_connection_limit 512\n", "orbit_event_connection_limit 256\n", "orbit_dispatch_workers 16\n", "orbit_dispatch_capacity 2048\n", "orbit_process_draining 1\n"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatal("missing runtime configuration", want)
|
||||
}
|
||||
|
||||
@@ -13,16 +13,20 @@ import (
|
||||
// RuntimeConfig 是节点启动后实际生效的后台控制参数。
|
||||
// 它只用于固定基数观测，不参与业务授权或持久化状态判断。
|
||||
type RuntimeConfig struct {
|
||||
WorkflowTickInterval time.Duration
|
||||
ScheduleTickInterval time.Duration
|
||||
RecoveryTickInterval time.Duration
|
||||
BatchLimit int
|
||||
ShutdownTimeout time.Duration
|
||||
WorkflowTickInterval time.Duration
|
||||
ScheduleTickInterval time.Duration
|
||||
RecoveryTickInterval time.Duration
|
||||
BatchLimit int
|
||||
ShutdownTimeout time.Duration
|
||||
WorkerConnectionLimit int // Worker WebSocket 单节点连接上限。
|
||||
EventConnectionLimit int // Event WebSocket 单节点连接上限。
|
||||
DispatchWorkers int // 每个 Hub 同时执行数据库领取调用的协程上限。
|
||||
DispatchCapacity int // 每个 Hub 领取调用的有界内存排队槽位。
|
||||
}
|
||||
|
||||
// RecordRuntimeConfig records the effective bounded controller settings.
|
||||
func (r *Registry) RecordRuntimeConfig(config RuntimeConfig) {
|
||||
if r == nil || config.WorkflowTickInterval <= 0 || config.ScheduleTickInterval <= 0 || config.RecoveryTickInterval <= 0 || config.BatchLimit <= 0 || config.ShutdownTimeout <= 0 {
|
||||
if r == nil || config.WorkflowTickInterval <= 0 || config.ScheduleTickInterval <= 0 || config.RecoveryTickInterval <= 0 || config.BatchLimit <= 0 || config.ShutdownTimeout <= 0 || config.WorkerConnectionLimit <= 0 || config.EventConnectionLimit <= 0 || config.DispatchWorkers <= 0 || config.DispatchCapacity <= 0 {
|
||||
return
|
||||
}
|
||||
r.SetGauge("orbit_runtime_tick_interval_seconds", config.WorkflowTickInterval.Seconds(), Labels{"component": "workflow"})
|
||||
@@ -30,6 +34,10 @@ func (r *Registry) RecordRuntimeConfig(config RuntimeConfig) {
|
||||
r.SetGauge("orbit_runtime_tick_interval_seconds", config.RecoveryTickInterval.Seconds(), Labels{"component": "execution_recovery"})
|
||||
r.SetGauge("orbit_runtime_batch_limit", float64(config.BatchLimit), nil)
|
||||
r.SetGauge("orbit_shutdown_timeout_seconds", config.ShutdownTimeout.Seconds(), nil)
|
||||
r.SetGauge("orbit_worker_connection_limit", float64(config.WorkerConnectionLimit), nil)
|
||||
r.SetGauge("orbit_event_connection_limit", float64(config.EventConnectionLimit), nil)
|
||||
r.SetGauge("orbit_dispatch_workers", float64(config.DispatchWorkers), nil)
|
||||
r.SetGauge("orbit_dispatch_capacity", float64(config.DispatchCapacity), nil)
|
||||
}
|
||||
|
||||
// RecordDraining records whether this process has entered graceful shutdown.
|
||||
|
||||
@@ -31,10 +31,11 @@ type EventHub struct {
|
||||
poll time.Duration
|
||||
}
|
||||
|
||||
func NewEventHub(parent context.Context, service *events.Service) *EventHub {
|
||||
func NewEventHub(parent context.Context, service *events.Service, options ...HubOptions) *EventHub {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
h := &EventHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, 128), poll: time.Second}
|
||||
h.dispatch = dispatch.New(ctx, 4, 128)
|
||||
limits := resolveHubOptions(options)
|
||||
h := &EventHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, limits.ConnectionLimit), poll: time.Second}
|
||||
h.dispatch = dispatch.New(ctx, limits.DispatchWorkers, limits.DispatchCapacity)
|
||||
h.wg.Add(1)
|
||||
go func() { defer h.wg.Done(); <-h.dispatch.Done() }()
|
||||
return h
|
||||
|
||||
48
internal/transport/httpserver/hub_options.go
Normal file
48
internal/transport/httpserver/hub_options.go
Normal file
@@ -0,0 +1,48 @@
|
||||
package httpserver
|
||||
|
||||
// HubOptions 是节点内 Worker/Event WebSocket Hub 的有界运行参数。
|
||||
// 这些参数只控制本节点内存和领取并发，不改变 MySQL 中的授权、租约或配额事实。
|
||||
type HubOptions struct {
|
||||
// ConnectionLimit 是该 Hub 允许的同时连接数；超过后返回 429。
|
||||
ConnectionLimit int
|
||||
// DispatchWorkers 是本 Hub 同时执行数据库领取调用的 Go 协程上限。
|
||||
DispatchWorkers int
|
||||
// DispatchCapacity 是领取调用的有界内存排队槽位；满额时连接等待下一轮扫描。
|
||||
DispatchCapacity int
|
||||
}
|
||||
|
||||
const (
|
||||
defaultHubConnectionLimit = 128
|
||||
defaultHubDispatchWorkers = 4
|
||||
defaultHubDispatchCapacity = 128
|
||||
maxHubConnectionLimit = 4096
|
||||
maxHubDispatchWorkers = 256
|
||||
maxHubDispatchCapacity = 65536
|
||||
)
|
||||
|
||||
func resolveHubOptions(options []HubOptions) HubOptions {
|
||||
out := HubOptions{
|
||||
ConnectionLimit: defaultHubConnectionLimit,
|
||||
DispatchWorkers: defaultHubDispatchWorkers,
|
||||
DispatchCapacity: defaultHubDispatchCapacity,
|
||||
}
|
||||
if len(options) > 1 {
|
||||
panic("at most one HubOptions value is allowed")
|
||||
}
|
||||
if len(options) == 1 {
|
||||
in := options[0]
|
||||
if in.ConnectionLimit != 0 {
|
||||
out.ConnectionLimit = in.ConnectionLimit
|
||||
}
|
||||
if in.DispatchWorkers != 0 {
|
||||
out.DispatchWorkers = in.DispatchWorkers
|
||||
}
|
||||
if in.DispatchCapacity != 0 {
|
||||
out.DispatchCapacity = in.DispatchCapacity
|
||||
}
|
||||
}
|
||||
if out.ConnectionLimit < 1 || out.ConnectionLimit > maxHubConnectionLimit || out.DispatchWorkers < 1 || out.DispatchWorkers > maxHubDispatchWorkers || out.DispatchCapacity < 1 || out.DispatchCapacity > maxHubDispatchCapacity {
|
||||
panic("HubOptions values exceed bounded range")
|
||||
}
|
||||
return out
|
||||
}
|
||||
42
internal/transport/httpserver/hub_options_test.go
Normal file
42
internal/transport/httpserver/hub_options_test.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package httpserver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/events"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
func TestHubOptionsAreBoundedAndApplied(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
workerOptions := resolveHubOptions([]HubOptions{{ConnectionLimit: 7, DispatchWorkers: 3, DispatchCapacity: 11}})
|
||||
if workerOptions.ConnectionLimit != 7 || workerOptions.DispatchWorkers != 3 || workerOptions.DispatchCapacity != 11 {
|
||||
t.Fatalf("worker hub options not resolved: %+v", workerOptions)
|
||||
}
|
||||
workerHub := NewWorkerHub(ctx, worker.New(nil), workerOptions)
|
||||
if cap(workerHub.slots) != 7 {
|
||||
t.Fatalf("worker connection limit not applied: %d", cap(workerHub.slots))
|
||||
}
|
||||
eventOptions := resolveHubOptions([]HubOptions{{ConnectionLimit: 5, DispatchWorkers: 2, DispatchCapacity: 9}})
|
||||
eventHub := NewEventHub(ctx, events.New(nil), eventOptions)
|
||||
if cap(eventHub.slots) != 5 {
|
||||
t.Fatalf("event connection limit not applied: %d", cap(eventHub.slots))
|
||||
}
|
||||
workerHub.Shutdown(context.Background())
|
||||
eventHub.Shutdown(context.Background())
|
||||
}
|
||||
|
||||
func TestHubOptionsRejectInvalidValues(t *testing.T) {
|
||||
for _, options := range []HubOptions{{ConnectionLimit: -1}, {ConnectionLimit: maxHubConnectionLimit + 1}, {DispatchWorkers: -1}, {DispatchWorkers: maxHubDispatchWorkers + 1}, {DispatchCapacity: -1}, {DispatchCapacity: maxHubDispatchCapacity + 1}} {
|
||||
t.Run("invalid", func(t *testing.T) {
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Fatal("invalid hub options did not panic")
|
||||
}
|
||||
}()
|
||||
resolveHubOptions([]HubOptions{options})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -38,10 +38,11 @@ type WorkerHub struct {
|
||||
emptyPolls map[[32]byte]time.Time
|
||||
}
|
||||
|
||||
func NewWorkerHub(parent context.Context, service *worker.Service) *WorkerHub {
|
||||
func NewWorkerHub(parent context.Context, service *worker.Service, options ...HubOptions) *WorkerHub {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
h := &WorkerHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, 128), poll: time.Second, emptyPolls: make(map[[32]byte]time.Time)}
|
||||
h.dispatch = dispatch.New(ctx, 4, 128)
|
||||
limits := resolveHubOptions(options)
|
||||
h := &WorkerHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, limits.ConnectionLimit), poll: time.Second, emptyPolls: make(map[[32]byte]time.Time)}
|
||||
h.dispatch = dispatch.New(ctx, limits.DispatchWorkers, limits.DispatchCapacity)
|
||||
h.wg.Add(1)
|
||||
go func() { defer h.wg.Done(); <-h.dispatch.Done() }()
|
||||
return h
|
||||
|
||||
Reference in New Issue
Block a user