feat: measure bounded worker execution with durable verification
This commit is contained in:
1
docs/reports/data/phase-6-execution-load/deadline.json
Normal file
1
docs/reports/data/phase-6-execution-load/deadline.json
Normal file
@@ -0,0 +1 @@
|
||||
{"run_id":"a7b51ef05252222cba0c3a458a2a330c","go_version":"go1.25.14","goos":"linux","goarch":"arm64","started_at":"2026-09-19T18:25:48.034952468Z","admission_finished_at":"2026-09-19T18:25:48.127721385Z","finished_at":"2026-09-19T18:25:50.044512177Z","requests":24,"concurrency":4,"workers":1,"capacity_per_worker":1,"durations_ms":[1000],"run_timeout_ms":2000,"poll_interval_ms":250,"http_timeout_ms":5000,"queue":{"max_unfinished_tasks":100000,"unfinished_tasks":"0","id":"61e5494b4cbf4c7126454a074a56412d","namespace_id":"68983a8088c43ef337aa69bbd349686e","name":"execution","paused":false,"max_concurrency":4,"rate_per_second":10,"revision":"1"},"attempted":24,"not_submitted":0,"accepted":24,"admission_errors":0,"verified_success":0,"unverified":24,"handler_calls":1,"duplicate_invocations":0,"unconfirmed_handler_calls":0,"worker_started":1,"worker_reconnects":0,"worker_renewals_confirmed":6,"admission":{"count":24,"p50_ms":12.970542,"p95_ms":20.052458,"p99_ms":22.639583,"max_ms":22.639583},"queue_wait":{"count":0,"p50_ms":null,"p95_ms":null,"p99_ms":null,"max_ms":null},"authorized_execution":{"count":0,"p50_ms":null,"p95_ms":null,"p99_ms":null,"max_ms":null},"task_completion":{"count":0,"p50_ms":null,"p95_ms":null,"p99_ms":null,"max_ms":null},"errors":{"INVOCATION_MISMATCH":23,"RUN_TIMEOUT":1,"TASK_NEEDS_REVIEW":1,"VERIFY_TIMEOUT":23},"complete":false}
|
||||
55
docs/reports/data/phase-6-execution-load/environment.json
Normal file
55
docs/reports/data/phase-6-execution-load/environment.json
Normal file
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"date_local": "2026-09-20",
|
||||
"timezone": "Asia/Shanghai",
|
||||
"test_go": "go1.25.14",
|
||||
"declared_go": "1.27.1",
|
||||
"temporary_modfile": true,
|
||||
"mysql": "8.4.10",
|
||||
"innodb_flush_log_at_trx_commit": 1,
|
||||
"sync_binlog": 1,
|
||||
"mysql_global_transaction_isolation": "REPEATABLE-READ",
|
||||
"orbit_transaction_isolation": "READ COMMITTED",
|
||||
"mysql_max_connections": 151,
|
||||
"innodb_buffer_pool_size_bytes": 134217728,
|
||||
"orbit_db_max_open": 20,
|
||||
"orbit_db_max_idle": 5,
|
||||
"docker_vm_cpus": 12,
|
||||
"docker_vm_memory_bytes": 12529274880,
|
||||
"architecture": "arm64",
|
||||
"orbit_processes": 1,
|
||||
"mysql_nodes": 1,
|
||||
"worker_processes": 1,
|
||||
"worker_connections_normal": 2,
|
||||
"worker_capacity_each": 4,
|
||||
"queue_max_concurrency": 4,
|
||||
"queue_rate_per_second": 10,
|
||||
"metrics_enabled": false,
|
||||
"cpu_memory_limits": "No dedicated limits; shared local Docker VM, other workloads possible",
|
||||
"profiles_in_order": [
|
||||
"short",
|
||||
"mixed",
|
||||
"long",
|
||||
"deadline"
|
||||
],
|
||||
"prior_task_counts": [
|
||||
0,
|
||||
96,
|
||||
192,
|
||||
198
|
||||
],
|
||||
"fresh_disposable_database": true,
|
||||
"network": "Go runner shares MySQL container network namespace; loopback HTTP/WSS and MySQL",
|
||||
"schema_verification": {
|
||||
"schema_version": 28,
|
||||
"migration_count": 28,
|
||||
"foreign_key_count": 0,
|
||||
"required_index_count": 6,
|
||||
"complete": true
|
||||
},
|
||||
"source_sha256": {
|
||||
"sdk/go/cmd/orbit-exec-load/main.go": "bcaecce6be9fe8aca7a0d93b4671cbc460b1083232a77d13a04642a96ae45d81",
|
||||
"sdk/go/cmd/orbit-exec-load/measure.go": "ea0a22e6598e12e175f281fd45d511db32e241e532f5bdcf40ca615392fa586e",
|
||||
"sdk/go/cmd/orbit-exec-load/measure_test.go": "bda2ce1e946745e149c7471462d511ad3bcf66352a8d14fab57b2aa1ef280c3d",
|
||||
"sdk/go/cmd/orbit-exec-load/report.go": "3d1da9acc1e63142bd3b4b37041d609e61e172eef56ff27467f1803dee857272"
|
||||
}
|
||||
}
|
||||
1
docs/reports/data/phase-6-execution-load/long.json
Normal file
1
docs/reports/data/phase-6-execution-load/long.json
Normal file
@@ -0,0 +1 @@
|
||||
{"run_id":"980e0bccdca1cb9fbb1bdcc78536cc90","go_version":"go1.25.14","goos":"linux","goarch":"arm64","started_at":"2026-09-19T18:24:34.92203642Z","admission_finished_at":"2026-09-19T18:24:34.965149878Z","finished_at":"2026-09-19T18:25:48.023298384Z","requests":6,"concurrency":3,"workers":2,"capacity_per_worker":4,"durations_ms":[35000],"run_timeout_ms":120000,"poll_interval_ms":250,"http_timeout_ms":5000,"queue":{"max_unfinished_tasks":100000,"unfinished_tasks":"0","id":"61e5494b4cbf4c7126454a074a56412d","namespace_id":"68983a8088c43ef337aa69bbd349686e","name":"execution","paused":false,"max_concurrency":4,"rate_per_second":10,"revision":"1"},"attempted":6,"not_submitted":0,"accepted":6,"admission_errors":0,"verified_success":6,"unverified":0,"handler_calls":6,"duplicate_invocations":0,"unconfirmed_handler_calls":0,"worker_started":6,"worker_reconnects":0,"worker_renewals_confirmed":18,"admission":{"count":6,"p50_ms":13.193791,"p95_ms":28.028083,"p99_ms":28.028083,"max_ms":28.028083},"queue_wait":{"count":6,"p50_ms":1058.772,"p95_ms":38015.493,"p99_ms":38015.493,"max_ms":38015.493},"authorized_execution":{"count":6,"p50_ms":35013.932,"p95_ms":35024.585,"p99_ms":35024.585,"max_ms":35024.585},"task_completion":{"count":6,"p50_ms":36067.492,"p95_ms":73040.078,"p99_ms":73040.078,"max_ms":73040.078},"errors":{},"complete":true}
|
||||
1
docs/reports/data/phase-6-execution-load/mixed.json
Normal file
1
docs/reports/data/phase-6-execution-load/mixed.json
Normal file
@@ -0,0 +1 @@
|
||||
{"run_id":"e78c366e6d6acb6cce8159dae900e8ca","go_version":"go1.25.14","goos":"linux","goarch":"arm64","started_at":"2026-09-19T18:23:52.454385095Z","admission_finished_at":"2026-09-19T18:23:52.759812553Z","finished_at":"2026-09-19T18:24:34.90756817Z","requests":96,"concurrency":8,"workers":2,"capacity_per_worker":4,"durations_ms":[0,100,1000],"run_timeout_ms":120000,"poll_interval_ms":250,"http_timeout_ms":5000,"queue":{"max_unfinished_tasks":100000,"unfinished_tasks":"0","id":"61e5494b4cbf4c7126454a074a56412d","namespace_id":"68983a8088c43ef337aa69bbd349686e","name":"execution","paused":false,"max_concurrency":4,"rate_per_second":10,"revision":"1"},"attempted":96,"not_submitted":0,"accepted":96,"admission_errors":0,"verified_success":96,"unverified":0,"handler_calls":96,"duplicate_invocations":0,"unconfirmed_handler_calls":0,"worker_started":96,"worker_reconnects":0,"worker_renewals_confirmed":0,"admission":{"count":96,"p50_ms":23.680834,"p95_ms":36.472167,"p99_ms":40.274167,"max_ms":40.274167},"queue_wait":{"count":96,"p50_ms":20883.756,"p95_ms":38754.793,"p99_ms":41024.973,"max_ms":41024.973},"authorized_execution":{"count":96,"p50_ms":109.912,"p95_ms":1021.918,"p99_ms":1042.773,"max_ms":1042.773},"task_completion":{"count":96,"p50_ms":20898.154,"p95_ms":39775.854,"p99_ms":42047.09,"max_ms":42047.09},"errors":{},"complete":true}
|
||||
1
docs/reports/data/phase-6-execution-load/short.json
Normal file
1
docs/reports/data/phase-6-execution-load/short.json
Normal file
@@ -0,0 +1 @@
|
||||
{"run_id":"e16f23365b9accaef3ce8b41349d77ea","go_version":"go1.25.14","goos":"linux","goarch":"arm64","started_at":"2026-09-19T18:23:38.326433797Z","admission_finished_at":"2026-09-19T18:23:38.671806422Z","finished_at":"2026-09-19T18:23:52.44758097Z","requests":96,"concurrency":8,"workers":2,"capacity_per_worker":4,"durations_ms":[0],"run_timeout_ms":120000,"poll_interval_ms":250,"http_timeout_ms":5000,"queue":{"max_unfinished_tasks":100000,"unfinished_tasks":"0","id":"61e5494b4cbf4c7126454a074a56412d","namespace_id":"68983a8088c43ef337aa69bbd349686e","name":"execution","paused":false,"max_concurrency":4,"rate_per_second":10,"revision":"1"},"attempted":96,"not_submitted":0,"accepted":96,"admission_errors":0,"verified_success":96,"unverified":0,"handler_calls":96,"duplicate_invocations":0,"unconfirmed_handler_calls":0,"worker_started":96,"worker_reconnects":0,"worker_renewals_confirmed":0,"admission":{"count":96,"p50_ms":26.13175,"p95_ms":38.935083,"p99_ms":54.67775,"max_ms":54.67775},"queue_wait":{"count":96,"p50_ms":5951.689,"p95_ms":12805.411,"p99_ms":13752.092,"max_ms":13752.092},"authorized_execution":{"count":96,"p50_ms":10.405,"p95_ms":23.504,"p99_ms":28.838,"max_ms":28.838},"task_completion":{"count":96,"p50_ms":5959.781,"p95_ms":12813.065,"p99_ms":13768.327,"max_ms":13768.327},"errors":{},"complete":true}
|
||||
71
docs/reports/phase-6-execution-load.md
Normal file
71
docs/reports/phase-6-execution-load.md
Normal file
@@ -0,0 +1,71 @@
|
||||
# P6：真实 Worker 任务执行测量
|
||||
|
||||
日期：2026-09-20（Asia/Shanghai，原始报告时间为UTC）。新增独立命令 `sdk/go/cmd/orbit-exec-load`，保留原 `orbit-load` 的纯受理边界。完整参数、权限、错误退出、分母和时钟定义见[执行测量手册](../api/execution-load.md)。本批交付短/混合/中长任务的可运行测量及正确性核验，不代表生产容量验收。
|
||||
|
||||
## 实现和核验
|
||||
|
||||
负载程序在业务侧独立SDK模块运行，与Orbit服务进程分离，只使用HTTP/WSS公共接口；没有数据库驱动或服务端内部依赖。每次使用独立处理器、固定序号/时长/输入、唯一提交键。HTTP提交不自动重试，任务最多一次尝试，Worker按现有协议心跳、续租、重连和收尾。
|
||||
|
||||
成功数量来自持久化任务及尝试历史的读取，逐项核验状态、输入/结果、队列/版本/业务键、单次尝试、授权/结束时间及单次处理器调用。SDK收到控制确认、处理器返回nil和本地事件都不能单独计为成功。截止或任一错配必须以 `complete=false` 和非零退出码结束；没有完成样本时百分位为null。
|
||||
|
||||
## 最终环境
|
||||
|
||||
原始配置及当前测量源码SHA-256保存在[环境清单](data/phase-6-execution-load/environment.json)。
|
||||
|
||||
| 项目 | 实际值 |
|
||||
| --- | --- |
|
||||
| 主机/运行环境 | macOS arm64、Docker Linux/arm64；共享VM可见12 CPU、12529274880字节内存，未独占硬件 |
|
||||
| 工具链 | 实际 `go1.25.14`，仅临时modfile替换Go指令；仓库仍声明Go 1.27.1 |
|
||||
| 拓扑 | 一次性MySQL 8.4.10，1个Orbit进程，1个负载进程内2条Worker连接，每条capacity=4 |
|
||||
| 网络 | Go运行容器共享MySQL容器网络命名空间；HTTP/WSS和MySQL均走loopback |
|
||||
| MySQL | `innodb_flush_log_at_trx_commit=1`、`sync_binlog=1`、buffer pool=128MiB、max_connections=151 |
|
||||
| 隔离级别 | MySQL全局REPEATABLE-READ；Orbit写事务显式READ COMMITTED |
|
||||
| Orbit | 数据库连接池max_open=20、max_idle=5；鉴权和持久事件/审计启用；metrics未启用 |
|
||||
| 队列 | max_concurrency=4、rate_per_second=10、max_unfinished_tasks=100000；工具未更改配置 |
|
||||
| 核验 | 每个HTTP请求5秒上限、单核验器250ms间隔，核验器最多与提交并发数相同 |
|
||||
| 数据量 | 空库初始化；short/mixed/long/deadline顺序运行，开始时已有任务数分别0/96/192/198；各运行身份隔离 |
|
||||
|
||||
最后 `orbit-admin verify` 返回 schema=28、migration_count=28、foreign_key_count=0、required_index_count=6、complete=true。测试容器退出后销毁一次性数据库和凭据。
|
||||
|
||||
## 原始结果
|
||||
|
||||
前三组均在2分钟整体预算内完整通过，退出0。三个阶段分布的分母分别为96、96、6；都没有重复处理器调用、无回执调用或Worker重连。
|
||||
|
||||
| 样本 | 任务/处理器时长 | 受理p99 ms | 排队p99 ms | 授权执行p99 ms | 任务完成p99 ms | 持久成功/计划数 |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| [short原始JSON](data/phase-6-execution-load/short.json) | 96项，0ms | 54.678 | 13752.092 | 28.838 | 13768.327 | 96/96 |
|
||||
| [mixed原始JSON](data/phase-6-execution-load/mixed.json) | 96项，0/100/1000ms循环，各32项 | 40.274 | 41024.973 | 1042.773 | 42047.090 | 96/96 |
|
||||
| [long原始JSON](data/phase-6-execution-load/long.json) | 6项，每项35000ms | 28.028 | 38015.493 | 35024.585 | 73040.078 | 6/6 |
|
||||
|
||||
长任务总计18次续租确认，最终仍为每项一次尝试、一次调用。该数字验证了此次实际续租路径，不表示节点/网络故障已经发生。
|
||||
|
||||
排队分布包含当前4并发/每秒10次开始的队列约束及调度扫描；Worker声明8个槽不会绕过队列限制。这组数字不能视为去掉配额后的最大吞吐。
|
||||
|
||||
`authorized_execution` 和 `task_completion` 用持久化数据库时间戳计算；终点是结束事务内部取时点，未包含其后COMMIT及确认延迟。已提交事实由后续读取核验。客户端核验顺序不计入这些分布，但查询负载会影响服务资源消耗。没有收集独占CPU/内存/磁盘/锁等待曲线，也没有多轮稳态和生产指标开启后的对照。
|
||||
|
||||
## 失败语义实测
|
||||
|
||||
[deadline原始JSON](data/phase-6-execution-load/deadline.json)：24项、4并发提交、1个Worker/1个槽、每项1秒、整体预算2秒。命令按预期退出1，`accepted=24`、`verified_success=0`、`unverified=24`、`complete=false`；本地只有1次处理器调用。读取观察到1项NEEDS_REVIEW，23项未能在截止前验证，三种完成阶段的分布均为 `count=0` / 百分位null。
|
||||
|
||||
错误计数保留 `RUN_TIMEOUT`、`TASK_NEEDS_REVIEW`、`VERIFY_TIMEOUT` 和 `INVOCATION_MISMATCH`。这些计数含运行级及交叠的核验错误，不能求和当成失败任务总数。工具没有删除、强制成功或重新提交这些任务；未观察到终态不等于数据库确认失败，剩余状态留在原身份下供核查。
|
||||
|
||||
## 复现和回归
|
||||
|
||||
按[手册](../api/execution-load.md)准备一次性数据库、当前迁移、带Worker权限的凭据及真实Orbit进程，然后在 `sdk/go` 运行：
|
||||
|
||||
```sh
|
||||
go run ./cmd/orbit-exec-load -requests 96 -concurrency 8 -workers 2 -capacity 4 -durations 0ms -run-timeout 2m
|
||||
go run ./cmd/orbit-exec-load -requests 96 -concurrency 8 -workers 2 -capacity 4 -durations 0ms,100ms,1s -run-timeout 2m
|
||||
go run ./cmd/orbit-exec-load -requests 6 -concurrency 3 -workers 2 -capacity 4 -durations 35s -run-timeout 2m
|
||||
# 此项应失败；必须读取JSON确认是预算不足，不应仅判断任意非零退出。
|
||||
go run ./cmd/orbit-exec-load -requests 24 -concurrency 4 -workers 1 -capacity 1 -durations 1s -run-timeout 2s
|
||||
```
|
||||
|
||||
本轮在上述Docker环境使用临时modfile构建真实Linux二进制，执行了四条等价命令及下列检查：
|
||||
|
||||
- SDK全模块 `go test -race ./...`、`go vet ./...`、`go build ./...` 通过；没有设置原SDK集成凭据的普通单测不冒充全量集成。
|
||||
- 新增单测覆盖：错配/缺失/空字段结果、未知效果、第二次尝试、悬空历史、时间逆序、提交超时/429不重报、整次取消、核查不算成功、读取失败、重复调用、无权限预检及错误脱敏。
|
||||
- SDK完整依赖图218项中，服务端内部/模型/模块和MySQL驱动依赖均为0。
|
||||
- `scripts/test-mysql.sh` 已加入6项真实执行命令冒烟，shell语法检查通过。本轮运行的是四个独立真实样本，未重新执行整个 `make integration`。
|
||||
|
||||
当前不把临时Go 1.25兼容回归升级为声明Go 1.27.1的正式发布结论。下一批继续定时集中触发、流程推进、长期积压、慢消费者/重连风暴及资源曲线，生产HA、复制切换与RPO/RTO仍需目标拓扑证据。
|
||||
@@ -17,6 +17,7 @@
|
||||
| SDK Worker/Event/Workflow 子进程故障 | 同一共享网络命名空间；容器内构建 Orbit 二进制；SDK `TestIntegration(Worker\|Event\|Workflow)Process` | PASS，136.394 秒；派发未送达、租约接管、ACK 提交回执丢失、重试未知结果、计划/优先级/键并发、流程暂停全停接管均通过 |
|
||||
| SDK 应用配额子进程故障 | 独立双命名空间应用配额凭据；共享网络命名空间；`TestIntegrationWorkerProcessApplicationBackpressure` | PASS，1.069 秒；重启后共享额度、原回执和跨命名空间背压保持 |
|
||||
| Orbit HTTP/metrics 子进程 | 同一共享网络命名空间；`TestIntegrationMetricsProcess`；metrics 启用/禁用各一组 | PASS，3.081 秒；监听隔离、隐私、后台采样和优雅退出均通过 |
|
||||
| 真实Worker执行测量 | `orbit-exec-load`；最终短/混合/中长及截止不足四组，环境及原始JSON见[测量报告](phase-6-execution-load.md) | 96+96+6项全部逐项核验，长任务18次续租；24项截止负例按预期非零退出。SDK全模块竞态/vet/build通过，生产容量未验收 |
|
||||
|
||||
## 当前环境仍未覆盖
|
||||
|
||||
|
||||
Reference in New Issue
Block a user