feat: sample recent business events

This commit is contained in:
2026-09-18 21:12:05 +08:00
parent 5284206731
commit 8ce9f4e1fa
14 changed files with 68 additions and 18 deletions

View File

@@ -81,7 +81,7 @@
- A04须截留真实task.assign并独立确认LEASED/未授权,再强杀旧节点、Wait确认退出、新节点等原租约自然到期;不能把已送达未开始替代未送达。COMMIT故障工具只在测试二进制:BEFORE阻止COMMIT,AFTER核验真实MySQL成功包后丢确认;独立连接回读,原身份/候选恢复,禁止在生产增加故障开关或将HTTP丢回执替代数据库提交不明。 - A04须截留真实task.assign并独立确认LEASED/未授权,再强杀旧节点、Wait确认退出、新节点等原租约自然到期;不能把已送达未开始替代未送达。COMMIT故障工具只在测试二进制:BEFORE阻止COMMIT,AFTER核验真实MySQL成功包后丢确认;独立连接回读,原身份/候选恢复,禁止在生产增加故障开关或将HTTP丢回执替代数据库提交不明。
- W16下一轮时钟只能在PENDING结果已保存等待后存在,三方竞争中的结果参与者是旧结果重放;新结果/未知/冲突另有在途查询回归。保持原受理时间、事件时间、截止与预算,不能凭有限入队顺序声称穷尽线程交错。 - W16下一轮时钟只能在PENDING结果已保存等待后存在,三方竞争中的结果参与者是旧结果重放;新结果/未知/冲突另有在途查询回归。保持原受理时间、事件时间、截止与预算,不能凭有限入队顺序声称穷尽线程交错。
- P6进程观测使用internal/observability固定指标目录,拒绝未知名称、类型和动态标签,不能引入任务/租户/业务键/凭据维度。ORBIT_METRICS_ADDR默认空(关闭),显式启用独立采集监听;业务端口不得暴露全局指标。渲染和网络I/O不持有全局记录锁,每个直方图局部快照保持桶/count/sum一致,最多2个并发抓取。启用指标后的5秒后台采样使用000026固定索引读取任务、交付和流程状态,每个状态最多10000条、900ms数据库截止;采样失败保留最后成功状态并独立记录错误/超时,不能影响业务事务或把有限样本当全局精确数。 - P6进程观测使用internal/observability固定指标目录,拒绝未知名称、类型和动态标签,不能引入任务/租户/业务键/凭据维度。ORBIT_METRICS_ADDR默认空(关闭),显式启用独立采集监听;业务端口不得暴露全局指标。渲染和网络I/O不持有全局记录锁,每个直方图局部快照保持桶/count/sum一致,最多2个并发抓取。启用指标后的5秒后台采样使用000026/000027固定索引读取任务、交付、流程状态和固定事件类别,每个状态或事件最多10000条、事件窗口5分钟、900ms数据库截止;采样失败保留最后成功状态并独立记录错误/超时,不能影响业务事务或把有限样本当全局精确数。
- HTTP成功升级在Hijack完成时记录握手,WS处理器存续计入独立连接指标;保持1xx临时响应、Flush、Hijack与原panic行为。DB池取本地Stats,抓取不查询业务库;后台批次success不是业务任务成功量。观测丢样或失败不得改变权威事务及恢复决定。 - HTTP成功升级在Hijack完成时记录握手,WS处理器存续计入独立连接指标;保持1xx临时响应、Flush、Hijack与原panic行为。DB池取本地Stats,抓取不查询业务库;后台批次success不是业务任务成功量。观测丢样或失败不得改变权威事务及恢复决定。
- 进程指标、固定状态采样、格式验证、告警规则和标准回归证据见docs/reports/phase-6-observability.md及docs/reports/phase-6-business-observability.md。仍须补受理/授权/完成与流程推进延迟、持久吞吐/积压的全局聚合、重试/核查/恢复量、发送队列、实际采集告警及负载开销,不能把进程观测通过写成P6已完成。普通清理仍须保护有效幂等、未确认事件、未知结果和未终结流程。 - 进程指标、固定状态采样、格式验证、告警规则和标准回归证据见docs/reports/phase-6-observability.md及docs/reports/phase-6-business-observability.md。仍须补受理/授权/完成与流程推进延迟、持久吞吐/积压的全局聚合、重试/核查/恢复量、发送队列、实际采集告警及负载开销,不能把进程观测通过写成P6已完成。普通清理仍须保护有效幂等、未确认事件、未知结果和未终结流程。
- 凭据轮换保留credential_id及订阅/Worker授权绑定,在同一事务替换摘要、固定到期时间、幂等回执和管理审计;新令牌必须重新建立会话,认证路径每次重新核对凭据状态。撤销必须让认证失效并保留原任务、订阅、交付、会话和审计证据,允许在应用或命名空间停用后执行;重复应用同一固定操作幂等返回。管理CLI先把固定操作文件写成0600,再提交;提交不明时保留文件并用原文件重放,不能重新生成秘密。日志、审计摘要、指标和业务表不得保存秘密原文或摘要。不要把轮换同一凭据ID解释成历史令牌仍有效,也不得级联删除订阅、任务或事件。 - 凭据轮换保留credential_id及订阅/Worker授权绑定,在同一事务替换摘要、固定到期时间、幂等回执和管理审计;新令牌必须重新建立会话,认证路径每次重新核对凭据状态。撤销必须让认证失效并保留原任务、订阅、交付、会话和审计证据,允许在应用或命名空间停用后执行;重复应用同一固定操作幂等返回。管理CLI先把固定操作文件写成0600,再提交;提交不明时保留文件并用原文件重放,不能重新生成秘密。日志、审计摘要、指标和业务表不得保存秘密原文或摘要。不要把轮换同一凭据ID解释成历史令牌仍有效,也不得级联删除订阅、任务或事件。

View File

@@ -200,6 +200,7 @@ func run() error {
TaskStates: snapshot.TaskStates, TaskOldestAgeSeconds: snapshot.TaskOldestAgeSeconds, TaskStates: snapshot.TaskStates, TaskOldestAgeSeconds: snapshot.TaskOldestAgeSeconds,
DeliveryStates: snapshot.DeliveryStates, DeliveryOldestAgeSeconds: snapshot.DeliveryOldestAgeSeconds, DeliveryStates: snapshot.DeliveryStates, DeliveryOldestAgeSeconds: snapshot.DeliveryOldestAgeSeconds,
WorkflowStates: snapshot.WorkflowStates, WorkflowOldestAgeSeconds: snapshot.WorkflowOldestAgeSeconds, WorkflowStates: snapshot.WorkflowStates, WorkflowOldestAgeSeconds: snapshot.WorkflowOldestAgeSeconds,
RecentEvents: snapshot.RecentEvents,
Truncated: snapshot.Truncated, Truncated: snapshot.Truncated,
}, started, err) }, started, err)
if err != nil && metricsCtx.Err() == nil { if err != nil && metricsCtx.Err() == nil {

View File

@@ -1,6 +1,6 @@
# 数据模型与实现基线 # 数据模型与实现基线
更新:2026-09-18。数据库统一禁止外键。当前迁移版本为26;000026只增加固定业务观测读取所需索引,不增加实体或字段。当前已实现任务执行、安全重试、运行中取消、计划调度、队列限速/背压和应用配额;优先级及节点公平领取已实现;可靠订阅核心、HTTP/WSS、管理与消费 SDK 已实现,独立 Inbox 场景已实现,流程编排继续开发。 更新:2026-09-18。数据库统一禁止外键。当前迁移版本为27;000026/000027只增加固定业务观测读取所需索引,不增加实体或字段。当前已实现任务执行、安全重试、运行中取消、计划调度、队列限速/背压和应用配额;优先级及节点公平领取已实现;可靠订阅核心、HTTP/WSS、管理与消费 SDK 已实现,独立 Inbox 场景已实现,流程编排继续开发。
## 1. 模块与依赖 ## 1. 模块与依赖
@@ -249,3 +249,5 @@ Event.WorkflowID为只读STORED生成列,从原不可变事件派生流程归
## 有界业务观测索引(000026) ## 有界业务观测索引(000026)
000026只增加三组读取索引:`pulse_tasks(state,available_at,id)`、`orbit_deliveries(state,available_at,id)`和`weave_instances(state,updated_at,id)`。它们服务于后台有界观测,不改变任何模型字段、状态机、事务边界或授权规则,也不包含外键或级联。采样器只使用代码固定的表、索引、状态和列名,每个状态最多读取10000条;结果达到上限时必须通过指标暴露截断,不能把样本数解释为全库精确数量。迁移后仍要求服务层 `CheckSchema` 验证零外键,真实升级见[有界业务采样报告](reports/phase-6-business-observability.md)。 000026只增加三组读取索引:`pulse_tasks(state,available_at,id)`、`orbit_deliveries(state,available_at,id)`和`weave_instances(state,updated_at,id)`。它们服务于后台有界观测,不改变任何模型字段、状态机、事务边界或授权规则,也不包含外键或级联。采样器只使用代码固定的表、索引、状态和列名,每个状态最多读取10000条;结果达到上限时必须通过指标暴露截断,不能把样本数解释为全库精确数量。迁移后仍要求服务层 `CheckSchema` 验证零外键,真实升级见[有界业务采样报告](reports/phase-6-business-observability.md)。
000027为`orbit_events(type,occurred_at,id)`增加固定事件窗口索引。采样器只读取代码列出的任务和流程事件类别、最近5分钟、每类最多10000条;未知事件类型不写入标签,事件指标只说明本节点有限趋势,不能代替审计查询、端到端阶段延迟或跨节点吞吐。索引不改变事件原文、不可变触发器、外键策略或事务边界。

View File

@@ -59,7 +59,7 @@ P4/P5定时等待与后续查询轮次的实现及回归记录见[时钟报告](
### P6 首批实施顺序 ### P6 首批实施顺序
1. 已补[首批可复现故障工具](reports/phase-6-fault-boundaries.md):A04“派发已提交但消息尚未送达”、W01/W13事务提交结果不明,以及W16最终回执/旧PENDING结果重放/下一轮竞争。每个切点必须有实际数据库或进程证据,不能用响应错误推断事务是否已提交,也不能用普通SIGNAL竞争替代最终查询场景。 1. 已补[首批可复现故障工具](reports/phase-6-fault-boundaries.md):A04“派发已提交但消息尚未送达”、W01/W13事务提交结果不明,以及W16最终回执/旧PENDING结果重放/下一轮竞争。每个切点必须有实际数据库或进程证据,不能用响应错误推断事务是否已提交,也不能用普通SIGNAL竞争替代最终查询场景。
2. 已实现[基础运行观测](api/observability.md)及[有界业务状态采样](reports/phase-6-business-observability.md):HTTP/WS、数据库连接池、后台批次、固定状态计数、最早年龄、截断/新鲜度指标和可离线验证的告警规则;独立采集监听默认关闭,固定指标目录拒绝动态标签。仍需受理/授权/完成阶段延迟与持久吞吐、流程推进延迟、重试/核查/恢复量、全局发送队列、多节点聚合及生产采集/通知验证。 2. 已实现[基础运行观测](api/observability.md)及[有界业务状态与事件采样](reports/phase-6-business-observability.md):HTTP/WS、数据库连接池、后台批次、固定状态计数、最早年龄、最近事件类别、截断/新鲜度指标和可离线验证的告警规则;独立采集监听默认关闭,固定指标目录拒绝动态标签。仍需受理/授权/完成端到端阶段延迟与全局持久吞吐、流程推进延迟、全局发送队列、多节点聚合及生产采集/通知验证。
3. 已补齐受保护管理环境中的固定凭据操作文件、轮换、撤销、幂等回执和脱敏管理审计;轮换保留授权绑定并要求新令牌重新建立会话,撤销允许在停用范围内执行且不改写历史记录。仍需补齐提交不明故障切点、多节点安全重绑演练、凭据最小权限配置核对和生产密钥托管/审计演练。 3. 已补齐受保护管理环境中的固定凭据操作文件、轮换、撤销、幂等回执和脱敏管理审计;轮换保留授权绑定并要求新令牌重新建立会话,撤销允许在停用范围内执行且不改写历史记录。仍需补齐提交不明故障切点、多节点安全重绑演练、凭据最小权限配置核对和生产密钥托管/审计演练。
4. 实现保留/归档与受保护记录核对,再执行备份恢复、兼容升级和六类容量负载。未确认交付、有效幂等身份、当前/历史执行证据和未终结流程不能被普通清理破坏。生产HA及RPO/RTO继续按实际拓扑验证,不缩减为本机多进程检查。 4. 实现保留/归档与受保护记录核对,再执行备份恢复、兼容升级和六类容量负载。未确认交付、有效幂等身份、当前/历史执行证据和未终结流程不能被普通清理破坏。生产HA及RPO/RTO继续按实际拓扑验证,不缩减为本机多进程检查。

View File

@@ -1,6 +1,6 @@
# 运行指标与告警接入 # 运行指标与告警接入
状态:P6第一批进程观测及有界业务状态采样已实现,指标、采集入口和下方规则均有回归。采样只提供每节点有限快照,完整生产观测尚未验收。 状态:P6第一批进程观测、有界业务状态采样和固定事件窗口采样已实现,指标、采集入口和下方规则均有回归。采样只提供每节点有限快照,完整生产观测尚未验收。
## 1. 启用独立采集端口 ## 1. 启用独立采集端口
@@ -25,7 +25,7 @@ curl -fsS http://127.0.0.1:9090/metrics
仅 `GET /metrics`(及标准HEAD)提供采集,查询参数返回400。正文使用[Prometheus文本格式0.0.4](https://prometheus.io/docs/instrumenting/exposition_formats/),Content-Type为 `text/plain; version=0.0.4; charset=utf-8`,Cache-Control为no-store。输出是本节点的运行快照,没有跨节点或跨指标原子快照保证。 仅 `GET /metrics`(及标准HEAD)提供采集,查询参数返回400。正文使用[Prometheus文本格式0.0.4](https://prometheus.io/docs/instrumenting/exposition_formats/),Content-Type为 `text/plain; version=0.0.4; charset=utf-8`,Cache-Control为no-store。输出是本节点的运行快照,没有跨节点或跨指标原子快照保证。
每节点同时最多2个采集请求,超出返回503及 `Retry-After: 1`,不排无限等待队列。独立HTTP服务器的读头/读/写/空闲超时分别为2/3/5/15秒,头部限制8 KiB。每次HTTP采集只访问进程内存,不查询业务表;DB池快照在启动时及之后每5秒读取 `database/sql.DBStats`,该调用不执行SQL。指标启用后同一个5秒后台采样器另外执行一次有界业务快照:查询固定状态、固定表和固定索引,单状态最多读取10000条,数据库调用使用900ms截止时间;失败只增加错误/超时计数并保留上次成功值,不改变业务事务。 每节点同时最多2个采集请求,超出返回503及 `Retry-After: 1`,不排无限等待队列。独立HTTP服务器的读头/读/写/空闲超时分别为2/3/5/15秒,头部限制8 KiB。每次HTTP采集只访问进程内存,不查询业务表;DB池快照在启动时及之后每5秒读取 `database/sql.DBStats`,该调用不执行SQL。指标启用后同一个5秒后台采样器另外执行一次有界业务快照:查询固定状态、固定表和固定索引,单状态或事件类别最多读取10000条;固定事件类别窗口为最近5分钟;数据库调用使用900ms截止时间;失败只增加错误/超时计数并保留上次成功值,不改变业务事务。
所有指标名称、类型、标签名和取值在代码目录中固定。未知名称、错误类型、非法标签、非有限数值和计数器倒退会被丢弃,并增加 `orbit_observability_rejected_samples_total`。新增指标需要修改目录和回归,不允许业务方动态注册标签。HTTP标签为8种方法、12种路由分类和6种状态分类,两组HTTP族各最多576种标签组合。禁止将任务/流程/租户ID、业务键、URL、参数、原始错误或凭据写入指标。 所有指标名称、类型、标签名和取值在代码目录中固定。未知名称、错误类型、非法标签、非有限数值和计数器倒退会被丢弃,并增加 `orbit_observability_rejected_samples_total`。新增指标需要修改目录和回归,不允许业务方动态注册标签。HTTP标签为8种方法、12种路由分类和6种状态分类,两组HTTP族各最多576种标签组合。禁止将任务/流程/租户ID、业务键、URL、参数、原始错误或凭据写入指标。
@@ -53,6 +53,7 @@ curl -fsS http://127.0.0.1:9090/metrics
| `orbit_result_delivery_oldest_age_seconds` | gauge | PENDING/LEASED交付样本中最早可处理时间的年龄 | | `orbit_result_delivery_oldest_age_seconds` | gauge | PENDING/LEASED交付样本中最早可处理时间的年龄 |
| `orbit_weave_workflows_state{state}` | gauge | Weave实例固定状态的有界计数 | | `orbit_weave_workflows_state{state}` | gauge | Weave实例固定状态的有界计数 |
| `orbit_weave_workflow_oldest_age_seconds` | gauge | RUNNING/WAITING/CANCELLING/NEEDS_REVIEW实例样本中最早更新时间的年龄 | | `orbit_weave_workflow_oldest_age_seconds` | gauge | RUNNING/WAITING/CANCELLING/NEEDS_REVIEW实例样本中最早更新时间的年龄 |
| `orbit_business_events_recent{event}` | gauge | 最近5分钟内固定事件类别的有界数量,用于观察受理、完成、核查、恢复和流程推进趋势 |
| `orbit_business_snapshot_total{outcome}` | counter | 有界业务快照成功、错误或超时次数 | | `orbit_business_snapshot_total{outcome}` | counter | 有界业务快照成功、错误或超时次数 |
| `orbit_business_snapshot_duration_seconds` | histogram | 有界业务快照耗时;包含数据库读取 | | `orbit_business_snapshot_duration_seconds` | histogram | 有界业务快照耗时;包含数据库读取 |
| `orbit_business_snapshot_last_success_timestamp_seconds` | gauge | 最近一次完整成功快照的Unix时间;失败不会推进 | | `orbit_business_snapshot_last_success_timestamp_seconds` | gauge | 最近一次完整成功快照的Unix时间;失败不会推进 |
@@ -100,4 +101,4 @@ promtool check rules deploy/monitoring/orbit.rules.yml
promtool test rules deploy/monitoring/orbit.rules.test.yml promtool test rules deploy/monitoring/orbit.rules.test.yml
``` ```
HTTP失败率和延迟可按route/status_class统计;受理接口响应成功与已完成业务应分别判断。业务状态指标是本节点有限样本,不能相加作为全局精确数量;应结合 `orbit_business_snapshot_last_success_timestamp_seconds` 和 `orbit_business_snapshot_truncated` 判断新鲜度与完整性。可靠结果量、排队/授权/完成阶段延迟、流程Inbox推进延迟、发送队列、磁盘/复制等仍须补充专门指标;具体当前证据见[进程观测报告](../reports/phase-6-observability.md)和[有界业务采样报告](../reports/phase-6-business-observability.md)。 HTTP失败率和延迟可按route/status_class统计;受理接口响应成功与已完成业务应分别判断。业务状态和事件窗口指标是本节点有限样本,不能相加作为全局精确数量;未知事件类型不会进入标签,应结合 `orbit_business_snapshot_last_success_timestamp_seconds` 和 `orbit_business_snapshot_truncated` 判断新鲜度与完整性。可靠结果量、排队/授权/完成阶段延迟、流程Inbox推进延迟、发送队列、磁盘/复制等仍须补充专门指标;具体当前证据见[进程观测报告](../reports/phase-6-observability.md)和[有界业务采样报告](../reports/phase-6-business-observability.md)。

View File

@@ -1,24 +1,25 @@
# P6:有界业务状态采样 # P6:有界业务状态与事件采样
日期:2026-09-18。本文记录第二批运行观测实现的边界和回归证据。它补充进程级批次指标,用于回答“本节点看到哪些固定状态、最早样本等待多久、采样是否完整”,不把有限样本当作全局业务统计,也不宣称生产容量或高可用已经验收。 日期:2026-09-18。本文记录第二批运行观测实现的边界和回归证据。它补充进程级批次指标,用于回答“本节点看到哪些固定状态、最近固定事件类别有多少、最早样本等待多久、采样是否完整”,不把有限样本当作全局业务统计,也不宣称生产容量或高可用已经验收。
## 已实现 ## 已实现
- 迁移 `000026_operational_metrics` 为 `pulse_tasks`、`orbit_deliveries`、`weave_instances` 增加固定前缀索引;不增加模型字段,不引入外键或级联。 - 迁移 `000026_operational_metrics` 为 `pulse_tasks`、`orbit_deliveries`、`weave_instances` 增加固定前缀索引;不增加模型字段,不引入外键或级联。
- [存储采样器](../../internal/storage/mysql/operational_metrics.go)只拼接代码内固定的表、索引、列名和状态集合。每个状态使用索引读取最多10001条,再将结果封顶到10000;至少一类封顶时 `orbit_business_snapshot_truncated=1`。 - 迁移 `000027_operational_event_metrics` 为不可变 `orbit_events` 增加 `(type,occurred_at,id)` 固定事件窗口索引;不增加模型字段,不引入外键或级联。
- 采样器先读取数据库UTC时间,再读取任务、结果交付和流程实例固定状态计数及最早未完成年龄。所有调用由独立900ms上下文限制;取消、数据库错误和超时都返回失败,不修改持久化数据。 - [存储采样器](../../internal/storage/mysql/operational_metrics.go)只拼接代码内固定的表、索引、列名、状态和事件集合。每个状态或事件类别使用索引读取最多10001条,再将结果封顶到10000;至少一类封顶时 `orbit_business_snapshot_truncated=1`。
- 采样器先读取数据库UTC时间,再读取任务、结果交付和流程实例固定状态计数、最早未完成年龄及最近5分钟受理/完成/核查/恢复/流程推进事件类别。所有调用由独立900ms上下文限制;取消、数据库错误和超时都返回失败,不修改持久化数据。
- [业务指标记录器](../../internal/observability/business.go)将成功、错误、超时分别计数并记录耗时。失败样本不覆盖最后一次成功状态,同时保留最后成功时间,方便告警区分“旧值可见”和“本轮采集失败”。标签基数和状态取值均在目录中固定。 - [业务指标记录器](../../internal/observability/business.go)将成功、错误、超时分别计数并记录耗时。失败样本不覆盖最后一次成功状态,同时保留最后成功时间,方便告警区分“旧值可见”和“本轮采集失败”。标签基数和状态取值均在目录中固定。
- [进程入口](../../cmd/orbit/main.go)仅在显式启用指标时每5秒后台采样;`GET /metrics`仍只渲染进程内存,不在抓取请求中执行SQL。采样错误进入脱敏结构化日志,不能改变任务、交付或流程事务。 - [进程入口](../../cmd/orbit/main.go)仅在显式启用指标时每5秒后台采样;`GET /metrics`仍只渲染进程内存,不在抓取请求中执行SQL。采样错误进入脱敏结构化日志,不能改变任务、交付或流程事务。
## 回归证据 ## 回归证据
- [业务记录器单测](../../internal/observability/metrics_test.go)验证成功状态、年龄、截断、成功计数和失败后保留最后成功值;超时计数单独增加。 - [业务记录器单测](../../internal/observability/metrics_test.go)验证成功状态、年龄、截断、成功计数和失败后保留最后成功值;超时计数单独增加。
- [真实MySQL专项](../../internal/storage/mysql/operational_metrics_integration_test.go)验证迁移索引真实存在、数据库外键为0、任务状态被采样、固定状态集合存在、空交付/流程状态不误报,并验证已取消上下文被拒绝。 - [真实MySQL专项](../../internal/storage/mysql/operational_metrics_integration_test.go)验证两次迁移索引真实存在、数据库外键为0、任务状态和`task.accepted`事件被采样、固定状态集合存在,并验证已取消上下文被拒绝。
- `go test ./...`覆盖编译和普通单测;真实MySQL专项由标准`GO=... DOCKER=... sh scripts/test-mysql.sh`执行,未设置`ORBIT_TEST_MYSQL_DSN`时专项会跳过,不能把跳过当通过。最终标准脚本在Go 1.27.1、darwin/arm64、Docker MySQL 8.4.10环境退出码为0;其中存储/迁移/事务组256.539秒,指标开关进程组5.204秒,流程运行器全停恢复49.333秒,SDK与两个独立业务示例均通过。专项日志确认`TestIntegrationOperationalSnapshot`、`TestIntegrationNoForeignKeysUpgrade`、`TestIntegrationCredentialOperations`及`TestIntegrationCredentialCommitFaults`均PASS。 - `go test ./...`覆盖编译和普通单测;真实MySQL专项由标准`GO=... DOCKER=... sh scripts/test-mysql.sh`执行,未设置`ORBIT_TEST_MYSQL_DSN`时专项会跳过,不能把跳过当通过。最新标准脚本在Go 1.27.1、darwin/arm64、Docker MySQL 8.4.10环境退出码为0;其中存储/迁移/事务组265.925秒,指标开关进程组4.548秒,流程运行器全停恢复48.263秒,SDK与两个独立业务示例均通过。专项日志确认`TestIntegrationOperationalSnapshot`、`TestIntegrationNoForeignKeysUpgrade`、`TestIntegrationCredentialOperations`及`TestIntegrationCredentialCommitFaults`均PASS,迁移账本到000027且dirty=false。
- 当前固定目录导出的指标夹具为929159字节,`promtool check metrics`成功;`promtool check rules deploy/monitoring/orbit.rules.yml`报告7条规则成功,`promtool test rules deploy/monitoring/orbit.rules.test.yml`成功。标准验证镜像仅用于离线检查,平台运行不依赖Prometheus。 - 当前固定目录导出的指标夹具为929298字节,包含事件标签回归;`promtool check metrics`成功;`promtool check rules deploy/monitoring/orbit.rules.yml`报告7条规则成功,`promtool test rules deploy/monitoring/orbit.rules.test.yml`成功。标准验证镜像仅用于离线检查,平台运行不依赖Prometheus。
## 解释和边界 ## 解释和边界
每个Orbit节点都读取自己的有限样本;多节点指标不能直接相加获得全局精确积压。达到10000上限、采样超时、节点重启或节点间时间不同都会影响解释,必须结合截断标记、最后成功时间、数据库权威查询和审计判断。最早年龄使用任务/交付的`available_at`及流程的`updated_at`,它是告警线索,不是端到端受理、授权或完成延迟。 每个Orbit节点都读取自己的有限样本;多节点指标不能直接相加获得全局精确积压或吞吐。达到10000上限、采样超时、节点重启或节点间时间不同都会影响解释,必须结合截断标记、最后成功时间、数据库权威查询和审计判断。事件窗口按`occurred_at`统计固定类别,不包含未知类型;它是趋势线索,不是端到端受理、授权或完成延迟。
当前仍未完成持久化吞吐与阶段延迟、流程Inbox推进延迟、重试/核查/恢复量、发送队列全局聚合、生产采集通知演练、锁/磁盘/复制指标和多故障域容量验证。此批实现不能替代保留、备份恢复、HA、RPO/RTO及发布验收。 当前仍未完成端到端阶段延迟、全局吞吐聚合、流程Inbox推进延迟、发送队列全局聚合、生产采集通知演练、锁/磁盘/复制指标和多故障域容量验证。此批实现不能替代保留、备份恢复、HA、RPO/RTO及发布验收。

View File

@@ -23,6 +23,8 @@ type BusinessSnapshot struct {
WorkflowOldestAgeSeconds float64 WorkflowOldestAgeSeconds float64
// Truncated 是否至少有一类状态达到采样上限。 // Truncated 是否至少有一类状态达到采样上限。
Truncated bool Truncated bool
// RecentEvents 固定事件类别在最近窗口内的有界数量。
RecentEvents map[string]uint64
} }
// RecordBusinessSnapshot records one bounded database sample. A failed sample // RecordBusinessSnapshot records one bounded database sample. A failed sample
@@ -53,6 +55,9 @@ func (r *Registry) RecordBusinessSnapshot(snapshot BusinessSnapshot, started tim
for _, state := range workflowStates { for _, state := range workflowStates {
r.SetGauge("orbit_weave_workflows_state", float64(snapshot.WorkflowStates[state]), Labels{"state": state}) r.SetGauge("orbit_weave_workflows_state", float64(snapshot.WorkflowStates[state]), Labels{"state": state})
} }
for _, eventType := range eventTypes {
r.SetGauge("orbit_business_events_recent", float64(snapshot.RecentEvents[eventType]), Labels{"event": eventType})
}
r.SetGauge("orbit_pulse_tasks_oldest_age_seconds", snapshot.TaskOldestAgeSeconds, nil) r.SetGauge("orbit_pulse_tasks_oldest_age_seconds", snapshot.TaskOldestAgeSeconds, nil)
r.SetGauge("orbit_result_delivery_oldest_age_seconds", snapshot.DeliveryOldestAgeSeconds, nil) r.SetGauge("orbit_result_delivery_oldest_age_seconds", snapshot.DeliveryOldestAgeSeconds, nil)
r.SetGauge("orbit_weave_workflow_oldest_age_seconds", snapshot.WorkflowOldestAgeSeconds, nil) r.SetGauge("orbit_weave_workflow_oldest_age_seconds", snapshot.WorkflowOldestAgeSeconds, nil)

View File

@@ -9,6 +9,7 @@ var components = []string{"workflow", "schedule", "execution_recovery"}
var taskStates = []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"} var taskStates = []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
var deliveryStates = []string{"PENDING", "LEASED", "ACKED", "EXPIRED"} var deliveryStates = []string{"PENDING", "LEASED", "ACKED", "EXPIRED"}
var workflowStates = []string{"RUNNING", "WAITING", "CANCELLING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"} var workflowStates = []string{"RUNNING", "WAITING", "CANCELLING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
var eventTypes = []string{"task.accepted", "task.started", "task.succeeded", "task.failed", "task.needs_review", "task.requeued", "workflow.started", "workflow.advanced", "workflow.recovered", "workflow.blocked", "workflow.stopped"}
// Every exported family is declared here, including each allowed label value. // Every exported family is declared here, including each allowed label value.
// Adding a metric therefore requires a code review; no user identifier is valid. // Adding a metric therefore requires a code review; no user identifier is valid.
@@ -42,6 +43,7 @@ func catalog() map[string]descriptor {
c["orbit_result_delivery_oldest_age_seconds"] = descriptor{gauge, "Age of the oldest sampled pending result delivery", nil} c["orbit_result_delivery_oldest_age_seconds"] = descriptor{gauge, "Age of the oldest sampled pending result delivery", nil}
c["orbit_weave_workflows_state"] = descriptor{gauge, "Bounded count of Weave instances by durable state", map[string][]string{"state": workflowStates}} c["orbit_weave_workflows_state"] = descriptor{gauge, "Bounded count of Weave instances by durable state", map[string][]string{"state": workflowStates}}
c["orbit_weave_workflow_oldest_age_seconds"] = descriptor{gauge, "Age of the oldest sampled unfinished Weave instance", nil} c["orbit_weave_workflow_oldest_age_seconds"] = descriptor{gauge, "Age of the oldest sampled unfinished Weave instance", nil}
c["orbit_business_events_recent"] = descriptor{gauge, "Bounded count of fixed event categories in the recent window", map[string][]string{"event": eventTypes}}
c["orbit_business_snapshot_total"] = descriptor{counter, "Completed bounded business-state snapshots by outcome", map[string][]string{"outcome": {"success", "error", "timeout"}}} c["orbit_business_snapshot_total"] = descriptor{counter, "Completed bounded business-state snapshots by outcome", map[string][]string{"outcome": {"success", "error", "timeout"}}}
c["orbit_business_snapshot_duration_seconds"] = descriptor{histogram, "Bounded business-state snapshot duration in seconds", nil} c["orbit_business_snapshot_duration_seconds"] = descriptor{histogram, "Bounded business-state snapshot duration in seconds", nil}
c["orbit_business_snapshot_last_success_timestamp_seconds"] = descriptor{gauge, "Unix time of the last successful bounded business-state snapshot", nil} c["orbit_business_snapshot_last_success_timestamp_seconds"] = descriptor{gauge, "Unix time of the last successful bounded business-state snapshot", nil}

View File

@@ -123,6 +123,7 @@ func TestBusinessSnapshotKeepsLastGoodStateOnFailure(t *testing.T) {
DeliveryOldestAgeSeconds: 8.25, DeliveryOldestAgeSeconds: 8.25,
WorkflowStates: map[string]uint64{"RUNNING": 1}, WorkflowStates: map[string]uint64{"RUNNING": 1},
WorkflowOldestAgeSeconds: 4.75, WorkflowOldestAgeSeconds: 4.75,
RecentEvents: map[string]uint64{"task.accepted": 5},
Truncated: true, Truncated: true,
}, started, nil) }, started, nil)
first := r.Render() first := r.Render()
@@ -130,6 +131,7 @@ func TestBusinessSnapshotKeepsLastGoodStateOnFailure(t *testing.T) {
`orbit_pulse_tasks_state{state="WAITING"} 3`, `orbit_pulse_tasks_state{state="WAITING"} 3`,
`orbit_result_deliveries_state{state="PENDING"} 2`, `orbit_result_deliveries_state{state="PENDING"} 2`,
`orbit_weave_workflows_state{state="RUNNING"} 1`, `orbit_weave_workflows_state{state="RUNNING"} 1`,
`orbit_business_events_recent{event="task.accepted"} 5`,
"orbit_pulse_tasks_oldest_age_seconds 12.5", "orbit_pulse_tasks_oldest_age_seconds 12.5",
"orbit_business_snapshot_truncated 1", "orbit_business_snapshot_truncated 1",
`orbit_business_snapshot_total{outcome="success"} 1`, `orbit_business_snapshot_total{outcome="success"} 1`,

View File

@@ -68,6 +68,7 @@ func TestIntegrationApplicationResourcesUpgrade(t *testing.T) {
f.exec(`ALTER TABLE pulse_tasks DROP INDEX ix_task_observe_state_available`) f.exec(`ALTER TABLE pulse_tasks DROP INDEX ix_task_observe_state_available`)
f.exec(`ALTER TABLE orbit_deliveries DROP INDEX ix_delivery_observe_state_available`) f.exec(`ALTER TABLE orbit_deliveries DROP INDEX ix_delivery_observe_state_available`)
f.exec(`ALTER TABLE weave_instances DROP INDEX ix_workflow_observe_state_updated`) f.exec(`ALTER TABLE weave_instances DROP INDEX ix_workflow_observe_state_updated`)
f.exec(`ALTER TABLE orbit_events DROP INDEX ix_event_observe_type_occurred`)
f.exec(`DROP TABLE weave_version_scopes`) f.exec(`DROP TABLE weave_version_scopes`)
f.exec(`ALTER TABLE weave_instances DROP INDEX ix_workflow_created,DROP INDEX ix_workflow_state_created,DROP INDEX ix_workflow_business_state_created,DROP INDEX ix_workflow_definition_created,DROP INDEX ix_workflow_definition_state_created,DROP INDEX ix_workflow_definition_business_created,DROP INDEX ix_workflow_definition_business_state_created`) f.exec(`ALTER TABLE weave_instances DROP INDEX ix_workflow_created,DROP INDEX ix_workflow_state_created,DROP INDEX ix_workflow_business_state_created,DROP INDEX ix_workflow_definition_created,DROP INDEX ix_workflow_definition_state_created,DROP INDEX ix_workflow_definition_business_created,DROP INDEX ix_workflow_definition_business_state_created`)
f.exec(`ALTER TABLE weave_signals DROP INDEX ix_signal_history`) f.exec(`ALTER TABLE weave_signals DROP INDEX ix_signal_history`)

View File

@@ -0,0 +1,4 @@
-- 为固定事件类别的短窗口运行观测提供有界索引。
-- 事件仍是不可变业务证据;索引不改变事件语义,也不引入外键或级联。
ALTER TABLE orbit_events
ADD KEY ix_event_observe_type_occurred (type, occurred_at, id);

View File

@@ -89,7 +89,7 @@ func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
indexes := func() []string { indexes := func() []string {
t.Helper() t.Helper()
// 排除后续新增/替换索引;原有项在此逐项比较,v24/v25新增项由对应Query/Timeline升级测试核对。 // 排除后续新增/替换索引;原有项在此逐项比较,v24/v25新增项由对应Query/Timeline升级测试核对。
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events','weave_version_scopes') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result','ix_signal_expiry','ix_signal_activation','uq_wait_winner_signal','ix_task_workflow_owner','ix_signal_history','ix_event_workflow_history','ix_workflow_created','ix_workflow_state_created','ix_workflow_business_state_created','ix_workflow_definition_created','ix_workflow_definition_state_created','ix_workflow_definition_business_created','ix_workflow_definition_business_state_created','ix_task_observe_state_available','ix_delivery_observe_state_available','ix_workflow_observe_state_updated') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`) rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events','weave_version_scopes') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result','ix_signal_expiry','ix_signal_activation','uq_wait_winner_signal','ix_task_workflow_owner','ix_signal_history','ix_event_workflow_history','ix_workflow_created','ix_workflow_state_created','ix_workflow_business_state_created','ix_workflow_definition_created','ix_workflow_definition_state_created','ix_workflow_definition_business_created','ix_workflow_definition_business_state_created','ix_task_observe_state_available','ix_delivery_observe_state_available','ix_workflow_observe_state_updated','ix_event_observe_type_occurred') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@@ -8,6 +8,7 @@ import (
) )
const operationalMetricSampleLimit = 10000 const operationalMetricSampleLimit = 10000
const operationalMetricEventWindow = 5 * time.Minute
// OperationalSnapshot 是进程观测使用的有限数据库快照。 // OperationalSnapshot 是进程观测使用的有限数据库快照。
// Counts 只包含固定状态的有界计数;达到上限时 Truncated 为 true,不能解释为精确总量。 // Counts 只包含固定状态的有界计数;达到上限时 Truncated 为 true,不能解释为精确总量。
@@ -26,12 +27,15 @@ type OperationalSnapshot struct {
WorkflowOldestAgeSeconds float64 WorkflowOldestAgeSeconds float64
// Truncated 表示至少一个状态达到有界采样上限。 // Truncated 表示至少一个状态达到有界采样上限。
Truncated bool Truncated bool
// RecentEvents 是固定事件类别在最近窗口内的有界数量。
RecentEvents map[string]uint64
} }
var ( var (
operationalTaskStates = []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"} operationalTaskStates = []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
operationalDeliveryStates = []string{"PENDING", "LEASED", "ACKED", "EXPIRED"} operationalDeliveryStates = []string{"PENDING", "LEASED", "ACKED", "EXPIRED"}
operationalWorkflowStates = []string{"RUNNING", "WAITING", "CANCELLING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"} operationalWorkflowStates = []string{"RUNNING", "WAITING", "CANCELLING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
operationalEventTypes = []string{"task.accepted", "task.started", "task.succeeded", "task.failed", "task.needs_review", "task.requeued", "workflow.started", "workflow.advanced", "workflow.recovered", "workflow.blocked", "workflow.stopped"}
) )
// ReadOperationalSnapshot performs only bounded, indexed reads. It is intended // ReadOperationalSnapshot performs only bounded, indexed reads. It is intended
@@ -68,11 +72,16 @@ func ReadOperationalSnapshot(ctx context.Context, db *sql.DB) (OperationalSnapsh
if err != nil { if err != nil {
return OperationalSnapshot{}, err return OperationalSnapshot{}, err
} }
events, eventTruncated, err := boundedRecentEventCounts(ctx, db, now)
if err != nil {
return OperationalSnapshot{}, err
}
return OperationalSnapshot{ return OperationalSnapshot{
TaskStates: tasks, TaskOldestAgeSeconds: taskAge, TaskStates: tasks, TaskOldestAgeSeconds: taskAge,
DeliveryStates: deliveries, DeliveryOldestAgeSeconds: deliveryAge, DeliveryStates: deliveries, DeliveryOldestAgeSeconds: deliveryAge,
WorkflowStates: workflows, WorkflowOldestAgeSeconds: workflowAge, WorkflowStates: workflows, WorkflowOldestAgeSeconds: workflowAge,
Truncated: taskTruncated || deliveryTruncated || workflowTruncated, RecentEvents: events,
Truncated: taskTruncated || deliveryTruncated || workflowTruncated || eventTruncated,
}, nil }, nil
} }
@@ -102,6 +111,25 @@ func boundedStateCounts(ctx context.Context, db *sql.DB, table, index string, st
return counts, truncated, nil return counts, truncated, nil
} }
func boundedRecentEventCounts(ctx context.Context, db *sql.DB, now time.Time) (map[string]uint64, bool, error) {
counts := make(map[string]uint64, len(operationalEventTypes))
truncated := false
cutoff := now.Add(-operationalMetricEventWindow)
for _, eventType := range operationalEventTypes {
query := "SELECT COUNT(*) FROM (SELECT id FROM orbit_events FORCE INDEX (ix_event_observe_type_occurred) WHERE type=? AND occurred_at>=? LIMIT ?) AS bounded"
var n uint64
if err := db.QueryRowContext(ctx, query, eventType, cutoff, operationalMetricSampleLimit+1).Scan(&n); err != nil {
return nil, false, err
}
if n > operationalMetricSampleLimit {
n = operationalMetricSampleLimit
truncated = true
}
counts[eventType] = n
}
return counts, truncated, nil
}
func oldestAge(ctx context.Context, db *sql.DB, table, index string, states []string, column string, now time.Time) (float64, error) { func oldestAge(ctx context.Context, db *sql.DB, table, index string, states []string, column string, now time.Time) (float64, error) {
placeholders := "?" placeholders := "?"
args := make([]any, 0, len(states)) args := make([]any, 0, len(states))

View File

@@ -60,6 +60,9 @@ func TestIntegrationOperationalSnapshot(t *testing.T) {
if got := snapshot.TaskStates["WAITING"]; got < 1 { if got := snapshot.TaskStates["WAITING"]; got < 1 {
t.Fatalf("waiting task was not observed: %#v", snapshot.TaskStates) t.Fatalf("waiting task was not observed: %#v", snapshot.TaskStates)
} }
if got := snapshot.RecentEvents["task.accepted"]; got < 1 {
t.Fatalf("accepted event was not observed: %#v", snapshot.RecentEvents)
}
if snapshot.TaskOldestAgeSeconds < 0 || snapshot.DeliveryOldestAgeSeconds < 0 || snapshot.WorkflowOldestAgeSeconds < 0 { if snapshot.TaskOldestAgeSeconds < 0 || snapshot.DeliveryOldestAgeSeconds < 0 || snapshot.WorkflowOldestAgeSeconds < 0 {
t.Fatalf("negative age in snapshot: %#v", snapshot) t.Fatalf("negative age in snapshot: %#v", snapshot)
} }