feat: alert on exhausted transaction retries
This commit is contained in:
@@ -55,6 +55,8 @@ tests:
|
||||
values: '0+0x400'
|
||||
- series: 'orbit_db_wait_duration_seconds_total{job="orbit",instance="node-a"}'
|
||||
values: '0+0x400'
|
||||
- series: 'orbit_db_transaction_retry_exhausted_total{job="orbit",instance="node-a"}'
|
||||
values: '0+0x400'
|
||||
- series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}'
|
||||
values: '0+0x400'
|
||||
- series: 'orbit_business_snapshot_last_success_timestamp_seconds{job="orbit",instance="node-a"}'
|
||||
@@ -66,6 +68,7 @@ tests:
|
||||
- {eval_time: 400s, alertname: OrbitRuntimeBatchStalled, exp_alerts: []}
|
||||
- {eval_time: 400s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []}
|
||||
- {eval_time: 400s, alertname: OrbitDBPoolWaiting, exp_alerts: []}
|
||||
- {eval_time: 400s, alertname: OrbitDBTransactionRetryExhausted, exp_alerts: []}
|
||||
- {eval_time: 400s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []}
|
||||
- {eval_time: 400s, alertname: OrbitBusinessSnapshotStalled, exp_alerts: []}
|
||||
- {eval_time: 400s, alertname: OrbitBusinessSnapshotTruncated, exp_alerts: []}
|
||||
@@ -76,6 +79,8 @@ tests:
|
||||
values: '0+1x400 400+0x400'
|
||||
- series: 'orbit_db_wait_duration_seconds_total{job="orbit",instance="node-a"}'
|
||||
values: '0+1x400 400+0x400'
|
||||
- series: 'orbit_db_transaction_retry_exhausted_total{job="orbit",instance="node-a"}'
|
||||
values: '0+1x400 400+0x400'
|
||||
- series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}'
|
||||
values: '0+1x400 400+0x400'
|
||||
- series: 'orbit_business_snapshot_last_success_timestamp_seconds{job="orbit",instance="node-a"}'
|
||||
@@ -97,6 +102,13 @@ tests:
|
||||
exp_annotations:
|
||||
summary: "Orbit is waiting for MySQL connections"
|
||||
description: "Correlate pool use with database locks, query latency and storage load before increasing the connection limit."
|
||||
- eval_time: 400s
|
||||
alertname: OrbitDBTransactionRetryExhausted
|
||||
exp_alerts:
|
||||
- exp_labels: {job: orbit, instance: node-a, severity: warning}
|
||||
exp_annotations:
|
||||
summary: "Orbit exhausted a bounded MySQL transaction retry"
|
||||
description: "Inspect lock waits, deadlocks and the original operation identity; this counter excludes ambiguous COMMIT responses and must not trigger blind replay."
|
||||
- eval_time: 400s
|
||||
alertname: OrbitInvalidMetricUpdates
|
||||
exp_alerts:
|
||||
@@ -113,5 +125,6 @@ tests:
|
||||
description: "State gauges are bounded samples, not exact global counts; use authoritative database queries and capacity review."
|
||||
- {eval_time: 800s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []}
|
||||
- {eval_time: 800s, alertname: OrbitDBPoolWaiting, exp_alerts: []}
|
||||
- {eval_time: 800s, alertname: OrbitDBTransactionRetryExhausted, exp_alerts: []}
|
||||
- {eval_time: 800s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []}
|
||||
- {eval_time: 800s, alertname: OrbitBusinessSnapshotTruncated, exp_alerts: []}
|
||||
|
||||
@@ -49,6 +49,14 @@ groups:
|
||||
annotations:
|
||||
summary: "Orbit is waiting for MySQL connections"
|
||||
description: "Correlate pool use with database locks, query latency and storage load before increasing the connection limit."
|
||||
- alert: OrbitDBTransactionRetryExhausted
|
||||
expr: increase(orbit_db_transaction_retry_exhausted_total{job="orbit"}[5m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Orbit exhausted a bounded MySQL transaction retry"
|
||||
description: "Inspect lock waits, deadlocks and the original operation identity; this counter excludes ambiguous COMMIT responses and must not trigger blind replay."
|
||||
- alert: OrbitInvalidMetricUpdates
|
||||
expr: increase(orbit_observability_rejected_samples_total{job="orbit"}[5m]) > 0
|
||||
for: 1m
|
||||
|
||||
@@ -101,7 +101,7 @@ scrape_configs:
|
||||
|
||||
示例地址仅适用于同机采集；远程采集需替换为受保护的内部目标。平台运行不依赖采集器、Prometheus或额外数据库。
|
||||
|
||||
[七条告警规则](../../deploy/monitoring/orbit.rules.yml)和[离线规则测试](../../deploy/monitoring/orbit.rules.test.yml)包含以下场景：
|
||||
[八条告警规则](../../deploy/monitoring/orbit.rules.yml)和[离线规则测试](../../deploy/monitoring/orbit.rules.test.yml)包含以下场景：
|
||||
|
||||
- 目标连续2分钟采集失败：检查指标监听、网络和进程，不直接判断业务任务停止。
|
||||
- 启动60秒后，某后台组件超过30秒没有成功批次并持续1分钟：结合readiness、批次错误和数据库状态调查。
|
||||
@@ -109,6 +109,7 @@ scrape_configs:
|
||||
- 有界业务快照连续没有成功超过30秒（启动宽限60秒）：检查采样超时/错误和MySQL健康；已有状态值可能已经陈旧。
|
||||
- 固定状态采样连续达到10000条上限5分钟：状态指标只是有限样本，转用权威数据库查询并进行容量核查。
|
||||
- 池累计等待时间增长超过0.1秒/秒并持续5分钟：联查锁等待、慢SQL、CPU和存储，不自动提高连接数。
|
||||
- 提交前 MySQL 事务重试达到有界上限：检查锁等待、死锁和原操作身份，不按告警盲目重放；提交回执不明不计入此规则。
|
||||
- 内部非法指标更新持续出现：检查观测代码契约，不能忽略丢弃的样本。
|
||||
|
||||
这些阈值是可修改的运行基线，尚未通过生产负载调优；不构成SLO或可用性承诺。规则测试覆盖触发、启动宽限、健康不报警和错误停止后的解除，未发送实际通知或创建外部监控服务。
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
- [业务记录器单测](../../internal/observability/metrics_test.go)验证成功状态、年龄、截断、成功计数和失败后保留最后成功值；超时计数单独增加。
|
||||
- [真实MySQL专项](../../internal/storage/mysql/operational_metrics_integration_test.go)验证三次迁移索引真实存在、数据库外键为0、任务状态、交付状态、流程状态、Inbox状态、`task.accepted`事件和终态耗时被采样、固定状态集合存在，并验证已取消上下文被拒绝。
|
||||
- `go test ./...`覆盖编译和普通单测；真实MySQL专项由标准`GO=... DOCKER=... sh scripts/test-mysql.sh`执行，未设置`ORBIT_TEST_MYSQL_DSN`时专项会跳过，不能把跳过当通过。最新000028标准脚本在Go 1.27.1、darwin/arm64、Docker MySQL 8.4.10环境退出码为0；存储/迁移/事务组、指标开关进程组、流程运行器全停恢复、SDK与两个独立业务示例均通过。日志确认`TestIntegrationOperationalSnapshot`、`TestIntegrationNoForeignKeysUpgrade`、`TestIntegrationCredentialOperations`、`TestIntegrationCredentialCommitFaults`、`TestIntegrationMetricsProcess`及`TestIntegrationWorkflowRuntimeProcess`均PASS，迁移账本到000028且dirty=false；脚本新增的`orbit-admin verify`实际输出`schema_version=28`、`migration_count=28`、`foreign_key_count=0`、`required_index_count=6`、`complete=true`。
|
||||
- 当前固定目录导出的指标夹具为932029字节，包含事件、Inbox和阶段耗时指标回归；`promtool check metrics`成功；`promtool check rules deploy/monitoring/orbit.rules.yml`报告7条规则成功，`promtool test rules deploy/monitoring/orbit.rules.test.yml`成功。标准验证镜像仅用于离线检查，平台运行不依赖Prometheus。
|
||||
- 当前固定目录导出的指标夹具为932029字节，包含事件、Inbox和阶段耗时指标回归；`promtool check metrics`成功；本报告原始验证记录7条规则，后续新增提交前事务重试耗尽规则后当前规则数为8，使用 `prom/prometheus:v3.5.0`（digest `sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996`）执行 `promtool check rules` 与 `promtool test rules` 均成功。标准验证镜像仅用于离线检查，平台运行不依赖Prometheus。
|
||||
|
||||
## 解释和边界
|
||||
|
||||
|
||||
Reference in New Issue
Block a user