From 351a7757ecdf63be5671278c9928fe180e30b4ec Mon Sep 17 00:00:00 2001 From: k2safe Date: Sun, 20 Sep 2026 22:17:17 +0800 Subject: [PATCH] feat: alert on exhausted transaction retries --- deploy/monitoring/orbit.rules.test.yml | 13 +++++++++++++ deploy/monitoring/orbit.rules.yml | 8 ++++++++ docs/api/observability.md | 3 ++- docs/reports/phase-6-business-observability.md | 2 +- 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/deploy/monitoring/orbit.rules.test.yml b/deploy/monitoring/orbit.rules.test.yml index 18ce0a0..c276626 100644 --- a/deploy/monitoring/orbit.rules.test.yml +++ b/deploy/monitoring/orbit.rules.test.yml @@ -55,6 +55,8 @@ tests: values: '0+0x400' - series: 'orbit_db_wait_duration_seconds_total{job="orbit",instance="node-a"}' values: '0+0x400' + - series: 'orbit_db_transaction_retry_exhausted_total{job="orbit",instance="node-a"}' + values: '0+0x400' - series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}' values: '0+0x400' - series: 'orbit_business_snapshot_last_success_timestamp_seconds{job="orbit",instance="node-a"}' @@ -66,6 +68,7 @@ tests: - {eval_time: 400s, alertname: OrbitRuntimeBatchStalled, exp_alerts: []} - {eval_time: 400s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []} - {eval_time: 400s, alertname: OrbitDBPoolWaiting, exp_alerts: []} + - {eval_time: 400s, alertname: OrbitDBTransactionRetryExhausted, exp_alerts: []} - {eval_time: 400s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []} - {eval_time: 400s, alertname: OrbitBusinessSnapshotStalled, exp_alerts: []} - {eval_time: 400s, alertname: OrbitBusinessSnapshotTruncated, exp_alerts: []} @@ -76,6 +79,8 @@ tests: values: '0+1x400 400+0x400' - series: 'orbit_db_wait_duration_seconds_total{job="orbit",instance="node-a"}' values: '0+1x400 400+0x400' + - series: 'orbit_db_transaction_retry_exhausted_total{job="orbit",instance="node-a"}' + values: '0+1x400 400+0x400' - series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}' values: '0+1x400 400+0x400' - series: 'orbit_business_snapshot_last_success_timestamp_seconds{job="orbit",instance="node-a"}' @@ -97,6 +102,13 @@ tests: exp_annotations: summary: "Orbit is waiting for MySQL connections" description: "Correlate pool use with database locks, query latency and storage load before increasing the connection limit." + - eval_time: 400s + alertname: OrbitDBTransactionRetryExhausted + exp_alerts: + - exp_labels: {job: orbit, instance: node-a, severity: warning} + exp_annotations: + summary: "Orbit exhausted a bounded MySQL transaction retry" + description: "Inspect lock waits, deadlocks and the original operation identity; this counter excludes ambiguous COMMIT responses and must not trigger blind replay." - eval_time: 400s alertname: OrbitInvalidMetricUpdates exp_alerts: @@ -113,5 +125,6 @@ tests: description: "State gauges are bounded samples, not exact global counts; use authoritative database queries and capacity review." - {eval_time: 800s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []} - {eval_time: 800s, alertname: OrbitDBPoolWaiting, exp_alerts: []} + - {eval_time: 800s, alertname: OrbitDBTransactionRetryExhausted, exp_alerts: []} - {eval_time: 800s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []} - {eval_time: 800s, alertname: OrbitBusinessSnapshotTruncated, exp_alerts: []} diff --git a/deploy/monitoring/orbit.rules.yml b/deploy/monitoring/orbit.rules.yml index 90292a3..948420b 100644 --- a/deploy/monitoring/orbit.rules.yml +++ b/deploy/monitoring/orbit.rules.yml @@ -49,6 +49,14 @@ groups: annotations: summary: "Orbit is waiting for MySQL connections" description: "Correlate pool use with database locks, query latency and storage load before increasing the connection limit." + - alert: OrbitDBTransactionRetryExhausted + expr: increase(orbit_db_transaction_retry_exhausted_total{job="orbit"}[5m]) > 0 + for: 1m + labels: + severity: warning + annotations: + summary: "Orbit exhausted a bounded MySQL transaction retry" + description: "Inspect lock waits, deadlocks and the original operation identity; this counter excludes ambiguous COMMIT responses and must not trigger blind replay." - alert: OrbitInvalidMetricUpdates expr: increase(orbit_observability_rejected_samples_total{job="orbit"}[5m]) > 0 for: 1m diff --git a/docs/api/observability.md b/docs/api/observability.md index e4e5823..74cc4d3 100644 --- a/docs/api/observability.md +++ b/docs/api/observability.md @@ -101,7 +101,7 @@ scrape_configs: 示例地址仅适用于同机采集；远程采集需替换为受保护的内部目标。平台运行不依赖采集器、Prometheus或额外数据库。 -[七条告警规则](../../deploy/monitoring/orbit.rules.yml)和[离线规则测试](../../deploy/monitoring/orbit.rules.test.yml)包含以下场景： +[八条告警规则](../../deploy/monitoring/orbit.rules.yml)和[离线规则测试](../../deploy/monitoring/orbit.rules.test.yml)包含以下场景： - 目标连续2分钟采集失败：检查指标监听、网络和进程，不直接判断业务任务停止。 - 启动60秒后，某后台组件超过30秒没有成功批次并持续1分钟：结合readiness、批次错误和数据库状态调查。 @@ -109,6 +109,7 @@ scrape_configs: - 有界业务快照连续没有成功超过30秒（启动宽限60秒）：检查采样超时/错误和MySQL健康；已有状态值可能已经陈旧。 - 固定状态采样连续达到10000条上限5分钟：状态指标只是有限样本，转用权威数据库查询并进行容量核查。 - 池累计等待时间增长超过0.1秒/秒并持续5分钟：联查锁等待、慢SQL、CPU和存储，不自动提高连接数。 +- 提交前 MySQL 事务重试达到有界上限：检查锁等待、死锁和原操作身份，不按告警盲目重放；提交回执不明不计入此规则。 - 内部非法指标更新持续出现：检查观测代码契约，不能忽略丢弃的样本。 这些阈值是可修改的运行基线，尚未通过生产负载调优；不构成SLO或可用性承诺。规则测试覆盖触发、启动宽限、健康不报警和错误停止后的解除，未发送实际通知或创建外部监控服务。 diff --git a/docs/reports/phase-6-business-observability.md b/docs/reports/phase-6-business-observability.md index 16f877b..1891e0b 100644 --- a/docs/reports/phase-6-business-observability.md +++ b/docs/reports/phase-6-business-observability.md @@ -20,7 +20,7 @@ - [业务记录器单测](../../internal/observability/metrics_test.go)验证成功状态、年龄、截断、成功计数和失败后保留最后成功值；超时计数单独增加。 - [真实MySQL专项](../../internal/storage/mysql/operational_metrics_integration_test.go)验证三次迁移索引真实存在、数据库外键为0、任务状态、交付状态、流程状态、Inbox状态、`task.accepted`事件和终态耗时被采样、固定状态集合存在，并验证已取消上下文被拒绝。 - `go test ./...`覆盖编译和普通单测；真实MySQL专项由标准`GO=... DOCKER=... sh scripts/test-mysql.sh`执行，未设置`ORBIT_TEST_MYSQL_DSN`时专项会跳过，不能把跳过当通过。最新000028标准脚本在Go 1.27.1、darwin/arm64、Docker MySQL 8.4.10环境退出码为0；存储/迁移/事务组、指标开关进程组、流程运行器全停恢复、SDK与两个独立业务示例均通过。日志确认`TestIntegrationOperationalSnapshot`、`TestIntegrationNoForeignKeysUpgrade`、`TestIntegrationCredentialOperations`、`TestIntegrationCredentialCommitFaults`、`TestIntegrationMetricsProcess`及`TestIntegrationWorkflowRuntimeProcess`均PASS，迁移账本到000028且dirty=false；脚本新增的`orbit-admin verify`实际输出`schema_version=28`、`migration_count=28`、`foreign_key_count=0`、`required_index_count=6`、`complete=true`。 -- 当前固定目录导出的指标夹具为932029字节，包含事件、Inbox和阶段耗时指标回归；`promtool check metrics`成功；`promtool check rules deploy/monitoring/orbit.rules.yml`报告7条规则成功，`promtool test rules deploy/monitoring/orbit.rules.test.yml`成功。标准验证镜像仅用于离线检查，平台运行不依赖Prometheus。 +- 当前固定目录导出的指标夹具为932029字节，包含事件、Inbox和阶段耗时指标回归；`promtool check metrics`成功；本报告原始验证记录7条规则，后续新增提交前事务重试耗尽规则后当前规则数为8，使用 `prom/prometheus:v3.5.0`（digest `sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996`）执行 `promtool check rules` 与 `promtool test rules` 均成功。标准验证镜像仅用于离线检查，平台运行不依赖Prometheus。 ## 解释和边界