k2safe/orbit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: alert on exhausted transaction retries

Browse Source
This commit is contained in:
k2safe
2026-09-20 22:17:17 +08:00
parent 1c3751fc7c
commit 351a7757ec
4 changed files with 24 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
deploy/monitoring/orbit.rules.test.yml
View File

@@ -55,6 +55,8 @@ tests:
values: '0+0x400'
- series: 'orbit_db_wait_duration_seconds_total{job="orbit",instance="node-a"}'
values: '0+0x400'
- series: 'orbit_db_transaction_retry_exhausted_total{job="orbit",instance="node-a"}'
values: '0+0x400'
- series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}'
values: '0+0x400'
- series: 'orbit_business_snapshot_last_success_timestamp_seconds{job="orbit",instance="node-a"}'
@@ -66,6 +68,7 @@ tests:
- {eval_time: 400s, alertname: OrbitRuntimeBatchStalled, exp_alerts: []}
- {eval_time: 400s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []}
- {eval_time: 400s, alertname: OrbitDBPoolWaiting, exp_alerts: []}
- {eval_time: 400s, alertname: OrbitDBTransactionRetryExhausted, exp_alerts: []}
- {eval_time: 400s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []}
- {eval_time: 400s, alertname: OrbitBusinessSnapshotStalled, exp_alerts: []}
- {eval_time: 400s, alertname: OrbitBusinessSnapshotTruncated, exp_alerts: []}
@@ -76,6 +79,8 @@ tests:
values: '0+1x400 400+0x400'
- series: 'orbit_db_wait_duration_seconds_total{job="orbit",instance="node-a"}'
values: '0+1x400 400+0x400'
- series: 'orbit_db_transaction_retry_exhausted_total{job="orbit",instance="node-a"}'
values: '0+1x400 400+0x400'
- series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}'
values: '0+1x400 400+0x400'
- series: 'orbit_business_snapshot_last_success_timestamp_seconds{job="orbit",instance="node-a"}'
@@ -97,6 +102,13 @@ tests:
exp_annotations:
summary: "Orbit is waiting for MySQL connections"
description: "Correlate pool use with database locks, query latency and storage load before increasing the connection limit."
- eval_time: 400s
alertname: OrbitDBTransactionRetryExhausted
exp_alerts:
- exp_labels: {job: orbit, instance: node-a, severity: warning}
exp_annotations:
summary: "Orbit exhausted a bounded MySQL transaction retry"
description: "Inspect lock waits, deadlocks and the original operation identity; this counter excludes ambiguous COMMIT responses and must not trigger blind replay."
- eval_time: 400s
alertname: OrbitInvalidMetricUpdates
exp_alerts:
@@ -113,5 +125,6 @@ tests:
description: "State gauges are bounded samples, not exact global counts; use authoritative database queries and capacity review."
- {eval_time: 800s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []}
- {eval_time: 800s, alertname: OrbitDBPoolWaiting, exp_alerts: []}
- {eval_time: 800s, alertname: OrbitDBTransactionRetryExhausted, exp_alerts: []}
- {eval_time: 800s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []}
- {eval_time: 800s, alertname: OrbitBusinessSnapshotTruncated, exp_alerts: []}

8
deploy/monitoring/orbit.rules.yml
View File

@@ -49,6 +49,14 @@ groups:
annotations:
summary: "Orbit is waiting for MySQL connections"
description: "Correlate pool use with database locks, query latency and storage load before increasing the connection limit."
- alert: OrbitDBTransactionRetryExhausted
expr: increase(orbit_db_transaction_retry_exhausted_total{job="orbit"}[5m]) > 0
for: 1m
labels:
severity: warning
annotations:
summary: "Orbit exhausted a bounded MySQL transaction retry"
description: "Inspect lock waits, deadlocks and the original operation identity; this counter excludes ambiguous COMMIT responses and must not trigger blind replay."
- alert: OrbitInvalidMetricUpdates
expr: increase(orbit_observability_rejected_samples_total{job="orbit"}[5m]) > 0
for: 1m