feat: add bounded business state observability
This commit is contained in:
@@ -25,6 +25,22 @@ groups:
|
||||
annotations:
|
||||
summary: "Orbit background batches are failing"
|
||||
description: "Use process logs and durable task/workflow audit to investigate. Batch errors are not business failures."
|
||||
- alert: OrbitBusinessSnapshotStalled
|
||||
expr: (time() - orbit_business_snapshot_last_success_timestamp_seconds{job="orbit"} > 30) and on (job, instance) (time() - orbit_process_start_time_seconds{job="orbit"} > 60)
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Orbit has no recent successful business-state snapshot"
|
||||
description: "Inspect the bounded sampler timeout/error counter and MySQL health; the last state gauges may be stale."
|
||||
- alert: OrbitBusinessSnapshotTruncated
|
||||
expr: orbit_business_snapshot_truncated{job="orbit"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Orbit business-state snapshot reached its sample cap"
|
||||
description: "State gauges are bounded samples, not exact global counts; use authoritative database queries and capacity review."
|
||||
- alert: OrbitDBPoolWaiting
|
||||
expr: rate(orbit_db_wait_duration_seconds_total{job="orbit"}[5m]) > 0.1
|
||||
for: 5m
|
||||
|
||||
Reference in New Issue
Block a user