groups: - name: orbit.process rules: - alert: OrbitMetricsUnavailable expr: up{job="orbit"} == 0 for: 2m labels: severity: warning annotations: summary: "Orbit metrics target is unavailable" description: "Check the metrics listener, private network and process health. This does not prove that business tasks have stopped." - alert: OrbitRuntimeBatchStalled expr: (time() - orbit_runtime_batch_last_success_timestamp_seconds{job="orbit"} > 30) and on (job, instance) (time() - orbit_process_start_time_seconds{job="orbit"} > 60) for: 1m labels: severity: warning annotations: summary: "Orbit background component has no recent successful batch" description: "Inspect database availability, schema readiness and batch errors before changing limits or retrying business effects." - alert: OrbitRuntimeBatchErrors expr: rate(orbit_runtime_batch_total{job="orbit",outcome="error"}[5m]) > 0 for: 2m labels: severity: warning annotations: summary: "Orbit background batches are failing" description: "Use process logs and durable task/workflow audit to investigate. Batch errors are not business failures." - alert: OrbitBusinessSnapshotStalled expr: (time() - orbit_business_snapshot_last_success_timestamp_seconds{job="orbit"} > 30) and on (job, instance) (time() - orbit_process_start_time_seconds{job="orbit"} > 60) for: 1m labels: severity: warning annotations: summary: "Orbit has no recent successful business-state snapshot" description: "Inspect the bounded sampler timeout/error counter and MySQL health; the last state gauges may be stale." - alert: OrbitBusinessSnapshotTruncated expr: orbit_business_snapshot_truncated{job="orbit"} == 1 for: 5m labels: severity: warning annotations: summary: "Orbit business-state snapshot reached its sample cap" description: "State gauges are bounded samples, not exact global counts; use authoritative database queries and capacity review." - alert: OrbitDBPoolWaiting expr: rate(orbit_db_wait_duration_seconds_total{job="orbit"}[5m]) > 0.1 for: 5m labels: severity: warning annotations: summary: "Orbit is waiting for MySQL connections" description: "Correlate pool use with database locks, query latency and storage load before increasing the connection limit." - alert: OrbitDBTransactionRetryExhausted expr: increase(orbit_db_transaction_retry_exhausted_total{job="orbit"}[5m]) > 0 for: 1m labels: severity: warning annotations: summary: "Orbit exhausted a bounded MySQL transaction retry" description: "Inspect lock waits, deadlocks and the original operation identity; this counter excludes ambiguous COMMIT responses and must not trigger blind replay." - alert: OrbitInvalidMetricUpdates expr: increase(orbit_observability_rejected_samples_total{job="orbit"}[5m]) > 0 for: 1m labels: severity: warning annotations: summary: "Orbit rejected an internal metrics update" description: "Check instrumentation changes for unsupported names, types, labels or counter regressions."