feat: add bounded business state observability

deploy/monitoring/orbit.rules.test.yml

@@ -22,6 +22,8 @@ tests:
         values: '0+0x180'
       - series: 'orbit_runtime_batch_last_success_timestamp_seconds{job="orbit",instance="node-a",component="workflow"}'
         values: '0+0x180'
+      - series: 'orbit_business_snapshot_last_success_timestamp_seconds{job="orbit",instance="node-a"}'
+        values: '0+0x180'
     alert_rule_test:
       - eval_time: 60s
         alertname: OrbitRuntimeBatchStalled
@@ -33,6 +35,13 @@ tests:
             exp_annotations:
               summary: "Orbit background component has no recent successful batch"
               description: "Inspect database availability, schema readiness and batch errors before changing limits or retrying business effects."
+      - eval_time: 130s
+        alertname: OrbitBusinessSnapshotStalled
+        exp_alerts:
+          - exp_labels: {job: orbit, instance: node-a, severity: warning}
+            exp_annotations:
+              summary: "Orbit has no recent successful business-state snapshot"
+              description: "Inspect the bounded sampler timeout/error counter and MySQL health; the last state gauges may be stale."
   - name: healthy_batches_and_waits
     interval: 1s
     input_series:
@@ -48,12 +57,18 @@ tests:
         values: '0+0x400'
       - series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}'
         values: '0+0x400'
+      - series: 'orbit_business_snapshot_last_success_timestamp_seconds{job="orbit",instance="node-a"}'
+        values: '0+1x400'
+      - series: 'orbit_business_snapshot_truncated{job="orbit",instance="node-a"}'
+        values: '0+0x400'
     alert_rule_test:
       - {eval_time: 400s, alertname: OrbitMetricsUnavailable, exp_alerts: []}
       - {eval_time: 400s, alertname: OrbitRuntimeBatchStalled, exp_alerts: []}
       - {eval_time: 400s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []}
       - {eval_time: 400s, alertname: OrbitDBPoolWaiting, exp_alerts: []}
       - {eval_time: 400s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []}
+      - {eval_time: 400s, alertname: OrbitBusinessSnapshotStalled, exp_alerts: []}
+      - {eval_time: 400s, alertname: OrbitBusinessSnapshotTruncated, exp_alerts: []}
   - name: failing_batches_waits_and_invalid_samples_then_recover
     interval: 1s
     input_series:
@@ -63,6 +78,10 @@ tests:
         values: '0+1x400 400+0x400'
       - series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}'
         values: '0+1x400 400+0x400'
+      - series: 'orbit_business_snapshot_last_success_timestamp_seconds{job="orbit",instance="node-a"}'
+        values: '0+0x800'
+      - series: 'orbit_business_snapshot_truncated{job="orbit",instance="node-a"}'
+        values: '1+0x400 0+0x400'
     alert_rule_test:
       - eval_time: 400s
         alertname: OrbitRuntimeBatchErrors
@@ -85,6 +104,14 @@ tests:
             exp_annotations:
               summary: "Orbit rejected an internal metrics update"
               description: "Check instrumentation changes for unsupported names, types, labels or counter regressions."
+      - eval_time: 400s
+        alertname: OrbitBusinessSnapshotTruncated
+        exp_alerts:
+          - exp_labels: {job: orbit, instance: node-a, severity: warning}
+            exp_annotations:
+              summary: "Orbit business-state snapshot reached its sample cap"
+              description: "State gauges are bounded samples, not exact global counts; use authoritative database queries and capacity review."
       - {eval_time: 800s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []}
       - {eval_time: 800s, alertname: OrbitDBPoolWaiting, exp_alerts: []}
       - {eval_time: 800s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []}
+      - {eval_time: 800s, alertname: OrbitBusinessSnapshotTruncated, exp_alerts: []}

deploy/monitoring/orbit.rules.yml

@@ -25,6 +25,22 @@ groups:
         annotations:
           summary: "Orbit background batches are failing"
           description: "Use process logs and durable task/workflow audit to investigate. Batch errors are not business failures."
+      - alert: OrbitBusinessSnapshotStalled
+        expr: (time() - orbit_business_snapshot_last_success_timestamp_seconds{job="orbit"} > 30) and on (job, instance) (time() - orbit_process_start_time_seconds{job="orbit"} > 60)
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Orbit has no recent successful business-state snapshot"
+          description: "Inspect the bounded sampler timeout/error counter and MySQL health; the last state gauges may be stale."
+      - alert: OrbitBusinessSnapshotTruncated
+        expr: orbit_business_snapshot_truncated{job="orbit"} == 1
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Orbit business-state snapshot reached its sample cap"
+          description: "State gauges are bounded samples, not exact global counts; use authoritative database queries and capacity review."
       - alert: OrbitDBPoolWaiting
         expr: rate(orbit_db_wait_duration_seconds_total{job="orbit"}[5m]) > 0.1
         for: 5m