feat: add bounded process metrics and private monitoring endpoint

deploy/monitoring/orbit.rules.test.yml

@@ -0,0 +1,90 @@
+rule_files:
+  - orbit.rules.yml
+evaluation_interval: 1s
+tests:
+  - name: unavailable_target
+    interval: 1s
+    input_series:
+      - series: 'up{job="orbit",instance="node-a"}'
+        values: '0+0x180'
+    alert_rule_test:
+      - eval_time: 130s
+        alertname: OrbitMetricsUnavailable
+        exp_alerts:
+          - exp_labels: {job: orbit, instance: node-a, severity: warning}
+            exp_annotations:
+              summary: "Orbit metrics target is unavailable"
+              description: "Check the metrics listener, private network and process health. This does not prove that business tasks have stopped."
+  - name: startup_grace_and_stalled_component
+    interval: 1s
+    input_series:
+      - series: 'orbit_process_start_time_seconds{job="orbit",instance="node-a"}'
+        values: '0+0x180'
+      - series: 'orbit_runtime_batch_last_success_timestamp_seconds{job="orbit",instance="node-a",component="workflow"}'
+        values: '0+0x180'
+    alert_rule_test:
+      - eval_time: 60s
+        alertname: OrbitRuntimeBatchStalled
+        exp_alerts: []
+      - eval_time: 130s
+        alertname: OrbitRuntimeBatchStalled
+        exp_alerts:
+          - exp_labels: {job: orbit, instance: node-a, component: workflow, severity: warning}
+            exp_annotations:
+              summary: "Orbit background component has no recent successful batch"
+              description: "Inspect database availability, schema readiness and batch errors before changing limits or retrying business effects."
+  - name: healthy_batches_and_waits
+    interval: 1s
+    input_series:
+      - series: 'up{job="orbit",instance="node-a"}'
+        values: '1+0x400'
+      - series: 'orbit_process_start_time_seconds{job="orbit",instance="node-a"}'
+        values: '0+0x400'
+      - series: 'orbit_runtime_batch_last_success_timestamp_seconds{job="orbit",instance="node-a",component="workflow"}'
+        values: '0+1x400'
+      - series: 'orbit_runtime_batch_total{job="orbit",instance="node-a",component="workflow",outcome="error"}'
+        values: '0+0x400'
+      - series: 'orbit_db_wait_duration_seconds_total{job="orbit",instance="node-a"}'
+        values: '0+0x400'
+      - series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}'
+        values: '0+0x400'
+    alert_rule_test:
+      - {eval_time: 400s, alertname: OrbitMetricsUnavailable, exp_alerts: []}
+      - {eval_time: 400s, alertname: OrbitRuntimeBatchStalled, exp_alerts: []}
+      - {eval_time: 400s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []}
+      - {eval_time: 400s, alertname: OrbitDBPoolWaiting, exp_alerts: []}
+      - {eval_time: 400s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []}
+  - name: failing_batches_waits_and_invalid_samples_then_recover
+    interval: 1s
+    input_series:
+      - series: 'orbit_runtime_batch_total{job="orbit",instance="node-a",component="workflow",outcome="error"}'
+        values: '0+1x400 400+0x400'
+      - series: 'orbit_db_wait_duration_seconds_total{job="orbit",instance="node-a"}'
+        values: '0+1x400 400+0x400'
+      - series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}'
+        values: '0+1x400 400+0x400'
+    alert_rule_test:
+      - eval_time: 400s
+        alertname: OrbitRuntimeBatchErrors
+        exp_alerts:
+          - exp_labels: {job: orbit, instance: node-a, component: workflow, outcome: error, severity: warning}
+            exp_annotations:
+              summary: "Orbit background batches are failing"
+              description: "Use process logs and durable task/workflow audit to investigate. Batch errors are not business failures."
+      - eval_time: 400s
+        alertname: OrbitDBPoolWaiting
+        exp_alerts:
+          - exp_labels: {job: orbit, instance: node-a, severity: warning}
+            exp_annotations:
+              summary: "Orbit is waiting for MySQL connections"
+              description: "Correlate pool use with database locks, query latency and storage load before increasing the connection limit."
+      - eval_time: 400s
+        alertname: OrbitInvalidMetricUpdates
+        exp_alerts:
+          - exp_labels: {job: orbit, instance: node-a, severity: warning}
+            exp_annotations:
+              summary: "Orbit rejected an internal metrics update"
+              description: "Check instrumentation changes for unsupported names, types, labels or counter regressions."
+      - {eval_time: 800s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []}
+      - {eval_time: 800s, alertname: OrbitDBPoolWaiting, exp_alerts: []}
+      - {eval_time: 800s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []}

deploy/monitoring/orbit.rules.yml

@@ -0,0 +1,43 @@
+groups:
+  - name: orbit.process
+    rules:
+      - alert: OrbitMetricsUnavailable
+        expr: up{job="orbit"} == 0
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Orbit metrics target is unavailable"
+          description: "Check the metrics listener, private network and process health. This does not prove that business tasks have stopped."
+      - alert: OrbitRuntimeBatchStalled
+        expr: (time() - orbit_runtime_batch_last_success_timestamp_seconds{job="orbit"} > 30) and on (job, instance) (time() - orbit_process_start_time_seconds{job="orbit"} > 60)
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Orbit background component has no recent successful batch"
+          description: "Inspect database availability, schema readiness and batch errors before changing limits or retrying business effects."
+      - alert: OrbitRuntimeBatchErrors
+        expr: rate(orbit_runtime_batch_total{job="orbit",outcome="error"}[5m]) > 0
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Orbit background batches are failing"
+          description: "Use process logs and durable task/workflow audit to investigate. Batch errors are not business failures."
+      - alert: OrbitDBPoolWaiting
+        expr: rate(orbit_db_wait_duration_seconds_total{job="orbit"}[5m]) > 0.1
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Orbit is waiting for MySQL connections"
+          description: "Correlate pool use with database locks, query latency and storage load before increasing the connection limit."
+      - alert: OrbitInvalidMetricUpdates
+        expr: increase(orbit_observability_rejected_samples_total{job="orbit"}[5m]) > 0
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Orbit rejected an internal metrics update"
+          description: "Check instrumentation changes for unsupported names, types, labels or counter regressions."