rule_files: - orbit.rules.yml evaluation_interval: 1s tests: - name: unavailable_target interval: 1s input_series: - series: 'up{job="orbit",instance="node-a"}' values: '0+0x180' alert_rule_test: - eval_time: 130s alertname: OrbitMetricsUnavailable exp_alerts: - exp_labels: {job: orbit, instance: node-a, severity: warning} exp_annotations: summary: "Orbit metrics target is unavailable" description: "Check the metrics listener, private network and process health. This does not prove that business tasks have stopped." - name: startup_grace_and_stalled_component interval: 1s input_series: - series: 'orbit_process_start_time_seconds{job="orbit",instance="node-a"}' values: '0+0x180' - series: 'orbit_runtime_batch_last_success_timestamp_seconds{job="orbit",instance="node-a",component="workflow"}' values: '0+0x180' - series: 'orbit_business_snapshot_last_success_timestamp_seconds{job="orbit",instance="node-a"}' values: '0+0x180' alert_rule_test: - eval_time: 60s alertname: OrbitRuntimeBatchStalled exp_alerts: [] - eval_time: 130s alertname: OrbitRuntimeBatchStalled exp_alerts: - exp_labels: {job: orbit, instance: node-a, component: workflow, severity: warning} exp_annotations: summary: "Orbit background component has no recent successful batch" description: "Inspect database availability, schema readiness and batch errors before changing limits or retrying business effects." - eval_time: 130s alertname: OrbitBusinessSnapshotStalled exp_alerts: - exp_labels: {job: orbit, instance: node-a, severity: warning} exp_annotations: summary: "Orbit has no recent successful business-state snapshot" description: "Inspect the bounded sampler timeout/error counter and MySQL health; the last state gauges may be stale." - name: healthy_batches_and_waits interval: 1s input_series: - series: 'up{job="orbit",instance="node-a"}' values: '1+0x400' - series: 'orbit_process_start_time_seconds{job="orbit",instance="node-a"}' values: '0+0x400' - series: 'orbit_runtime_batch_last_success_timestamp_seconds{job="orbit",instance="node-a",component="workflow"}' values: '0+1x400' - series: 'orbit_runtime_batch_total{job="orbit",instance="node-a",component="workflow",outcome="error"}' values: '0+0x400' - series: 'orbit_db_wait_duration_seconds_total{job="orbit",instance="node-a"}' values: '0+0x400' - series: 'orbit_db_transaction_retry_exhausted_total{job="orbit",instance="node-a"}' values: '0+0x400' - series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}' values: '0+0x400' - series: 'orbit_business_snapshot_last_success_timestamp_seconds{job="orbit",instance="node-a"}' values: '0+1x400' - series: 'orbit_business_snapshot_truncated{job="orbit",instance="node-a"}' values: '0+0x400' alert_rule_test: - {eval_time: 400s, alertname: OrbitMetricsUnavailable, exp_alerts: []} - {eval_time: 400s, alertname: OrbitRuntimeBatchStalled, exp_alerts: []} - {eval_time: 400s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []} - {eval_time: 400s, alertname: OrbitDBPoolWaiting, exp_alerts: []} - {eval_time: 400s, alertname: OrbitDBTransactionRetryExhausted, exp_alerts: []} - {eval_time: 400s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []} - {eval_time: 400s, alertname: OrbitBusinessSnapshotStalled, exp_alerts: []} - {eval_time: 400s, alertname: OrbitBusinessSnapshotTruncated, exp_alerts: []} - name: failing_batches_waits_and_invalid_samples_then_recover interval: 1s input_series: - series: 'orbit_runtime_batch_total{job="orbit",instance="node-a",component="workflow",outcome="error"}' values: '0+1x400 400+0x400' - series: 'orbit_db_wait_duration_seconds_total{job="orbit",instance="node-a"}' values: '0+1x400 400+0x400' - series: 'orbit_db_transaction_retry_exhausted_total{job="orbit",instance="node-a"}' values: '0+1x400 400+0x400' - series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}' values: '0+1x400 400+0x400' - series: 'orbit_business_snapshot_last_success_timestamp_seconds{job="orbit",instance="node-a"}' values: '0+0x800' - series: 'orbit_business_snapshot_truncated{job="orbit",instance="node-a"}' values: '1+0x400 0+0x400' alert_rule_test: - eval_time: 400s alertname: OrbitRuntimeBatchErrors exp_alerts: - exp_labels: {job: orbit, instance: node-a, component: workflow, outcome: error, severity: warning} exp_annotations: summary: "Orbit background batches are failing" description: "Use process logs and durable task/workflow audit to investigate. Batch errors are not business failures." - eval_time: 400s alertname: OrbitDBPoolWaiting exp_alerts: - exp_labels: {job: orbit, instance: node-a, severity: warning} exp_annotations: summary: "Orbit is waiting for MySQL connections" description: "Correlate pool use with database locks, query latency and storage load before increasing the connection limit." - eval_time: 400s alertname: OrbitDBTransactionRetryExhausted exp_alerts: - exp_labels: {job: orbit, instance: node-a, severity: warning} exp_annotations: summary: "Orbit exhausted a bounded MySQL transaction retry" description: "Inspect lock waits, deadlocks and the original operation identity; this counter excludes ambiguous COMMIT responses and must not trigger blind replay." - eval_time: 400s alertname: OrbitInvalidMetricUpdates exp_alerts: - exp_labels: {job: orbit, instance: node-a, severity: warning} exp_annotations: summary: "Orbit rejected an internal metrics update" description: "Check instrumentation changes for unsupported names, types, labels or counter regressions." - eval_time: 400s alertname: OrbitBusinessSnapshotTruncated exp_alerts: - exp_labels: {job: orbit, instance: node-a, severity: warning} exp_annotations: summary: "Orbit business-state snapshot reached its sample cap" description: "State gauges are bounded samples, not exact global counts; use authoritative database queries and capacity review." - {eval_time: 800s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []} - {eval_time: 800s, alertname: OrbitDBPoolWaiting, exp_alerts: []} - {eval_time: 800s, alertname: OrbitDBTransactionRetryExhausted, exp_alerts: []} - {eval_time: 800s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []} - {eval_time: 800s, alertname: OrbitBusinessSnapshotTruncated, exp_alerts: []}