91 lines
4.5 KiB
YAML
91 lines
4.5 KiB
YAML
rule_files:
|
|
- orbit.rules.yml
|
|
evaluation_interval: 1s
|
|
tests:
|
|
- name: unavailable_target
|
|
interval: 1s
|
|
input_series:
|
|
- series: 'up{job="orbit",instance="node-a"}'
|
|
values: '0+0x180'
|
|
alert_rule_test:
|
|
- eval_time: 130s
|
|
alertname: OrbitMetricsUnavailable
|
|
exp_alerts:
|
|
- exp_labels: {job: orbit, instance: node-a, severity: warning}
|
|
exp_annotations:
|
|
summary: "Orbit metrics target is unavailable"
|
|
description: "Check the metrics listener, private network and process health. This does not prove that business tasks have stopped."
|
|
- name: startup_grace_and_stalled_component
|
|
interval: 1s
|
|
input_series:
|
|
- series: 'orbit_process_start_time_seconds{job="orbit",instance="node-a"}'
|
|
values: '0+0x180'
|
|
- series: 'orbit_runtime_batch_last_success_timestamp_seconds{job="orbit",instance="node-a",component="workflow"}'
|
|
values: '0+0x180'
|
|
alert_rule_test:
|
|
- eval_time: 60s
|
|
alertname: OrbitRuntimeBatchStalled
|
|
exp_alerts: []
|
|
- eval_time: 130s
|
|
alertname: OrbitRuntimeBatchStalled
|
|
exp_alerts:
|
|
- exp_labels: {job: orbit, instance: node-a, component: workflow, severity: warning}
|
|
exp_annotations:
|
|
summary: "Orbit background component has no recent successful batch"
|
|
description: "Inspect database availability, schema readiness and batch errors before changing limits or retrying business effects."
|
|
- name: healthy_batches_and_waits
|
|
interval: 1s
|
|
input_series:
|
|
- series: 'up{job="orbit",instance="node-a"}'
|
|
values: '1+0x400'
|
|
- series: 'orbit_process_start_time_seconds{job="orbit",instance="node-a"}'
|
|
values: '0+0x400'
|
|
- series: 'orbit_runtime_batch_last_success_timestamp_seconds{job="orbit",instance="node-a",component="workflow"}'
|
|
values: '0+1x400'
|
|
- series: 'orbit_runtime_batch_total{job="orbit",instance="node-a",component="workflow",outcome="error"}'
|
|
values: '0+0x400'
|
|
- series: 'orbit_db_wait_duration_seconds_total{job="orbit",instance="node-a"}'
|
|
values: '0+0x400'
|
|
- series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}'
|
|
values: '0+0x400'
|
|
alert_rule_test:
|
|
- {eval_time: 400s, alertname: OrbitMetricsUnavailable, exp_alerts: []}
|
|
- {eval_time: 400s, alertname: OrbitRuntimeBatchStalled, exp_alerts: []}
|
|
- {eval_time: 400s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []}
|
|
- {eval_time: 400s, alertname: OrbitDBPoolWaiting, exp_alerts: []}
|
|
- {eval_time: 400s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []}
|
|
- name: failing_batches_waits_and_invalid_samples_then_recover
|
|
interval: 1s
|
|
input_series:
|
|
- series: 'orbit_runtime_batch_total{job="orbit",instance="node-a",component="workflow",outcome="error"}'
|
|
values: '0+1x400 400+0x400'
|
|
- series: 'orbit_db_wait_duration_seconds_total{job="orbit",instance="node-a"}'
|
|
values: '0+1x400 400+0x400'
|
|
- series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}'
|
|
values: '0+1x400 400+0x400'
|
|
alert_rule_test:
|
|
- eval_time: 400s
|
|
alertname: OrbitRuntimeBatchErrors
|
|
exp_alerts:
|
|
- exp_labels: {job: orbit, instance: node-a, component: workflow, outcome: error, severity: warning}
|
|
exp_annotations:
|
|
summary: "Orbit background batches are failing"
|
|
description: "Use process logs and durable task/workflow audit to investigate. Batch errors are not business failures."
|
|
- eval_time: 400s
|
|
alertname: OrbitDBPoolWaiting
|
|
exp_alerts:
|
|
- exp_labels: {job: orbit, instance: node-a, severity: warning}
|
|
exp_annotations:
|
|
summary: "Orbit is waiting for MySQL connections"
|
|
description: "Correlate pool use with database locks, query latency and storage load before increasing the connection limit."
|
|
- eval_time: 400s
|
|
alertname: OrbitInvalidMetricUpdates
|
|
exp_alerts:
|
|
- exp_labels: {job: orbit, instance: node-a, severity: warning}
|
|
exp_annotations:
|
|
summary: "Orbit rejected an internal metrics update"
|
|
description: "Check instrumentation changes for unsupported names, types, labels or counter regressions."
|
|
- {eval_time: 800s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []}
|
|
- {eval_time: 800s, alertname: OrbitDBPoolWaiting, exp_alerts: []}
|
|
- {eval_time: 800s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []}
|