k2safe/orbit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add bounded business state observability

Browse Source
This commit is contained in:
k2safe
2026-09-18 20:44:41 +08:00
parent bea8efa790
commit 5284206731
18 changed files with 450 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
deploy/monitoring/orbit.rules.test.yml
View File

@@ -22,6 +22,8 @@ tests:
values: '0+0x180'
- series: 'orbit_runtime_batch_last_success_timestamp_seconds{job="orbit",instance="node-a",component="workflow"}'
values: '0+0x180'
- series: 'orbit_business_snapshot_last_success_timestamp_seconds{job="orbit",instance="node-a"}'
values: '0+0x180'
alert_rule_test:
- eval_time: 60s
alertname: OrbitRuntimeBatchStalled
@@ -33,6 +35,13 @@ tests:
exp_annotations:
summary: "Orbit background component has no recent successful batch"
description: "Inspect database availability, schema readiness and batch errors before changing limits or retrying business effects."
- eval_time: 130s
alertname: OrbitBusinessSnapshotStalled
exp_alerts:
- exp_labels: {job: orbit, instance: node-a, severity: warning}
exp_annotations:
summary: "Orbit has no recent successful business-state snapshot"
description: "Inspect the bounded sampler timeout/error counter and MySQL health; the last state gauges may be stale."
- name: healthy_batches_and_waits
interval: 1s
input_series:
@@ -48,12 +57,18 @@ tests:
values: '0+0x400'
- series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}'
values: '0+0x400'
- series: 'orbit_business_snapshot_last_success_timestamp_seconds{job="orbit",instance="node-a"}'
values: '0+1x400'
- series: 'orbit_business_snapshot_truncated{job="orbit",instance="node-a"}'
values: '0+0x400'
alert_rule_test:
- {eval_time: 400s, alertname: OrbitMetricsUnavailable, exp_alerts: []}
- {eval_time: 400s, alertname: OrbitRuntimeBatchStalled, exp_alerts: []}
- {eval_time: 400s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []}
- {eval_time: 400s, alertname: OrbitDBPoolWaiting, exp_alerts: []}
- {eval_time: 400s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []}
- {eval_time: 400s, alertname: OrbitBusinessSnapshotStalled, exp_alerts: []}
- {eval_time: 400s, alertname: OrbitBusinessSnapshotTruncated, exp_alerts: []}
- name: failing_batches_waits_and_invalid_samples_then_recover
interval: 1s
input_series:
@@ -63,6 +78,10 @@ tests:
values: '0+1x400 400+0x400'
- series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}'
values: '0+1x400 400+0x400'
- series: 'orbit_business_snapshot_last_success_timestamp_seconds{job="orbit",instance="node-a"}'
values: '0+0x800'
- series: 'orbit_business_snapshot_truncated{job="orbit",instance="node-a"}'
values: '1+0x400 0+0x400'
alert_rule_test:
- eval_time: 400s
alertname: OrbitRuntimeBatchErrors
@@ -85,6 +104,14 @@ tests:
exp_annotations:
summary: "Orbit rejected an internal metrics update"
description: "Check instrumentation changes for unsupported names, types, labels or counter regressions."
- eval_time: 400s
alertname: OrbitBusinessSnapshotTruncated
exp_alerts:
- exp_labels: {job: orbit, instance: node-a, severity: warning}
exp_annotations:
summary: "Orbit business-state snapshot reached its sample cap"
description: "State gauges are bounded samples, not exact global counts; use authoritative database queries and capacity review."
- {eval_time: 800s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []}
- {eval_time: 800s, alertname: OrbitDBPoolWaiting, exp_alerts: []}
- {eval_time: 800s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []}
- {eval_time: 800s, alertname: OrbitBusinessSnapshotTruncated, exp_alerts: []}