feat: add bounded business state observability

internal/observability/business.go

@@ -0,0 +1,65 @@
+package observability
+
+import (
+	"context"
+	"errors"
+	"time"
+)
+
+// BusinessSnapshot 是存储层返回的固定基数业务快照。
+// Map 的键只由存储层固定状态集合产生，Registry 仍只向目录中的标签写值。
+type BusinessSnapshot struct {
+	// TaskStates Pulse 任务状态的有界计数。
+	TaskStates map[string]uint64
+	// TaskOldestAgeSeconds 未完成任务中最早可处理记录的年龄。
+	TaskOldestAgeSeconds float64
+	// DeliveryStates 结果交付状态的有界计数。
+	DeliveryStates map[string]uint64
+	// DeliveryOldestAgeSeconds 待处理交付中最早可处理记录的年龄。
+	DeliveryOldestAgeSeconds float64
+	// WorkflowStates Weave 实例状态的有界计数。
+	WorkflowStates map[string]uint64
+	// WorkflowOldestAgeSeconds 未终结流程中最早更新时间的年龄。
+	WorkflowOldestAgeSeconds float64
+	// Truncated 是否至少有一类状态达到采样上限。
+	Truncated bool
+}
+
+// RecordBusinessSnapshot records one bounded database sample. A failed sample
+// does not overwrite the last successful state, so stale data remains visible
+// together with an error counter and alertable last-success timestamp.
+func (r *Registry) RecordBusinessSnapshot(snapshot BusinessSnapshot, started time.Time, err error) {
+	if r == nil {
+		return
+	}
+	outcome := "success"
+	if err != nil {
+		outcome = "error"
+		if errors.Is(err, context.DeadlineExceeded) {
+			outcome = "timeout"
+		}
+	}
+	r.AddCounter("orbit_business_snapshot_total", 1, Labels{"outcome": outcome})
+	r.ObserveHistogram("orbit_business_snapshot_duration_seconds", time.Since(started).Seconds(), nil)
+	if err != nil {
+		return
+	}
+	for _, state := range taskStates {
+		r.SetGauge("orbit_pulse_tasks_state", float64(snapshot.TaskStates[state]), Labels{"state": state})
+	}
+	for _, state := range deliveryStates {
+		r.SetGauge("orbit_result_deliveries_state", float64(snapshot.DeliveryStates[state]), Labels{"state": state})
+	}
+	for _, state := range workflowStates {
+		r.SetGauge("orbit_weave_workflows_state", float64(snapshot.WorkflowStates[state]), Labels{"state": state})
+	}
+	r.SetGauge("orbit_pulse_tasks_oldest_age_seconds", snapshot.TaskOldestAgeSeconds, nil)
+	r.SetGauge("orbit_result_delivery_oldest_age_seconds", snapshot.DeliveryOldestAgeSeconds, nil)
+	r.SetGauge("orbit_weave_workflow_oldest_age_seconds", snapshot.WorkflowOldestAgeSeconds, nil)
+	truncated := 0.0
+	if snapshot.Truncated {
+		truncated = 1
+	}
+	r.SetGauge("orbit_business_snapshot_truncated", truncated, nil)
+	r.SetGauge("orbit_business_snapshot_last_success_timestamp_seconds", float64(time.Now().Unix()), nil)
+}

internal/observability/catalog.go

@@ -6,6 +6,9 @@ var methods = []string{"GET", "POST", "PATCH", "PUT", "DELETE", "HEAD", "OPTIONS
 var routes = []string{"health_live", "health_ready", "worker_connect", "event_connect", "tasks", "queues", "workflows", "workflow_definitions", "schedules", "subscriptions", "application_resources", "other"}
 var statusClasses = []string{"1xx", "2xx", "3xx", "4xx", "5xx", "other"}
 var components = []string{"workflow", "schedule", "execution_recovery"}
+var taskStates = []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
+var deliveryStates = []string{"PENDING", "LEASED", "ACKED", "EXPIRED"}
+var workflowStates = []string{"RUNNING", "WAITING", "CANCELLING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
 
 // Every exported family is declared here, including each allowed label value.
 // Adding a metric therefore requires a code review; no user identifier is valid.
@@ -33,5 +36,15 @@ func catalog() map[string]descriptor {
 	}
 	c["orbit_db_wait_count_total"] = descriptor{counter, "Cumulative waits for a MySQL connection", nil}
 	c["orbit_db_wait_duration_seconds_total"] = descriptor{counter, "Cumulative seconds waiting for a MySQL connection", nil}
+	c["orbit_pulse_tasks_state"] = descriptor{gauge, "Bounded count of Pulse tasks by durable state", map[string][]string{"state": taskStates}}
+	c["orbit_pulse_tasks_oldest_age_seconds"] = descriptor{gauge, "Age of the oldest sampled unfinished Pulse task", nil}
+	c["orbit_result_deliveries_state"] = descriptor{gauge, "Bounded count of result deliveries by durable state", map[string][]string{"state": deliveryStates}}
+	c["orbit_result_delivery_oldest_age_seconds"] = descriptor{gauge, "Age of the oldest sampled pending result delivery", nil}
+	c["orbit_weave_workflows_state"] = descriptor{gauge, "Bounded count of Weave instances by durable state", map[string][]string{"state": workflowStates}}
+	c["orbit_weave_workflow_oldest_age_seconds"] = descriptor{gauge, "Age of the oldest sampled unfinished Weave instance", nil}
+	c["orbit_business_snapshot_total"] = descriptor{counter, "Completed bounded business-state snapshots by outcome", map[string][]string{"outcome": {"success", "error", "timeout"}}}
+	c["orbit_business_snapshot_duration_seconds"] = descriptor{histogram, "Bounded business-state snapshot duration in seconds", nil}
+	c["orbit_business_snapshot_last_success_timestamp_seconds"] = descriptor{gauge, "Unix time of the last successful bounded business-state snapshot", nil}
+	c["orbit_business_snapshot_truncated"] = descriptor{gauge, "Whether a bounded business-state sample hit its per-state cap", nil}
 	return c
 }

internal/observability/metrics_test.go

@@ -112,6 +112,42 @@ func TestRuntimeObservationDoesNotExposeErrors(t *testing.T) {
 	disabled.RecordBatch("workflow", time.Now(), nil)
 	disabled.RecordDatabase(sql.DBStats{})
 }
+
+func TestBusinessSnapshotKeepsLastGoodStateOnFailure(t *testing.T) {
+	r := NewRegistry()
+	started := time.Now().Add(-25 * time.Millisecond)
+	r.RecordBusinessSnapshot(BusinessSnapshot{
+		TaskStates:               map[string]uint64{"WAITING": 3},
+		TaskOldestAgeSeconds:     12.5,
+		DeliveryStates:           map[string]uint64{"PENDING": 2},
+		DeliveryOldestAgeSeconds: 8.25,
+		WorkflowStates:           map[string]uint64{"RUNNING": 1},
+		WorkflowOldestAgeSeconds: 4.75,
+		Truncated:                true,
+	}, started, nil)
+	first := r.Render()
+	for _, want := range []string{
+		`orbit_pulse_tasks_state{state="WAITING"} 3`,
+		`orbit_result_deliveries_state{state="PENDING"} 2`,
+		`orbit_weave_workflows_state{state="RUNNING"} 1`,
+		"orbit_pulse_tasks_oldest_age_seconds 12.5",
+		"orbit_business_snapshot_truncated 1",
+		`orbit_business_snapshot_total{outcome="success"} 1`,
+	} {
+		if !strings.Contains(first, want) {
+			t.Fatalf("missing business sample %q", want)
+		}
+	}
+	lastSuccess := r.families["orbit_business_snapshot_last_success_timestamp_seconds"].series[""].value.Load()
+	r.RecordBusinessSnapshot(BusinessSnapshot{TaskStates: map[string]uint64{"WAITING": 99}}, time.Now(), context.DeadlineExceeded)
+	second := r.Render()
+	if !strings.Contains(second, `orbit_business_snapshot_total{outcome="timeout"} 1`) || !strings.Contains(second, `orbit_pulse_tasks_state{state="WAITING"} 3`) {
+		t.Fatal("failed sample overwrote good state")
+	}
+	if got := r.families["orbit_business_snapshot_last_success_timestamp_seconds"].series[""].value.Load(); got != lastSuccess {
+		t.Fatal("failed sample changed last success timestamp")
+	}
+}
 func TestHTTPBoundedLabelsAndPanicPreserved(t *testing.T) {
 	r := NewRegistry()
 	h := r.Middleware(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(202) }))