feat: observe workflow inbox backlog

internal/observability/business.go

@@ -21,6 +21,10 @@ type BusinessSnapshot struct {
 	WorkflowStates map[string]uint64
 	// WorkflowOldestAgeSeconds 未终结流程中最早更新时间的年龄。
 	WorkflowOldestAgeSeconds float64
+	// InboxStates Weave 内部事件状态的有界计数。
+	InboxStates map[string]uint64
+	// InboxOldestAgeSeconds 待消费内部事件中最早可处理记录的年龄。
+	InboxOldestAgeSeconds float64
 	// Truncated 是否至少有一类状态达到采样上限。
 	Truncated bool
 	// RecentEvents 固定事件类别在最近窗口内的有界数量。
@@ -63,6 +67,9 @@ func (r *Registry) RecordBusinessSnapshot(snapshot BusinessSnapshot, started tim
 	for _, state := range workflowStates {
 		r.SetGauge("orbit_weave_workflows_state", float64(snapshot.WorkflowStates[state]), Labels{"state": state})
 	}
+	for _, state := range inboxStates {
+		r.SetGauge("orbit_weave_inbox_state", float64(snapshot.InboxStates[state]), Labels{"state": state})
+	}
 	for _, eventType := range eventTypes {
 		r.SetGauge("orbit_business_events_recent", float64(snapshot.RecentEvents[eventType]), Labels{"event": eventType})
 	}
@@ -73,6 +80,7 @@ func (r *Registry) RecordBusinessSnapshot(snapshot BusinessSnapshot, started tim
 	r.SetGauge("orbit_pulse_tasks_oldest_age_seconds", snapshot.TaskOldestAgeSeconds, nil)
 	r.SetGauge("orbit_result_delivery_oldest_age_seconds", snapshot.DeliveryOldestAgeSeconds, nil)
 	r.SetGauge("orbit_weave_workflow_oldest_age_seconds", snapshot.WorkflowOldestAgeSeconds, nil)
+	r.SetGauge("orbit_weave_inbox_oldest_age_seconds", snapshot.InboxOldestAgeSeconds, nil)
 	truncated := 0.0
 	if snapshot.Truncated {
 		truncated = 1

internal/observability/catalog.go

@@ -9,6 +9,7 @@ var components = []string{"workflow", "schedule", "execution_recovery"}
 var taskStates = []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
 var deliveryStates = []string{"PENDING", "LEASED", "ACKED", "EXPIRED"}
 var workflowStates = []string{"RUNNING", "WAITING", "CANCELLING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
+var inboxStates = []string{"PENDING", "LEASED", "PROCESSED", "QUARANTINED"}
 var eventTypes = []string{"task.accepted", "task.started", "task.succeeded", "task.failed", "task.needs_review", "task.requeued", "workflow.started", "workflow.advanced", "workflow.recovered", "workflow.blocked", "workflow.stopped"}
 
 // Every exported family is declared here, including each allowed label value.
@@ -43,6 +44,8 @@ func catalog() map[string]descriptor {
 	c["orbit_result_delivery_oldest_age_seconds"] = descriptor{gauge, "Age of the oldest sampled pending result delivery", nil}
 	c["orbit_weave_workflows_state"] = descriptor{gauge, "Bounded count of Weave instances by durable state", map[string][]string{"state": workflowStates}}
 	c["orbit_weave_workflow_oldest_age_seconds"] = descriptor{gauge, "Age of the oldest sampled unfinished Weave instance", nil}
+	c["orbit_weave_inbox_state"] = descriptor{gauge, "Bounded count of Weave internal inbox records by durable state", map[string][]string{"state": inboxStates}}
+	c["orbit_weave_inbox_oldest_age_seconds"] = descriptor{gauge, "Age of the oldest sampled pending Weave inbox record", nil}
 	c["orbit_business_events_recent"] = descriptor{gauge, "Bounded count of fixed event categories in the recent window", map[string][]string{"event": eventTypes}}
 	c["orbit_pulse_task_terminal_latency_seconds"] = descriptor{gauge, "Average sampled Pulse task create-to-terminal latency in seconds", nil}
 	c["orbit_pulse_task_terminal_latency_max_seconds"] = descriptor{gauge, "Maximum sampled Pulse task create-to-terminal latency in seconds", nil}

internal/observability/metrics_test.go

@@ -123,6 +123,8 @@ func TestBusinessSnapshotKeepsLastGoodStateOnFailure(t *testing.T) {
 		DeliveryOldestAgeSeconds:   8.25,
 		WorkflowStates:             map[string]uint64{"RUNNING": 1},
 		WorkflowOldestAgeSeconds:   4.75,
+		InboxStates:                map[string]uint64{"PENDING": 6},
+		InboxOldestAgeSeconds:      3.25,
 		RecentEvents:               map[string]uint64{"task.accepted": 5},
 		TaskTerminalLatencySeconds: 2.5, TaskTerminalLatencyMaxSeconds: 7.5,
 		WorkflowTerminalLatencySeconds: 3.5, WorkflowTerminalLatencyMaxSeconds: 9.5,
@@ -133,10 +135,12 @@ func TestBusinessSnapshotKeepsLastGoodStateOnFailure(t *testing.T) {
 		`orbit_pulse_tasks_state{state="WAITING"} 3`,
 		`orbit_result_deliveries_state{state="PENDING"} 2`,
 		`orbit_weave_workflows_state{state="RUNNING"} 1`,
+		`orbit_weave_inbox_state{state="PENDING"} 6`,
 		`orbit_business_events_recent{event="task.accepted"} 5`,
 		"orbit_pulse_task_terminal_latency_seconds 2.5",
 		"orbit_weave_workflow_terminal_latency_max_seconds 9.5",
 		"orbit_pulse_tasks_oldest_age_seconds 12.5",
+		"orbit_weave_inbox_oldest_age_seconds 3.25",
 		"orbit_business_snapshot_truncated 1",
 		`orbit_business_snapshot_total{outcome="success"} 1`,
 	} {