feat: add bounded business state observability

cmd/orbit/main.go

@@ -192,6 +192,25 @@ func run() error {
 				case <-ticker.C:
 					metrics.RecordDatabase(db.Stats())
 					metrics.RecordProcess()
+					started := time.Now()
+					call, cancel := context.WithTimeout(metricsCtx, 900*time.Millisecond)
+					snapshot, err := mysql.ReadOperationalSnapshot(call, db)
+					cancel()
+					metrics.RecordBusinessSnapshot(observability.BusinessSnapshot{
+						TaskStates: snapshot.TaskStates, TaskOldestAgeSeconds: snapshot.TaskOldestAgeSeconds,
+						DeliveryStates: snapshot.DeliveryStates, DeliveryOldestAgeSeconds: snapshot.DeliveryOldestAgeSeconds,
+						WorkflowStates: snapshot.WorkflowStates, WorkflowOldestAgeSeconds: snapshot.WorkflowOldestAgeSeconds,
+						Truncated: snapshot.Truncated,
+					}, started, err)
+					if err != nil && metricsCtx.Err() == nil {
+						reason := "query_error"
+						if errors.Is(err, context.DeadlineExceeded) {
+							reason = "timeout"
+						} else if errors.Is(err, context.Canceled) {
+							reason = "cancelled"
+						}
+						slog.Error("business metrics snapshot failed", "node_id", node, "reason", reason)
+					}
 				}
 			}
 		}()