k2safe/orbit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: observe workflow inbox backlog

Browse Source
This commit is contained in:
k2safe
2026-09-19 17:42:49 +08:00
parent da5874b17a
commit 0ac3f2f3f8
10 changed files with 46 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
internal/storage/mysql/operational_metrics.go
View File

@@ -25,6 +25,10 @@ type OperationalSnapshot struct {
WorkflowStates map[string]uint64
// WorkflowOldestAgeSeconds 是未终结流程中最早 updated_at 的年龄。
WorkflowOldestAgeSeconds float64
// InboxStates 是 Weave 内部事件按状态的有界计数。
InboxStates map[string]uint64
// InboxOldestAgeSeconds 是待消费内部事件中最早 available_at 的年龄。
InboxOldestAgeSeconds float64
// Truncated 表示至少一个状态达到有界采样上限。
Truncated bool
// RecentEvents 是固定事件类别在最近窗口内的有界数量。
@@ -43,6 +47,7 @@ var (
operationalTaskStates = []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
operationalDeliveryStates = []string{"PENDING", "LEASED", "ACKED", "EXPIRED"}
operationalWorkflowStates = []string{"RUNNING", "WAITING", "CANCELLING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
operationalInboxStates = []string{"PENDING", "LEASED", "PROCESSED", "QUARANTINED"}
operationalEventTypes = []string{"task.accepted", "task.started", "task.succeeded", "task.failed", "task.needs_review", "task.requeued", "workflow.started", "workflow.advanced", "workflow.recovered", "workflow.blocked", "workflow.stopped"}
)
@@ -68,10 +73,18 @@ func ReadOperationalSnapshot(ctx context.Context, db *sql.DB) (OperationalSnapsh
if err != nil {
return OperationalSnapshot{}, err
}
inbox, inboxTruncated, err := boundedStateCounts(ctx, db, "weave_inbox", "ix_inbox_due", operationalInboxStates)
if err != nil {
return OperationalSnapshot{}, err
}
taskAge, err := oldestAge(ctx, db, "pulse_tasks", "ix_task_observe_state_available", []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW"}, "available_at", now)
if err != nil {
return OperationalSnapshot{}, err
}
inboxAge, err := oldestAge(ctx, db, "weave_inbox", "ix_inbox_due", []string{"PENDING", "LEASED"}, "available_at", now)
if err != nil {
return OperationalSnapshot{}, err
}
deliveryAge, err := oldestAge(ctx, db, "orbit_deliveries", "ix_delivery_observe_state_available", []string{"PENDING", "LEASED"}, "available_at", now)
if err != nil {
return OperationalSnapshot{}, err
@@ -96,10 +109,11 @@ func ReadOperationalSnapshot(ctx context.Context, db *sql.DB) (OperationalSnapsh
TaskStates: tasks, TaskOldestAgeSeconds: taskAge,
DeliveryStates: deliveries, DeliveryOldestAgeSeconds: deliveryAge,
WorkflowStates: workflows, WorkflowOldestAgeSeconds: workflowAge,
InboxStates: inbox, InboxOldestAgeSeconds: inboxAge,
RecentEvents: events,
TaskTerminalLatencySeconds: taskLatency.average, TaskTerminalLatencyMaxSeconds: taskLatency.maximum,
WorkflowTerminalLatencySeconds: workflowLatency.average, WorkflowTerminalLatencyMaxSeconds: workflowLatency.maximum,
Truncated: taskTruncated || deliveryTruncated || workflowTruncated || eventTruncated || taskLatency.truncated || workflowLatency.truncated,
Truncated: taskTruncated || deliveryTruncated || workflowTruncated || inboxTruncated || eventTruncated || taskLatency.truncated || workflowLatency.truncated,
}, nil
}

7
internal/storage/mysql/operational_metrics_integration_test.go
View File

@@ -70,7 +70,7 @@ func TestIntegrationOperationalSnapshot(t *testing.T) {
if got := snapshot.RecentEvents["task.accepted"]; got < 1 {
t.Fatalf("accepted event was not observed: %#v", snapshot.RecentEvents)
}
if snapshot.TaskOldestAgeSeconds < 0 || snapshot.DeliveryOldestAgeSeconds < 0 || snapshot.WorkflowOldestAgeSeconds < 0 {
if snapshot.TaskOldestAgeSeconds < 0 || snapshot.DeliveryOldestAgeSeconds < 0 || snapshot.WorkflowOldestAgeSeconds < 0 || snapshot.InboxOldestAgeSeconds < 0 {
t.Fatalf("negative age in snapshot: %#v", snapshot)
}
for _, state := range []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"} {
@@ -84,6 +84,11 @@ func TestIntegrationOperationalSnapshot(t *testing.T) {
if _, ok := snapshot.WorkflowStates["RUNNING"]; !ok {
t.Fatal("missing running workflow state")
}
for _, state := range []string{"PENDING", "LEASED", "PROCESSED", "QUARANTINED"} {
if _, ok := snapshot.InboxStates[state]; !ok {
t.Fatalf("missing fixed inbox state %q", state)
}
}
f.exec("UPDATE pulse_tasks SET state='SUCCEEDED', completed_at=UTC_TIMESTAMP(6), updated_at=UTC_TIMESTAMP(6), revision=revision+1 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID)
completed, err := ReadOperationalSnapshot(ctx, db)
if err != nil {