k2safe/orbit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: observe workflow inbox backlog

Browse Source
This commit is contained in:
k2safe
2026-09-19 17:42:49 +08:00
parent da5874b17a
commit 0ac3f2f3f8
10 changed files with 46 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
internal/storage/mysql/operational_metrics.go
View File

@@ -25,6 +25,10 @@ type OperationalSnapshot struct {
WorkflowStates map[string]uint64
// WorkflowOldestAgeSeconds 是未终结流程中最早 updated_at 的年龄。
WorkflowOldestAgeSeconds float64
// InboxStates 是 Weave 内部事件按状态的有界计数。
InboxStates map[string]uint64
// InboxOldestAgeSeconds 是待消费内部事件中最早 available_at 的年龄。
InboxOldestAgeSeconds float64
// Truncated 表示至少一个状态达到有界采样上限。
Truncated bool
// RecentEvents 是固定事件类别在最近窗口内的有界数量。
@@ -43,6 +47,7 @@ var (
operationalTaskStates = []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
operationalDeliveryStates = []string{"PENDING", "LEASED", "ACKED", "EXPIRED"}
operationalWorkflowStates = []string{"RUNNING", "WAITING", "CANCELLING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
operationalInboxStates = []string{"PENDING", "LEASED", "PROCESSED", "QUARANTINED"}
operationalEventTypes = []string{"task.accepted", "task.started", "task.succeeded", "task.failed", "task.needs_review", "task.requeued", "workflow.started", "workflow.advanced", "workflow.recovered", "workflow.blocked", "workflow.stopped"}
)
@@ -68,10 +73,18 @@ func ReadOperationalSnapshot(ctx context.Context, db *sql.DB) (OperationalSnapsh
if err != nil {
return OperationalSnapshot{}, err
}
inbox, inboxTruncated, err := boundedStateCounts(ctx, db, "weave_inbox", "ix_inbox_due", operationalInboxStates)
if err != nil {
return OperationalSnapshot{}, err
}
taskAge, err := oldestAge(ctx, db, "pulse_tasks", "ix_task_observe_state_available", []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW"}, "available_at", now)
if err != nil {
return OperationalSnapshot{}, err
}
inboxAge, err := oldestAge(ctx, db, "weave_inbox", "ix_inbox_due", []string{"PENDING", "LEASED"}, "available_at", now)
if err != nil {
return OperationalSnapshot{}, err
}
deliveryAge, err := oldestAge(ctx, db, "orbit_deliveries", "ix_delivery_observe_state_available", []string{"PENDING", "LEASED"}, "available_at", now)
if err != nil {
return OperationalSnapshot{}, err
@@ -96,10 +109,11 @@ func ReadOperationalSnapshot(ctx context.Context, db *sql.DB) (OperationalSnapsh
TaskStates: tasks, TaskOldestAgeSeconds: taskAge,
DeliveryStates: deliveries, DeliveryOldestAgeSeconds: deliveryAge,
WorkflowStates: workflows, WorkflowOldestAgeSeconds: workflowAge,
InboxStates: inbox, InboxOldestAgeSeconds: inboxAge,
RecentEvents: events,
TaskTerminalLatencySeconds: taskLatency.average, TaskTerminalLatencyMaxSeconds: taskLatency.maximum,
WorkflowTerminalLatencySeconds: workflowLatency.average, WorkflowTerminalLatencyMaxSeconds: workflowLatency.maximum,
Truncated: taskTruncated || deliveryTruncated || workflowTruncated || eventTruncated || taskLatency.truncated || workflowLatency.truncated,
Truncated: taskTruncated || deliveryTruncated || workflowTruncated || inboxTruncated || eventTruncated || taskLatency.truncated || workflowLatency.truncated,
}, nil
}