feat: sample recent business events
This commit is contained in:
@@ -23,6 +23,8 @@ type BusinessSnapshot struct {
|
||||
WorkflowOldestAgeSeconds float64
|
||||
// Truncated 是否至少有一类状态达到采样上限。
|
||||
Truncated bool
|
||||
// RecentEvents 固定事件类别在最近窗口内的有界数量。
|
||||
RecentEvents map[string]uint64
|
||||
}
|
||||
|
||||
// RecordBusinessSnapshot records one bounded database sample. A failed sample
|
||||
@@ -53,6 +55,9 @@ func (r *Registry) RecordBusinessSnapshot(snapshot BusinessSnapshot, started tim
|
||||
for _, state := range workflowStates {
|
||||
r.SetGauge("orbit_weave_workflows_state", float64(snapshot.WorkflowStates[state]), Labels{"state": state})
|
||||
}
|
||||
for _, eventType := range eventTypes {
|
||||
r.SetGauge("orbit_business_events_recent", float64(snapshot.RecentEvents[eventType]), Labels{"event": eventType})
|
||||
}
|
||||
r.SetGauge("orbit_pulse_tasks_oldest_age_seconds", snapshot.TaskOldestAgeSeconds, nil)
|
||||
r.SetGauge("orbit_result_delivery_oldest_age_seconds", snapshot.DeliveryOldestAgeSeconds, nil)
|
||||
r.SetGauge("orbit_weave_workflow_oldest_age_seconds", snapshot.WorkflowOldestAgeSeconds, nil)
|
||||
|
||||
@@ -9,6 +9,7 @@ var components = []string{"workflow", "schedule", "execution_recovery"}
|
||||
var taskStates = []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
|
||||
var deliveryStates = []string{"PENDING", "LEASED", "ACKED", "EXPIRED"}
|
||||
var workflowStates = []string{"RUNNING", "WAITING", "CANCELLING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
|
||||
var eventTypes = []string{"task.accepted", "task.started", "task.succeeded", "task.failed", "task.needs_review", "task.requeued", "workflow.started", "workflow.advanced", "workflow.recovered", "workflow.blocked", "workflow.stopped"}
|
||||
|
||||
// Every exported family is declared here, including each allowed label value.
|
||||
// Adding a metric therefore requires a code review; no user identifier is valid.
|
||||
@@ -42,6 +43,7 @@ func catalog() map[string]descriptor {
|
||||
c["orbit_result_delivery_oldest_age_seconds"] = descriptor{gauge, "Age of the oldest sampled pending result delivery", nil}
|
||||
c["orbit_weave_workflows_state"] = descriptor{gauge, "Bounded count of Weave instances by durable state", map[string][]string{"state": workflowStates}}
|
||||
c["orbit_weave_workflow_oldest_age_seconds"] = descriptor{gauge, "Age of the oldest sampled unfinished Weave instance", nil}
|
||||
c["orbit_business_events_recent"] = descriptor{gauge, "Bounded count of fixed event categories in the recent window", map[string][]string{"event": eventTypes}}
|
||||
c["orbit_business_snapshot_total"] = descriptor{counter, "Completed bounded business-state snapshots by outcome", map[string][]string{"outcome": {"success", "error", "timeout"}}}
|
||||
c["orbit_business_snapshot_duration_seconds"] = descriptor{histogram, "Bounded business-state snapshot duration in seconds", nil}
|
||||
c["orbit_business_snapshot_last_success_timestamp_seconds"] = descriptor{gauge, "Unix time of the last successful bounded business-state snapshot", nil}
|
||||
|
||||
@@ -123,6 +123,7 @@ func TestBusinessSnapshotKeepsLastGoodStateOnFailure(t *testing.T) {
|
||||
DeliveryOldestAgeSeconds: 8.25,
|
||||
WorkflowStates: map[string]uint64{"RUNNING": 1},
|
||||
WorkflowOldestAgeSeconds: 4.75,
|
||||
RecentEvents: map[string]uint64{"task.accepted": 5},
|
||||
Truncated: true,
|
||||
}, started, nil)
|
||||
first := r.Render()
|
||||
@@ -130,6 +131,7 @@ func TestBusinessSnapshotKeepsLastGoodStateOnFailure(t *testing.T) {
|
||||
`orbit_pulse_tasks_state{state="WAITING"} 3`,
|
||||
`orbit_result_deliveries_state{state="PENDING"} 2`,
|
||||
`orbit_weave_workflows_state{state="RUNNING"} 1`,
|
||||
`orbit_business_events_recent{event="task.accepted"} 5`,
|
||||
"orbit_pulse_tasks_oldest_age_seconds 12.5",
|
||||
"orbit_business_snapshot_truncated 1",
|
||||
`orbit_business_snapshot_total{outcome="success"} 1`,
|
||||
|
||||
@@ -68,6 +68,7 @@ func TestIntegrationApplicationResourcesUpgrade(t *testing.T) {
|
||||
f.exec(`ALTER TABLE pulse_tasks DROP INDEX ix_task_observe_state_available`)
|
||||
f.exec(`ALTER TABLE orbit_deliveries DROP INDEX ix_delivery_observe_state_available`)
|
||||
f.exec(`ALTER TABLE weave_instances DROP INDEX ix_workflow_observe_state_updated`)
|
||||
f.exec(`ALTER TABLE orbit_events DROP INDEX ix_event_observe_type_occurred`)
|
||||
f.exec(`DROP TABLE weave_version_scopes`)
|
||||
f.exec(`ALTER TABLE weave_instances DROP INDEX ix_workflow_created,DROP INDEX ix_workflow_state_created,DROP INDEX ix_workflow_business_state_created,DROP INDEX ix_workflow_definition_created,DROP INDEX ix_workflow_definition_state_created,DROP INDEX ix_workflow_definition_business_created,DROP INDEX ix_workflow_definition_business_state_created`)
|
||||
f.exec(`ALTER TABLE weave_signals DROP INDEX ix_signal_history`)
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
-- 为固定事件类别的短窗口运行观测提供有界索引。
|
||||
-- 事件仍是不可变业务证据;索引不改变事件语义,也不引入外键或级联。
|
||||
ALTER TABLE orbit_events
|
||||
ADD KEY ix_event_observe_type_occurred (type, occurred_at, id);
|
||||
@@ -89,7 +89,7 @@ func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
|
||||
indexes := func() []string {
|
||||
t.Helper()
|
||||
// 排除后续新增/替换索引;原有项在此逐项比较,v24/v25新增项由对应Query/Timeline升级测试核对。
|
||||
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events','weave_version_scopes') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result','ix_signal_expiry','ix_signal_activation','uq_wait_winner_signal','ix_task_workflow_owner','ix_signal_history','ix_event_workflow_history','ix_workflow_created','ix_workflow_state_created','ix_workflow_business_state_created','ix_workflow_definition_created','ix_workflow_definition_state_created','ix_workflow_definition_business_created','ix_workflow_definition_business_state_created','ix_task_observe_state_available','ix_delivery_observe_state_available','ix_workflow_observe_state_updated') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
|
||||
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events','weave_version_scopes') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result','ix_signal_expiry','ix_signal_activation','uq_wait_winner_signal','ix_task_workflow_owner','ix_signal_history','ix_event_workflow_history','ix_workflow_created','ix_workflow_state_created','ix_workflow_business_state_created','ix_workflow_definition_created','ix_workflow_definition_state_created','ix_workflow_definition_business_created','ix_workflow_definition_business_state_created','ix_task_observe_state_available','ix_delivery_observe_state_available','ix_workflow_observe_state_updated','ix_event_observe_type_occurred') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
)
|
||||
|
||||
const operationalMetricSampleLimit = 10000
|
||||
const operationalMetricEventWindow = 5 * time.Minute
|
||||
|
||||
// OperationalSnapshot 是进程观测使用的有限数据库快照。
|
||||
// Counts 只包含固定状态的有界计数;达到上限时 Truncated 为 true,不能解释为精确总量。
|
||||
@@ -26,12 +27,15 @@ type OperationalSnapshot struct {
|
||||
WorkflowOldestAgeSeconds float64
|
||||
// Truncated 表示至少一个状态达到有界采样上限。
|
||||
Truncated bool
|
||||
// RecentEvents 是固定事件类别在最近窗口内的有界数量。
|
||||
RecentEvents map[string]uint64
|
||||
}
|
||||
|
||||
var (
|
||||
operationalTaskStates = []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
|
||||
operationalDeliveryStates = []string{"PENDING", "LEASED", "ACKED", "EXPIRED"}
|
||||
operationalWorkflowStates = []string{"RUNNING", "WAITING", "CANCELLING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
|
||||
operationalEventTypes = []string{"task.accepted", "task.started", "task.succeeded", "task.failed", "task.needs_review", "task.requeued", "workflow.started", "workflow.advanced", "workflow.recovered", "workflow.blocked", "workflow.stopped"}
|
||||
)
|
||||
|
||||
// ReadOperationalSnapshot performs only bounded, indexed reads. It is intended
|
||||
@@ -68,11 +72,16 @@ func ReadOperationalSnapshot(ctx context.Context, db *sql.DB) (OperationalSnapsh
|
||||
if err != nil {
|
||||
return OperationalSnapshot{}, err
|
||||
}
|
||||
events, eventTruncated, err := boundedRecentEventCounts(ctx, db, now)
|
||||
if err != nil {
|
||||
return OperationalSnapshot{}, err
|
||||
}
|
||||
return OperationalSnapshot{
|
||||
TaskStates: tasks, TaskOldestAgeSeconds: taskAge,
|
||||
DeliveryStates: deliveries, DeliveryOldestAgeSeconds: deliveryAge,
|
||||
WorkflowStates: workflows, WorkflowOldestAgeSeconds: workflowAge,
|
||||
Truncated: taskTruncated || deliveryTruncated || workflowTruncated,
|
||||
RecentEvents: events,
|
||||
Truncated: taskTruncated || deliveryTruncated || workflowTruncated || eventTruncated,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -102,6 +111,25 @@ func boundedStateCounts(ctx context.Context, db *sql.DB, table, index string, st
|
||||
return counts, truncated, nil
|
||||
}
|
||||
|
||||
func boundedRecentEventCounts(ctx context.Context, db *sql.DB, now time.Time) (map[string]uint64, bool, error) {
|
||||
counts := make(map[string]uint64, len(operationalEventTypes))
|
||||
truncated := false
|
||||
cutoff := now.Add(-operationalMetricEventWindow)
|
||||
for _, eventType := range operationalEventTypes {
|
||||
query := "SELECT COUNT(*) FROM (SELECT id FROM orbit_events FORCE INDEX (ix_event_observe_type_occurred) WHERE type=? AND occurred_at>=? LIMIT ?) AS bounded"
|
||||
var n uint64
|
||||
if err := db.QueryRowContext(ctx, query, eventType, cutoff, operationalMetricSampleLimit+1).Scan(&n); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if n > operationalMetricSampleLimit {
|
||||
n = operationalMetricSampleLimit
|
||||
truncated = true
|
||||
}
|
||||
counts[eventType] = n
|
||||
}
|
||||
return counts, truncated, nil
|
||||
}
|
||||
|
||||
func oldestAge(ctx context.Context, db *sql.DB, table, index string, states []string, column string, now time.Time) (float64, error) {
|
||||
placeholders := "?"
|
||||
args := make([]any, 0, len(states))
|
||||
|
||||
@@ -60,6 +60,9 @@ func TestIntegrationOperationalSnapshot(t *testing.T) {
|
||||
if got := snapshot.TaskStates["WAITING"]; got < 1 {
|
||||
t.Fatalf("waiting task was not observed: %#v", snapshot.TaskStates)
|
||||
}
|
||||
if got := snapshot.RecentEvents["task.accepted"]; got < 1 {
|
||||
t.Fatalf("accepted event was not observed: %#v", snapshot.RecentEvents)
|
||||
}
|
||||
if snapshot.TaskOldestAgeSeconds < 0 || snapshot.DeliveryOldestAgeSeconds < 0 || snapshot.WorkflowOldestAgeSeconds < 0 {
|
||||
t.Fatalf("negative age in snapshot: %#v", snapshot)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user