376 lines
16 KiB
Go
376 lines
16 KiB
Go
package mysql
|
||
|
||
import (
|
||
"context"
|
||
"database/sql"
|
||
"errors"
|
||
"sort"
|
||
"time"
|
||
)
|
||
|
||
const operationalMetricSampleLimit = 10000
|
||
const operationalMetricEventWindow = 5 * time.Minute
|
||
|
||
// OperationalSnapshot 是进程观测使用的有限数据库快照。
|
||
// Counts 只包含固定状态的有界计数；达到上限时 Truncated 为 true，不能解释为精确总量。
|
||
type OperationalSnapshot struct {
|
||
// TaskStates 是 Pulse 任务按状态的有界计数。
|
||
TaskStates map[string]uint64 `json:"task_states"`
|
||
// TaskOldestAgeSeconds 是运行中任务中最早 available_at 的年龄。
|
||
TaskOldestAgeSeconds float64 `json:"task_oldest_age_seconds"`
|
||
// DeliveryStates 是结果交付按状态的有界计数。
|
||
DeliveryStates map[string]uint64 `json:"delivery_states"`
|
||
// DeliveryOldestAgeSeconds 是待处理交付中最早 available_at 的年龄。
|
||
DeliveryOldestAgeSeconds float64 `json:"delivery_oldest_age_seconds"`
|
||
// WorkflowStates 是 Weave 实例按状态的有界计数。
|
||
WorkflowStates map[string]uint64 `json:"workflow_states"`
|
||
// WorkflowOldestAgeSeconds 是未终结流程中最早 updated_at 的年龄。
|
||
WorkflowOldestAgeSeconds float64 `json:"workflow_oldest_age_seconds"`
|
||
// InboxStates 是 Weave 内部事件按状态的有界计数。
|
||
InboxStates map[string]uint64 `json:"inbox_states"`
|
||
// InboxOldestAgeSeconds 是待消费内部事件中最早 available_at 的年龄。
|
||
InboxOldestAgeSeconds float64 `json:"inbox_oldest_age_seconds"`
|
||
// Truncated 表示至少一个状态达到有界采样上限。
|
||
Truncated bool `json:"truncated"`
|
||
// RecentEvents 是固定事件类别在最近窗口内的有界数量。
|
||
RecentEvents map[string]uint64 `json:"recent_events"`
|
||
// TaskTerminalLatencySeconds 是最近窗口终态任务从创建到完成的平均耗时样本。
|
||
TaskTerminalLatencySeconds float64 `json:"task_terminal_latency_seconds"`
|
||
// TaskTerminalLatencyMaxSeconds 是最近窗口终态任务样本的最大耗时。
|
||
TaskTerminalLatencyMaxSeconds float64 `json:"task_terminal_latency_max_seconds"`
|
||
// WorkflowTerminalLatencySeconds 是最近窗口终态流程从创建到完成的平均耗时样本。
|
||
WorkflowTerminalLatencySeconds float64 `json:"workflow_terminal_latency_seconds"`
|
||
// WorkflowTerminalLatencyMaxSeconds 是最近窗口终态流程样本的最大耗时。
|
||
WorkflowTerminalLatencyMaxSeconds float64 `json:"workflow_terminal_latency_max_seconds"`
|
||
// TaskAcceptToStartLatencySeconds 是最近窗口任务从受理事件到开始事件的平均耗时样本。
|
||
TaskAcceptToStartLatencySeconds float64 `json:"task_accept_to_start_latency_seconds"`
|
||
// TaskAcceptToStartLatencyMaxSeconds 是最近窗口任务受理到开始耗时样本的最大值。
|
||
TaskAcceptToStartLatencyMaxSeconds float64 `json:"task_accept_to_start_latency_max_seconds"`
|
||
// TaskAcceptToTerminalLatencySeconds 是最近窗口任务从受理到确定终态事件的平均耗时样本。
|
||
TaskAcceptToTerminalLatencySeconds float64 `json:"task_accept_to_terminal_latency_seconds"`
|
||
// TaskAcceptToTerminalLatencyMaxSeconds 是最近窗口任务受理到确定终态耗时样本的最大值。
|
||
TaskAcceptToTerminalLatencyMaxSeconds float64 `json:"task_accept_to_terminal_latency_max_seconds"`
|
||
// WorkflowStartToTerminalLatencySeconds 是最近窗口流程从开始到停止事件的平均耗时样本。
|
||
WorkflowStartToTerminalLatencySeconds float64 `json:"workflow_start_to_terminal_latency_seconds"`
|
||
// WorkflowStartToTerminalLatencyMaxSeconds 是最近窗口流程开始到停止耗时样本的最大值。
|
||
WorkflowStartToTerminalLatencyMaxSeconds float64 `json:"workflow_start_to_terminal_latency_max_seconds"`
|
||
}
|
||
|
||
var (
|
||
operationalTaskStates = []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
|
||
operationalDeliveryStates = []string{"PENDING", "LEASED", "ACKED", "EXPIRED"}
|
||
operationalWorkflowStates = []string{"RUNNING", "WAITING", "CANCELLING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
|
||
operationalInboxStates = []string{"PENDING", "LEASED", "PROCESSED", "QUARANTINED"}
|
||
operationalEventTypes = []string{"task.accepted", "task.started", "task.succeeded", "task.failed", "task.cancelled", "task.needs_review", "task.requeued", "workflow.started", "workflow.advanced", "workflow.recovered", "workflow.blocked", "workflow.stopped"}
|
||
phaseEventTypes = []string{"task.accepted", "task.started", "task.succeeded", "task.failed", "task.cancelled", "workflow.started", "workflow.stopped"}
|
||
)
|
||
|
||
// ReadOperationalSnapshot performs only bounded, indexed reads. It is intended
|
||
// for a background sampler with a short context deadline, never for a request.
|
||
func ReadOperationalSnapshot(ctx context.Context, db *sql.DB) (OperationalSnapshot, error) {
|
||
if db == nil {
|
||
return OperationalSnapshot{}, errors.New("database is required")
|
||
}
|
||
now, err := readDatabaseNow(ctx, db)
|
||
if err != nil {
|
||
return OperationalSnapshot{}, err
|
||
}
|
||
tasks, taskTruncated, err := boundedStateCounts(ctx, db, "pulse_tasks", "ix_task_observe_state_available", operationalTaskStates)
|
||
if err != nil {
|
||
return OperationalSnapshot{}, err
|
||
}
|
||
deliveries, deliveryTruncated, err := boundedStateCounts(ctx, db, "orbit_deliveries", "ix_delivery_observe_state_available", operationalDeliveryStates)
|
||
if err != nil {
|
||
return OperationalSnapshot{}, err
|
||
}
|
||
workflows, workflowTruncated, err := boundedStateCounts(ctx, db, "weave_instances", "ix_workflow_observe_state_updated", operationalWorkflowStates)
|
||
if err != nil {
|
||
return OperationalSnapshot{}, err
|
||
}
|
||
inbox, inboxTruncated, err := boundedStateCounts(ctx, db, "weave_inbox", "ix_inbox_due", operationalInboxStates)
|
||
if err != nil {
|
||
return OperationalSnapshot{}, err
|
||
}
|
||
taskAge, err := oldestAge(ctx, db, "pulse_tasks", "ix_task_observe_state_available", []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW"}, "available_at", now)
|
||
if err != nil {
|
||
return OperationalSnapshot{}, err
|
||
}
|
||
inboxAge, err := oldestAge(ctx, db, "weave_inbox", "ix_inbox_due", []string{"PENDING", "LEASED"}, "available_at", now)
|
||
if err != nil {
|
||
return OperationalSnapshot{}, err
|
||
}
|
||
deliveryAge, err := oldestAge(ctx, db, "orbit_deliveries", "ix_delivery_observe_state_available", []string{"PENDING", "LEASED"}, "available_at", now)
|
||
if err != nil {
|
||
return OperationalSnapshot{}, err
|
||
}
|
||
workflowAge, err := oldestAge(ctx, db, "weave_instances", "ix_workflow_observe_state_updated", []string{"RUNNING", "WAITING", "CANCELLING", "NEEDS_REVIEW"}, "updated_at", now)
|
||
if err != nil {
|
||
return OperationalSnapshot{}, err
|
||
}
|
||
events, eventTruncated, err := boundedRecentEventCounts(ctx, db, now)
|
||
if err != nil {
|
||
return OperationalSnapshot{}, err
|
||
}
|
||
taskLatency, err := sampledTerminalLatency(ctx, db, "pulse_tasks", "ix_task_observe_terminal_completed", []string{"SUCCEEDED", "FAILED", "CANCELLED"}, now.Add(-operationalMetricEventWindow))
|
||
if err != nil {
|
||
return OperationalSnapshot{}, err
|
||
}
|
||
workflowLatency, err := sampledTerminalLatency(ctx, db, "weave_instances", "ix_workflow_observe_terminal_completed", []string{"SUCCEEDED", "FAILED", "CANCELLED"}, now.Add(-operationalMetricEventWindow))
|
||
if err != nil {
|
||
return OperationalSnapshot{}, err
|
||
}
|
||
phaseLatency, err := sampledPhaseLatency(ctx, db, now.Add(-operationalMetricEventWindow))
|
||
if err != nil {
|
||
return OperationalSnapshot{}, err
|
||
}
|
||
return OperationalSnapshot{
|
||
TaskStates: tasks, TaskOldestAgeSeconds: taskAge,
|
||
DeliveryStates: deliveries, DeliveryOldestAgeSeconds: deliveryAge,
|
||
WorkflowStates: workflows, WorkflowOldestAgeSeconds: workflowAge,
|
||
InboxStates: inbox, InboxOldestAgeSeconds: inboxAge,
|
||
RecentEvents: events,
|
||
TaskTerminalLatencySeconds: taskLatency.average, TaskTerminalLatencyMaxSeconds: taskLatency.maximum,
|
||
WorkflowTerminalLatencySeconds: workflowLatency.average, WorkflowTerminalLatencyMaxSeconds: workflowLatency.maximum,
|
||
TaskAcceptToStartLatencySeconds: phaseLatency.taskAcceptToStart.average, TaskAcceptToStartLatencyMaxSeconds: phaseLatency.taskAcceptToStart.maximum,
|
||
TaskAcceptToTerminalLatencySeconds: phaseLatency.taskAcceptToTerminal.average, TaskAcceptToTerminalLatencyMaxSeconds: phaseLatency.taskAcceptToTerminal.maximum,
|
||
WorkflowStartToTerminalLatencySeconds: phaseLatency.workflowStartToTerminal.average, WorkflowStartToTerminalLatencyMaxSeconds: phaseLatency.workflowStartToTerminal.maximum,
|
||
Truncated: taskTruncated || deliveryTruncated || workflowTruncated || inboxTruncated || eventTruncated || taskLatency.truncated || workflowLatency.truncated || phaseLatency.truncated,
|
||
}, nil
|
||
}
|
||
|
||
func readDatabaseNow(ctx context.Context, db *sql.DB) (time.Time, error) {
|
||
var now time.Time
|
||
if err := db.QueryRowContext(ctx, "SELECT UTC_TIMESTAMP(6)").Scan(&now); err != nil {
|
||
return time.Time{}, err
|
||
}
|
||
return now.UTC(), nil
|
||
}
|
||
|
||
func boundedStateCounts(ctx context.Context, db *sql.DB, table, index string, states []string) (map[string]uint64, bool, error) {
|
||
counts := make(map[string]uint64, len(states))
|
||
truncated := false
|
||
for _, state := range states {
|
||
query := "SELECT COUNT(*) FROM (SELECT id FROM " + table + " FORCE INDEX (" + index + ") WHERE state=? LIMIT ?) AS bounded"
|
||
var n uint64
|
||
if err := db.QueryRowContext(ctx, query, state, operationalMetricSampleLimit+1).Scan(&n); err != nil {
|
||
return nil, false, err
|
||
}
|
||
if n > operationalMetricSampleLimit {
|
||
n = operationalMetricSampleLimit
|
||
truncated = true
|
||
}
|
||
counts[state] = n
|
||
}
|
||
return counts, truncated, nil
|
||
}
|
||
|
||
func boundedRecentEventCounts(ctx context.Context, db *sql.DB, now time.Time) (map[string]uint64, bool, error) {
|
||
counts := make(map[string]uint64, len(operationalEventTypes))
|
||
truncated := false
|
||
cutoff := now.Add(-operationalMetricEventWindow)
|
||
for _, eventType := range operationalEventTypes {
|
||
query := "SELECT COUNT(*) FROM (SELECT id FROM orbit_events FORCE INDEX (ix_event_observe_type_occurred) WHERE type=? AND occurred_at>=? LIMIT ?) AS bounded"
|
||
var n uint64
|
||
if err := db.QueryRowContext(ctx, query, eventType, cutoff, operationalMetricSampleLimit+1).Scan(&n); err != nil {
|
||
return nil, false, err
|
||
}
|
||
if n > operationalMetricSampleLimit {
|
||
n = operationalMetricSampleLimit
|
||
truncated = true
|
||
}
|
||
counts[eventType] = n
|
||
}
|
||
return counts, truncated, nil
|
||
}
|
||
|
||
type terminalLatencySample struct {
|
||
average float64
|
||
maximum float64
|
||
truncated bool
|
||
}
|
||
|
||
type phaseLatencySamples struct {
|
||
taskAcceptToStart terminalLatencySample
|
||
taskAcceptToTerminal terminalLatencySample
|
||
workflowStartToTerminal terminalLatencySample
|
||
truncated bool
|
||
}
|
||
|
||
type phaseEvent struct {
|
||
kind string
|
||
id string
|
||
typeName string
|
||
at time.Time
|
||
}
|
||
|
||
// sampledPhaseLatency pairs only fixed event types in the recent bounded
|
||
// window. Missing history, retries and events outside the window are excluded;
|
||
// the result is a trend sample rather than an end-to-end SLO.
|
||
func sampledPhaseLatency(ctx context.Context, db *sql.DB, cutoff time.Time) (phaseLatencySamples, error) {
|
||
placeholders := "?"
|
||
args := make([]any, 0, len(phaseEventTypes)+2)
|
||
args = append(args, phaseEventTypes[0])
|
||
for _, eventType := range phaseEventTypes[1:] {
|
||
placeholders += ",?"
|
||
args = append(args, eventType)
|
||
}
|
||
args = append([]any{cutoff}, args...)
|
||
args = append(args, operationalMetricSampleLimit+1)
|
||
query := "SELECT aggregate_kind,aggregate_id,type,occurred_at FROM orbit_events FORCE INDEX (ix_event_observe_type_occurred) WHERE occurred_at>=? AND type IN (" + placeholders + ") LIMIT ?"
|
||
rows, err := db.QueryContext(ctx, query, args...)
|
||
if err != nil {
|
||
return phaseLatencySamples{}, err
|
||
}
|
||
defer rows.Close()
|
||
tasks := make(map[string][]phaseEvent)
|
||
workflows := make(map[string][]phaseEvent)
|
||
read := 0
|
||
for rows.Next() {
|
||
read++
|
||
var event phaseEvent
|
||
var id []byte
|
||
if err := rows.Scan(&event.kind, &id, &event.typeName, &event.at); err != nil {
|
||
return phaseLatencySamples{}, err
|
||
}
|
||
if read <= operationalMetricSampleLimit {
|
||
event.id = string(id)
|
||
switch event.kind {
|
||
case "task":
|
||
tasks[event.id] = append(tasks[event.id], event)
|
||
case "workflow":
|
||
workflows[event.id] = append(workflows[event.id], event)
|
||
}
|
||
}
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return phaseLatencySamples{}, err
|
||
}
|
||
result := phaseLatencySamples{truncated: read > operationalMetricSampleLimit}
|
||
result.taskAcceptToStart = pairPhaseLatencies(tasks, "task.accepted", "task.started")
|
||
result.taskAcceptToTerminal = pairPhaseLatencies(tasks, "task.accepted", "task.succeeded", "task.failed", "task.cancelled")
|
||
result.workflowStartToTerminal = pairPhaseLatencies(workflows, "workflow.started", "workflow.stopped")
|
||
result.taskAcceptToStart.truncated = false
|
||
result.taskAcceptToTerminal.truncated = false
|
||
result.workflowStartToTerminal.truncated = false
|
||
return result, nil
|
||
}
|
||
|
||
func pairPhaseLatencies(groups map[string][]phaseEvent, from string, to ...string) terminalLatencySample {
|
||
targets := make(map[string]struct{}, len(to))
|
||
for _, eventType := range to {
|
||
targets[eventType] = struct{}{}
|
||
}
|
||
var total, maximum float64
|
||
var count int
|
||
for _, events := range groups {
|
||
sort.Slice(events, func(i, j int) bool {
|
||
if events[i].at.Equal(events[j].at) {
|
||
return events[i].id < events[j].id
|
||
}
|
||
return events[i].at.Before(events[j].at)
|
||
})
|
||
var start *time.Time
|
||
var finish *time.Time
|
||
for _, event := range events {
|
||
if event.typeName == from && start == nil {
|
||
at := event.at.UTC()
|
||
start = &at
|
||
continue
|
||
}
|
||
if start != nil {
|
||
if _, ok := targets[event.typeName]; ok && !event.at.Before(*start) {
|
||
at := event.at.UTC()
|
||
finish = &at
|
||
break
|
||
}
|
||
}
|
||
}
|
||
if start == nil || finish == nil {
|
||
continue
|
||
}
|
||
seconds := finish.Sub(*start).Seconds()
|
||
if seconds < 0 {
|
||
seconds = 0
|
||
}
|
||
total += seconds
|
||
if seconds > maximum {
|
||
maximum = seconds
|
||
}
|
||
count++
|
||
}
|
||
result := terminalLatencySample{maximum: maximum}
|
||
if count > 0 {
|
||
result.average = total / float64(count)
|
||
}
|
||
return result
|
||
}
|
||
|
||
func sampledTerminalLatency(ctx context.Context, db *sql.DB, table, index string, states []string, cutoff time.Time) (terminalLatencySample, error) {
|
||
placeholders := "?"
|
||
args := make([]any, 0, len(states)+2)
|
||
args = append(args, states[0])
|
||
for _, state := range states[1:] {
|
||
placeholders += ",?"
|
||
args = append(args, state)
|
||
}
|
||
args = append(args, cutoff, operationalMetricSampleLimit+1)
|
||
query := "SELECT created_at,completed_at FROM " + table + " FORCE INDEX (" + index + ") WHERE state IN (" + placeholders + ") AND completed_at>=? ORDER BY completed_at,id LIMIT ?"
|
||
rows, err := db.QueryContext(ctx, query, args...)
|
||
if err != nil {
|
||
return terminalLatencySample{}, err
|
||
}
|
||
defer rows.Close()
|
||
var total, maximum float64
|
||
var read, valid int
|
||
for rows.Next() {
|
||
read++
|
||
if read > operationalMetricSampleLimit {
|
||
continue
|
||
}
|
||
var created, completed time.Time
|
||
if err := rows.Scan(&created, &completed); err != nil {
|
||
return terminalLatencySample{}, err
|
||
}
|
||
seconds := completed.Sub(created).Seconds()
|
||
if seconds < 0 {
|
||
seconds = 0
|
||
}
|
||
total += seconds
|
||
if seconds > maximum {
|
||
maximum = seconds
|
||
}
|
||
valid++
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return terminalLatencySample{}, err
|
||
}
|
||
result := terminalLatencySample{maximum: maximum, truncated: read > operationalMetricSampleLimit}
|
||
if valid > 0 {
|
||
result.average = total / float64(valid)
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
func oldestAge(ctx context.Context, db *sql.DB, table, index string, states []string, column string, now time.Time) (float64, error) {
|
||
placeholders := "?"
|
||
args := make([]any, 0, len(states))
|
||
args = append(args, states[0])
|
||
for _, state := range states[1:] {
|
||
placeholders += ",?"
|
||
args = append(args, state)
|
||
}
|
||
query := "SELECT " + column + " FROM " + table + " FORCE INDEX (" + index + ") WHERE state IN (" + placeholders + ") ORDER BY " + column + ",id LIMIT 1"
|
||
var occurred time.Time
|
||
if err := db.QueryRowContext(ctx, query, args...).Scan(&occurred); err != nil {
|
||
if err == sql.ErrNoRows {
|
||
return 0, nil
|
||
}
|
||
return 0, err
|
||
}
|
||
age := now.Sub(occurred.UTC()).Seconds()
|
||
if age < 0 {
|
||
return 0, nil
|
||
}
|
||
return age, nil
|
||
}
|