feat: sample terminal business latency

internal/storage/mysql/operational_metrics.go

@@ -29,6 +29,14 @@ type OperationalSnapshot struct {
 	Truncated bool
 	// RecentEvents 是固定事件类别在最近窗口内的有界数量。
 	RecentEvents map[string]uint64
+	// TaskTerminalLatencySeconds 是最近窗口终态任务从创建到完成的平均耗时样本。
+	TaskTerminalLatencySeconds float64
+	// TaskTerminalLatencyMaxSeconds 是最近窗口终态任务样本的最大耗时。
+	TaskTerminalLatencyMaxSeconds float64
+	// WorkflowTerminalLatencySeconds 是最近窗口终态流程从创建到完成的平均耗时样本。
+	WorkflowTerminalLatencySeconds float64
+	// WorkflowTerminalLatencyMaxSeconds 是最近窗口终态流程样本的最大耗时。
+	WorkflowTerminalLatencyMaxSeconds float64
 }
 
 var (
@@ -76,12 +84,22 @@ func ReadOperationalSnapshot(ctx context.Context, db *sql.DB) (OperationalSnapsh
 	if err != nil {
 		return OperationalSnapshot{}, err
 	}
+	taskLatency, err := sampledTerminalLatency(ctx, db, "pulse_tasks", "ix_task_observe_terminal_completed", []string{"SUCCEEDED", "FAILED", "CANCELLED"}, now.Add(-operationalMetricEventWindow))
+	if err != nil {
+		return OperationalSnapshot{}, err
+	}
+	workflowLatency, err := sampledTerminalLatency(ctx, db, "weave_instances", "ix_workflow_observe_terminal_completed", []string{"SUCCEEDED", "FAILED", "CANCELLED"}, now.Add(-operationalMetricEventWindow))
+	if err != nil {
+		return OperationalSnapshot{}, err
+	}
 	return OperationalSnapshot{
 		TaskStates: tasks, TaskOldestAgeSeconds: taskAge,
 		DeliveryStates: deliveries, DeliveryOldestAgeSeconds: deliveryAge,
 		WorkflowStates: workflows, WorkflowOldestAgeSeconds: workflowAge,
-		RecentEvents: events,
-		Truncated:    taskTruncated || deliveryTruncated || workflowTruncated || eventTruncated,
+		RecentEvents:               events,
+		TaskTerminalLatencySeconds: taskLatency.average, TaskTerminalLatencyMaxSeconds: taskLatency.maximum,
+		WorkflowTerminalLatencySeconds: workflowLatency.average, WorkflowTerminalLatencyMaxSeconds: workflowLatency.maximum,
+		Truncated: taskTruncated || deliveryTruncated || workflowTruncated || eventTruncated || taskLatency.truncated || workflowLatency.truncated,
 	}, nil
 }
 
@@ -130,6 +148,58 @@ func boundedRecentEventCounts(ctx context.Context, db *sql.DB, now time.Time) (m
 	return counts, truncated, nil
 }
 
+type terminalLatencySample struct {
+	average   float64
+	maximum   float64
+	truncated bool
+}
+
+func sampledTerminalLatency(ctx context.Context, db *sql.DB, table, index string, states []string, cutoff time.Time) (terminalLatencySample, error) {
+	placeholders := "?"
+	args := make([]any, 0, len(states)+2)
+	args = append(args, states[0])
+	for _, state := range states[1:] {
+		placeholders += ",?"
+		args = append(args, state)
+	}
+	args = append(args, cutoff, operationalMetricSampleLimit+1)
+	query := "SELECT created_at,completed_at FROM " + table + " FORCE INDEX (" + index + ") WHERE state IN (" + placeholders + ") AND completed_at>=? ORDER BY completed_at,id LIMIT ?"
+	rows, err := db.QueryContext(ctx, query, args...)
+	if err != nil {
+		return terminalLatencySample{}, err
+	}
+	defer rows.Close()
+	var total, maximum float64
+	var read, valid int
+	for rows.Next() {
+		read++
+		if read > operationalMetricSampleLimit {
+			continue
+		}
+		var created, completed time.Time
+		if err := rows.Scan(&created, &completed); err != nil {
+			return terminalLatencySample{}, err
+		}
+		seconds := completed.Sub(created).Seconds()
+		if seconds < 0 {
+			seconds = 0
+		}
+		total += seconds
+		if seconds > maximum {
+			maximum = seconds
+		}
+		valid++
+	}
+	if err := rows.Err(); err != nil {
+		return terminalLatencySample{}, err
+	}
+	result := terminalLatencySample{maximum: maximum, truncated: read > operationalMetricSampleLimit}
+	if valid > 0 {
+		result.average = total / float64(valid)
+	}
+	return result, nil
+}
+
 func oldestAge(ctx context.Context, db *sql.DB, table, index string, states []string, column string, now time.Time) (float64, error) {
 	placeholders := "?"
 	args := make([]any, 0, len(states))