feat: add bounded process metrics and private monitoring endpoint

2026-09-18 13:17:33 +08:00
parent 1f1d833f69
commit a7be32f661
24 changed files with 1467 additions and 19 deletions
--- a/.env.example
+++ b/.env.example
@@ -4,3 +4,6 @@ ORBIT_HTTP_ADDR=127.0.0.1:8080
 ORBIT_MYSQL_DSN=
 ORBIT_DB_MAX_OPEN=20
 ORBIT_DB_MAX_IDLE=5
+# Empty disables metrics. To enable a separate private listener, use e.g.
+# ORBIT_METRICS_ADDR=127.0.0.1:9090. Never expose it on the public API ingress.
+ORBIT_METRICS_ADDR=
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -76,7 +76,11 @@

 - 竞争回归通过performance_schema确认参与者实际进入数据库锁等待，禁止将请求已发出或sleep当作并发证据。W09以原数据库截止自然到达放行；PAUSE以后不得新建任务/尝试或授权，已授权结果仍需收集。公共SDK暂停全停场景须在全部旧节点Wait确认退出后才启动新节点；保留原回执、等待身份、版本、查询轮次与截止。普通SIGNAL竞争不能替代最终查询/回执/下一轮的独立故障边界；COMMIT不明和A04已补独立故障工具及真实回归；生产HA与容量等继续按P6/P7验收。

- 版本/暂停竞争本批已通过21个真实存储子场景、公共SDK两旧节点全停及两新节点恢复、23个SDK示例/2个业务模块的标准全量回归。证据见docs/reports/phase-4-5-races.md；A04/W01/W13/W16精确故障也已补，见docs/reports/phase-6-fault-boundaries.md；下一批按docs/08-development-plan.md推进运行观测，再做凭据轮换，不重复将上述已验证场景列为未做。全平台目标继续保持，不能以本批通过替代观测、凭据轮换、归档/备份、生产拓扑和发布交付。
+- 版本/暂停竞争本批已通过21个真实存储子场景、公共SDK两旧节点全停及两新节点恢复、23个SDK示例/2个业务模块的标准全量回归。证据见docs/reports/phase-4-5-races.md；A04/W01/W13/W16精确故障也已补，见docs/reports/phase-6-fault-boundaries.md；进程观测基础已实现，下一批按docs/08-development-plan.md继续持久积压/阶段延迟与发送队列，再做凭据轮换，不重复将上述已验证场景列为未做。全平台目标继续保持，不能以本批通过替代观测、凭据轮换、归档/备份、生产拓扑和发布交付。

 - A04须截留真实task.assign并独立确认LEASED/未授权，再强杀旧节点、Wait确认退出、新节点等原租约自然到期；不能把已送达未开始替代未送达。COMMIT故障工具只在测试二进制：BEFORE阻止COMMIT，AFTER核验真实MySQL成功包后丢确认；独立连接回读，原身份/候选恢复，禁止在生产增加故障开关或将HTTP丢回执替代数据库提交不明。
 - W16下一轮时钟只能在PENDING结果已保存等待后存在，三方竞争中的结果参与者是旧结果重放；新结果/未知/冲突另有在途查询回归。保持原受理时间、事件时间、截止与预算，不能凭有限入队顺序声称穷尽线程交错。
+
+- P6进程观测使用internal/observability固定指标目录，拒绝未知名称、类型和动态标签，不能引入任务/租户/业务键/凭据维度。ORBIT_METRICS_ADDR默认空（关闭），显式启用独立采集监听；业务端口不得暴露全局指标。渲染和网络I/O不持有全局记录锁，每个直方图局部快照保持桶/count/sum一致，最多2个并发抓取。
+- HTTP成功升级在Hijack完成时记录握手，WS处理器存续计入独立连接指标；保持1xx临时响应、Flush、Hijack与原panic行为。DB池取本地Stats，抓取不查询业务库；后台批次success不是业务任务成功量。观测丢样或失败不得改变权威事务及恢复决定。
+- 进程指标、格式验证、告警规则和标准回归证据见docs/reports/phase-6-observability.md。仍须补受理/授权/完成与流程推进延迟、持久吞吐/积压、重试/核查/恢复量、发送队列、实际采集告警及负载开销，不能把进程观测通过写成P6已完成。普通清理仍须保护有效幂等、未确认事件、未知结果和未终结流程。
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@ Orbit 是面向多个业务项目的通用任务调度与流程编排平台，

 Weave已提供[定义发布](docs/api/workflow-definitions.md)、[实例启动、步骤详情、暂停/继续/取消、信号和操作审计](docs/api/workflows.md)，以及[六类人工核查恢复](docs/api/workflow-recovery.md)的HTTP与独立Go SDK。持久运行器支持顺序、分支、有限并行、TIMER、SIGNAL及有界POLL，恢复沿用原预算与截止。[并行作业与回执示例](docs/examples/workflow.md)提供完整调用链；[历史激活/轮次/信号及信号审计](docs/api/workflow-history.md)已接入；[实例筛选分页](docs/api/workflow-queries.md)已接入，[状态事件时间线](docs/api/workflow-events.md)已接入，独立[轮询](docs/examples/workflow-poll.md)与[恢复](docs/examples/workflow-recovery.md)示例已接入，已补[版本/暂停竞争/全停回归](docs/reports/phase-4-5-races.md)，功能阶段证据见[P4/P5核对](docs/reports/phase-4-5.md)，提交断线及派发未送达见[故障边界](docs/reports/phase-6-fault-boundaries.md)；[任务流程来源](docs/api/task-source.md#5-流程固定来源)已接入，P4/P5功能阶段已通过，P6生产验证和P7发布交付继续推进。

-数据库禁止外键，关联一致性通过服务端事务校验，索引、唯一约束和 CHECK 保留；readiness 同时检查迁移及零外键要求。这不表示完整平台已具备生产服务能力。验证证据见 [P0](docs/reports/phase-0.md)、[P1](docs/reports/phase-1.md)、[P2 阶段验收](docs/reports/phase-2.md)、[安全重试](docs/reports/phase-3-safe-retry.md)、[计划调度](docs/reports/phase-3-schedules.md)、[取消与无外键回归](docs/reports/phase-3-cancellation.md) 及 [队列资源回归](docs/reports/phase-3-queue-resources.md)。
+数据库禁止外键，关联一致性通过服务端事务校验，索引、唯一约束和 CHECK 保留；readiness 同时检查迁移及零外键要求。[运行观测](docs/api/observability.md)提供固定基数 HTTP、数据库池和后台批次指标。这不表示完整平台已具备生产服务能力。验证证据见 [P0](docs/reports/phase-0.md)、[P1](docs/reports/phase-1.md)、[P2 阶段验收](docs/reports/phase-2.md)、[安全重试](docs/reports/phase-3-safe-retry.md)、[计划调度](docs/reports/phase-3-schedules.md)、[取消与无外键回归](docs/reports/phase-3-cancellation.md) 及 [队列资源回归](docs/reports/phase-3-queue-resources.md)。

 ## 目录

--- a/cmd/orbit/main.go
+++ b/cmd/orbit/main.go
@@ -12,6 +12,7 @@ import (
 	"time"

 	"git.yuntuo.hk/k2safe/orbit/internal/config"
+	"git.yuntuo.hk/k2safe/orbit/internal/observability"
 	"git.yuntuo.hk/k2safe/orbit/internal/storage/mysql"
 	"git.yuntuo.hk/k2safe/orbit/internal/transport/httpserver"
 	core "git.yuntuo.hk/k2safe/orbit/model"
@@ -53,12 +54,24 @@ func run() error {
 		return err
 	}
 	cancel()
-	server := httpserver.New(cfg.HTTPAddr, func(ctx context.Context) error {
+	var metrics *observability.Registry
+	var metricsListener net.Listener
+	if cfg.MetricsAddr != "" {
+		metricsListener, err = net.Listen("tcp", cfg.MetricsAddr)
+		if err != nil {
+			return errors.New("metrics listener could not bind")
+		}
+		defer metricsListener.Close()
+		metrics = observability.NewRegistry()
+		metrics.RecordDatabase(db.Stats())
+	}
+	ready := func(ctx context.Context) error {
 		if err := db.PingContext(ctx); err != nil {
 			return err
 		}
 		return mysql.CheckSchema(ctx, db)
-	}, service.New(mysql.NewTaskStore(db)))
+	}
+	server := httpserver.New(cfg.HTTPAddr, ready, service.New(mysql.NewTaskStore(db)))
 	httpserver.RegisterApplicationResources(server, resources.New(mysql.NewTaskStore(db)))
 	node, err := core.NewID()
 	if err != nil {
@@ -86,6 +99,7 @@ func run() error {
 	httpserver.RegisterSchedules(server, schedule.New(schedules))
 	httpserver.RegisterWorkflowDefinitions(server, definition.New(mysql.NewWorkflowDefinitionStore(db)))
 	httpserver.RegisterWorkflows(server, instance.New(mysql.NewWorkflowInstanceStore(db)), control.New(mysql.NewWorkflowControlStore(db)), weavesignal.New(mysql.NewWorkflowSignalStore(db)), recovery.New(mysql.NewWorkflowRecoveryStore(db)))
+	httpserver.WrapMetrics(server, metrics)
 	workflows, err := mysql.NewWorkflowRuntime(db, node)
 	if err != nil {
 		return err
@@ -101,9 +115,11 @@ func run() error {
 			case <-workflowCtx.Done():
 				return
 			case <-ticker.C:
+				started := time.Now()
 				call, cancel := context.WithTimeout(workflowCtx, 5*time.Second)
 				_, err := workflows.Tick(call, 100)
 				cancel()
+				metrics.RecordBatch("workflow", started, err)
 				if err != nil && workflowCtx.Err() == nil {
 					slog.Error("workflow batch failed", "node_id", node)
 				}
@@ -122,9 +138,11 @@ func run() error {
 			case <-scheduleCtx.Done():
 				return
 			case <-ticker.C:
+				started := time.Now()
 				call, cancel := context.WithTimeout(scheduleCtx, 5*time.Second)
 				_, err := schedules.Tick(call, 100)
 				cancel()
+				metrics.RecordBatch("schedule", started, err)
 				if err != nil && scheduleCtx.Err() == nil {
 					slog.Error("schedule batch failed", "node_id", node)
 				}
@@ -148,9 +166,11 @@ func run() error {
 			case <-recoveryCtx.Done():
 				return
 			case <-ticker.C:
+				started := time.Now()
 				call, cancel := context.WithTimeout(recoveryCtx, 5*time.Second)
 				_, err := executions.Recover(call, 100)
 				cancel()
+				metrics.RecordBatch("execution_recovery", started, err)
 				if err != nil && recoveryCtx.Err() == nil {
 					slog.Error("execution recovery batch failed", "node_id", node)
 				}
@@ -158,11 +178,38 @@ func run() error {
 		}
 	}()
 	defer func() { stopRecovery(); <-recoveryDone }()
+	if metrics != nil {
+		metricsCtx, stopMetrics := context.WithCancel(root)
+		metricsDone := make(chan struct{})
+		go func() {
+			defer close(metricsDone)
+			ticker := time.NewTicker(5 * time.Second)
+			defer ticker.Stop()
+			for {
+				select {
+				case <-metricsCtx.Done():
+					return
+				case <-ticker.C:
+					metrics.RecordDatabase(db.Stats())
+					metrics.RecordProcess()
+				}
+			}
+		}()
+		defer func() { stopMetrics(); <-metricsDone }()
+	}
 	listener, err := net.Listen("tcp", cfg.HTTPAddr)
 	if err != nil {
 		return err
 	}
-	done := make(chan error, 1)
+	defer server.Close()
+	var metricsServer *http.Server
+	done := make(chan error, 2)
+	if metrics != nil {
+		metricsServer = httpserver.NewMetricsServer(cfg.MetricsAddr, metrics)
+		defer metricsServer.Close()
+		go func() { done <- metricsServer.Serve(metricsListener) }()
+		slog.Info("Orbit metrics server starting", "metrics_address", metricsListener.Addr().String())
+	}
 	go func() { done <- server.Serve(listener) }()
 	slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task, queue, schedule, subscription, workflow and Worker APIs", "node_id", node)
 	select {
@@ -178,10 +225,20 @@ func run() error {
 			_ = server.Close()
 			return err
 		}
-		err := <-done
-		if errors.Is(err, http.ErrServerClosed) {
-			return nil
+		if metricsServer != nil {
+			if err := metricsServer.Shutdown(shutdown); err != nil {
+				return err
+			}
 		}
-		return err
+		count := 1
+		if metricsServer != nil {
+			count++
+		}
+		for i := 0; i < count; i++ {
+			if err := <-done; !errors.Is(err, http.ErrServerClosed) {
+				return err
+			}
+		}
+		return nil
 	}
 }
--- a/cmd/orbit/metrics_integration_test.go
+++ b/cmd/orbit/metrics_integration_test.go
@@ -0,0 +1,162 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+)
+
+// TestIntegrationMetricsProcess uses the same compiled binary and disposable
+// database as the full regression. It never reads an existing business DSN.
+func TestIntegrationMetricsProcess(t *testing.T) {
+	binary, dsn := os.Getenv("ORBIT_TEST_SERVER_BINARY"), os.Getenv("ORBIT_TEST_MYSQL_DSN")
+	if binary == "" || dsn == "" {
+		t.Skip("requires compiled Orbit and disposable database")
+	}
+	for _, enabled := range []bool{true, false} {
+		t.Run(map[bool]string{true: "enabled", false: "disabled"}[enabled], func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+			defer cancel()
+			logPath := filepath.Join(t.TempDir(), "server.log")
+			log, err := os.Create(logPath)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer log.Close()
+			cmd := exec.CommandContext(ctx, binary)
+			for _, entry := range os.Environ() {
+				if strings.HasPrefix(entry, "ORBIT_MYSQL_DSN=") || strings.HasPrefix(entry, "ORBIT_HTTP_ADDR=") || strings.HasPrefix(entry, "ORBIT_METRICS_ADDR=") {
+					continue
+				}
+				cmd.Env = append(cmd.Env, entry)
+			}
+			addr := ""
+			if enabled {
+				addr = "127.0.0.1:0"
+			}
+			cmd.Env = append(cmd.Env, "ORBIT_MYSQL_DSN="+dsn, "ORBIT_HTTP_ADDR=127.0.0.1:0", "ORBIT_METRICS_ADDR="+addr)
+			cmd.Stdout = log
+			cmd.Stderr = log
+			if err = cmd.Start(); err != nil {
+				t.Fatal(err)
+			}
+			done := make(chan struct{})
+			var exit error
+			go func() { exit = cmd.Wait(); close(done) }()
+			defer func() { cancel(); <-done }()
+			var api, metrics string
+			for api == "" || (enabled && metrics == "") {
+				data, _ := os.ReadFile(logPath)
+				for _, line := range strings.Split(string(data), "\n") {
+					var event struct {
+						Message        string `json:"msg"`
+						Address        string `json:"address"`
+						MetricsAddress string `json:"metrics_address"`
+					}
+					if json.Unmarshal([]byte(line), &event) != nil {
+						continue
+					}
+					if event.Message == "Orbit HTTP server starting" {
+						api = "http://" + event.Address
+					}
+					if event.Message == "Orbit metrics server starting" {
+						metrics = "http://" + event.MetricsAddress
+					}
+				}
+				select {
+				case <-done:
+					t.Fatal("server exited before listeners became ready")
+				case <-ctx.Done():
+					t.Fatal("listener startup deadline")
+				case <-time.After(10 * time.Millisecond):
+				}
+			}
+			client := &http.Client{Timeout: 2 * time.Second}
+			get := func(url string, want int) string {
+				t.Helper()
+				resp, err := client.Get(url)
+				if err != nil {
+					t.Fatal(err)
+				}
+				defer resp.Body.Close()
+				raw, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
+				if err != nil || resp.StatusCode != want {
+					t.Fatalf("unexpected response: status=%d error=%v", resp.StatusCode, err)
+				}
+				return string(raw)
+			}
+			get(api+"/health/ready", 200)
+			get(api+"/v1/tasks/private-business-id?business_key=private-business-key", 401)
+			get(api+"/metrics", 404)
+			if enabled {
+				if api == metrics {
+					t.Fatal("metrics shares business listener")
+				}
+				get(metrics+"/health/ready", 404)
+				get(metrics+"/v1/tasks", 404)
+				get(metrics+"/metrics?token=private-key", 400)
+				for {
+					body := get(metrics+"/metrics", 200)
+					for _, secret := range []string{dsn, "private-business", "private-key"} {
+						if strings.Contains(body, secret) {
+							t.Fatal("metric data contains request data or credentials")
+						}
+					}
+					if !strings.Contains(body, "orbit_http_requests_total{") || !strings.Contains(body, "orbit_db_connections_max_open 20\n") {
+						t.Fatal("process metrics not connected")
+					}
+					observed := false
+					for _, line := range strings.Split(body, "\n") {
+						prefix := `orbit_runtime_batch_total{component="execution_recovery",outcome="success"} `
+						if strings.HasPrefix(line, prefix) {
+							n, err := strconv.ParseFloat(strings.TrimPrefix(line, prefix), 64)
+							observed = err == nil && n > 0
+						}
+					}
+					if observed {
+						break
+					}
+					select {
+					case <-ctx.Done():
+						t.Fatal("background batch was not observed")
+					case <-time.After(50 * time.Millisecond):
+					}
+				}
+			} else if metrics != "" {
+				t.Fatal("disabled metrics listener was started")
+			}
+			if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
+				t.Fatal(err)
+			}
+			select {
+			case <-done:
+				if exit != nil {
+					t.Fatal("graceful exit failed", exit)
+				}
+			case <-ctx.Done():
+				t.Fatal("shutdown hung")
+			}
+			for _, url := range []string{api, metrics} {
+				if url == "" {
+					continue
+				}
+				c, err := net.DialTimeout("tcp", strings.TrimPrefix(url, "http://"), time.Second)
+				if err == nil {
+					c.Close()
+					t.Fatal("listener remained after shutdown")
+				}
+			}
+			t.Log("real process: metrics enable/disable, separate listeners, HTTP/database/background samples, request privacy and graceful closure verified")
+		})
+	}
+}
--- a/deploy/monitoring/orbit.rules.test.yml
+++ b/deploy/monitoring/orbit.rules.test.yml
@@ -0,0 +1,90 @@
+rule_files:
+  - orbit.rules.yml
+evaluation_interval: 1s
+tests:
+  - name: unavailable_target
+    interval: 1s
+    input_series:
+      - series: 'up{job="orbit",instance="node-a"}'
+        values: '0+0x180'
+    alert_rule_test:
+      - eval_time: 130s
+        alertname: OrbitMetricsUnavailable
+        exp_alerts:
+          - exp_labels: {job: orbit, instance: node-a, severity: warning}
+            exp_annotations:
+              summary: "Orbit metrics target is unavailable"
+              description: "Check the metrics listener, private network and process health. This does not prove that business tasks have stopped."
+  - name: startup_grace_and_stalled_component
+    interval: 1s
+    input_series:
+      - series: 'orbit_process_start_time_seconds{job="orbit",instance="node-a"}'
+        values: '0+0x180'
+      - series: 'orbit_runtime_batch_last_success_timestamp_seconds{job="orbit",instance="node-a",component="workflow"}'
+        values: '0+0x180'
+    alert_rule_test:
+      - eval_time: 60s
+        alertname: OrbitRuntimeBatchStalled
+        exp_alerts: []
+      - eval_time: 130s
+        alertname: OrbitRuntimeBatchStalled
+        exp_alerts:
+          - exp_labels: {job: orbit, instance: node-a, component: workflow, severity: warning}
+            exp_annotations:
+              summary: "Orbit background component has no recent successful batch"
+              description: "Inspect database availability, schema readiness and batch errors before changing limits or retrying business effects."
+  - name: healthy_batches_and_waits
+    interval: 1s
+    input_series:
+      - series: 'up{job="orbit",instance="node-a"}'
+        values: '1+0x400'
+      - series: 'orbit_process_start_time_seconds{job="orbit",instance="node-a"}'
+        values: '0+0x400'
+      - series: 'orbit_runtime_batch_last_success_timestamp_seconds{job="orbit",instance="node-a",component="workflow"}'
+        values: '0+1x400'
+      - series: 'orbit_runtime_batch_total{job="orbit",instance="node-a",component="workflow",outcome="error"}'
+        values: '0+0x400'
+      - series: 'orbit_db_wait_duration_seconds_total{job="orbit",instance="node-a"}'
+        values: '0+0x400'
+      - series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}'
+        values: '0+0x400'
+    alert_rule_test:
+      - {eval_time: 400s, alertname: OrbitMetricsUnavailable, exp_alerts: []}
+      - {eval_time: 400s, alertname: OrbitRuntimeBatchStalled, exp_alerts: []}
+      - {eval_time: 400s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []}
+      - {eval_time: 400s, alertname: OrbitDBPoolWaiting, exp_alerts: []}
+      - {eval_time: 400s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []}
+  - name: failing_batches_waits_and_invalid_samples_then_recover
+    interval: 1s
+    input_series:
+      - series: 'orbit_runtime_batch_total{job="orbit",instance="node-a",component="workflow",outcome="error"}'
+        values: '0+1x400 400+0x400'
+      - series: 'orbit_db_wait_duration_seconds_total{job="orbit",instance="node-a"}'
+        values: '0+1x400 400+0x400'
+      - series: 'orbit_observability_rejected_samples_total{job="orbit",instance="node-a"}'
+        values: '0+1x400 400+0x400'
+    alert_rule_test:
+      - eval_time: 400s
+        alertname: OrbitRuntimeBatchErrors
+        exp_alerts:
+          - exp_labels: {job: orbit, instance: node-a, component: workflow, outcome: error, severity: warning}
+            exp_annotations:
+              summary: "Orbit background batches are failing"
+              description: "Use process logs and durable task/workflow audit to investigate. Batch errors are not business failures."
+      - eval_time: 400s
+        alertname: OrbitDBPoolWaiting
+        exp_alerts:
+          - exp_labels: {job: orbit, instance: node-a, severity: warning}
+            exp_annotations:
+              summary: "Orbit is waiting for MySQL connections"
+              description: "Correlate pool use with database locks, query latency and storage load before increasing the connection limit."
+      - eval_time: 400s
+        alertname: OrbitInvalidMetricUpdates
+        exp_alerts:
+          - exp_labels: {job: orbit, instance: node-a, severity: warning}
+            exp_annotations:
+              summary: "Orbit rejected an internal metrics update"
+              description: "Check instrumentation changes for unsupported names, types, labels or counter regressions."
+      - {eval_time: 800s, alertname: OrbitRuntimeBatchErrors, exp_alerts: []}
+      - {eval_time: 800s, alertname: OrbitDBPoolWaiting, exp_alerts: []}
+      - {eval_time: 800s, alertname: OrbitInvalidMetricUpdates, exp_alerts: []}
--- a/deploy/monitoring/orbit.rules.yml
+++ b/deploy/monitoring/orbit.rules.yml
@@ -0,0 +1,43 @@
+groups:
+  - name: orbit.process
+    rules:
+      - alert: OrbitMetricsUnavailable
+        expr: up{job="orbit"} == 0
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Orbit metrics target is unavailable"
+          description: "Check the metrics listener, private network and process health. This does not prove that business tasks have stopped."
+      - alert: OrbitRuntimeBatchStalled
+        expr: (time() - orbit_runtime_batch_last_success_timestamp_seconds{job="orbit"} > 30) and on (job, instance) (time() - orbit_process_start_time_seconds{job="orbit"} > 60)
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Orbit background component has no recent successful batch"
+          description: "Inspect database availability, schema readiness and batch errors before changing limits or retrying business effects."
+      - alert: OrbitRuntimeBatchErrors
+        expr: rate(orbit_runtime_batch_total{job="orbit",outcome="error"}[5m]) > 0
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Orbit background batches are failing"
+          description: "Use process logs and durable task/workflow audit to investigate. Batch errors are not business failures."
+      - alert: OrbitDBPoolWaiting
+        expr: rate(orbit_db_wait_duration_seconds_total{job="orbit"}[5m]) > 0.1
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Orbit is waiting for MySQL connections"
+          description: "Correlate pool use with database locks, query latency and storage load before increasing the connection limit."
+      - alert: OrbitInvalidMetricUpdates
+        expr: increase(orbit_observability_rejected_samples_total{job="orbit"}[5m]) > 0
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Orbit rejected an internal metrics update"
+          description: "Check instrumentation changes for unsupported names, types, labels or counter regressions."
--- a/docs/08-development-plan.md
+++ b/docs/08-development-plan.md
@@ -30,7 +30,7 @@
 | P3 Pulse 完整能力 | 安全重试、延时、Cron/固定间隔、漏跑策略、计划版本、运行中取消、并发许可、配额/背压、可靠结果订阅、批量受理、独立任务核查/人工重试、查询筛选与事务Outbox接入 | 多实例领取/触发竞争、重试上限、取消竞争、队列隔离、积压恢复、事件重复交付 | 已完成平台功能阶段验收：[原范围逐项核对](reports/phase-3.md)、[本轮完整回归](reports/phase-3-delay-recovery.md)。补齐相对延迟、改参数关联示例和四种重试全停恢复场景；20个可运行示例全部通过。全平台生产门槛及SDK首版交付剩余项仍按P6–P7推进 |
 | P4 Weave 主流程 | SDK 定义构建与发布校验、实例、顺序/分支/结构化并行汇合、信号/定时等待、版本绑定 | 重启接续、早到/重复信号、并行竞争、内部事件重放、发布版本兼容；端到端通用流程 | 已完成平台功能阶段验收：[原功能及W01–W17逐项核对](reports/phase-4-5.md)、[版本/暂停/全停回归](reports/phase-4-5-races.md)、[提交故障边界](reports/phase-6-fault-boundaries.md)。研发发布、固定版本、顺序/分支/有限并行、信号/时间等待、公共API/SDK及可运行示例均已交付；生产拓扑与发布兼容继续P6/P7 |
 | P5 Weave 恢复与查询 | 有界查询、暂停继续、取消、人工核查恢复、控制门控与审计 | 轮次与尝试独立预算；暂停/开始/回执/超时竞争；恢复保留预算；未知结果不盲目重复 | 已完成平台功能阶段验收：[原功能核对](reports/phase-4-5.md)。有界查询、独立预算、暂停继续/取消、六类恢复、历史/筛选/事件/来源与SDK场景均已交付；真实竞争、COMMIT断线及进程全停保留原身份、截止和预算。运维、容量与首版发布继续P6/P7 |
-| P6 生产验证与运维 | 完整监控、凭据轮换、备份恢复、滚动升级、运行手册、可复现负载和故障工具 | 对照生产验收清单执行，报告环境/指标/遗留项；容量、RPO/RTO 按真实拓扑验证 | 进行中；已补[首批精确故障边界](reports/phase-6-fault-boundaries.md)，继续运行观测、凭据轮换及其余生产门槛 |
+| P6 生产验证与运维 | 完整监控、凭据轮换、备份恢复、滚动升级、运行手册、可复现负载和故障工具 | 对照生产验收清单执行，报告环境/指标/遗留项；容量、RPO/RTO 按真实拓扑验证 | 进行中；已补[精确故障边界](reports/phase-6-fault-boundaries.md)和[进程观测](reports/phase-6-observability.md)；继续业务积压/阶段延迟、凭据轮换及其余生产门槛 |
 | P7 首版交付核对 | SDK 兼容与发布准备、完整 API 手册、场景示例、全量回归及交付清单 | 新接入者按文档跑通全部示例；所有首版能力可追溯到实现与测试；报告实际生产边界 | 待开发 |

 SDK、文档与示例随 P1–P5 同步开发，P7 是全量核对，不是最后才开始补文档。
@@ -59,7 +59,7 @@ P4/P5定时等待与后续查询轮次的实现及回归记录见[时钟报告](
 ### P6 首批实施顺序

 1. 已补[首批可复现故障工具](reports/phase-6-fault-boundaries.md)：A04“派发已提交但消息尚未送达”、W01/W13事务提交结果不明，以及W16最终回执/旧PENDING结果重放/下一轮竞争。每个切点必须有实际数据库或进程证据，不能用响应错误推断事务是否已提交，也不能用普通SIGNAL竞争替代最终查询场景。
-2. 实现运行观测：受理/授权/完成及流程推进延迟、重试/核查/恢复、待处理与到期积压、连接/发送队列和数据库连接池。明确采集范围与开销；任务、流程、业务键和凭据不得作为无界指标标签。
+2. 已实现[基础运行观测](api/observability.md)：HTTP/WS、数据库连接池、后台批次和进程指标，以及可离线验证的告警规则；独立采集监听默认关闭，固定指标目录拒绝动态标签。仍需受理/授权/完成阶段延迟与持久吞吐、流程推进延迟、重试/核查/恢复量、待处理与到期积压、发送队列及生产采集/通知验证。
 3. 补齐凭据签发、轮换、撤销及Worker/订阅安全重绑，保持最小权限、管理审计与未知回执可恢复。
 4. 实现保留/归档与受保护记录核对，再执行备份恢复、兼容升级和六类容量负载。未确认交付、有效幂等身份、当前/历史执行证据和未终结流程不能被普通清理破坏。生产HA及RPO/RTO继续按实际拓扑验证，不缩减为本机多进程检查。

--- a/docs/README.md
+++ b/docs/README.md
@@ -2,7 +2,7 @@

 - 文档版本：0.2
 - 整理日期：2026-09-17
- 阶段：P0–P3平台功能已通过阶段回归；[P3逐项验收](reports/phase-3.md)保留原范围与证据边界。[相对延迟及重试全停恢复](reports/phase-3-delay-recovery.md)已通过完整回归。继续P4–P7，尚未完成全平台生产验收，数据库保持无外键。
+- 阶段：P0–P5平台功能已通过阶段回归；[P4/P5原范围核对](reports/phase-4-5.md)和[首批故障边界](reports/phase-6-fault-boundaries.md)保留证据边界。P6生产观测与运维、P7首版交付继续推进，尚未完成全平台生产验收，数据库保持无外键。
 - 适用范围：Orbit 统一平台，包含 Pulse 任务调度模块、Weave 流程编排模块与公共能力。平台面向多个业务系统复用。

 ## 阅读顺序
@@ -39,6 +39,7 @@
 | 数据库关联 | 禁止外键及级联删除，关联存在性与归属由服务端事务验证；保留必要索引、唯一约束和 CHECK |
 | 通信方向 | HTTP + WebSocket；生产使用 HTTPS / WSS |
 | HTTP 职责 | 任务提交、流程启动与信号、查询、计划管理、配置与管理操作 |
+| 运行观测 | 独立监听的`/metrics`提供无业务标识的固定基数运行指标；默认关闭，生产访问需内部网络控制 |
 | WS 职责 | Worker 接入、能力与容量上报、任务派发、续租、结果上报、取消通知 |
 | 执行位置 | 业务处理器位于业务项目自己的 Worker 中 |
 | 外部依赖范围 | 首版以 MySQL 为必需的外部存储依赖；运行不要求 Redis、独立消息队列或独立协调服务 |
--- a/docs/api/README.md
+++ b/docs/api/README.md
@@ -8,6 +8,7 @@
 | --- | --- | --- |
 | `GET /health/live` | 已实现 | [健康检查](health.md) |
 | `GET /health/ready` | 已实现，验证数据库、迁移记录及零外键要求 | [健康检查](health.md) |
+| 独立监听的 `GET /metrics` | 已实现基础运行指标，默认关闭 | [运行观测](observability.md) |
 | `/v1/tasks` 提交、查询、分页与待执行取消 | 已实现，P3扩展业务键与创建时间筛选 | [任务接口](tasks.md) / [查询契约](task-queries.md) |
 | `GET /v1/tasks/{task_id}/source` | 已实现普通、人工重试、计划固定版本及流程历史激活/轮次来源 | [来源查询](task-source.md) |
 | `POST /v1/tasks:batch` | P3 已实现逐项受理、独立幂等、部分结果与补交 | [批量任务](batch-tasks.md) |
--- a/docs/api/health.md
+++ b/docs/api/health.md
@@ -42,4 +42,4 @@ curl -i http://127.0.0.1:8080/health/ready

 对应 `Client.Liveness(ctx)` 和 `Client.Readiness(ctx)`，见 [SDK 接入](../sdk/README.md)。HTTP 503 返回 `*orbit.HTTPError`。

-ready 只表示数据库与迁移基础可用；不检查 Worker、任务积压、流程执行或业务效果。P1 启动时先校验迁移，缺失或 dirty 会拒绝启动；运行后数据库不可用时 ready 返回 503。SDK 默认请求超时为 10 秒，调用方可通过 context 设置更短期限。
+ready 只表示数据库与迁移基础可用；不检查 Worker、任务积压、流程执行或业务效果。P1 启动时先校验迁移，缺失或 dirty 会拒绝启动；运行后数据库不可用时 ready 返回 503。SDK 默认请求超时为 10 秒，调用方可通过 context 设置更短期限。运行指标见[观测接口](observability.md)，指标端点是否启用由 `ORBIT_METRICS_ADDR` 控制。
--- a/docs/api/observability.md
+++ b/docs/api/observability.md
@@ -0,0 +1,91 @@
+# 运行指标与告警接入
+
+状态：P6第一批进程观测已实现，指标、采集入口和下方规则均有回归。任务/流程的持久积压和业务阶段延迟仍待补齐；完整生产观测尚未验收。
+
+## 1. 启用独立采集端口
+
+默认不启动指标监听，也不创建指标注册器。显式配置后，业务端口与采集端口分别监听：
+
+```sh
+export ORBIT_HTTP_ADDR=127.0.0.1:8080
+export ORBIT_METRICS_ADDR=127.0.0.1:9090
+# 数据库凭据按现有秘密配置注入；先完成迁移。
+go run ./cmd/orbit
+```
+
+```sh
+curl -fsS http://127.0.0.1:9090/metrics
+```
+
+`ORBIT_METRICS_ADDR` 为空或未设置即关闭，`127.0.0.1:0` 可用于测试时分配空闲端口。非法地址、端口超界或绑定失败会拒绝启动；绑定失败不能静默降级为“看起来已经监控”。进程日志用 `metrics_address` 报告实际采集地址。业务端口 `/metrics` 返回404；指标端口不注册业务或健康接口。
+
+指标监听不使用业务凭据，没有多租户查询接口；只面向平台运维人员。部署到远程采集网络时，显式选择内部地址并配置网络策略或带鉴权/TLS的内部代理，不将此端口发布到公网。SDK不封装该端点；业务读取、核查和审计仍使用原有鉴权API。
+
+## 2. HTTP契约和数据范围
+
+仅 `GET /metrics`（及标准HEAD）提供采集，查询参数返回400。正文使用[Prometheus文本格式0.0.4](https://prometheus.io/docs/instrumenting/exposition_formats/)，Content-Type为 `text/plain; version=0.0.4; charset=utf-8`，Cache-Control为no-store。输出是本节点的运行快照，没有跨节点或跨指标原子快照保证。
+
+每节点同时最多2个采集请求，超出返回503及 `Retry-After: 1`，不排无限等待队列。独立HTTP服务器的读头/读/写/空闲超时分别为2/3/5/15秒，头部限制8 KiB。采集只访问进程内存，不查询业务表；DB池快照在启动时及之后每5秒读取 `database/sql.DBStats`，该调用不执行SQL。
+
+所有指标名称、类型、标签名和取值在代码目录中固定。未知名称、错误类型、非法标签、非有限数值和计数器倒退会被丢弃，并增加 `orbit_observability_rejected_samples_total`。新增指标需要修改目录和回归，不允许业务方动态注册标签。HTTP标签为8种方法、12种路由分类和6种状态分类，两组HTTP族各最多576种标签组合。禁止将任务/流程/租户ID、业务键、URL、参数、原始错误或凭据写入指标。
+
+格式化输出前只短暂复制注册器引用；不在全局记录锁下排序、格式化或写网络。直方图的桶、count、sum在同一局部锁下获取，保持单条直方图快照一致。指标在重启后重置，不能用于账务审计、幂等判断或恢复授权。
+
+## 3. 指标目录
+
+| 名称 | 类型 | 口径 |
+| --- | --- | --- |
+| `orbit_http_requests_total` | counter | HTTP响应完成次数；成功WS握手在Hijack完成时计入 |
+| `orbit_http_request_duration_seconds` | histogram | HTTP处理/升级握手耗时，不包含成功升级后的连接存续时间 |
+| `orbit_http_requests_in_flight` | gauge | 尚未返回或成功升级的HTTP处理器数量 |
+| `orbit_http_panics_total` | counter | 非正常返回的HTTP处理器次数；无响应时按5xx观测分类，但不声称客户端收到500；保留原panic传播及已发响应 |
+| `orbit_websocket_connections` | gauge | 成功升级且处理器尚未退出的WS连接，按worker_connect/event_connect区分 |
+| `orbit_websocket_connections_total` | counter | WS成功升级累计次数；并不表示协议hello、业务鉴权或任务执行成功 |
+| `orbit_runtime_batch_total` | counter | 后台批次返回次数，标签component及outcome |
+| `orbit_runtime_batch_duration_seconds` | histogram | 后台批次调用耗时，标签component |
+| `orbit_runtime_batch_last_success_timestamp_seconds` | gauge | 最后一次无错误批次返回的Unix时间；启动后未成功时为0 |
+| `orbit_db_connections_open` / `orbit_db_connections_in_use` / `orbit_db_connections_idle` | gauge | 本节点MySQL连接池当前连接数 |
+| `orbit_db_connections_max_open` | gauge | 池配置的最大连接数 |
+| `orbit_db_wait_count_total` / `orbit_db_wait_duration_seconds_total` | counter | 连接池累计等待次数/秒数，不是数据库锁等待 |
+| `orbit_process_start_time_seconds` / `orbit_process_uptime_seconds` | gauge | 本次指标注册器初始化Unix时间/单调时钟经过的秒数；初始化发生在数据库启动校验之后 |
+| `orbit_go_goroutines` | gauge | Go协程数，按5秒采样 |
+| `orbit_build_info` | gauge | 恒为1，go_version仅取当前编译运行时版本；不是Orbit发布版本 |
+| `orbit_observability_rejected_samples_total` | counter | 内部不合法指标更新次数，正常运行应为0 |
+
+HTTP方法为GET/POST/PATCH/PUT/DELETE/HEAD/OPTIONS/OTHER；路由为health_live、health_ready、worker_connect、event_connect、tasks、queues、workflows、workflow_definitions、schedules、subscriptions、application_resources、other；状态为1xx–5xx及other。路径值不会进入标签；分类不表示路由存在或授权通过。独立采集请求不混入业务HTTP计数。
+
+后台component为workflow、schedule、execution_recovery；outcome为success、error、cancelled。context.Canceled归cancelled，超时与其他错误归error，错误文本不导出。success只说明一批调用无错误返回，允许空批次；不能将它解释为持久任务成功量。已提交但确认不明的状态仍须按原身份核对。
+
+直方图有限桶边界固定为0.005、0.01、0.025、0.05、0.1、0.25、0.5、1、2.5、5、10秒，另有+Inf桶和sum/count。不得将高于10秒的有限分位估计当作精确延迟，后续容量验证需结合样本及合适的桶配置。
+
+## 4. 采集与告警例子
+
+如果已有兼容采集器，可配置job名为orbit（规则按此名筛选）：
+
+```yaml
+scrape_configs:
+  - job_name: orbit
+    scrape_interval: 15s
+    scrape_timeout: 3s
+    static_configs:
+      - targets: ["127.0.0.1:9090"]
+```
+
+示例地址仅适用于同机采集；远程采集需替换为受保护的内部目标。平台运行不依赖采集器、Prometheus或额外数据库。
+
+[五条告警规则](../../deploy/monitoring/orbit.rules.yml)和[离线规则测试](../../deploy/monitoring/orbit.rules.test.yml)包含以下场景：
+
+- 目标连续2分钟采集失败：检查指标监听、网络和进程，不直接判断业务任务停止。
+- 启动60秒后，某后台组件超过30秒没有成功批次并持续1分钟：结合readiness、批次错误和数据库状态调查。
+- 批次错误速率持续为正2分钟：查日志和原任务/流程审计，不把数据库故障改判成业务失败。
+- 池累计等待时间增长超过0.1秒/秒并持续5分钟：联查锁等待、慢SQL、CPU和存储，不自动提高连接数。
+- 内部非法指标更新持续出现：检查观测代码契约，不能忽略丢弃的样本。
+
+这些阈值是可修改的运行基线，尚未通过生产负载调优；不构成SLO或可用性承诺。规则测试覆盖触发、启动宽限、健康不报警和错误停止后的解除，未发送实际通知或创建外部监控服务。
+
+```sh
+promtool check rules deploy/monitoring/orbit.rules.yml
+promtool test rules deploy/monitoring/orbit.rules.test.yml
+```
+
+HTTP失败率和延迟可按route/status_class统计；受理接口响应成功与已完成业务应分别判断。可靠结果量、排队/授权/完成延迟、流程Inbox积压、发送队列、磁盘/复制等仍须补充专门指标；具体当前证据见[验证报告](../reports/phase-6-observability.md)。
--- a/docs/reports/phase-6-observability.md
+++ b/docs/reports/phase-6-observability.md
@@ -0,0 +1,83 @@
+# P6：进程指标、采集隔离与告警规则
+
+日期：2026-09-18。范围为运行观测第一批，交付HTTP/WS、连接池、后台批次、进程指标和可离线验证的告警规则。P6仍在进行，任务/流程持久积压和阶段延迟没有被本批替代。
+
+## 实现和边界
+
+- [指标目录与注册器](../../internal/observability/metrics.go)只接受[代码中固定的名称、类型及标签集合](../../internal/observability/catalog.go)。没有通用动态注册接口；非法更新丢弃并计数，未知指标不创建族，同名不同类型不会触发空指针或混写。
+- [HTTP包装器](../../internal/observability/http.go)保留原始状态、流式写入、Hijack及panic传播。成功WS升级在握手完成时计一次HTTP，连接存续单独计数；普通HTTP延迟不包含长连接生命期。103等临时响应不阻止最终响应。
+- [运行采样](../../internal/observability/runtime.go)读取本进程池状态，不在采集路径执行SQL；批次开始/返回时间只观测原调用，不改变事务、重试或状态语义。计数器初始零值和最后成功时间支持“尚未成功”和“随后停滞”的判断。
+- [进程入口](../../cmd/orbit/main.go)使用独立指标监听，默认关闭；启用后业务端口仍无`/metrics`，采集端口不提供业务API。服务启动、绑定失败和SIGTERM退出明确处理两种监听。
+- [接口及运维手册](../api/observability.md)说明所有名称、单位、标签、采样间隔、超时、限流和剩余范围。[告警规则](../../deploy/monitoring/orbit.rules.yml)是可修改的起始阈值，未对接或发送实际外部通知。
+
+模型仍为36个、直接持久化字段312个、迁移版本25；未改模型/迁移、没有外键或级联。没有新增Go模块依赖、SDK耦合或运行必需服务。官方promtool仅作为一次性离线验证器。
+
+## 针对性证据
+
+[指标竞态与约束回归](../../internal/observability/metrics_test.go)覆盖16个并发写入者及同时采集；每条直方图的桶/count/sum保持一致。3000个动态名称和3000个动态标签均被拒绝；每种HTTP方法/路由/状态组合都写入后，原序列仍继续累计。类型冲突、计数器倒退、非有限数字和错误文本泄露均有直接断言。
+
+[真实TLS WebSocket回归](../../internal/observability/http_test.go)通过现有coder/websocket实际升级，保持连接未关闭时确认HTTP计数完成、HTTP在途为0、WS在途为1；关闭后WS归0且HTTP不重计。另有真实HTTP 103→201及Flush完整响应检查。
+
+[进程专项](../../cmd/orbit/metrics_integration_test.go)使用一次性MySQL和编译为race模式的Orbit二进制：分别启用/关闭指标，验证readiness、业务请求、独立采集、DB池及自然运行的后台批次、请求数据不泄露，最后SIGTERM并确认监听全部关闭。分配优化后的定向入口退出0，5.696秒；这是测试耗时，不是延迟指标。
+
+最终`make fmt test vet build`退出0。未配置外部测试环境时的普通go test会跳过MySQL/进程场景，不能用这些跳过项代替专项与标准集成执行。
+
+## 独立格式与规则验证
+
+全量有限HTTP组合的文本夹具为926666字节，使用本地已有官方promtool 3.5.0（linux/arm64，revision 8be3a9560fbdd18a94dedec4b747c35178177202）执行`check metrics`，退出0且无格式/命名告警。固定验证镜像摘要为`prom/prometheus@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996`，运行时关闭网络；此版本仅描述实际验证器，不是生产部署版本建议。
+
+五条告警规则通过`check rules`；[规则测试](../../deploy/monitoring/orbit.rules.test.yml)通过`test rules`，验证目标失联、启动宽限、组件停滞、健康不报警、错误/等待/非法采样触发及恢复后的解除。
+
+```sh
+# 不要求平台在运行，可导出有限指标组合供独立校验。
+ORBIT_TEST_METRICS_OUTPUT=/tmp/orbit-metrics.txt go test -count=1 -run '^TestExportExpositionFixture$' ./internal/observability
+promtool check metrics < /tmp/orbit-metrics.txt
+promtool check rules deploy/monitoring/orbit.rules.yml
+promtool test rules deploy/monitoring/orbit.rules.test.yml
+```
+
+格式契约以[官方文本说明](https://prometheus.io/docs/instrumenting/exposition_formats/)为依据，框架自身不要求部署Prometheus。
+
+## 全量回归与开销记录
+
+标准`make integration`对应的`GO=… DOCKER=… sh scripts/test-mysql.sh`执行完毕，退出0。标准脚本显式为测试子进程分配独立随机指标端口，让真实SDK/WSS及进程恢复链路在指标启用时执行；进程专项另验证默认关闭场景。环境为Go 1.27.1、darwin/arm64、Docker MySQL 8.4.10。
+
+| 范围 | 本次终态及耗时 |
+| --- | --- |
+| 存储、升级、事务竞争和COMMIT切点 | PASS，280.152秒 |
+| 指标独立监听开/关与退出 | PASS，4.620秒 |
+| 流程运行器真实节点恢复 | PASS，48.262秒 |
+| SDK任务/Worker、调度、队列、批量、查询、订阅/消费、恢复和流程接口 | 各组PASS；23个SDK示例均运行成功 |
+| 独立业务Inbox / Outbox模块及demo | PASS，36.005 / 34.409秒；demo成功 |
+| Worker / 事件消费进程故障组 | PASS，127.765秒；包含A04未送达派发、自然租约到期、ACK丢确认和四种重试全停恢复 |
+| 公共SDK流程暂停全停恢复 | PASS，13.789秒；两旧节点全部退出后由两新节点接续 |
+
+无外键升级专项再次确认：v7的70条历史外键全部移除，323项原索引组成不变，原任务保留，启动检查拒绝外键漂移。当前schema版本25；业务Inbox和Outbox库也保持零外键。早期迁移文件保留原校验和，通过000008前向移除，禁止将历史DDL误解为当前运行结构。
+
+全量执行结束后的微基准发现标签转义器重复分配，随后唯一生产代码调整是复用一个并发安全的`strings.Replacer`，转义规则不变。该调整后重新通过`make fmt test vet build`，并重新导出完整指标夹具校验格式；真实race二进制的监听/采集/退出专项复验通过，退出0、5.696秒。上表为该分配优化之前的全量结果，不将它写成优化后的重复全量执行。
+
+可复现中间件微基准：
+
+```sh
+go test -run '^$' -bench '^BenchmarkHTTPMiddleware$' -benchmem -benchtime=1s -count=3 -cpu=1,8 ./internal/observability
+```
+
+Apple M2 Max、12核、32 GiB；全量集成停止后单独运行，退出0。空204处理器复用请求和响应对象，使用`RunParallel`，下表为三次测量范围。
+
+| 指标启用 | GOMAXPROCS | ns/op | B/op | allocs/op |
+| --- | --- | --- | --- | --- |
+| 否 | 1 | 1.997–2.122 | 0 | 0 |
+| 否 | 8 | 0.2581–0.2747 | 0 | 0 |
+| 是 | 1 | 1090–1126 | 704 | 19 |
+| 是 | 8 | 1404–1482 | 704 | 19 |
+
+优化前启用时为41272 B/op、61 allocs/op；共享转义器消除了每次构建查找表的分配。并行ns/op是总工作量的摊销值，不是单请求延迟；启用后的锁竞争仍需结合真实负载检查。这只是中间件本机开销，不包含网络、MySQL、并发抓取或业务处理，不能换算成受理吞吐或百万企业容量。
+
+文档检查通过：1204个本地链接、68段JSON示例，未发现失效链接、非法JSON或接入方项目标识；`git diff --check`通过。
+
+## 后续原范围
+
+1. 受理/授权/完成的持久吞吐、阶段延迟、重试/核查/恢复量、流程Inbox至推进延迟；区分提交成功、回执丢失、重报和事件重放，不能把HTTP成功数直接当作业务量。
+2. 持久待处理、到期积压与发送队列。使用有上限/超时的后台采样及明确的新鲜度/失败状态，不能让每次抓取扫描全库；状态指标和多节点重复采集的聚合口径须先明确。
+3. 慢消费者、重连风暴、业务负载下观测开销、真实采集/通知演练，以及数据库锁/磁盘/复制指标和跨故障域验证。
+4. 凭据轮换、保留/归档、备份恢复、容量六类负载和P7版本发布继续按[持续计划](../08-development-plan.md)执行。
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -9,15 +9,16 @@ import (
 )

 type Config struct {
-	HTTPAddr string
-	MySQLDSN string
-	MaxOpen  int
-	MaxIdle  int
+	HTTPAddr    string
+	MySQLDSN    string
+	MaxOpen     int
+	MaxIdle     int
+	MetricsAddr string
 }

 func Load() (Config, error) { return FromEnv(os.Getenv) }
 func FromEnv(get func(string) string) (Config, error) {
-	c := Config{HTTPAddr: get("ORBIT_HTTP_ADDR"), MySQLDSN: get("ORBIT_MYSQL_DSN"), MaxOpen: 20, MaxIdle: 5}
+	c := Config{HTTPAddr: get("ORBIT_HTTP_ADDR"), MySQLDSN: get("ORBIT_MYSQL_DSN"), MaxOpen: 20, MaxIdle: 5, MetricsAddr: get("ORBIT_METRICS_ADDR")}
 	if c.HTTPAddr == "" {
 		c.HTTPAddr = "127.0.0.1:8080"
 	}
@@ -42,5 +43,12 @@ func FromEnv(get func(string) string) (Config, error) {
 	if c.MaxOpen <= 0 || c.MaxIdle > c.MaxOpen {
 		return Config{}, errors.New("database pool requires 0 <= max_idle <= max_open and max_open > 0")
 	}
+	if c.MetricsAddr != "" {
+		_, port, err := net.SplitHostPort(c.MetricsAddr)
+		n, parseErr := strconv.Atoi(port)
+		if err != nil || parseErr != nil || n < 0 || n > 65535 {
+			return Config{}, errors.New("invalid ORBIT_METRICS_ADDR")
+		}
+	}
 	return c, nil
 }
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -0,0 +1,25 @@
+package config
+
+import "testing"
+
+func TestMetricsAddressIsExplicitAndValidated(t *testing.T) {
+	base := map[string]string{"ORBIT_MYSQL_DSN": "private-dsn"}
+	get := func(k string) string { return base[k] }
+	c, err := FromEnv(get)
+	if err != nil || c.MetricsAddr != "" {
+		t.Fatal("metrics must be disabled by default", err)
+	}
+	for _, addr := range []string{"127.0.0.1:0", "[::1]:9090", "0.0.0.0:9090"} {
+		base["ORBIT_METRICS_ADDR"] = addr
+		c, err = FromEnv(get)
+		if err != nil || c.MetricsAddr != addr {
+			t.Fatal("valid explicit address rejected", addr, err)
+		}
+	}
+	for _, addr := range []string{"bad", ":secret", "127.0.0.1:-1", ":65536", "http://127.0.0.1:9000"} {
+		base["ORBIT_METRICS_ADDR"] = addr
+		if _, err = FromEnv(get); err == nil {
+			t.Fatal("invalid address accepted", addr)
+		}
+	}
+}
--- a/internal/observability/catalog.go
+++ b/internal/observability/catalog.go
@@ -0,0 +1,37 @@
+package observability
+
+import "runtime"
+
+var methods = []string{"GET", "POST", "PATCH", "PUT", "DELETE", "HEAD", "OPTIONS", "OTHER"}
+var routes = []string{"health_live", "health_ready", "worker_connect", "event_connect", "tasks", "queues", "workflows", "workflow_definitions", "schedules", "subscriptions", "application_resources", "other"}
+var statusClasses = []string{"1xx", "2xx", "3xx", "4xx", "5xx", "other"}
+var components = []string{"workflow", "schedule", "execution_recovery"}
+
+// Every exported family is declared here, including each allowed label value.
+// Adding a metric therefore requires a code review; no user identifier is valid.
+func catalog() map[string]descriptor {
+	httpLabels := map[string][]string{"method": methods, "route": routes, "status_class": statusClasses}
+	c := map[string]descriptor{
+		"orbit_http_requests_total":                          {counter, "Completed HTTP responses, including successful upgrade handshakes", httpLabels},
+		"orbit_http_request_duration_seconds":                {histogram, "HTTP response or upgrade handshake duration in seconds", httpLabels},
+		"orbit_http_requests_in_flight":                      {gauge, "HTTP handlers before completion or successful hijack", nil},
+		"orbit_http_panics_total":                            {counter, "HTTP handlers which panicked, before or after upgrade", nil},
+		"orbit_websocket_connections":                        {gauge, "Upgraded WebSocket handlers currently active", map[string][]string{"route": {"worker_connect", "event_connect"}}},
+		"orbit_websocket_connections_total":                  {counter, "Successful WebSocket upgrades", map[string][]string{"route": {"worker_connect", "event_connect"}}},
+		"orbit_runtime_batch_total":                          {counter, "Completed background batches by outcome", map[string][]string{"component": components, "outcome": {"success", "error", "cancelled"}}},
+		"orbit_runtime_batch_duration_seconds":               {histogram, "Background batch duration in seconds", map[string][]string{"component": components}},
+		"orbit_runtime_batch_last_success_timestamp_seconds": {gauge, "Unix time of last successful background batch, zero before first success", map[string][]string{"component": components}},
+		"orbit_process_start_time_seconds":                   {gauge, "Unix time when the Orbit metrics registry was started", nil},
+		"orbit_process_uptime_seconds":                       {gauge, "Monotonic elapsed time since metrics registry startup", nil},
+		"orbit_go_goroutines":                                {gauge, "Current Go goroutine count", nil},
+		"orbit_build_info":                                   {gauge, "Orbit Go runtime build information", map[string][]string{"go_version": {runtime.Version()}}},
+	}
+	for name, help := range map[string]string{
+		"open": "Open MySQL connections", "in_use": "MySQL connections currently in use", "idle": "Idle MySQL connections", "max_open": "Configured maximum open MySQL connections",
+	} {
+		c["orbit_db_connections_"+name] = descriptor{gauge, help, nil}
+	}
+	c["orbit_db_wait_count_total"] = descriptor{counter, "Cumulative waits for a MySQL connection", nil}
+	c["orbit_db_wait_duration_seconds_total"] = descriptor{counter, "Cumulative seconds waiting for a MySQL connection", nil}
+	return c
+}
--- a/internal/observability/http.go
+++ b/internal/observability/http.go
@@ -0,0 +1,154 @@
+package observability
+
+import (
+	"bufio"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Middleware counts HTTP completion once. A successful upgrade completes HTTP
+// at Hijack; the still-running WebSocket handler is counted separately.
+func (r *Registry) Middleware(next http.Handler) http.Handler {
+	if r == nil {
+		return next
+	}
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		started := time.Now()
+		route := routeClass(req.URL.Path)
+		r.AddGauge("orbit_http_requests_in_flight", 1, nil)
+		cw := &captureWriter{ResponseWriter: w}
+		completed := false
+		ws := false
+		complete := func(status int) {
+			if completed {
+				return
+			}
+			completed = true
+			class := "other"
+			if status >= 100 && status < 600 {
+				class = strconv.Itoa(status/100) + "xx"
+			}
+			labels := Labels{"method": boundedMethod(req.Method), "route": route, "status_class": class}
+			r.AddGauge("orbit_http_requests_in_flight", -1, nil)
+			r.AddCounter("orbit_http_requests_total", 1, labels)
+			r.ObserveHistogram("orbit_http_request_duration_seconds", time.Since(started).Seconds(), labels)
+		}
+		cw.onHijack = func() {
+			complete(cw.status)
+			if cw.status == 101 && (route == "worker_connect" || route == "event_connect") {
+				ws = true
+				r.AddGauge("orbit_websocket_connections", 1, Labels{"route": route})
+				r.AddCounter("orbit_websocket_connections_total", 1, Labels{"route": route})
+			}
+		}
+		returned := false
+		defer func() {
+			status := cw.status
+			if !returned {
+				r.AddCounter("orbit_http_panics_total", 1, nil)
+				if status == 0 {
+					status = 500
+				}
+			}
+			if status == 0 {
+				status = 200
+			}
+			complete(status)
+			if ws {
+				r.AddGauge("orbit_websocket_connections", -1, Labels{"route": route})
+			}
+			// No recover: net/http retains its panic/ErrAbortHandler behavior.
+		}()
+		next.ServeHTTP(cw, req)
+		returned = true
+	})
+}
+func boundedMethod(method string) string {
+	for _, v := range methods {
+		if method == v {
+			return v
+		}
+	}
+	return "OTHER"
+}
+func routeClass(path string) string {
+	switch path {
+	case "/health/live":
+		return "health_live"
+	case "/health/ready":
+		return "health_ready"
+	case "/v1/worker/connect":
+		return "worker_connect"
+	case "/v1/events/connect":
+		return "event_connect"
+	}
+	for _, p := range []struct{ prefix, route string }{
+		{"/v1/tasks", "tasks"}, {"/v1/queues", "queues"}, {"/v1/workflows", "workflows"},
+		{"/v1/workflow-definitions", "workflow_definitions"}, {"/v1/schedules", "schedules"},
+		{"/v1/subscriptions", "subscriptions"}, {"/v1/application/resources", "application_resources"},
+	} {
+		if path == p.prefix || strings.HasPrefix(path, p.prefix+"/") || strings.HasPrefix(path, p.prefix+":") {
+			return p.route
+		}
+	}
+	return "other"
+}
+
+type captureWriter struct {
+	http.ResponseWriter
+	status   int
+	onHijack func()
+}
+
+func (w *captureWriter) WriteHeader(status int) {
+	if w.status != 0 {
+		return
+	}
+	// Forward informational responses without preventing a later final response.
+	if status >= 100 && status < 200 && status != 101 {
+		w.ResponseWriter.WriteHeader(status)
+		return
+	}
+	w.ResponseWriter.WriteHeader(status)
+	w.status = status
+}
+func (w *captureWriter) Write(p []byte) (int, error) {
+	if w.status == 0 {
+		w.WriteHeader(200)
+	}
+	return w.ResponseWriter.Write(p)
+}
+func (w *captureWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter }
+func (w *captureWriter) FlushError() error {
+	if w.status == 0 {
+		w.WriteHeader(200)
+	}
+	return http.NewResponseController(w.ResponseWriter).Flush()
+}
+func (w *captureWriter) Flush() { _ = w.FlushError() }
+func (w *captureWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	c, b, err := http.NewResponseController(w.ResponseWriter).Hijack()
+	if err == nil && w.onHijack != nil {
+		w.onHijack()
+	}
+	return c, b, err
+}
+func (w *captureWriter) Push(target string, opts *http.PushOptions) error {
+	if p, ok := w.ResponseWriter.(http.Pusher); ok {
+		return p.Push(target, opts)
+	}
+	return http.ErrNotSupported
+}
+func (w *captureWriter) ReadFrom(src io.Reader) (int64, error) {
+	if w.status == 0 {
+		w.WriteHeader(200)
+	}
+	if rf, ok := w.ResponseWriter.(io.ReaderFrom); ok {
+		return rf.ReadFrom(src)
+	}
+	return io.Copy(w.ResponseWriter, src)
+}
--- a/internal/observability/http_test.go
+++ b/internal/observability/http_test.go
@@ -0,0 +1,93 @@
+package observability
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/coder/websocket"
+)
+
+func TestWebSocketHandshakeIsNotConnectionLifetime(t *testing.T) {
+	r := NewRegistry()
+	released := make(chan struct{})
+	done := make(chan struct{})
+	h := r.Middleware(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		defer close(done)
+		c, err := websocket.Accept(w, req, nil)
+		if err != nil {
+			return
+		}
+		defer c.CloseNow()
+		<-released
+	}))
+	server := httptest.NewTLSServer(h)
+	defer server.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	c, _, err := websocket.Dial(ctx, strings.Replace(server.URL, "https://", "wss://", 1)+"/v1/worker/connect", &websocket.DialOptions{HTTPClient: server.Client()})
+	if err != nil {
+		close(released)
+		t.Fatal(err)
+	}
+	defer c.CloseNow()
+	// Dial returns after the upgrade response, while the server handler stays open.
+	deadline := time.Now().Add(time.Second)
+	for !strings.Contains(r.Render(), `orbit_websocket_connections{route="worker_connect"} 1`) {
+		if time.Now().After(deadline) {
+			close(released)
+			t.Fatal("upgrade not observed")
+		}
+		time.Sleep(time.Millisecond)
+	}
+	out := r.Render()
+	close(released)
+	<-done
+	if !strings.Contains(out, `orbit_http_requests_total{method="GET",route="worker_connect",status_class="1xx"} 1`) || !strings.Contains(out, "orbit_http_requests_in_flight 0\n") {
+		t.Fatal("upgrade counted as unfinished HTTP")
+	}
+	// done closes inside the handler; wait for middleware's defer to publish closure.
+	deadline = time.Now().Add(time.Second)
+	for !strings.Contains(r.Render(), `orbit_websocket_connections{route="worker_connect"} 0`) {
+		if time.Now().After(deadline) {
+			t.Fatal("connection gauge leaked")
+		}
+		time.Sleep(time.Millisecond)
+	}
+	if !strings.Contains(r.Render(), `orbit_http_requests_total{method="GET",route="worker_connect",status_class="1xx"} 1`) {
+		t.Fatal("upgrade counted twice")
+	}
+}
+func TestInformationalResponseAndStreamingPreserveHTTP(t *testing.T) {
+	r := NewRegistry()
+	server := httptest.NewServer(r.Middleware(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		w.WriteHeader(103)
+		w.WriteHeader(201)
+		_, _ = io.WriteString(w, "first")
+		if err := http.NewResponseController(w).Flush(); err != nil {
+			panic(err)
+		}
+		_, _ = io.WriteString(w, "second")
+	})))
+	defer server.Close()
+	resp, err := server.Client().Get(server.URL + "/v1/tasks")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil || resp.StatusCode != 201 || string(body) != "firstsecond" {
+		t.Fatal("wrapper changed HTTP response", resp.StatusCode, err)
+	}
+	deadline := time.Now().Add(time.Second)
+	for !strings.Contains(r.Render(), `route="tasks",status_class="2xx"} 1`) {
+		if time.Now().After(deadline) {
+			t.Fatal("103 blocked final status metric")
+		}
+		time.Sleep(time.Millisecond)
+	}
+}
--- a/internal/observability/metrics.go
+++ b/internal/observability/metrics.go
@@ -0,0 +1,272 @@
+// Package observability exposes process-local operational metrics. It does not
+// participate in durable state, authorization or execution decisions.
+package observability
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type Labels map[string]string
+
+// Registry accepts only the closed catalog below: neither names nor label
+// values can grow from caller-controlled request or business data.
+type Registry struct {
+	mu       sync.Mutex
+	families map[string]*family
+	rejected atomic.Uint64
+	scrape   chan struct{}
+	started  time.Time
+}
+type metricKind string
+
+const (
+	counter   metricKind = "counter"
+	gauge     metricKind = "gauge"
+	histogram metricKind = "histogram"
+)
+
+type descriptor struct {
+	kind   metricKind
+	help   string
+	labels map[string][]string
+}
+type family struct {
+	desc   descriptor
+	series map[string]*series
+}
+type series struct {
+	labels string // Immutable, already escaped and validated against the catalog.
+	value  atomic.Uint64
+	hist   distribution
+}
+type distribution struct {
+	mu      sync.Mutex
+	buckets [11]uint64
+	count   uint64
+	sum     float64
+}
+
+var bounds = [...]float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}
+
+func NewRegistry() *Registry {
+	r := &Registry{families: make(map[string]*family), scrape: make(chan struct{}, 2), started: time.Now()}
+	for name, d := range catalog() {
+		f := &family{desc: d, series: make(map[string]*series)}
+		if len(d.labels) == 0 {
+			f.series[""] = &series{}
+		}
+		r.families[name] = f
+	}
+	r.initializeProcess()
+	return r
+}
+func finite(v float64) bool { return !math.IsNaN(v) && !math.IsInf(v, 0) }
+func (r *Registry) lookup(name string, kind metricKind, labels Labels) *series {
+	if r == nil {
+		return nil
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	f, ok := r.families[name]
+	if !ok || f.desc.kind != kind || len(labels) != len(f.desc.labels) {
+		r.rejected.Add(1)
+		return nil
+	}
+	keys := make([]string, 0, len(labels))
+	for k, v := range labels {
+		valid := false
+		for _, allowed := range f.desc.labels[k] {
+			if v == allowed {
+				valid = true
+				break
+			}
+		}
+		if !valid {
+			r.rejected.Add(1)
+			return nil
+		}
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	var parts []string
+	for _, k := range keys {
+		parts = append(parts, k+`="`+escape(labels[k])+`"`)
+	}
+	encoded := ""
+	if len(parts) > 0 {
+		encoded = "{" + strings.Join(parts, ",") + "}"
+	}
+	s := f.series[encoded]
+	if s == nil {
+		s = &series{labels: encoded}
+		f.series[encoded] = s
+	}
+	return s
+}
+func (r *Registry) reject() {
+	if r != nil {
+		r.rejected.Add(1)
+	}
+}
+func (r *Registry) AddCounter(name string, delta float64, labels Labels) {
+	if !finite(delta) || delta < 0 {
+		r.reject()
+		return
+	}
+	if s := r.lookup(name, counter, labels); s != nil {
+		add(&s.value, delta)
+	}
+}
+
+// SetCounter only accepts a monotone source, such as database/sql.DBStats.
+func (r *Registry) SetCounter(name string, value float64, labels Labels) {
+	if !finite(value) || value < 0 {
+		r.reject()
+		return
+	}
+	if s := r.lookup(name, counter, labels); s != nil {
+		for {
+			old := s.value.Load()
+			if value < math.Float64frombits(old) {
+				r.reject()
+				return
+			}
+			if s.value.CompareAndSwap(old, math.Float64bits(value)) {
+				return
+			}
+		}
+	}
+}
+func (r *Registry) AddGauge(name string, delta float64, labels Labels) {
+	if !finite(delta) {
+		r.reject()
+		return
+	}
+	if s := r.lookup(name, gauge, labels); s != nil {
+		add(&s.value, delta)
+	}
+}
+func (r *Registry) SetGauge(name string, value float64, labels Labels) {
+	if !finite(value) {
+		r.reject()
+		return
+	}
+	if s := r.lookup(name, gauge, labels); s != nil {
+		s.value.Store(math.Float64bits(value))
+	}
+}
+func (r *Registry) ObserveHistogram(name string, value float64, labels Labels) {
+	if !finite(value) || value < 0 {
+		r.reject()
+		return
+	}
+	s := r.lookup(name, histogram, labels)
+	if s == nil {
+		return
+	}
+	h := &s.hist
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	for i, b := range bounds {
+		if value <= b {
+			h.buckets[i]++
+		}
+	}
+	h.count++
+	h.sum += value
+}
+func add(dst *atomic.Uint64, delta float64) {
+	for {
+		old := dst.Load()
+		next := math.Float64frombits(old) + delta
+		if !finite(next) {
+			return
+		}
+		if dst.CompareAndSwap(old, math.Float64bits(next)) {
+			return
+		}
+	}
+}
+func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	w.Header().Set("Cache-Control", "no-store")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	if req.URL.RawQuery != "" {
+		http.Error(w, "metrics does not accept query parameters", 400)
+		return
+	}
+	select {
+	case r.scrape <- struct{}{}:
+		defer func() { <-r.scrape }()
+	default:
+		w.Header().Set("Retry-After", "1")
+		http.Error(w, "metrics busy", 503)
+		return
+	}
+	w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+	_, _ = io.WriteString(w, r.Render())
+}
+
+// Render snapshots pointers under the registry lock; formatting and network I/O
+// never hold that lock. Each histogram's buckets/count/sum use one local snapshot.
+func (r *Registry) Render() string {
+	type view struct {
+		name   string
+		desc   descriptor
+		series []*series
+	}
+	var views []view
+	r.mu.Lock()
+	for n, f := range r.families {
+		v := view{name: n, desc: f.desc}
+		for _, s := range f.series {
+			v.series = append(v.series, s)
+		}
+		views = append(views, v)
+	}
+	r.mu.Unlock()
+	sort.Slice(views, func(i, j int) bool { return views[i].name < views[j].name })
+	var b strings.Builder
+	for _, v := range views {
+		fmt.Fprintf(&b, "# HELP %s %s\n# TYPE %s %s\n", v.name, v.desc.help, v.name, v.desc.kind)
+		sort.Slice(v.series, func(i, j int) bool { return v.series[i].labels < v.series[j].labels })
+		for _, s := range v.series {
+			if v.desc.kind != histogram {
+				fmt.Fprintf(&b, "%s%s %s\n", v.name, s.labels, number(math.Float64frombits(s.value.Load())))
+				continue
+			}
+			h := &s.hist
+			h.mu.Lock()
+			buckets, count, sum := h.buckets, h.count, h.sum
+			h.mu.Unlock()
+			for i, bound := range bounds {
+				fmt.Fprintf(&b, "%s_bucket%s %d\n", v.name, withLabel(s.labels, "le", number(bound)), buckets[i])
+			}
+			fmt.Fprintf(&b, "%s_bucket%s %d\n%s_sum%s %s\n%s_count%s %d\n", v.name, withLabel(s.labels, "le", "+Inf"), count, v.name, s.labels, number(sum), v.name, s.labels, count)
+		}
+	}
+	b.WriteString("# HELP orbit_observability_rejected_samples_total Invalid internal metric updates rejected\n# TYPE orbit_observability_rejected_samples_total counter\n")
+	fmt.Fprintf(&b, "orbit_observability_rejected_samples_total %d\n", r.rejected.Load())
+	return b.String()
+}
+
+// Replacer is safe for concurrent use. Construct it once rather than allocating
+// its lookup tables for every label of every observed request.
+var labelEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`)
+
+func escape(s string) string  { return labelEscaper.Replace(s) }
+func number(v float64) string { return strconv.FormatFloat(v, 'g', -1, 64) }
+func withLabel(labels, k, v string) string {
+	if labels == "" {
+		return "{" + k + `="` + escape(v) + `"}`
+	}
+	return labels[:len(labels)-1] + "," + k + `="` + escape(v) + `"}`
+}
--- a/internal/observability/metrics_test.go
+++ b/internal/observability/metrics_test.go
@@ -0,0 +1,208 @@
+package observability
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"math"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestConcurrentScrapesAndUpdates(t *testing.T) {
+	r := NewRegistry()
+	labels := Labels{"method": "GET", "route": "tasks", "status_class": "2xx"}
+	var wg sync.WaitGroup
+	for i := 0; i < 16; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for n := 0; n < 100; n++ {
+				r.AddCounter("orbit_http_requests_total", 1, labels)
+				r.ObserveHistogram("orbit_http_request_duration_seconds", .01, labels)
+				if n%10 == 0 {
+					r.Render()
+				}
+			}
+		}()
+	}
+	wg.Wait()
+	out := r.Render()
+	for _, want := range []string{
+		`orbit_http_requests_total{method="GET",route="tasks",status_class="2xx"} 1600`,
+		`orbit_http_request_duration_seconds_count{method="GET",route="tasks",status_class="2xx"} 1600`,
+		`orbit_http_request_duration_seconds_bucket{method="GET",route="tasks",status_class="2xx",le="0.005"} 0`,
+		`orbit_http_request_duration_seconds_bucket{method="GET",route="tasks",status_class="2xx",le="+Inf"} 1600`,
+	} {
+		if !strings.Contains(out, want) {
+			t.Fatal("lost samples or inconsistent histogram", want)
+		}
+	}
+}
+func TestCatalogRejectsNamesTypesAndAllDynamicLabels(t *testing.T) {
+	r := NewRegistry()
+	families := len(r.families)
+	for i := 0; i < 3000; i++ {
+		r.AddCounter("untrusted_"+strconv.Itoa(i), 1, nil)
+		r.AddCounter("orbit_http_requests_total", 1, Labels{"method": "GET", "route": "private-" + strconv.Itoa(i), "status_class": "2xx"})
+	}
+	r.ObserveHistogram("orbit_http_requests_in_flight", 1, nil) // Must not panic on a type mismatch.
+	r.SetGauge("orbit_http_requests_total", 1, nil)
+	r.SetGauge("orbit_go_goroutines", 1, Labels{"tenant": "private"})
+	if len(r.families) != families || r.rejected.Load() != 6003 {
+		t.Fatal("unbounded family creation or invalid samples accepted")
+	}
+	if out := r.Render(); strings.Contains(out, "private") || strings.Contains(out, "untrusted") {
+		t.Fatal("dynamic data exposed")
+	}
+	// Exercise every possible HTTP combination, then verify known series continue updating.
+	for _, method := range methods {
+		for _, route := range routes {
+			for _, status := range statusClasses {
+				labels := Labels{"method": method, "route": route, "status_class": status}
+				r.AddCounter("orbit_http_requests_total", 1, labels)
+				r.ObserveHistogram("orbit_http_request_duration_seconds", 1, labels)
+			}
+		}
+	}
+	labels := Labels{"method": "GET", "route": "tasks", "status_class": "2xx"}
+	r.AddCounter("orbit_http_requests_total", 1, labels)
+	if n := len(r.families["orbit_http_requests_total"].series); n != len(methods)*len(routes)*len(statusClasses) {
+		t.Fatal("wrong series bound", n)
+	}
+	if !strings.Contains(r.Render(), `orbit_http_requests_total{method="GET",route="tasks",status_class="2xx"} 2`) {
+		t.Fatal("existing series stopped updating")
+	}
+}
+func TestInvalidNumbersAndMonotoneCounters(t *testing.T) {
+	r := NewRegistry()
+	r.SetCounter("orbit_db_wait_count_total", 10, nil)
+	for _, value := range []float64{-1, math.NaN(), math.Inf(1)} {
+		r.AddCounter("orbit_db_wait_count_total", value, nil)
+		r.ObserveHistogram("orbit_http_request_duration_seconds", value, nil)
+	}
+	r.SetCounter("orbit_db_wait_count_total", 9, nil)
+	if !strings.Contains(r.Render(), "orbit_db_wait_count_total 10\n") || r.rejected.Load() != 7 {
+		t.Fatal("invalid counter update")
+	}
+}
+func TestRuntimeObservationDoesNotExposeErrors(t *testing.T) {
+	r := NewRegistry()
+	r.RecordBatch("workflow", time.Now().Add(-time.Millisecond), nil)
+	r.RecordBatch("workflow", time.Now(), errors.New("private-dsn"))
+	r.RecordBatch("schedule", time.Now(), context.Canceled)
+	r.RecordDatabase(sql.DBStats{MaxOpenConnections: 20, OpenConnections: 4, InUse: 3, Idle: 1, WaitCount: 9, WaitDuration: 2 * time.Second})
+	out := r.Render()
+	for _, want := range []string{`orbit_runtime_batch_total{component="workflow",outcome="success"} 1`, `orbit_runtime_batch_total{component="workflow",outcome="error"} 1`, `orbit_runtime_batch_total{component="schedule",outcome="cancelled"} 1`, "orbit_db_connections_in_use 3\n", "orbit_db_wait_duration_seconds_total 2\n"} {
+		if !strings.Contains(out, want) {
+			t.Fatal("missing runtime evidence", want)
+		}
+	}
+	if strings.Contains(out, "private-dsn") {
+		t.Fatal("error data leaked")
+	}
+	var disabled *Registry
+	disabled.RecordBatch("workflow", time.Now(), nil)
+	disabled.RecordDatabase(sql.DBStats{})
+}
+func TestHTTPBoundedLabelsAndPanicPreserved(t *testing.T) {
+	r := NewRegistry()
+	h := r.Middleware(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(202) }))
+	for i := 0; i < 100; i++ {
+		h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", fmt.Sprintf("/v1/tasks/private-%d?token=private-key", i), nil))
+	}
+	out := r.Render()
+	if strings.Contains(out, "private-") || strings.Count(out, "orbit_http_requests_total{") != 1 {
+		t.Fatal("request label cardinality or privacy")
+	}
+	panics := r.Middleware(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { panic("original panic") }))
+	func() {
+		defer func() {
+			if recover() != "original panic" {
+				t.Fatal("panic changed")
+			}
+		}()
+		panics.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/v1/queues", nil))
+	}()
+	out = r.Render()
+	if !strings.Contains(out, "orbit_http_requests_in_flight 0\n") || !strings.Contains(out, "orbit_http_panics_total 1\n") || !strings.Contains(out, `route="queues",status_class="5xx"} 1`) {
+		t.Fatal("panic leaked in-flight state")
+	}
+}
+func TestScrapeConcurrencyRejectsRatherThanBlockingBusiness(t *testing.T) {
+	r := NewRegistry()
+	r.scrape <- struct{}{}
+	r.scrape <- struct{}{}
+	rr := httptest.NewRecorder()
+	r.ServeHTTP(rr, httptest.NewRequest("GET", "/metrics", nil))
+	if rr.Code != 503 || rr.Header().Get("Retry-After") != "1" {
+		t.Fatal("unbounded scrape")
+	}
+	// Registry recording is independent of the scrape admission slots.
+	r.RecordBatch("workflow", time.Now(), nil)
+	<-r.scrape
+	<-r.scrape
+	rr = httptest.NewRecorder()
+	r.ServeHTTP(rr, httptest.NewRequest("GET", "/metrics", nil))
+	if rr.Code != 200 || !strings.HasSuffix(rr.Body.String(), "\n") {
+		t.Fatal("scrape did not recover")
+	}
+}
+
+// TestExportExpositionFixture optionally exports the full finite HTTP catalog
+// for an independent Prometheus promtool check; ordinary tests need no tool.
+func TestExportExpositionFixture(t *testing.T) {
+	r := NewRegistry()
+	for _, method := range methods {
+		for _, route := range routes {
+			for _, status := range statusClasses {
+				labels := Labels{"method": method, "route": route, "status_class": status}
+				r.AddCounter("orbit_http_requests_total", 1, labels)
+				r.ObserveHistogram("orbit_http_request_duration_seconds", .02, labels)
+			}
+		}
+	}
+	r.RecordDatabase(sql.DBStats{MaxOpenConnections: 20})
+	r.RecordBatch("workflow", time.Now(), nil)
+	body := r.Render()
+	if len(body) > 2<<20 {
+		t.Fatal("finite catalog exceeded bounded output size")
+	}
+	if path := os.Getenv("ORBIT_TEST_METRICS_OUTPUT"); path != "" {
+		if err := os.WriteFile(path, []byte(body), 0600); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+type discardWriter struct{ header http.Header }
+
+func (w *discardWriter) Header() http.Header         { return w.header }
+func (w *discardWriter) Write(p []byte) (int, error) { return len(p), nil }
+func (w *discardWriter) WriteHeader(int)             {}
+func BenchmarkHTTPMiddleware(b *testing.B) {
+	for _, enabled := range []bool{false, true} {
+		b.Run(strconv.FormatBool(enabled), func(b *testing.B) {
+			var h http.Handler = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(204) })
+			if enabled {
+				h = NewRegistry().Middleware(h)
+			}
+			b.ReportAllocs()
+			b.ResetTimer()
+			b.RunParallel(func(pb *testing.PB) {
+				req := httptest.NewRequest("GET", "/v1/tasks/id", nil)
+				w := &discardWriter{header: make(http.Header)}
+				for pb.Next() {
+					h.ServeHTTP(w, req)
+				}
+			})
+		})
+	}
+}
--- a/internal/observability/runtime.go
+++ b/internal/observability/runtime.go
@@ -0,0 +1,64 @@
+package observability
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"runtime"
+	"time"
+)
+
+// initializeProcess is called once by the registry constructor.
+func (r *Registry) initializeProcess() {
+	if r == nil {
+		return
+	}
+	r.SetGauge("orbit_process_start_time_seconds", float64(r.started.Unix()), nil)
+	r.SetGauge("orbit_build_info", 1, Labels{"go_version": runtime.Version()})
+	r.RecordProcess()
+	for _, component := range components {
+		labels := Labels{"component": component}
+		r.SetGauge("orbit_runtime_batch_last_success_timestamp_seconds", 0, labels)
+		for _, outcome := range []string{"success", "error", "cancelled"} {
+			r.AddCounter("orbit_runtime_batch_total", 0, Labels{"component": component, "outcome": outcome})
+		}
+	}
+	for _, route := range []string{"worker_connect", "event_connect"} {
+		r.SetGauge("orbit_websocket_connections", 0, Labels{"route": route})
+		r.AddCounter("orbit_websocket_connections_total", 0, Labels{"route": route})
+	}
+}
+func (r *Registry) RecordProcess() {
+	if r == nil {
+		return
+	}
+	r.SetGauge("orbit_process_uptime_seconds", time.Since(r.started).Seconds(), nil)
+	r.SetGauge("orbit_go_goroutines", float64(runtime.NumGoroutine()), nil)
+}
+func (r *Registry) RecordBatch(component string, started time.Time, err error) {
+	if r == nil {
+		return
+	}
+	outcome := "success"
+	if err != nil {
+		outcome = "error"
+		if errors.Is(err, context.Canceled) {
+			outcome = "cancelled"
+		}
+	}
+	r.AddCounter("orbit_runtime_batch_total", 1, Labels{"component": component, "outcome": outcome})
+	r.ObserveHistogram("orbit_runtime_batch_duration_seconds", time.Since(started).Seconds(), Labels{"component": component})
+	if err == nil {
+		r.SetGauge("orbit_runtime_batch_last_success_timestamp_seconds", float64(time.Now().Unix()), Labels{"component": component})
+	}
+}
+
+// RecordDatabase reads the local connection-pool snapshot; it issues no SQL.
+func (r *Registry) RecordDatabase(s sql.DBStats) {
+	r.SetGauge("orbit_db_connections_open", float64(s.OpenConnections), nil)
+	r.SetGauge("orbit_db_connections_in_use", float64(s.InUse), nil)
+	r.SetGauge("orbit_db_connections_idle", float64(s.Idle), nil)
+	r.SetGauge("orbit_db_connections_max_open", float64(s.MaxOpenConnections), nil)
+	r.SetCounter("orbit_db_wait_count_total", float64(s.WaitCount), nil)
+	r.SetCounter("orbit_db_wait_duration_seconds_total", s.WaitDuration.Seconds(), nil)
+}
--- a/internal/transport/httpserver/server.go
+++ b/internal/transport/httpserver/server.go
@@ -4,9 +4,11 @@ package httpserver
 import (
 	"context"
 	"encoding/json"
-	"git.yuntuo.hk/k2safe/orbit/pulse/service"
 	"net/http"
 	"time"
+
+	"git.yuntuo.hk/k2safe/orbit/internal/observability"
+	"git.yuntuo.hk/k2safe/orbit/pulse/service"
 )

 type Checker func(context.Context) error
@@ -36,3 +38,23 @@ func New(addr string, ready Checker, tasks ...*service.Service) *http.Server {
 	}
 	return &http.Server{Addr: addr, Handler: mux, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 10 * time.Second, WriteTimeout: 15 * time.Second, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 16 << 10}
 }
+
+// WrapMetrics must run after all Register* functions because those functions
+// intentionally register routes on the underlying ServeMux.
+func WrapMetrics(server *http.Server, metrics *observability.Registry) {
+	if server == nil || metrics == nil {
+		return
+	}
+	server.Handler = metrics.Middleware(server.Handler)
+}
+
+// NewMetricsServer uses a distinct listener so public API routing cannot expose
+// operational aggregates. It does not provide business or database endpoints.
+func NewMetricsServer(addr string, metrics *observability.Registry) *http.Server {
+	mux := http.NewServeMux()
+	if metrics != nil {
+		mux.Handle("GET /metrics", metrics)
+	}
+	return &http.Server{Addr: addr, Handler: mux, ReadHeaderTimeout: 2 * time.Second,
+		ReadTimeout: 3 * time.Second, WriteTimeout: 5 * time.Second, IdleTimeout: 15 * time.Second, MaxHeaderBytes: 8 << 10}
+}
--- a/internal/transport/httpserver/server_test.go
+++ b/internal/transport/httpserver/server_test.go
@@ -7,6 +7,8 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"git.yuntuo.hk/k2safe/orbit/internal/observability"
 )

 func TestHealthSeparatesLivenessAndReadiness(t *testing.T) {
@@ -19,3 +21,28 @@ func TestHealthSeparatesLivenessAndReadiness(t *testing.T) {
 		}
 	}
 }
+
+func TestMetricsListenerIsSeparateAndOptional(t *testing.T) {
+	metrics := observability.NewRegistry()
+	business := New("", nil)
+	WrapMetrics(business, metrics)
+	private := NewMetricsServer("", metrics)
+	for _, tc := range []struct {
+		handler http.Handler
+		path    string
+		want    int
+	}{
+		{business.Handler, "/metrics", 404}, {private.Handler, "/v1/tasks", 404},
+		{private.Handler, "/metrics", 200}, {NewMetricsServer("", nil).Handler, "/metrics", 404},
+		{private.Handler, "/metrics?secret=value", 400},
+	} {
+		rr := httptest.NewRecorder()
+		tc.handler.ServeHTTP(rr, httptest.NewRequest("GET", tc.path, nil))
+		if rr.Code != tc.want {
+			t.Fatalf("%s: %d", tc.path, rr.Code)
+		}
+		if strings.Contains(rr.Body.String(), "secret") {
+			t.Fatal("query leaked")
+		}
+	}
+}
--- a/scripts/test-mysql.sh
+++ b/scripts/test-mysql.sh
@@ -37,6 +37,9 @@ export ORBIT_TEST_MYSQL_DSN="root:$TASK_DB_PASSWORD@tcp(127.0.0.1:$TASK_DB_PORT)
 "$TASK_GO" test -race -count=1 -run Integration -v ./internal/storage/mysql

 "$TASK_GO" build -o "$TASK_TEST_DIR/orbit" ./cmd/orbit
+# Exercise metrics on every integration process, using a fresh private port.
+export ORBIT_METRICS_ADDR=127.0.0.1:0
+ORBIT_TEST_SERVER_BINARY="$TASK_TEST_DIR/orbit" "$TASK_GO" test -race -count=1 -run '^TestIntegrationMetricsProcess$' -v ./cmd/orbit
 ORBIT_TEST_SERVER_BINARY="$TASK_TEST_DIR/orbit" "$TASK_GO" test -race -count=1 -run '^TestIntegrationWorkflowRuntimeProcess$' -v ./internal/storage/mysql
 "$TASK_GO" build -o "$TASK_TEST_DIR/orbit-migrate" ./cmd/orbit-migrate
 "$TASK_GO" build -o "$TASK_TEST_DIR/orbit-admin" ./cmd/orbit-admin