feat: add bounded process metrics and private monitoring endpoint

2026-09-18 13:17:33 +08:00
parent 1f1d833f69
commit a7be32f661
24 changed files with 1467 additions and 19 deletions
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -9,15 +9,16 @@ import (
 )

 type Config struct {
-	HTTPAddr string
-	MySQLDSN string
-	MaxOpen  int
-	MaxIdle  int
+	HTTPAddr    string
+	MySQLDSN    string
+	MaxOpen     int
+	MaxIdle     int
+	MetricsAddr string
 }

 func Load() (Config, error) { return FromEnv(os.Getenv) }
 func FromEnv(get func(string) string) (Config, error) {
-	c := Config{HTTPAddr: get("ORBIT_HTTP_ADDR"), MySQLDSN: get("ORBIT_MYSQL_DSN"), MaxOpen: 20, MaxIdle: 5}
+	c := Config{HTTPAddr: get("ORBIT_HTTP_ADDR"), MySQLDSN: get("ORBIT_MYSQL_DSN"), MaxOpen: 20, MaxIdle: 5, MetricsAddr: get("ORBIT_METRICS_ADDR")}
 	if c.HTTPAddr == "" {
 		c.HTTPAddr = "127.0.0.1:8080"
 	}
@@ -42,5 +43,12 @@ func FromEnv(get func(string) string) (Config, error) {
 	if c.MaxOpen <= 0 || c.MaxIdle > c.MaxOpen {
 		return Config{}, errors.New("database pool requires 0 <= max_idle <= max_open and max_open > 0")
 	}
+	if c.MetricsAddr != "" {
+		_, port, err := net.SplitHostPort(c.MetricsAddr)
+		n, parseErr := strconv.Atoi(port)
+		if err != nil || parseErr != nil || n < 0 || n > 65535 {
+			return Config{}, errors.New("invalid ORBIT_METRICS_ADDR")
+		}
+	}
 	return c, nil
 }
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -0,0 +1,25 @@
+package config
+
+import "testing"
+
+func TestMetricsAddressIsExplicitAndValidated(t *testing.T) {
+	base := map[string]string{"ORBIT_MYSQL_DSN": "private-dsn"}
+	get := func(k string) string { return base[k] }
+	c, err := FromEnv(get)
+	if err != nil || c.MetricsAddr != "" {
+		t.Fatal("metrics must be disabled by default", err)
+	}
+	for _, addr := range []string{"127.0.0.1:0", "[::1]:9090", "0.0.0.0:9090"} {
+		base["ORBIT_METRICS_ADDR"] = addr
+		c, err = FromEnv(get)
+		if err != nil || c.MetricsAddr != addr {
+			t.Fatal("valid explicit address rejected", addr, err)
+		}
+	}
+	for _, addr := range []string{"bad", ":secret", "127.0.0.1:-1", ":65536", "http://127.0.0.1:9000"} {
+		base["ORBIT_METRICS_ADDR"] = addr
+		if _, err = FromEnv(get); err == nil {
+			t.Fatal("invalid address accepted", addr)
+		}
+	}
+}
--- a/internal/observability/catalog.go
+++ b/internal/observability/catalog.go
@@ -0,0 +1,37 @@
+package observability
+
+import "runtime"
+
+var methods = []string{"GET", "POST", "PATCH", "PUT", "DELETE", "HEAD", "OPTIONS", "OTHER"}
+var routes = []string{"health_live", "health_ready", "worker_connect", "event_connect", "tasks", "queues", "workflows", "workflow_definitions", "schedules", "subscriptions", "application_resources", "other"}
+var statusClasses = []string{"1xx", "2xx", "3xx", "4xx", "5xx", "other"}
+var components = []string{"workflow", "schedule", "execution_recovery"}
+
+// Every exported family is declared here, including each allowed label value.
+// Adding a metric therefore requires a code review; no user identifier is valid.
+func catalog() map[string]descriptor {
+	httpLabels := map[string][]string{"method": methods, "route": routes, "status_class": statusClasses}
+	c := map[string]descriptor{
+		"orbit_http_requests_total":                          {counter, "Completed HTTP responses, including successful upgrade handshakes", httpLabels},
+		"orbit_http_request_duration_seconds":                {histogram, "HTTP response or upgrade handshake duration in seconds", httpLabels},
+		"orbit_http_requests_in_flight":                      {gauge, "HTTP handlers before completion or successful hijack", nil},
+		"orbit_http_panics_total":                            {counter, "HTTP handlers which panicked, before or after upgrade", nil},
+		"orbit_websocket_connections":                        {gauge, "Upgraded WebSocket handlers currently active", map[string][]string{"route": {"worker_connect", "event_connect"}}},
+		"orbit_websocket_connections_total":                  {counter, "Successful WebSocket upgrades", map[string][]string{"route": {"worker_connect", "event_connect"}}},
+		"orbit_runtime_batch_total":                          {counter, "Completed background batches by outcome", map[string][]string{"component": components, "outcome": {"success", "error", "cancelled"}}},
+		"orbit_runtime_batch_duration_seconds":               {histogram, "Background batch duration in seconds", map[string][]string{"component": components}},
+		"orbit_runtime_batch_last_success_timestamp_seconds": {gauge, "Unix time of last successful background batch, zero before first success", map[string][]string{"component": components}},
+		"orbit_process_start_time_seconds":                   {gauge, "Unix time when the Orbit metrics registry was started", nil},
+		"orbit_process_uptime_seconds":                       {gauge, "Monotonic elapsed time since metrics registry startup", nil},
+		"orbit_go_goroutines":                                {gauge, "Current Go goroutine count", nil},
+		"orbit_build_info":                                   {gauge, "Orbit Go runtime build information", map[string][]string{"go_version": {runtime.Version()}}},
+	}
+	for name, help := range map[string]string{
+		"open": "Open MySQL connections", "in_use": "MySQL connections currently in use", "idle": "Idle MySQL connections", "max_open": "Configured maximum open MySQL connections",
+	} {
+		c["orbit_db_connections_"+name] = descriptor{gauge, help, nil}
+	}
+	c["orbit_db_wait_count_total"] = descriptor{counter, "Cumulative waits for a MySQL connection", nil}
+	c["orbit_db_wait_duration_seconds_total"] = descriptor{counter, "Cumulative seconds waiting for a MySQL connection", nil}
+	return c
+}
--- a/internal/observability/http.go
+++ b/internal/observability/http.go
@@ -0,0 +1,154 @@
+package observability
+
+import (
+	"bufio"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Middleware counts HTTP completion once. A successful upgrade completes HTTP
+// at Hijack; the still-running WebSocket handler is counted separately.
+func (r *Registry) Middleware(next http.Handler) http.Handler {
+	if r == nil {
+		return next
+	}
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		started := time.Now()
+		route := routeClass(req.URL.Path)
+		r.AddGauge("orbit_http_requests_in_flight", 1, nil)
+		cw := &captureWriter{ResponseWriter: w}
+		completed := false
+		ws := false
+		complete := func(status int) {
+			if completed {
+				return
+			}
+			completed = true
+			class := "other"
+			if status >= 100 && status < 600 {
+				class = strconv.Itoa(status/100) + "xx"
+			}
+			labels := Labels{"method": boundedMethod(req.Method), "route": route, "status_class": class}
+			r.AddGauge("orbit_http_requests_in_flight", -1, nil)
+			r.AddCounter("orbit_http_requests_total", 1, labels)
+			r.ObserveHistogram("orbit_http_request_duration_seconds", time.Since(started).Seconds(), labels)
+		}
+		cw.onHijack = func() {
+			complete(cw.status)
+			if cw.status == 101 && (route == "worker_connect" || route == "event_connect") {
+				ws = true
+				r.AddGauge("orbit_websocket_connections", 1, Labels{"route": route})
+				r.AddCounter("orbit_websocket_connections_total", 1, Labels{"route": route})
+			}
+		}
+		returned := false
+		defer func() {
+			status := cw.status
+			if !returned {
+				r.AddCounter("orbit_http_panics_total", 1, nil)
+				if status == 0 {
+					status = 500
+				}
+			}
+			if status == 0 {
+				status = 200
+			}
+			complete(status)
+			if ws {
+				r.AddGauge("orbit_websocket_connections", -1, Labels{"route": route})
+			}
+			// No recover: net/http retains its panic/ErrAbortHandler behavior.
+		}()
+		next.ServeHTTP(cw, req)
+		returned = true
+	})
+}
+func boundedMethod(method string) string {
+	for _, v := range methods {
+		if method == v {
+			return v
+		}
+	}
+	return "OTHER"
+}
+func routeClass(path string) string {
+	switch path {
+	case "/health/live":
+		return "health_live"
+	case "/health/ready":
+		return "health_ready"
+	case "/v1/worker/connect":
+		return "worker_connect"
+	case "/v1/events/connect":
+		return "event_connect"
+	}
+	for _, p := range []struct{ prefix, route string }{
+		{"/v1/tasks", "tasks"}, {"/v1/queues", "queues"}, {"/v1/workflows", "workflows"},
+		{"/v1/workflow-definitions", "workflow_definitions"}, {"/v1/schedules", "schedules"},
+		{"/v1/subscriptions", "subscriptions"}, {"/v1/application/resources", "application_resources"},
+	} {
+		if path == p.prefix || strings.HasPrefix(path, p.prefix+"/") || strings.HasPrefix(path, p.prefix+":") {
+			return p.route
+		}
+	}
+	return "other"
+}
+
+type captureWriter struct {
+	http.ResponseWriter
+	status   int
+	onHijack func()
+}
+
+func (w *captureWriter) WriteHeader(status int) {
+	if w.status != 0 {
+		return
+	}
+	// Forward informational responses without preventing a later final response.
+	if status >= 100 && status < 200 && status != 101 {
+		w.ResponseWriter.WriteHeader(status)
+		return
+	}
+	w.ResponseWriter.WriteHeader(status)
+	w.status = status
+}
+func (w *captureWriter) Write(p []byte) (int, error) {
+	if w.status == 0 {
+		w.WriteHeader(200)
+	}
+	return w.ResponseWriter.Write(p)
+}
+func (w *captureWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter }
+func (w *captureWriter) FlushError() error {
+	if w.status == 0 {
+		w.WriteHeader(200)
+	}
+	return http.NewResponseController(w.ResponseWriter).Flush()
+}
+func (w *captureWriter) Flush() { _ = w.FlushError() }
+func (w *captureWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	c, b, err := http.NewResponseController(w.ResponseWriter).Hijack()
+	if err == nil && w.onHijack != nil {
+		w.onHijack()
+	}
+	return c, b, err
+}
+func (w *captureWriter) Push(target string, opts *http.PushOptions) error {
+	if p, ok := w.ResponseWriter.(http.Pusher); ok {
+		return p.Push(target, opts)
+	}
+	return http.ErrNotSupported
+}
+func (w *captureWriter) ReadFrom(src io.Reader) (int64, error) {
+	if w.status == 0 {
+		w.WriteHeader(200)
+	}
+	if rf, ok := w.ResponseWriter.(io.ReaderFrom); ok {
+		return rf.ReadFrom(src)
+	}
+	return io.Copy(w.ResponseWriter, src)
+}
--- a/internal/observability/http_test.go
+++ b/internal/observability/http_test.go
@@ -0,0 +1,93 @@
+package observability
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/coder/websocket"
+)
+
+func TestWebSocketHandshakeIsNotConnectionLifetime(t *testing.T) {
+	r := NewRegistry()
+	released := make(chan struct{})
+	done := make(chan struct{})
+	h := r.Middleware(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		defer close(done)
+		c, err := websocket.Accept(w, req, nil)
+		if err != nil {
+			return
+		}
+		defer c.CloseNow()
+		<-released
+	}))
+	server := httptest.NewTLSServer(h)
+	defer server.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	c, _, err := websocket.Dial(ctx, strings.Replace(server.URL, "https://", "wss://", 1)+"/v1/worker/connect", &websocket.DialOptions{HTTPClient: server.Client()})
+	if err != nil {
+		close(released)
+		t.Fatal(err)
+	}
+	defer c.CloseNow()
+	// Dial returns after the upgrade response, while the server handler stays open.
+	deadline := time.Now().Add(time.Second)
+	for !strings.Contains(r.Render(), `orbit_websocket_connections{route="worker_connect"} 1`) {
+		if time.Now().After(deadline) {
+			close(released)
+			t.Fatal("upgrade not observed")
+		}
+		time.Sleep(time.Millisecond)
+	}
+	out := r.Render()
+	close(released)
+	<-done
+	if !strings.Contains(out, `orbit_http_requests_total{method="GET",route="worker_connect",status_class="1xx"} 1`) || !strings.Contains(out, "orbit_http_requests_in_flight 0\n") {
+		t.Fatal("upgrade counted as unfinished HTTP")
+	}
+	// done closes inside the handler; wait for middleware's defer to publish closure.
+	deadline = time.Now().Add(time.Second)
+	for !strings.Contains(r.Render(), `orbit_websocket_connections{route="worker_connect"} 0`) {
+		if time.Now().After(deadline) {
+			t.Fatal("connection gauge leaked")
+		}
+		time.Sleep(time.Millisecond)
+	}
+	if !strings.Contains(r.Render(), `orbit_http_requests_total{method="GET",route="worker_connect",status_class="1xx"} 1`) {
+		t.Fatal("upgrade counted twice")
+	}
+}
+func TestInformationalResponseAndStreamingPreserveHTTP(t *testing.T) {
+	r := NewRegistry()
+	server := httptest.NewServer(r.Middleware(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		w.WriteHeader(103)
+		w.WriteHeader(201)
+		_, _ = io.WriteString(w, "first")
+		if err := http.NewResponseController(w).Flush(); err != nil {
+			panic(err)
+		}
+		_, _ = io.WriteString(w, "second")
+	})))
+	defer server.Close()
+	resp, err := server.Client().Get(server.URL + "/v1/tasks")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil || resp.StatusCode != 201 || string(body) != "firstsecond" {
+		t.Fatal("wrapper changed HTTP response", resp.StatusCode, err)
+	}
+	deadline := time.Now().Add(time.Second)
+	for !strings.Contains(r.Render(), `route="tasks",status_class="2xx"} 1`) {
+		if time.Now().After(deadline) {
+			t.Fatal("103 blocked final status metric")
+		}
+		time.Sleep(time.Millisecond)
+	}
+}
--- a/internal/observability/metrics.go
+++ b/internal/observability/metrics.go
@@ -0,0 +1,272 @@
+// Package observability exposes process-local operational metrics. It does not
+// participate in durable state, authorization or execution decisions.
+package observability
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type Labels map[string]string
+
+// Registry accepts only the closed catalog below: neither names nor label
+// values can grow from caller-controlled request or business data.
+type Registry struct {
+	mu       sync.Mutex
+	families map[string]*family
+	rejected atomic.Uint64
+	scrape   chan struct{}
+	started  time.Time
+}
+type metricKind string
+
+const (
+	counter   metricKind = "counter"
+	gauge     metricKind = "gauge"
+	histogram metricKind = "histogram"
+)
+
+type descriptor struct {
+	kind   metricKind
+	help   string
+	labels map[string][]string
+}
+type family struct {
+	desc   descriptor
+	series map[string]*series
+}
+type series struct {
+	labels string // Immutable, already escaped and validated against the catalog.
+	value  atomic.Uint64
+	hist   distribution
+}
+type distribution struct {
+	mu      sync.Mutex
+	buckets [11]uint64
+	count   uint64
+	sum     float64
+}
+
+var bounds = [...]float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}
+
+func NewRegistry() *Registry {
+	r := &Registry{families: make(map[string]*family), scrape: make(chan struct{}, 2), started: time.Now()}
+	for name, d := range catalog() {
+		f := &family{desc: d, series: make(map[string]*series)}
+		if len(d.labels) == 0 {
+			f.series[""] = &series{}
+		}
+		r.families[name] = f
+	}
+	r.initializeProcess()
+	return r
+}
+func finite(v float64) bool { return !math.IsNaN(v) && !math.IsInf(v, 0) }
+func (r *Registry) lookup(name string, kind metricKind, labels Labels) *series {
+	if r == nil {
+		return nil
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	f, ok := r.families[name]
+	if !ok || f.desc.kind != kind || len(labels) != len(f.desc.labels) {
+		r.rejected.Add(1)
+		return nil
+	}
+	keys := make([]string, 0, len(labels))
+	for k, v := range labels {
+		valid := false
+		for _, allowed := range f.desc.labels[k] {
+			if v == allowed {
+				valid = true
+				break
+			}
+		}
+		if !valid {
+			r.rejected.Add(1)
+			return nil
+		}
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	var parts []string
+	for _, k := range keys {
+		parts = append(parts, k+`="`+escape(labels[k])+`"`)
+	}
+	encoded := ""
+	if len(parts) > 0 {
+		encoded = "{" + strings.Join(parts, ",") + "}"
+	}
+	s := f.series[encoded]
+	if s == nil {
+		s = &series{labels: encoded}
+		f.series[encoded] = s
+	}
+	return s
+}
+func (r *Registry) reject() {
+	if r != nil {
+		r.rejected.Add(1)
+	}
+}
+func (r *Registry) AddCounter(name string, delta float64, labels Labels) {
+	if !finite(delta) || delta < 0 {
+		r.reject()
+		return
+	}
+	if s := r.lookup(name, counter, labels); s != nil {
+		add(&s.value, delta)
+	}
+}
+
+// SetCounter only accepts a monotone source, such as database/sql.DBStats.
+func (r *Registry) SetCounter(name string, value float64, labels Labels) {
+	if !finite(value) || value < 0 {
+		r.reject()
+		return
+	}
+	if s := r.lookup(name, counter, labels); s != nil {
+		for {
+			old := s.value.Load()
+			if value < math.Float64frombits(old) {
+				r.reject()
+				return
+			}
+			if s.value.CompareAndSwap(old, math.Float64bits(value)) {
+				return
+			}
+		}
+	}
+}
+func (r *Registry) AddGauge(name string, delta float64, labels Labels) {
+	if !finite(delta) {
+		r.reject()
+		return
+	}
+	if s := r.lookup(name, gauge, labels); s != nil {
+		add(&s.value, delta)
+	}
+}
+func (r *Registry) SetGauge(name string, value float64, labels Labels) {
+	if !finite(value) {
+		r.reject()
+		return
+	}
+	if s := r.lookup(name, gauge, labels); s != nil {
+		s.value.Store(math.Float64bits(value))
+	}
+}
+func (r *Registry) ObserveHistogram(name string, value float64, labels Labels) {
+	if !finite(value) || value < 0 {
+		r.reject()
+		return
+	}
+	s := r.lookup(name, histogram, labels)
+	if s == nil {
+		return
+	}
+	h := &s.hist
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	for i, b := range bounds {
+		if value <= b {
+			h.buckets[i]++
+		}
+	}
+	h.count++
+	h.sum += value
+}
+func add(dst *atomic.Uint64, delta float64) {
+	for {
+		old := dst.Load()
+		next := math.Float64frombits(old) + delta
+		if !finite(next) {
+			return
+		}
+		if dst.CompareAndSwap(old, math.Float64bits(next)) {
+			return
+		}
+	}
+}
+func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	w.Header().Set("Cache-Control", "no-store")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	if req.URL.RawQuery != "" {
+		http.Error(w, "metrics does not accept query parameters", 400)
+		return
+	}
+	select {
+	case r.scrape <- struct{}{}:
+		defer func() { <-r.scrape }()
+	default:
+		w.Header().Set("Retry-After", "1")
+		http.Error(w, "metrics busy", 503)
+		return
+	}
+	w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+	_, _ = io.WriteString(w, r.Render())
+}
+
+// Render snapshots pointers under the registry lock; formatting and network I/O
+// never hold that lock. Each histogram's buckets/count/sum use one local snapshot.
+func (r *Registry) Render() string {
+	type view struct {
+		name   string
+		desc   descriptor
+		series []*series
+	}
+	var views []view
+	r.mu.Lock()
+	for n, f := range r.families {
+		v := view{name: n, desc: f.desc}
+		for _, s := range f.series {
+			v.series = append(v.series, s)
+		}
+		views = append(views, v)
+	}
+	r.mu.Unlock()
+	sort.Slice(views, func(i, j int) bool { return views[i].name < views[j].name })
+	var b strings.Builder
+	for _, v := range views {
+		fmt.Fprintf(&b, "# HELP %s %s\n# TYPE %s %s\n", v.name, v.desc.help, v.name, v.desc.kind)
+		sort.Slice(v.series, func(i, j int) bool { return v.series[i].labels < v.series[j].labels })
+		for _, s := range v.series {
+			if v.desc.kind != histogram {
+				fmt.Fprintf(&b, "%s%s %s\n", v.name, s.labels, number(math.Float64frombits(s.value.Load())))
+				continue
+			}
+			h := &s.hist
+			h.mu.Lock()
+			buckets, count, sum := h.buckets, h.count, h.sum
+			h.mu.Unlock()
+			for i, bound := range bounds {
+				fmt.Fprintf(&b, "%s_bucket%s %d\n", v.name, withLabel(s.labels, "le", number(bound)), buckets[i])
+			}
+			fmt.Fprintf(&b, "%s_bucket%s %d\n%s_sum%s %s\n%s_count%s %d\n", v.name, withLabel(s.labels, "le", "+Inf"), count, v.name, s.labels, number(sum), v.name, s.labels, count)
+		}
+	}
+	b.WriteString("# HELP orbit_observability_rejected_samples_total Invalid internal metric updates rejected\n# TYPE orbit_observability_rejected_samples_total counter\n")
+	fmt.Fprintf(&b, "orbit_observability_rejected_samples_total %d\n", r.rejected.Load())
+	return b.String()
+}
+
+// Replacer is safe for concurrent use. Construct it once rather than allocating
+// its lookup tables for every label of every observed request.
+var labelEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`)
+
+func escape(s string) string  { return labelEscaper.Replace(s) }
+func number(v float64) string { return strconv.FormatFloat(v, 'g', -1, 64) }
+func withLabel(labels, k, v string) string {
+	if labels == "" {
+		return "{" + k + `="` + escape(v) + `"}`
+	}
+	return labels[:len(labels)-1] + "," + k + `="` + escape(v) + `"}`
+}
--- a/internal/observability/metrics_test.go
+++ b/internal/observability/metrics_test.go
@@ -0,0 +1,208 @@
+package observability
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"math"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestConcurrentScrapesAndUpdates(t *testing.T) {
+	r := NewRegistry()
+	labels := Labels{"method": "GET", "route": "tasks", "status_class": "2xx"}
+	var wg sync.WaitGroup
+	for i := 0; i < 16; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for n := 0; n < 100; n++ {
+				r.AddCounter("orbit_http_requests_total", 1, labels)
+				r.ObserveHistogram("orbit_http_request_duration_seconds", .01, labels)
+				if n%10 == 0 {
+					r.Render()
+				}
+			}
+		}()
+	}
+	wg.Wait()
+	out := r.Render()
+	for _, want := range []string{
+		`orbit_http_requests_total{method="GET",route="tasks",status_class="2xx"} 1600`,
+		`orbit_http_request_duration_seconds_count{method="GET",route="tasks",status_class="2xx"} 1600`,
+		`orbit_http_request_duration_seconds_bucket{method="GET",route="tasks",status_class="2xx",le="0.005"} 0`,
+		`orbit_http_request_duration_seconds_bucket{method="GET",route="tasks",status_class="2xx",le="+Inf"} 1600`,
+	} {
+		if !strings.Contains(out, want) {
+			t.Fatal("lost samples or inconsistent histogram", want)
+		}
+	}
+}
+func TestCatalogRejectsNamesTypesAndAllDynamicLabels(t *testing.T) {
+	r := NewRegistry()
+	families := len(r.families)
+	for i := 0; i < 3000; i++ {
+		r.AddCounter("untrusted_"+strconv.Itoa(i), 1, nil)
+		r.AddCounter("orbit_http_requests_total", 1, Labels{"method": "GET", "route": "private-" + strconv.Itoa(i), "status_class": "2xx"})
+	}
+	r.ObserveHistogram("orbit_http_requests_in_flight", 1, nil) // Must not panic on a type mismatch.
+	r.SetGauge("orbit_http_requests_total", 1, nil)
+	r.SetGauge("orbit_go_goroutines", 1, Labels{"tenant": "private"})
+	if len(r.families) != families || r.rejected.Load() != 6003 {
+		t.Fatal("unbounded family creation or invalid samples accepted")
+	}
+	if out := r.Render(); strings.Contains(out, "private") || strings.Contains(out, "untrusted") {
+		t.Fatal("dynamic data exposed")
+	}
+	// Exercise every possible HTTP combination, then verify known series continue updating.
+	for _, method := range methods {
+		for _, route := range routes {
+			for _, status := range statusClasses {
+				labels := Labels{"method": method, "route": route, "status_class": status}
+				r.AddCounter("orbit_http_requests_total", 1, labels)
+				r.ObserveHistogram("orbit_http_request_duration_seconds", 1, labels)
+			}
+		}
+	}
+	labels := Labels{"method": "GET", "route": "tasks", "status_class": "2xx"}
+	r.AddCounter("orbit_http_requests_total", 1, labels)
+	if n := len(r.families["orbit_http_requests_total"].series); n != len(methods)*len(routes)*len(statusClasses) {
+		t.Fatal("wrong series bound", n)
+	}
+	if !strings.Contains(r.Render(), `orbit_http_requests_total{method="GET",route="tasks",status_class="2xx"} 2`) {
+		t.Fatal("existing series stopped updating")
+	}
+}
+func TestInvalidNumbersAndMonotoneCounters(t *testing.T) {
+	r := NewRegistry()
+	r.SetCounter("orbit_db_wait_count_total", 10, nil)
+	for _, value := range []float64{-1, math.NaN(), math.Inf(1)} {
+		r.AddCounter("orbit_db_wait_count_total", value, nil)
+		r.ObserveHistogram("orbit_http_request_duration_seconds", value, nil)
+	}
+	r.SetCounter("orbit_db_wait_count_total", 9, nil)
+	if !strings.Contains(r.Render(), "orbit_db_wait_count_total 10\n") || r.rejected.Load() != 7 {
+		t.Fatal("invalid counter update")
+	}
+}
+func TestRuntimeObservationDoesNotExposeErrors(t *testing.T) {
+	r := NewRegistry()
+	r.RecordBatch("workflow", time.Now().Add(-time.Millisecond), nil)
+	r.RecordBatch("workflow", time.Now(), errors.New("private-dsn"))
+	r.RecordBatch("schedule", time.Now(), context.Canceled)
+	r.RecordDatabase(sql.DBStats{MaxOpenConnections: 20, OpenConnections: 4, InUse: 3, Idle: 1, WaitCount: 9, WaitDuration: 2 * time.Second})
+	out := r.Render()
+	for _, want := range []string{`orbit_runtime_batch_total{component="workflow",outcome="success"} 1`, `orbit_runtime_batch_total{component="workflow",outcome="error"} 1`, `orbit_runtime_batch_total{component="schedule",outcome="cancelled"} 1`, "orbit_db_connections_in_use 3\n", "orbit_db_wait_duration_seconds_total 2\n"} {
+		if !strings.Contains(out, want) {
+			t.Fatal("missing runtime evidence", want)
+		}
+	}
+	if strings.Contains(out, "private-dsn") {
+		t.Fatal("error data leaked")
+	}
+	var disabled *Registry
+	disabled.RecordBatch("workflow", time.Now(), nil)
+	disabled.RecordDatabase(sql.DBStats{})
+}
+func TestHTTPBoundedLabelsAndPanicPreserved(t *testing.T) {
+	r := NewRegistry()
+	h := r.Middleware(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(202) }))
+	for i := 0; i < 100; i++ {
+		h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", fmt.Sprintf("/v1/tasks/private-%d?token=private-key", i), nil))
+	}
+	out := r.Render()
+	if strings.Contains(out, "private-") || strings.Count(out, "orbit_http_requests_total{") != 1 {
+		t.Fatal("request label cardinality or privacy")
+	}
+	panics := r.Middleware(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { panic("original panic") }))
+	func() {
+		defer func() {
+			if recover() != "original panic" {
+				t.Fatal("panic changed")
+			}
+		}()
+		panics.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/v1/queues", nil))
+	}()
+	out = r.Render()
+	if !strings.Contains(out, "orbit_http_requests_in_flight 0\n") || !strings.Contains(out, "orbit_http_panics_total 1\n") || !strings.Contains(out, `route="queues",status_class="5xx"} 1`) {
+		t.Fatal("panic leaked in-flight state")
+	}
+}
+func TestScrapeConcurrencyRejectsRatherThanBlockingBusiness(t *testing.T) {
+	r := NewRegistry()
+	r.scrape <- struct{}{}
+	r.scrape <- struct{}{}
+	rr := httptest.NewRecorder()
+	r.ServeHTTP(rr, httptest.NewRequest("GET", "/metrics", nil))
+	if rr.Code != 503 || rr.Header().Get("Retry-After") != "1" {
+		t.Fatal("unbounded scrape")
+	}
+	// Registry recording is independent of the scrape admission slots.
+	r.RecordBatch("workflow", time.Now(), nil)
+	<-r.scrape
+	<-r.scrape
+	rr = httptest.NewRecorder()
+	r.ServeHTTP(rr, httptest.NewRequest("GET", "/metrics", nil))
+	if rr.Code != 200 || !strings.HasSuffix(rr.Body.String(), "\n") {
+		t.Fatal("scrape did not recover")
+	}
+}
+
+// TestExportExpositionFixture optionally exports the full finite HTTP catalog
+// for an independent Prometheus promtool check; ordinary tests need no tool.
+func TestExportExpositionFixture(t *testing.T) {
+	r := NewRegistry()
+	for _, method := range methods {
+		for _, route := range routes {
+			for _, status := range statusClasses {
+				labels := Labels{"method": method, "route": route, "status_class": status}
+				r.AddCounter("orbit_http_requests_total", 1, labels)
+				r.ObserveHistogram("orbit_http_request_duration_seconds", .02, labels)
+			}
+		}
+	}
+	r.RecordDatabase(sql.DBStats{MaxOpenConnections: 20})
+	r.RecordBatch("workflow", time.Now(), nil)
+	body := r.Render()
+	if len(body) > 2<<20 {
+		t.Fatal("finite catalog exceeded bounded output size")
+	}
+	if path := os.Getenv("ORBIT_TEST_METRICS_OUTPUT"); path != "" {
+		if err := os.WriteFile(path, []byte(body), 0600); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+type discardWriter struct{ header http.Header }
+
+func (w *discardWriter) Header() http.Header         { return w.header }
+func (w *discardWriter) Write(p []byte) (int, error) { return len(p), nil }
+func (w *discardWriter) WriteHeader(int)             {}
+func BenchmarkHTTPMiddleware(b *testing.B) {
+	for _, enabled := range []bool{false, true} {
+		b.Run(strconv.FormatBool(enabled), func(b *testing.B) {
+			var h http.Handler = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(204) })
+			if enabled {
+				h = NewRegistry().Middleware(h)
+			}
+			b.ReportAllocs()
+			b.ResetTimer()
+			b.RunParallel(func(pb *testing.PB) {
+				req := httptest.NewRequest("GET", "/v1/tasks/id", nil)
+				w := &discardWriter{header: make(http.Header)}
+				for pb.Next() {
+					h.ServeHTTP(w, req)
+				}
+			})
+		})
+	}
+}
--- a/internal/observability/runtime.go
+++ b/internal/observability/runtime.go
@@ -0,0 +1,64 @@
+package observability
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"runtime"
+	"time"
+)
+
+// initializeProcess is called once by the registry constructor.
+func (r *Registry) initializeProcess() {
+	if r == nil {
+		return
+	}
+	r.SetGauge("orbit_process_start_time_seconds", float64(r.started.Unix()), nil)
+	r.SetGauge("orbit_build_info", 1, Labels{"go_version": runtime.Version()})
+	r.RecordProcess()
+	for _, component := range components {
+		labels := Labels{"component": component}
+		r.SetGauge("orbit_runtime_batch_last_success_timestamp_seconds", 0, labels)
+		for _, outcome := range []string{"success", "error", "cancelled"} {
+			r.AddCounter("orbit_runtime_batch_total", 0, Labels{"component": component, "outcome": outcome})
+		}
+	}
+	for _, route := range []string{"worker_connect", "event_connect"} {
+		r.SetGauge("orbit_websocket_connections", 0, Labels{"route": route})
+		r.AddCounter("orbit_websocket_connections_total", 0, Labels{"route": route})
+	}
+}
+func (r *Registry) RecordProcess() {
+	if r == nil {
+		return
+	}
+	r.SetGauge("orbit_process_uptime_seconds", time.Since(r.started).Seconds(), nil)
+	r.SetGauge("orbit_go_goroutines", float64(runtime.NumGoroutine()), nil)
+}
+func (r *Registry) RecordBatch(component string, started time.Time, err error) {
+	if r == nil {
+		return
+	}
+	outcome := "success"
+	if err != nil {
+		outcome = "error"
+		if errors.Is(err, context.Canceled) {
+			outcome = "cancelled"
+		}
+	}
+	r.AddCounter("orbit_runtime_batch_total", 1, Labels{"component": component, "outcome": outcome})
+	r.ObserveHistogram("orbit_runtime_batch_duration_seconds", time.Since(started).Seconds(), Labels{"component": component})
+	if err == nil {
+		r.SetGauge("orbit_runtime_batch_last_success_timestamp_seconds", float64(time.Now().Unix()), Labels{"component": component})
+	}
+}
+
+// RecordDatabase reads the local connection-pool snapshot; it issues no SQL.
+func (r *Registry) RecordDatabase(s sql.DBStats) {
+	r.SetGauge("orbit_db_connections_open", float64(s.OpenConnections), nil)
+	r.SetGauge("orbit_db_connections_in_use", float64(s.InUse), nil)
+	r.SetGauge("orbit_db_connections_idle", float64(s.Idle), nil)
+	r.SetGauge("orbit_db_connections_max_open", float64(s.MaxOpenConnections), nil)
+	r.SetCounter("orbit_db_wait_count_total", float64(s.WaitCount), nil)
+	r.SetCounter("orbit_db_wait_duration_seconds_total", s.WaitDuration.Seconds(), nil)
+}
--- a/internal/transport/httpserver/server.go
+++ b/internal/transport/httpserver/server.go
@@ -4,9 +4,11 @@ package httpserver
 import (
 	"context"
 	"encoding/json"
-	"git.yuntuo.hk/k2safe/orbit/pulse/service"
 	"net/http"
 	"time"
+
+	"git.yuntuo.hk/k2safe/orbit/internal/observability"
+	"git.yuntuo.hk/k2safe/orbit/pulse/service"
 )

 type Checker func(context.Context) error
@@ -36,3 +38,23 @@ func New(addr string, ready Checker, tasks ...*service.Service) *http.Server {
 	}
 	return &http.Server{Addr: addr, Handler: mux, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 10 * time.Second, WriteTimeout: 15 * time.Second, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 16 << 10}
 }
+
+// WrapMetrics must run after all Register* functions because those functions
+// intentionally register routes on the underlying ServeMux.
+func WrapMetrics(server *http.Server, metrics *observability.Registry) {
+	if server == nil || metrics == nil {
+		return
+	}
+	server.Handler = metrics.Middleware(server.Handler)
+}
+
+// NewMetricsServer uses a distinct listener so public API routing cannot expose
+// operational aggregates. It does not provide business or database endpoints.
+func NewMetricsServer(addr string, metrics *observability.Registry) *http.Server {
+	mux := http.NewServeMux()
+	if metrics != nil {
+		mux.Handle("GET /metrics", metrics)
+	}
+	return &http.Server{Addr: addr, Handler: mux, ReadHeaderTimeout: 2 * time.Second,
+		ReadTimeout: 3 * time.Second, WriteTimeout: 5 * time.Second, IdleTimeout: 15 * time.Second, MaxHeaderBytes: 8 << 10}
+}
--- a/internal/transport/httpserver/server_test.go
+++ b/internal/transport/httpserver/server_test.go
@@ -7,6 +7,8 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"git.yuntuo.hk/k2safe/orbit/internal/observability"
 )

 func TestHealthSeparatesLivenessAndReadiness(t *testing.T) {
@@ -19,3 +21,28 @@ func TestHealthSeparatesLivenessAndReadiness(t *testing.T) {
 		}
 	}
 }
+
+func TestMetricsListenerIsSeparateAndOptional(t *testing.T) {
+	metrics := observability.NewRegistry()
+	business := New("", nil)
+	WrapMetrics(business, metrics)
+	private := NewMetricsServer("", metrics)
+	for _, tc := range []struct {
+		handler http.Handler
+		path    string
+		want    int
+	}{
+		{business.Handler, "/metrics", 404}, {private.Handler, "/v1/tasks", 404},
+		{private.Handler, "/metrics", 200}, {NewMetricsServer("", nil).Handler, "/metrics", 404},
+		{private.Handler, "/metrics?secret=value", 400},
+	} {
+		rr := httptest.NewRecorder()
+		tc.handler.ServeHTTP(rr, httptest.NewRequest("GET", tc.path, nil))
+		if rr.Code != tc.want {
+			t.Fatalf("%s: %d", tc.path, rr.Code)
+		}
+		if strings.Contains(rr.Body.String(), "secret") {
+			t.Fatal("query leaked")
+		}
+	}
+}