test: capture backlog resource samples

docs/api/schedule-load.md

@@ -22,6 +22,8 @@ go run ./cmd/orbit-schedule-load \
 
 要观察持久化积压，可在一次性数据库使用 `-schedules 32 -requests 8 -workers 1 -capacity 1 -start-delay 5s -run-timeout 180s`。该配置会让256个触发点集中进入队列，报告预期非零并保留未核验分母；独立连接应每2秒读取 `pulse_tasks` 的 `WAITING/LEASED/RUNNING/SUCCEEDED` 计数，示例结果见[长期积压报告](../reports/phase-6-backlog.md)。仓库还提供[`scripts/test-schedule-backlog.sh`](../../scripts/test-schedule-backlog.sh)自动创建/清理一次性 MySQL、采样状态和执行最终外键核验。
 
+积压脚本同时启用临时指标监听，每2秒把服务进程 CPU/RSS、MySQL 容器 CPU/内存、DB 连接池使用量、连接池等待累计值、等待/运行任务和业务快照耗时计数写入 `metrics.jsonl`。设置 `ORBIT_TEST_KEEP_ARTIFACTS=1`（可配合 `ORBIT_TEST_KEEP_ARTIFACTS_PATH`）会保留该文件；它是单节点采样原始证据，不是跨节点聚合或生产容量报告。
+
 ## 参数边界
 
 | 参数 | 默认值 | 限制/语义 |

docs/reports/phase-6-backlog.md

@@ -36,7 +36,7 @@
 | 后段 | 98 | 158 | 256 | 32 |
 | 截止前最后点 | 83 | 173 | 256 | 32 |
 
-本批没有开启指标监听，也没有独占 CPU、内存、磁盘或数据库资源；因此曲线只证明持久状态和收尾行为，不证明资源消耗曲线。后续目标拓扑容量测试需要同时采集数据库连接池、锁等待、CPU、内存、磁盘、复制延迟和队列阶段延迟，并进行多轮稳态与恢复对照。
+本批原始样本没有开启指标监听，也没有独占 CPU、内存、磁盘或数据库资源；因此已保存曲线只证明持久状态和收尾行为，不证明资源消耗曲线。随后脚本已加入临时指标端口和 `metrics.jsonl` 采样，会在下一次复现中留存服务进程 CPU/RSS、MySQL 容器资源、数据库连接池等待和业务快照计数；新文件仍只是单节点原始采样。目标拓扑容量测试还需要采集锁等待、复制延迟和队列阶段延迟，并进行多轮稳态与恢复对照。
 
 ## 环境与复现
 

scripts/test-schedule-backlog.sh

@@ -69,17 +69,19 @@ ORBIT_MYSQL_DSN="$TASK_DSN" "$TASK_DIR/orbit-admin" bootstrap \
 	--application schedule-backlog --namespace test --environment test --queue schedule \
 	--worker-access --schedule-access --output "$TASK_DIR/credentials.json" >/dev/null
 
-ORBIT_MYSQL_DSN="$TASK_DSN" ORBIT_HTTP_ADDR=127.0.0.1:0 "$TASK_DIR/orbit" >"$TASK_DIR/server.log" 2>&1 &
+ORBIT_MYSQL_DSN="$TASK_DSN" ORBIT_HTTP_ADDR=127.0.0.1:0 ORBIT_METRICS_ADDR=127.0.0.1:0 "$TASK_DIR/orbit" >"$TASK_DIR/server.log" 2>&1 &
 TASK_SERVER_PID=$!
 TASK_HTTP_PORT=
+TASK_METRICS_PORT=
 for task_attempt in $(seq 1 30); do
 	TASK_HTTP_PORT=$(sed -n 's/.*"address":"127.0.0.1:\([0-9]*\)".*/\1/p' "$TASK_DIR/server.log" | head -n 1)
-	if [ -n "$TASK_HTTP_PORT" ]; then break; fi
+	TASK_METRICS_PORT=$(sed -n 's/.*"metrics_address":"127.0.0.1:\([0-9]*\)".*/\1/p' "$TASK_DIR/server.log" | head -n 1)
+	if [ -n "$TASK_HTTP_PORT" ] && [ -n "$TASK_METRICS_PORT" ]; then break; fi
 	if ! kill -0 "$TASK_SERVER_PID" 2>/dev/null; then cat "$TASK_DIR/server.log" >&2; exit 1; fi
 	sleep 1
 done
-if [ -z "$TASK_HTTP_PORT" ]; then
+if [ -z "$TASK_HTTP_PORT" ] || [ -z "$TASK_METRICS_PORT" ]; then
-	printf '%s\n' 'Orbit HTTP server failed to start.' >&2
+	printf '%s\n' 'Orbit HTTP or metrics server failed to start.' >&2
 	exit 1
 fi
 
@@ -88,6 +90,7 @@ query() {
 		mysql -uroot --protocol=tcp --batch --skip-column-names orbit_test -e "$1"
 }
 : >"$TASK_DIR/curve.jsonl"
+: >"$TASK_DIR/metrics.jsonl"
 export ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_DIR/credentials.json"
 set +e
 "$TASK_DIR/orbit-schedule-load" -schedules 32 -requests 8 -workers 1 -capacity 1 \
@@ -102,6 +105,48 @@ while kill -0 "$TASK_LOAD_PID" 2>/dev/null; do
 	if [ "$#" -eq 6 ]; then
 		printf '{"at":"%s","waiting":%s,"leased":%s,"running":%s,"succeeded":%s,"occurrences":%s,"unpaused_schedules":%s}\n' "$TASK_AT" "$1" "$2" "$3" "$4" "$5" "$6" >>"$TASK_DIR/curve.jsonl"
 	fi
+	if curl -fsS --max-time 3 "http://127.0.0.1:$TASK_METRICS_PORT/metrics" >"$TASK_DIR/metrics.current" 2>/dev/null; then
+		TASK_PROCESS_CPU=$(ps -p "$TASK_SERVER_PID" -o %cpu= | awk '{print $1}')
+		TASK_PROCESS_RSS=$(ps -p "$TASK_SERVER_PID" -o rss= | awk '{print $1}')
+		TASK_DB_STATS=$("$TASK_DOCKER" stats --no-stream --format '{{.CPUPerc}}|{{.MemUsage}}|{{.MemPerc}}' "$TASK_NAME" 2>/dev/null || true)
+		METRICS_AT="$TASK_AT" PROCESS_CPU="$TASK_PROCESS_CPU" PROCESS_RSS_KB="$TASK_PROCESS_RSS" DB_CONTAINER_STATS="$TASK_DB_STATS" python3 - "$TASK_DIR/metrics.current" >>"$TASK_DIR/metrics.jsonl" <<'PY'
+import json
+import os
+import sys
+
+with open(sys.argv[1], encoding="utf-8") as f:
+    lines = f.read().splitlines()
+values = {}
+for line in lines:
+    if line.startswith("#") or " " not in line:
+        continue
+    name, value = line.rsplit(" ", 1)
+    try:
+        values[name] = float(value)
+    except ValueError:
+        pass
+stats = os.environ.get("DB_CONTAINER_STATS", "")
+cpu, memory, memory_pct = (stats.split("|", 2) + ["", "", ""])[:3]
+row = {
+    "at": os.environ["METRICS_AT"],
+    "server_cpu_percent": float(os.environ.get("PROCESS_CPU") or 0),
+    "server_rss_kb": int(float(os.environ.get("PROCESS_RSS_KB") or 0)),
+    "mysql_cpu_percent": cpu,
+    "mysql_memory": memory,
+    "mysql_memory_percent": memory_pct,
+    "db_connections_open": values.get("orbit_db_connections_open", 0),
+    "db_connections_in_use": values.get("orbit_db_connections_in_use", 0),
+    "db_connections_idle": values.get("orbit_db_connections_idle", 0),
+    "db_wait_count_total": values.get("orbit_db_wait_count_total", 0),
+    "db_wait_duration_seconds_total": values.get("orbit_db_wait_duration_seconds_total", 0),
+    "waiting_tasks": values.get('orbit_pulse_tasks_state{state="WAITING"}', 0),
+    "running_tasks": values.get('orbit_pulse_tasks_state{state="RUNNING"}', 0),
+    "business_snapshot_duration_count": values.get("orbit_business_snapshot_duration_seconds_count", 0),
+    "business_snapshot_duration_sum": values.get("orbit_business_snapshot_duration_seconds_sum", 0),
+}
+print(json.dumps(row, separators=(",", ":")))
+PY
+	fi
 	sleep 2
 done
 set +e
@@ -137,6 +182,7 @@ if [ "${ORBIT_TEST_KEEP_ARTIFACTS:-0}" = 1 ]; then
 	mkdir -p "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}"
 	cp "$TASK_DIR/report.json" "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}/sample-256-deadline.json"
 	cp "$TASK_DIR/curve.jsonl" "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}/curve.jsonl"
+	cp "$TASK_DIR/metrics.jsonl" "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}/metrics.jsonl"
 	printf '%s\n' "$TASK_FINAL" >"${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}/database.json"
 fi
 printf '{"tool_exit":%s,"verify":%s,"database":%s,"curve_points":%s}\n' "$TASK_LOAD_CODE" "$TASK_VERIFY" "$TASK_FINAL" "$TASK_POINTS"