diff --git a/docs/api/schedule-load.md b/docs/api/schedule-load.md index c68a8d7..694b1cd 100644 --- a/docs/api/schedule-load.md +++ b/docs/api/schedule-load.md @@ -22,6 +22,8 @@ go run ./cmd/orbit-schedule-load \ 要观察持久化积压,可在一次性数据库使用 `-schedules 32 -requests 8 -workers 1 -capacity 1 -start-delay 5s -run-timeout 180s`。该配置会让256个触发点集中进入队列,报告预期非零并保留未核验分母;独立连接应每2秒读取 `pulse_tasks` 的 `WAITING/LEASED/RUNNING/SUCCEEDED` 计数,示例结果见[长期积压报告](../reports/phase-6-backlog.md)。仓库还提供[`scripts/test-schedule-backlog.sh`](../../scripts/test-schedule-backlog.sh)自动创建/清理一次性 MySQL、采样状态和执行最终外键核验。 +积压脚本同时启用临时指标监听,每2秒把服务进程 CPU/RSS、MySQL 容器 CPU/内存、DB 连接池使用量、连接池等待累计值、等待/运行任务和业务快照耗时计数写入 `metrics.jsonl`。设置 `ORBIT_TEST_KEEP_ARTIFACTS=1`(可配合 `ORBIT_TEST_KEEP_ARTIFACTS_PATH`)会保留该文件;它是单节点采样原始证据,不是跨节点聚合或生产容量报告。 + ## 参数边界 | 参数 | 默认值 | 限制/语义 | diff --git a/docs/reports/phase-6-backlog.md b/docs/reports/phase-6-backlog.md index 405dbf4..bb3ac4c 100644 --- a/docs/reports/phase-6-backlog.md +++ b/docs/reports/phase-6-backlog.md @@ -36,7 +36,7 @@ | 后段 | 98 | 158 | 256 | 32 | | 截止前最后点 | 83 | 173 | 256 | 32 | -本批没有开启指标监听,也没有独占 CPU、内存、磁盘或数据库资源;因此曲线只证明持久状态和收尾行为,不证明资源消耗曲线。后续目标拓扑容量测试需要同时采集数据库连接池、锁等待、CPU、内存、磁盘、复制延迟和队列阶段延迟,并进行多轮稳态与恢复对照。 +本批原始样本没有开启指标监听,也没有独占 CPU、内存、磁盘或数据库资源;因此已保存曲线只证明持久状态和收尾行为,不证明资源消耗曲线。随后脚本已加入临时指标端口和 `metrics.jsonl` 采样,会在下一次复现中留存服务进程 CPU/RSS、MySQL 容器资源、数据库连接池等待和业务快照计数;新文件仍只是单节点原始采样。目标拓扑容量测试还需要采集锁等待、复制延迟和队列阶段延迟,并进行多轮稳态与恢复对照。 ## 环境与复现 diff --git a/scripts/test-schedule-backlog.sh b/scripts/test-schedule-backlog.sh index dfbc88b..e0f8dda 100755 --- a/scripts/test-schedule-backlog.sh +++ b/scripts/test-schedule-backlog.sh @@ -69,17 +69,19 @@ ORBIT_MYSQL_DSN="$TASK_DSN" "$TASK_DIR/orbit-admin" bootstrap \ --application schedule-backlog --namespace test --environment test --queue schedule \ --worker-access --schedule-access --output "$TASK_DIR/credentials.json" >/dev/null -ORBIT_MYSQL_DSN="$TASK_DSN" ORBIT_HTTP_ADDR=127.0.0.1:0 "$TASK_DIR/orbit" >"$TASK_DIR/server.log" 2>&1 & +ORBIT_MYSQL_DSN="$TASK_DSN" ORBIT_HTTP_ADDR=127.0.0.1:0 ORBIT_METRICS_ADDR=127.0.0.1:0 "$TASK_DIR/orbit" >"$TASK_DIR/server.log" 2>&1 & TASK_SERVER_PID=$! TASK_HTTP_PORT= +TASK_METRICS_PORT= for task_attempt in $(seq 1 30); do TASK_HTTP_PORT=$(sed -n 's/.*"address":"127.0.0.1:\([0-9]*\)".*/\1/p' "$TASK_DIR/server.log" | head -n 1) - if [ -n "$TASK_HTTP_PORT" ]; then break; fi + TASK_METRICS_PORT=$(sed -n 's/.*"metrics_address":"127.0.0.1:\([0-9]*\)".*/\1/p' "$TASK_DIR/server.log" | head -n 1) + if [ -n "$TASK_HTTP_PORT" ] && [ -n "$TASK_METRICS_PORT" ]; then break; fi if ! kill -0 "$TASK_SERVER_PID" 2>/dev/null; then cat "$TASK_DIR/server.log" >&2; exit 1; fi sleep 1 done -if [ -z "$TASK_HTTP_PORT" ]; then - printf '%s\n' 'Orbit HTTP server failed to start.' >&2 +if [ -z "$TASK_HTTP_PORT" ] || [ -z "$TASK_METRICS_PORT" ]; then + printf '%s\n' 'Orbit HTTP or metrics server failed to start.' >&2 exit 1 fi @@ -88,6 +90,7 @@ query() { mysql -uroot --protocol=tcp --batch --skip-column-names orbit_test -e "$1" } : >"$TASK_DIR/curve.jsonl" +: >"$TASK_DIR/metrics.jsonl" export ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_DIR/credentials.json" set +e "$TASK_DIR/orbit-schedule-load" -schedules 32 -requests 8 -workers 1 -capacity 1 \ @@ -102,6 +105,48 @@ while kill -0 "$TASK_LOAD_PID" 2>/dev/null; do if [ "$#" -eq 6 ]; then printf '{"at":"%s","waiting":%s,"leased":%s,"running":%s,"succeeded":%s,"occurrences":%s,"unpaused_schedules":%s}\n' "$TASK_AT" "$1" "$2" "$3" "$4" "$5" "$6" >>"$TASK_DIR/curve.jsonl" fi + if curl -fsS --max-time 3 "http://127.0.0.1:$TASK_METRICS_PORT/metrics" >"$TASK_DIR/metrics.current" 2>/dev/null; then + TASK_PROCESS_CPU=$(ps -p "$TASK_SERVER_PID" -o %cpu= | awk '{print $1}') + TASK_PROCESS_RSS=$(ps -p "$TASK_SERVER_PID" -o rss= | awk '{print $1}') + TASK_DB_STATS=$("$TASK_DOCKER" stats --no-stream --format '{{.CPUPerc}}|{{.MemUsage}}|{{.MemPerc}}' "$TASK_NAME" 2>/dev/null || true) + METRICS_AT="$TASK_AT" PROCESS_CPU="$TASK_PROCESS_CPU" PROCESS_RSS_KB="$TASK_PROCESS_RSS" DB_CONTAINER_STATS="$TASK_DB_STATS" python3 - "$TASK_DIR/metrics.current" >>"$TASK_DIR/metrics.jsonl" <<'PY' +import json +import os +import sys + +with open(sys.argv[1], encoding="utf-8") as f: + lines = f.read().splitlines() +values = {} +for line in lines: + if line.startswith("#") or " " not in line: + continue + name, value = line.rsplit(" ", 1) + try: + values[name] = float(value) + except ValueError: + pass +stats = os.environ.get("DB_CONTAINER_STATS", "") +cpu, memory, memory_pct = (stats.split("|", 2) + ["", "", ""])[:3] +row = { + "at": os.environ["METRICS_AT"], + "server_cpu_percent": float(os.environ.get("PROCESS_CPU") or 0), + "server_rss_kb": int(float(os.environ.get("PROCESS_RSS_KB") or 0)), + "mysql_cpu_percent": cpu, + "mysql_memory": memory, + "mysql_memory_percent": memory_pct, + "db_connections_open": values.get("orbit_db_connections_open", 0), + "db_connections_in_use": values.get("orbit_db_connections_in_use", 0), + "db_connections_idle": values.get("orbit_db_connections_idle", 0), + "db_wait_count_total": values.get("orbit_db_wait_count_total", 0), + "db_wait_duration_seconds_total": values.get("orbit_db_wait_duration_seconds_total", 0), + "waiting_tasks": values.get('orbit_pulse_tasks_state{state="WAITING"}', 0), + "running_tasks": values.get('orbit_pulse_tasks_state{state="RUNNING"}', 0), + "business_snapshot_duration_count": values.get("orbit_business_snapshot_duration_seconds_count", 0), + "business_snapshot_duration_sum": values.get("orbit_business_snapshot_duration_seconds_sum", 0), +} +print(json.dumps(row, separators=(",", ":"))) +PY + fi sleep 2 done set +e @@ -137,6 +182,7 @@ if [ "${ORBIT_TEST_KEEP_ARTIFACTS:-0}" = 1 ]; then mkdir -p "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}" cp "$TASK_DIR/report.json" "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}/sample-256-deadline.json" cp "$TASK_DIR/curve.jsonl" "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}/curve.jsonl" + cp "$TASK_DIR/metrics.jsonl" "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}/metrics.jsonl" printf '%s\n' "$TASK_FINAL" >"${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}/database.json" fi printf '{"tool_exit":%s,"verify":%s,"database":%s,"curve_points":%s}\n' "$TASK_LOAD_CODE" "$TASK_VERIFY" "$TASK_FINAL" "$TASK_POINTS"