k2safe/orbit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: capture backlog resource samples

Browse Source
This commit is contained in:
k2safe
2026-09-20 15:03:19 +08:00
parent 465349cd6d
commit b057af21b1
3 changed files with 53 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
scripts/test-schedule-backlog.sh
View File

@@ -69,17 +69,19 @@ ORBIT_MYSQL_DSN="$TASK_DSN" "$TASK_DIR/orbit-admin" bootstrap \
--application schedule-backlog --namespace test --environment test --queue schedule \
--worker-access --schedule-access --output "$TASK_DIR/credentials.json" >/dev/null
ORBIT_MYSQL_DSN="$TASK_DSN" ORBIT_HTTP_ADDR=127.0.0.1:0 "$TASK_DIR/orbit" >"$TASK_DIR/server.log" 2>&1 &
ORBIT_MYSQL_DSN="$TASK_DSN" ORBIT_HTTP_ADDR=127.0.0.1:0 ORBIT_METRICS_ADDR=127.0.0.1:0 "$TASK_DIR/orbit" >"$TASK_DIR/server.log" 2>&1 &
TASK_SERVER_PID=$!
TASK_HTTP_PORT=
TASK_METRICS_PORT=
for task_attempt in $(seq 1 30); do
TASK_HTTP_PORT=$(sed -n 's/.*"address":"127.0.0.1:\([0-9]*\)".*/\1/p' "$TASK_DIR/server.log" | head -n 1)
if [ -n "$TASK_HTTP_PORT" ]; then break; fi
TASK_METRICS_PORT=$(sed -n 's/.*"metrics_address":"127.0.0.1:\([0-9]*\)".*/\1/p' "$TASK_DIR/server.log" | head -n 1)
if [ -n "$TASK_HTTP_PORT" ] && [ -n "$TASK_METRICS_PORT" ]; then break; fi
if ! kill -0 "$TASK_SERVER_PID" 2>/dev/null; then cat "$TASK_DIR/server.log" >&2; exit 1; fi
sleep 1
done
if [ -z "$TASK_HTTP_PORT" ]; then
printf '%s\n' 'Orbit HTTP server failed to start.' >&2
if [ -z "$TASK_HTTP_PORT" ] || [ -z "$TASK_METRICS_PORT" ]; then
printf '%s\n' 'Orbit HTTP or metrics server failed to start.' >&2
exit 1
fi
@@ -88,6 +90,7 @@ query() {
mysql -uroot --protocol=tcp --batch --skip-column-names orbit_test -e "$1"
}
: >"$TASK_DIR/curve.jsonl"
: >"$TASK_DIR/metrics.jsonl"
export ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_DIR/credentials.json"
set +e
"$TASK_DIR/orbit-schedule-load" -schedules 32 -requests 8 -workers 1 -capacity 1 \
@@ -102,6 +105,48 @@ while kill -0 "$TASK_LOAD_PID" 2>/dev/null; do
if [ "$#" -eq 6 ]; then
printf '{"at":"%s","waiting":%s,"leased":%s,"running":%s,"succeeded":%s,"occurrences":%s,"unpaused_schedules":%s}\n' "$TASK_AT" "$1" "$2" "$3" "$4" "$5" "$6" >>"$TASK_DIR/curve.jsonl"
fi
if curl -fsS --max-time 3 "http://127.0.0.1:$TASK_METRICS_PORT/metrics" >"$TASK_DIR/metrics.current" 2>/dev/null; then
TASK_PROCESS_CPU=$(ps -p "$TASK_SERVER_PID" -o %cpu= | awk '{print $1}')
TASK_PROCESS_RSS=$(ps -p "$TASK_SERVER_PID" -o rss= | awk '{print $1}')
TASK_DB_STATS=$("$TASK_DOCKER" stats --no-stream --format '{{.CPUPerc}}|{{.MemUsage}}|{{.MemPerc}}' "$TASK_NAME" 2>/dev/null || true)
METRICS_AT="$TASK_AT" PROCESS_CPU="$TASK_PROCESS_CPU" PROCESS_RSS_KB="$TASK_PROCESS_RSS" DB_CONTAINER_STATS="$TASK_DB_STATS" python3 - "$TASK_DIR/metrics.current" >>"$TASK_DIR/metrics.jsonl" <<'PY'
import json
import os
import sys
with open(sys.argv[1], encoding="utf-8") as f:
lines = f.read().splitlines()
values = {}
for line in lines:
if line.startswith("#") or " " not in line:
continue
name, value = line.rsplit(" ", 1)
try:
values[name] = float(value)
except ValueError:
pass
stats = os.environ.get("DB_CONTAINER_STATS", "")
cpu, memory, memory_pct = (stats.split("|", 2) + ["", "", ""])[:3]
row = {
"at": os.environ["METRICS_AT"],
"server_cpu_percent": float(os.environ.get("PROCESS_CPU") or 0),
"server_rss_kb": int(float(os.environ.get("PROCESS_RSS_KB") or 0)),
"mysql_cpu_percent": cpu,
"mysql_memory": memory,
"mysql_memory_percent": memory_pct,
"db_connections_open": values.get("orbit_db_connections_open", 0),
"db_connections_in_use": values.get("orbit_db_connections_in_use", 0),
"db_connections_idle": values.get("orbit_db_connections_idle", 0),
"db_wait_count_total": values.get("orbit_db_wait_count_total", 0),
"db_wait_duration_seconds_total": values.get("orbit_db_wait_duration_seconds_total", 0),
"waiting_tasks": values.get('orbit_pulse_tasks_state{state="WAITING"}', 0),
"running_tasks": values.get('orbit_pulse_tasks_state{state="RUNNING"}', 0),
"business_snapshot_duration_count": values.get("orbit_business_snapshot_duration_seconds_count", 0),
"business_snapshot_duration_sum": values.get("orbit_business_snapshot_duration_seconds_sum", 0),
}
print(json.dumps(row, separators=(",", ":")))
PY
fi
sleep 2
done
set +e
@@ -137,6 +182,7 @@ if [ "${ORBIT_TEST_KEEP_ARTIFACTS:-0}" = 1 ]; then
mkdir -p "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}"
cp "$TASK_DIR/report.json" "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}/sample-256-deadline.json"
cp "$TASK_DIR/curve.jsonl" "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}/curve.jsonl"
cp "$TASK_DIR/metrics.jsonl" "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}/metrics.jsonl"
printf '%s\n' "$TASK_FINAL" >"${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}/database.json"
fi
printf '{"tool_exit":%s,"verify":%s,"database":%s,"curve_points":%s}\n' "$TASK_LOAD_CODE" "$TASK_VERIFY" "$TASK_FINAL" "$TASK_POINTS"