diff --git a/docs/api/schedule-load.md b/docs/api/schedule-load.md index 8a68672..c68a8d7 100644 --- a/docs/api/schedule-load.md +++ b/docs/api/schedule-load.md @@ -20,7 +20,7 @@ go run ./cmd/orbit-schedule-load \ 可以把 `-schedules` 调到 32、`-requests` 设为 4,观察同一秒集中触发及现有队列速率限制下的排队。队列默认并发和每秒授权量仍由部署配置决定,工具只记录预检快照,不提高配额。另可使用 `-schedules 2 -requests 12 -start-delay 1s -run-timeout 5s` 验证截止不足的失败报告;它必须非零退出,未观察到的触发点不能算成功。 -要观察持久化积压,可在一次性数据库使用 `-schedules 32 -requests 8 -workers 1 -capacity 1 -start-delay 5s -run-timeout 180s`。该配置会让256个触发点集中进入队列,报告预期非零并保留未核验分母;独立连接应每2秒读取 `pulse_tasks` 的 `WAITING/LEASED/RUNNING/SUCCEEDED` 计数,示例结果见[长期积压报告](../reports/phase-6-backlog.md)。 +要观察持久化积压,可在一次性数据库使用 `-schedules 32 -requests 8 -workers 1 -capacity 1 -start-delay 5s -run-timeout 180s`。该配置会让256个触发点集中进入队列,报告预期非零并保留未核验分母;独立连接应每2秒读取 `pulse_tasks` 的 `WAITING/LEASED/RUNNING/SUCCEEDED` 计数,示例结果见[长期积压报告](../reports/phase-6-backlog.md)。仓库还提供[`scripts/test-schedule-backlog.sh`](../../scripts/test-schedule-backlog.sh)自动创建/清理一次性 MySQL、采样状态和执行最终外键核验。 ## 参数边界 diff --git a/docs/reports/phase-6-backlog.md b/docs/reports/phase-6-backlog.md index 6badbce..405dbf4 100644 --- a/docs/reports/phase-6-backlog.md +++ b/docs/reports/phase-6-backlog.md @@ -52,6 +52,6 @@ ORBIT_CREDENTIALS_FILE=/tmp/schedule-backlog.credentials.json \ -start-delay 5s -run-timeout 180s -poll-interval 100ms ``` -准备阶段必须使用一次性 MySQL、`orbit-migrate up`、隔离队列和 `worker-access`/`schedule-access` 权限;不要把这条命令指向生产队列。命令预期在本样本的速率和容量下以非零退出,并在报告中保留未核验分母。收尾后使用 `orbit-admin verify`,再从独立数据库连接检查任务状态和外键数量。完整启动/清理方式按[定时工具手册](../api/schedule-load.md)和仓库一次性 MySQL 脚本约定执行。 +准备阶段必须使用一次性 MySQL、`orbit-migrate up`、隔离队列和 `worker-access`/`schedule-access` 权限;不要把这条命令指向生产队列。命令预期在本样本的速率和容量下以非零退出,并在报告中保留未核验分母。收尾后使用 `orbit-admin verify`,再从独立数据库连接检查任务状态和外键数量。完整启动/清理方式按[定时工具手册](../api/schedule-load.md)执行;仓库提供[`scripts/test-schedule-backlog.sh`](../../scripts/test-schedule-backlog.sh),设置 `ORBIT_TEST_KEEP_ARTIFACTS=1` 可保留报告、曲线和数据库快照。脚本只使用一次性 MySQL,退出0表示预期的工具非零截止结果已核对。 本批没有验证长期运行数小时的存储增长、历史归档、跨节点公平、Worker 重连风暴、MySQL 故障切换、备份恢复或 RPO/RTO;这些仍属于 P6/P7 未完成门槛。 diff --git a/scripts/test-schedule-backlog.sh b/scripts/test-schedule-backlog.sh new file mode 100755 index 0000000..dfbc88b --- /dev/null +++ b/scripts/test-schedule-backlog.sh @@ -0,0 +1,142 @@ +#!/bin/sh +# 在一次性 MySQL 中复现集中触发后的持久任务积压曲线。 +# 样本故意以有界截止结束;退出 0 表示“预期的非零工具结果已核对”,不是任务全部完成。 +set -eu + +TASK_DOCKER=${DOCKER:-docker} +TASK_GO=${GO:-go} +TASK_MYSQL_IMAGE=${ORBIT_TEST_MYSQL_IMAGE:-mysql:8.4} +TASK_RUN_TIMEOUT=${ORBIT_BACKLOG_RUN_TIMEOUT:-180s} +TASK_NAME="orbit-schedule-backlog-$$" +TASK_DIR=$(mktemp -d "${TMPDIR:-/tmp}/orbit-schedule-backlog.XXXXXX") +TASK_SERVER_PID= +TASK_LOAD_PID= + +cleanup() { + if [ -n "$TASK_LOAD_PID" ]; then + kill "$TASK_LOAD_PID" >/dev/null 2>&1 || true + wait "$TASK_LOAD_PID" 2>/dev/null || true + fi + if [ -n "$TASK_SERVER_PID" ]; then + kill "$TASK_SERVER_PID" >/dev/null 2>&1 || true + wait "$TASK_SERVER_PID" 2>/dev/null || true + fi + "$TASK_DOCKER" container stop "$TASK_NAME" >/dev/null 2>&1 || true + "$TASK_DOCKER" container rm "$TASK_NAME" >/dev/null 2>&1 || true + if [ "${ORBIT_TEST_KEEP_ARTIFACTS:-0}" = 1 ]; then + TASK_KEEP_DIR=${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last} + mkdir -p "$TASK_KEEP_DIR" + for task_artifact in "$TASK_DIR"/*; do + if [ -f "$task_artifact" ]; then cp "$task_artifact" "$TASK_KEEP_DIR/"; fi + done + fi + for task_artifact in "$TASK_DIR"/*; do + if [ -f "$task_artifact" ]; then unlink "$task_artifact"; fi + done + rmdir "$TASK_DIR" 2>/dev/null || true +} +trap cleanup EXIT HUP INT TERM + +umask 077 +TASK_DB_PASSWORD=$(openssl rand -hex 24) +"$TASK_DOCKER" run -d --name "$TASK_NAME" \ + -e MYSQL_ROOT_PASSWORD="$TASK_DB_PASSWORD" \ + -e MYSQL_DATABASE=orbit_test \ + -p 127.0.0.1::3306 "$TASK_MYSQL_IMAGE" >/dev/null +TASK_READY=0 +for task_attempt in $(seq 1 60); do + if "$TASK_DOCKER" exec -e MYSQL_PWD="$TASK_DB_PASSWORD" "$TASK_NAME" \ + mysql -uroot --protocol=tcp -e 'SELECT 1' >/dev/null 2>&1; then + TASK_READY=1 + break + fi + sleep 1 +done +if [ "$TASK_READY" -ne 1 ]; then + printf '%s\n' 'Disposable MySQL failed to become ready.' >&2 + "$TASK_DOCKER" logs --tail 30 "$TASK_NAME" >&2 || true + exit 1 +fi +TASK_DB_PORT=$("$TASK_DOCKER" port "$TASK_NAME" 3306/tcp | sed 's/.*://') +TASK_DSN="root:$TASK_DB_PASSWORD@tcp(127.0.0.1:$TASK_DB_PORT)/orbit_test" + +"$TASK_GO" build -o "$TASK_DIR/orbit" ./cmd/orbit +"$TASK_GO" build -o "$TASK_DIR/orbit-migrate" ./cmd/orbit-migrate +"$TASK_GO" build -o "$TASK_DIR/orbit-admin" ./cmd/orbit-admin +(cd sdk/go && GOWORK=off "$TASK_GO" build -o "$TASK_DIR/orbit-schedule-load" ./cmd/orbit-schedule-load) +ORBIT_MYSQL_DSN="$TASK_DSN" "$TASK_DIR/orbit-migrate" up >/dev/null +ORBIT_MYSQL_DSN="$TASK_DSN" "$TASK_DIR/orbit-admin" bootstrap \ + --application schedule-backlog --namespace test --environment test --queue schedule \ + --worker-access --schedule-access --output "$TASK_DIR/credentials.json" >/dev/null + +ORBIT_MYSQL_DSN="$TASK_DSN" ORBIT_HTTP_ADDR=127.0.0.1:0 "$TASK_DIR/orbit" >"$TASK_DIR/server.log" 2>&1 & +TASK_SERVER_PID=$! +TASK_HTTP_PORT= +for task_attempt in $(seq 1 30); do + TASK_HTTP_PORT=$(sed -n 's/.*"address":"127.0.0.1:\([0-9]*\)".*/\1/p' "$TASK_DIR/server.log" | head -n 1) + if [ -n "$TASK_HTTP_PORT" ]; then break; fi + if ! kill -0 "$TASK_SERVER_PID" 2>/dev/null; then cat "$TASK_DIR/server.log" >&2; exit 1; fi + sleep 1 +done +if [ -z "$TASK_HTTP_PORT" ]; then + printf '%s\n' 'Orbit HTTP server failed to start.' >&2 + exit 1 +fi + +query() { + "$TASK_DOCKER" exec -e MYSQL_PWD="$TASK_DB_PASSWORD" "$TASK_NAME" \ + mysql -uroot --protocol=tcp --batch --skip-column-names orbit_test -e "$1" +} +: >"$TASK_DIR/curve.jsonl" +export ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_DIR/credentials.json" +set +e +"$TASK_DIR/orbit-schedule-load" -schedules 32 -requests 8 -workers 1 -capacity 1 \ + -start-delay 5s -run-timeout "$TASK_RUN_TIMEOUT" -poll-interval 100ms \ + >"$TASK_DIR/report.json" 2>"$TASK_DIR/tool.stderr" & +TASK_LOAD_PID=$! +set -e +while kill -0 "$TASK_LOAD_PID" 2>/dev/null; do + TASK_AT=$(date -u +%Y-%m-%dT%H:%M:%S.000Z) + TASK_ROW=$(query 'SELECT (SELECT COUNT(*) FROM pulse_tasks WHERE state="WAITING"),(SELECT COUNT(*) FROM pulse_tasks WHERE state="LEASED"),(SELECT COUNT(*) FROM pulse_tasks WHERE state="RUNNING"),(SELECT COUNT(*) FROM pulse_tasks WHERE state="SUCCEEDED"),(SELECT COUNT(*) FROM pulse_schedule_occurrences),(SELECT COUNT(*) FROM pulse_schedules WHERE paused=FALSE)' 2>/dev/null || true) + set -- $TASK_ROW + if [ "$#" -eq 6 ]; then + printf '{"at":"%s","waiting":%s,"leased":%s,"running":%s,"succeeded":%s,"occurrences":%s,"unpaused_schedules":%s}\n' "$TASK_AT" "$1" "$2" "$3" "$4" "$5" "$6" >>"$TASK_DIR/curve.jsonl" + fi + sleep 2 +done +set +e +wait "$TASK_LOAD_PID" +TASK_LOAD_CODE=$? +set -e +TASK_LOAD_PID= +if [ "$TASK_LOAD_CODE" -ne 1 ]; then + printf 'expected orbit-schedule-load exit 1, got %s\n' "$TASK_LOAD_CODE" >&2 + cat "$TASK_DIR/tool.stderr" >&2 || true + exit 1 +fi +TASK_FINAL=$(query 'SELECT JSON_OBJECT("foreign_key_count",(SELECT COUNT(*) FROM information_schema.referential_constraints WHERE constraint_schema=DATABASE()),"task_count",(SELECT COUNT(*) FROM pulse_tasks),"succeeded_count",(SELECT COUNT(*) FROM pulse_tasks WHERE state="SUCCEEDED"),"occurrence_count",(SELECT COUNT(*) FROM pulse_schedule_occurrences),"unpaused_schedules",(SELECT COUNT(*) FROM pulse_schedules WHERE paused=FALSE));') +TASK_VERIFY=$(ORBIT_MYSQL_DSN="$TASK_DSN" "$TASK_DIR/orbit-admin" verify) +python3 - "$TASK_DIR/report.json" "$TASK_FINAL" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as f: + report = json.load(f) +database = json.loads(sys.argv[2]) +if report["expected_occurrences"] != 256 or report["occurrences"] != 256 or report["fired"] != 256: + raise SystemExit("not all bounded schedule occurrences were persisted") +if report["complete"] or report["errors"].get("RUN_TIMEOUT") != 1 or report["unverified"] <= 0: + raise SystemExit("deadline negative sample did not preserve an incomplete denominator") +if not all(plan["pause_confirmed"] for plan in report["plans"]): + raise SystemExit("not every created schedule was confirmed paused") +if database["task_count"] != 256 or database["occurrence_count"] != 256 or database["unpaused_schedules"] != 0 or database["foreign_key_count"] != 0: + raise SystemExit("final database evidence is inconsistent") +PY +TASK_POINTS=$(wc -l <"$TASK_DIR/curve.jsonl" | tr -d ' ') +if [ "${ORBIT_TEST_KEEP_ARTIFACTS:-0}" = 1 ]; then + mkdir -p "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}" + cp "$TASK_DIR/report.json" "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}/sample-256-deadline.json" + cp "$TASK_DIR/curve.jsonl" "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}/curve.jsonl" + printf '%s\n' "$TASK_FINAL" >"${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-schedule-backlog-last}/database.json" +fi +printf '{"tool_exit":%s,"verify":%s,"database":%s,"curve_points":%s}\n' "$TASK_LOAD_CODE" "$TASK_VERIFY" "$TASK_FINAL" "$TASK_POINTS"