test: add mysql outage recovery rehearsal
This commit is contained in:
157
scripts/test-db-recovery.sh
Executable file
157
scripts/test-db-recovery.sh
Executable file
@@ -0,0 +1,157 @@
|
||||
#!/bin/sh
|
||||
# 验证一次性 MySQL 短暂不可用后的 readiness 和连接池重连。
|
||||
set -eu
|
||||
|
||||
TASK_DOCKER=${DOCKER:-docker}
|
||||
TASK_MYSQL_IMAGE=${ORBIT_TEST_MYSQL_IMAGE:-mysql:8.4}
|
||||
TASK_ORBIT=${ORBIT_BIN:-bin/orbit}
|
||||
TASK_MIGRATE=${ORBIT_MIGRATE_BIN:-bin/orbit-migrate}
|
||||
TASK_ADMIN=${ORBIT_ADMIN_BIN:-bin/orbit-admin}
|
||||
TASK_NAME="orbit-db-recovery-$$"
|
||||
TASK_VOLUME="orbit-db-recovery-volume-$$"
|
||||
TASK_DIR=$(mktemp -d "${TMPDIR:-/tmp}/orbit-db-recovery.XXXXXX")
|
||||
TASK_SERVER_PID=
|
||||
|
||||
cleanup() {
|
||||
if [ -n "$TASK_SERVER_PID" ]; then
|
||||
kill "$TASK_SERVER_PID" >/dev/null 2>&1 || true
|
||||
wait "$TASK_SERVER_PID" 2>/dev/null || true
|
||||
fi
|
||||
if [ "${ORBIT_TEST_KEEP_ARTIFACTS:-0}" = 1 ] && [ -f "$TASK_DIR/server.log" ]; then
|
||||
cp "$TASK_DIR/server.log" "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-db-recovery-last.log}"
|
||||
fi
|
||||
"$TASK_DOCKER" container stop "$TASK_NAME" >/dev/null 2>&1 || true
|
||||
"$TASK_DOCKER" container rm "$TASK_NAME" >/dev/null 2>&1 || true
|
||||
"$TASK_DOCKER" volume rm "$TASK_VOLUME" >/dev/null 2>&1 || true
|
||||
for task_artifact in "$TASK_DIR"/*; do
|
||||
if [ -f "$task_artifact" ]; then unlink "$task_artifact"; fi
|
||||
done
|
||||
rmdir "$TASK_DIR" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
umask 077
|
||||
TASK_DB_PASSWORD=$(openssl rand -hex 24)
|
||||
"$TASK_DOCKER" volume create "$TASK_VOLUME" >/dev/null
|
||||
"$TASK_DOCKER" run -d --name "$TASK_NAME" \
|
||||
-e MYSQL_ROOT_PASSWORD="$TASK_DB_PASSWORD" \
|
||||
-e MYSQL_DATABASE=orbit_test \
|
||||
-v "$TASK_VOLUME:/var/lib/mysql" \
|
||||
-p 127.0.0.1::3306 "$TASK_MYSQL_IMAGE" >/dev/null
|
||||
|
||||
TASK_READY=0
|
||||
for task_attempt in $(seq 1 60); do
|
||||
if "$TASK_DOCKER" exec "$TASK_NAME" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=tcp -e "SELECT 1" >/dev/null 2>&1'; then
|
||||
TASK_READY=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "$TASK_READY" -ne 1 ]; then
|
||||
printf '%s\n' 'Disposable MySQL failed to become ready.' >&2
|
||||
"$TASK_DOCKER" logs --tail 30 "$TASK_NAME" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TASK_DB_PORT=$("$TASK_DOCKER" port "$TASK_NAME" 3306/tcp | sed 's/.*://')
|
||||
export ORBIT_TEST_MYSQL_DSN="root:$TASK_DB_PASSWORD@tcp(127.0.0.1:$TASK_DB_PORT)/orbit_test"
|
||||
ORBIT_MYSQL_DSN="$ORBIT_TEST_MYSQL_DSN" "$TASK_MIGRATE" up >/dev/null
|
||||
ORBIT_MYSQL_DSN="$ORBIT_TEST_MYSQL_DSN" "$TASK_ADMIN" bootstrap --application db-recovery-example --namespace default --environment test --queue recovery --output "$TASK_DIR/credentials.json" >/dev/null
|
||||
|
||||
ORBIT_MYSQL_DSN="$ORBIT_TEST_MYSQL_DSN" ORBIT_HTTP_ADDR=127.0.0.1:0 "$TASK_ORBIT" >"$TASK_DIR/server.log" 2>&1 &
|
||||
TASK_SERVER_PID=$!
|
||||
TASK_HTTP_PORT=
|
||||
for task_attempt in $(seq 1 30); do
|
||||
TASK_HTTP_PORT=$(sed -n 's/.*"address":"127.0.0.1:\([0-9]*\)".*/\1/p' "$TASK_DIR/server.log" | head -n 1)
|
||||
if [ -n "$TASK_HTTP_PORT" ]; then break; fi
|
||||
if ! kill -0 "$TASK_SERVER_PID" 2>/dev/null; then
|
||||
cat "$TASK_DIR/server.log" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ -z "$TASK_HTTP_PORT" ]; then
|
||||
printf '%s\n' 'Orbit HTTP server failed to start.' >&2
|
||||
exit 1
|
||||
fi
|
||||
TASK_URL="http://127.0.0.1:$TASK_HTTP_PORT"
|
||||
|
||||
TASK_INITIAL_STATUS=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 3 "$TASK_URL/health/ready")
|
||||
if [ "$TASK_INITIAL_STATUS" != 200 ]; then
|
||||
printf 'initial readiness returned %s\n' "$TASK_INITIAL_STATUS" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"$TASK_DOCKER" container stop "$TASK_NAME" >/dev/null
|
||||
"$TASK_DOCKER" container rm "$TASK_NAME" >/dev/null
|
||||
TASK_DOWN_STATUS=
|
||||
for task_attempt in $(seq 1 30); do
|
||||
TASK_DOWN_STATUS=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 3 "$TASK_URL/health/ready" || true)
|
||||
if [ "$TASK_DOWN_STATUS" = 503 ]; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "$TASK_DOWN_STATUS" != 503 ]; then
|
||||
printf 'readiness did not report database outage, status=%s\n' "$TASK_DOWN_STATUS" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"$TASK_DOCKER" run -d --name "$TASK_NAME" \
|
||||
-e MYSQL_ROOT_PASSWORD="$TASK_DB_PASSWORD" \
|
||||
-e MYSQL_DATABASE=orbit_test \
|
||||
-v "$TASK_VOLUME:/var/lib/mysql" \
|
||||
-p "127.0.0.1:$TASK_DB_PORT:3306" "$TASK_MYSQL_IMAGE" >/dev/null
|
||||
TASK_READY=0
|
||||
for task_attempt in $(seq 1 60); do
|
||||
if "$TASK_DOCKER" exec "$TASK_NAME" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=tcp -e "SELECT 1" >/dev/null 2>&1'; then
|
||||
TASK_READY=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "$TASK_READY" -ne 1 ]; then
|
||||
printf '%s\n' 'MySQL did not recover after restart.' >&2
|
||||
exit 1
|
||||
fi
|
||||
ORBIT_MYSQL_DSN="$ORBIT_TEST_MYSQL_DSN" "$TASK_ADMIN" verify >/dev/null
|
||||
|
||||
TASK_RECOVERED_STATUS=
|
||||
for task_attempt in $(seq 1 30); do
|
||||
TASK_RECOVERED_STATUS=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 3 "$TASK_URL/health/ready" || true)
|
||||
if [ "$TASK_RECOVERED_STATUS" = 200 ]; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "$TASK_RECOVERED_STATUS" != 200 ]; then
|
||||
printf 'readiness did not recover, status=%s\n' "$TASK_RECOVERED_STATUS" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
read TASK_TOKEN TASK_QUEUE <<EOF
|
||||
$(python3 - "$TASK_DIR/credentials.json" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as f:
|
||||
value = json.load(f)
|
||||
print(value["token"], value["queue_id"])
|
||||
PY
|
||||
)
|
||||
EOF
|
||||
cat >"$TASK_DIR/task.json" <<EOF
|
||||
{"queue_id":"$TASK_QUEUE","handler":"db.recovery.noop","handler_version":"v1","payload_version":"v1","payload":{"source":"database-recovery"},"recovery_safety":"RECONCILE_ONLY","retry":{"max_attempts":1}}
|
||||
EOF
|
||||
TASK_RESPONSE=$(curl -sS --max-time 5 -X POST "$TASK_URL/v1/tasks" \
|
||||
-H "Authorization: Bearer $TASK_TOKEN" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Idempotency-Key: db-recovery-operation' \
|
||||
--data-binary @"$TASK_DIR/task.json")
|
||||
python3 - "$TASK_RESPONSE" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
value = json.loads(sys.argv[1])
|
||||
if not value.get("data", {}).get("task", {}).get("id"):
|
||||
raise SystemExit("recovered database did not accept a task")
|
||||
PY
|
||||
|
||||
printf '{"initial_readiness":%s,"during_database_outage":%s,"after_database_recovery":%s,"task_accepted_after_recovery":true}\n' \
|
||||
"$TASK_INITIAL_STATUS" "$TASK_DOWN_STATUS" "$TASK_RECOVERED_STATUS"
|
||||
Reference in New Issue
Block a user