orbit/scripts/test-db-recovery.sh

#!/bin/sh
# 验证一次性 MySQL 短暂不可用后的 readiness 和连接池重连。
set -eu

TASK_DOCKER=${DOCKER:-docker}
TASK_MYSQL_IMAGE=${ORBIT_TEST_MYSQL_IMAGE:-mysql:8.4}
TASK_ORBIT=${ORBIT_BIN:-bin/orbit}
TASK_MIGRATE=${ORBIT_MIGRATE_BIN:-bin/orbit-migrate}
TASK_ADMIN=${ORBIT_ADMIN_BIN:-bin/orbit-admin}
TASK_NAME="orbit-db-recovery-$$"
TASK_VOLUME="orbit-db-recovery-volume-$$"
TASK_DIR=$(mktemp -d "${TMPDIR:-/tmp}/orbit-db-recovery.XXXXXX")
TASK_SERVER_PID=

cleanup() {
	if [ -n "$TASK_SERVER_PID" ]; then
		kill "$TASK_SERVER_PID" >/dev/null 2>&1 || true
		wait "$TASK_SERVER_PID" 2>/dev/null || true
	fi
	if [ "${ORBIT_TEST_KEEP_ARTIFACTS:-0}" = 1 ] && [ -f "$TASK_DIR/server.log" ]; then
		cp "$TASK_DIR/server.log" "${ORBIT_TEST_KEEP_ARTIFACTS_PATH:-/tmp/orbit-db-recovery-last.log}"
	fi
	"$TASK_DOCKER" container stop "$TASK_NAME" >/dev/null 2>&1 || true
	"$TASK_DOCKER" container rm "$TASK_NAME" >/dev/null 2>&1 || true
	"$TASK_DOCKER" volume rm "$TASK_VOLUME" >/dev/null 2>&1 || true
	for task_artifact in "$TASK_DIR"/*; do
		if [ -f "$task_artifact" ]; then unlink "$task_artifact"; fi
	done
	rmdir "$TASK_DIR" 2>/dev/null || true
}
trap cleanup EXIT HUP INT TERM

umask 077
TASK_DB_PASSWORD=$(openssl rand -hex 24)
"$TASK_DOCKER" volume create "$TASK_VOLUME" >/dev/null
"$TASK_DOCKER" run -d --name "$TASK_NAME" \
	-e MYSQL_ROOT_PASSWORD="$TASK_DB_PASSWORD" \
	-e MYSQL_DATABASE=orbit_test \
	-v "$TASK_VOLUME:/var/lib/mysql" \
	-p 127.0.0.1::3306 "$TASK_MYSQL_IMAGE" >/dev/null

TASK_READY=0
for task_attempt in $(seq 1 60); do
	if "$TASK_DOCKER" exec "$TASK_NAME" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=tcp -e "SELECT 1" >/dev/null 2>&1'; then
		TASK_READY=1
		break
	fi
	sleep 1
done
if [ "$TASK_READY" -ne 1 ]; then
	printf '%s\n' 'Disposable MySQL failed to become ready.' >&2
	"$TASK_DOCKER" logs --tail 30 "$TASK_NAME" >&2
	exit 1
fi

TASK_DB_PORT=$("$TASK_DOCKER" port "$TASK_NAME" 3306/tcp | sed 's/.*://')
export ORBIT_TEST_MYSQL_DSN="root:$TASK_DB_PASSWORD@tcp(127.0.0.1:$TASK_DB_PORT)/orbit_test"
ORBIT_MYSQL_DSN="$ORBIT_TEST_MYSQL_DSN" "$TASK_MIGRATE" up >/dev/null
ORBIT_MYSQL_DSN="$ORBIT_TEST_MYSQL_DSN" "$TASK_ADMIN" bootstrap --application db-recovery-example --namespace default --environment test --queue recovery --output "$TASK_DIR/credentials.json" >/dev/null

ORBIT_MYSQL_DSN="$ORBIT_TEST_MYSQL_DSN" ORBIT_HTTP_ADDR=127.0.0.1:0 "$TASK_ORBIT" >"$TASK_DIR/server.log" 2>&1 &
TASK_SERVER_PID=$!
TASK_HTTP_PORT=
for task_attempt in $(seq 1 30); do
	TASK_HTTP_PORT=$(sed -n 's/.*"address":"127.0.0.1:\([0-9]*\)".*/\1/p' "$TASK_DIR/server.log" | head -n 1)
	if [ -n "$TASK_HTTP_PORT" ]; then break; fi
	if ! kill -0 "$TASK_SERVER_PID" 2>/dev/null; then
		cat "$TASK_DIR/server.log" >&2
		exit 1
	fi
	sleep 1
done
if [ -z "$TASK_HTTP_PORT" ]; then
	printf '%s\n' 'Orbit HTTP server failed to start.' >&2
	exit 1
fi
TASK_URL="http://127.0.0.1:$TASK_HTTP_PORT"

TASK_INITIAL_STATUS=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 3 "$TASK_URL/health/ready")
if [ "$TASK_INITIAL_STATUS" != 200 ]; then
	printf 'initial readiness returned %s\n' "$TASK_INITIAL_STATUS" >&2
	exit 1
fi

"$TASK_DOCKER" container stop "$TASK_NAME" >/dev/null
"$TASK_DOCKER" container rm "$TASK_NAME" >/dev/null
TASK_DOWN_STATUS=
for task_attempt in $(seq 1 30); do
	TASK_DOWN_STATUS=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 3 "$TASK_URL/health/ready" || true)
	if [ "$TASK_DOWN_STATUS" = 503 ]; then break; fi
	sleep 1
done
if [ "$TASK_DOWN_STATUS" != 503 ]; then
	printf 'readiness did not report database outage, status=%s\n' "$TASK_DOWN_STATUS" >&2
	exit 1
fi

"$TASK_DOCKER" run -d --name "$TASK_NAME" \
	-e MYSQL_ROOT_PASSWORD="$TASK_DB_PASSWORD" \
	-e MYSQL_DATABASE=orbit_test \
	-v "$TASK_VOLUME:/var/lib/mysql" \
	-p "127.0.0.1:$TASK_DB_PORT:3306" "$TASK_MYSQL_IMAGE" >/dev/null
TASK_READY=0
for task_attempt in $(seq 1 60); do
	if "$TASK_DOCKER" exec "$TASK_NAME" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=tcp -e "SELECT 1" >/dev/null 2>&1'; then
		TASK_READY=1
		break
	fi
	sleep 1
done
if [ "$TASK_READY" -ne 1 ]; then
	printf '%s\n' 'MySQL did not recover after restart.' >&2
	exit 1
fi
ORBIT_MYSQL_DSN="$ORBIT_TEST_MYSQL_DSN" "$TASK_ADMIN" verify >/dev/null

TASK_RECOVERED_STATUS=
for task_attempt in $(seq 1 30); do
	TASK_RECOVERED_STATUS=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 3 "$TASK_URL/health/ready" || true)
	if [ "$TASK_RECOVERED_STATUS" = 200 ]; then break; fi
	sleep 1
done
if [ "$TASK_RECOVERED_STATUS" != 200 ]; then
	printf 'readiness did not recover, status=%s\n' "$TASK_RECOVERED_STATUS" >&2
	exit 1
fi

read TASK_TOKEN TASK_QUEUE <<EOF
$(python3 - "$TASK_DIR/credentials.json" <<'PY'
import json
import sys

with open(sys.argv[1], encoding="utf-8") as f:
    value = json.load(f)
print(value["token"], value["queue_id"])
PY
)
EOF
cat >"$TASK_DIR/task.json" <<EOF
{"queue_id":"$TASK_QUEUE","handler":"db.recovery.noop","handler_version":"v1","payload_version":"v1","payload":{"source":"database-recovery"},"recovery_safety":"RECONCILE_ONLY","retry":{"max_attempts":1}}
EOF
TASK_RESPONSE=$(curl -sS --max-time 5 -X POST "$TASK_URL/v1/tasks" \
	-H "Authorization: Bearer $TASK_TOKEN" \
	-H 'Content-Type: application/json' \
	-H 'Idempotency-Key: db-recovery-operation' \
	--data-binary @"$TASK_DIR/task.json")
python3 - "$TASK_RESPONSE" <<'PY'
import json
import sys

value = json.loads(sys.argv[1])
if not value.get("data", {}).get("task", {}).get("id"):
    raise SystemExit("recovered database did not accept a task")
PY

printf '{"initial_readiness":%s,"during_database_outage":%s,"after_database_recovery":%s,"task_accepted_after_recovery":true}\n' \
	"$TASK_INITIAL_STATUS" "$TASK_DOWN_STATUS" "$TASK_RECOVERED_STATUS"