diff --git a/Makefile b/Makefile index d8fb44b..f5a571f 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ GO ?= go -.PHONY: fmt test vet build image integration mysql-replication mysql-compatibility sdk-check protocol-check docs-check schedule-backlog release-manifest release-local-check +.PHONY: fmt test vet build image integration console-e2e mysql-replication mysql-compatibility sdk-check protocol-check docs-check schedule-backlog release-manifest release-local-check fmt: $(GO) fmt ./... cd sdk/go && $(GO) fmt ./... @@ -32,6 +32,9 @@ image: integration: ./scripts/test-mysql.sh +console-e2e: + ./scripts/test-console-e2e.sh + mysql-replication: ./scripts/test-mysql-replication.sh diff --git a/README.md b/README.md index c1e139a..01b7785 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ make vet make build make image make integration +make console-e2e make protocol-check make docs-check ``` diff --git a/docs/08-development-plan.md b/docs/08-development-plan.md index 6101c02..b2bee09 100644 --- a/docs/08-development-plan.md +++ b/docs/08-development-plan.md @@ -36,7 +36,7 @@ ### 控制台第一版 -已将 [Orbit Console](console.md) 编译进服务端，入口为 `/console/`。第一版读取健康、版本、任务、计划、流程和结果订阅接口，支持本地凭据 JSON、响应式总览和有界自动刷新；不直接访问数据库，也不绕过现有 Bearer Token 与动作范围。独立管理员身份和跨命名空间聚合仍不在控制台内提供。 +已将 [Orbit Console](console.md) 编译进服务端，入口为 `/console/`。第一版读取健康、版本、任务、计划、流程和结果订阅接口，使用数据库控制台账号和 HttpOnly 会话，响应式总览和有界自动刷新；不直接访问数据库，也不绕过现有凭据与动作范围。首个管理员可以随 `orbit-admin bootstrap` 同事务创建，独立管理员身份和跨命名空间聚合仍不在控制台内提供。 控制台任务详情已接入 Pulse 的任务快照、执行尝试、操作审计、取消和安全重试接口。页面操作沿用 `expected_revision`、`Idempotency-Key`、原因、证据和服务端权限校验；`NEEDS_REVIEW` 仍要求通过核查 API/SDK 提交最终结论。流程实例详情已接入当前视图、步骤状态和有界事件时间线，并支持带 `control_revision` 的暂停、恢复和取消；对 `NEEDS_REVIEW` 实例进一步提供六类带证据的恢复入口（任务核查、步骤结论、继续、步骤重试、PENDING 接续、等待重建），页面只组装协议请求，当前修订、激活身份、权限、审计和状态门控由服务端负责。结果订阅详情已接入配置、积压窗口和交付历史，并支持启停及 EXPIRED 交付追加预算重试。权限不足时保留明确提示，不把页面操作视为绕过服务端事务边界的快捷入口。 @@ -46,6 +46,8 @@ 已完成一次真实链路回归：标准 `scripts/test-mysql.sh` 在一次性 MySQL 和真实 Orbit 进程中跑通任务、Worker、结果订阅、Weave 流程、进程恢复及独立业务示例；控制台五个入口、健康检查和详情所依赖的读取 API 已用真实结果核对，证据见 [Console 真实链路回归](reports/phase-7-console-e2e.md)。这关闭了本地闭环验证项，生产拓扑和发布门槛仍保持未验收。 +新增 `make console-e2e` 作为首次部署闭环门禁：在一次性 MySQL 和 Docker Orbit 中从空库迁移、初始化控制台管理员，完成登录、受保护页面、WS Worker 成功执行、明确失败重试、定时任务、控制台任务读取和登出后的 401 校验，证据见[首次部署与控制台任务闭环](reports/phase-8-console-e2e.md)。该门禁证明功能链路可重复，不替代目标生产拓扑、容量和 RPO/RTO 验收。 + ## 2. 持续推进方式 每个阶段按以下顺序推进： diff --git a/docs/README.md b/docs/README.md index e0b3f7c..a820b09 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,6 +20,7 @@ | [19 容量模型](19-capacity-model.md) | 企业规模到任务、流程、数据库与连接负载的推导及生产验收方法 | | [P7 协议兼容矩阵](reports/phase-7-compatibility.md) | 当前协议门禁、历史提交双向任务 API 演练与正式标签矩阵边界 | | [P7 Console 真实链路回归](reports/phase-7-console-e2e.md) | 真实 MySQL、Worker、流程、结果订阅与控制台读取闭环 | +| [首次部署与控制台任务闭环](reports/phase-8-console-e2e.md) | 从空库初始化管理员、Docker 启动、控制台登录、WS Worker、重试和定时任务 | | [20 运行与发布手册](20-operations-runbook.md) | 启动、发布、滚动停止、数据库恢复、备份和事故处理 | | [21 容器发布说明](21-container-deployment.md) | 最小镜像、迁移/核验/启动顺序、探针和发布边界 | | [09 可靠结果订阅](09-result-subscription-core.md) | 提交水位、冻结结果与可靠交付 | diff --git a/docs/reports/phase-8-console-e2e.md b/docs/reports/phase-8-console-e2e.md new file mode 100644 index 0000000..d217a0b --- /dev/null +++ b/docs/reports/phase-8-console-e2e.md @@ -0,0 +1,33 @@ +# 首次部署与控制台任务闭环 + +这份回归验证 Orbit 从空数据库开始的第一条可运行链路：迁移、初始化管理员、启动服务、控制台登录、任务提交、Worker WebSocket 执行、结果回传、控制台查询、明确失败重试和定时计划。 + +## 一键执行 + +在仓库根目录运行： + +```sh +make console-e2e +``` + +脚本会创建一次性 Docker 网络、MySQL 容器和 Orbit 容器。所有应用、队列、凭据和管理员账号只写入临时数据库；脚本退出后删除容器、网络和临时文件，不连接已有本地数据库。 + +## 验收内容 + +| 阶段 | 验证内容 | +| --- | --- | +| 空库初始化 | 使用 `orbit-migrate up` 创建当前迁移链，数据库从零开始 | +| 管理员初始化 | `orbit-admin bootstrap` 同事务创建业务资源、Worker/计划权限和首个控制台管理员 | +| 服务启动 | Docker 启动 Orbit，`/health/ready` 返回就绪 | +| 控制台登录 | HttpOnly 会话登录，受保护页面和默认队列配置可读取 | +| WS 执行 | Go SDK 自动 Worker 通过 `pulse.worker.v1` 接收 3 个任务并确认结果 | +| 可靠重试 | 第一次明确失败且可重试，第二次执行成功，历史包含 2 次尝试 | +| 定时任务 | 计划生成固定版本任务，Worker 执行成功且来源为 `SCHEDULE` | +| 页面数据 | 控制台权限范围内能读取成功任务 | +| 退出保护 | 登出后业务 API 返回 401，会话不能继续调用任务接口 | + +通过时输出 `console-e2e=passed`。脚本中的密码只保存在临时文件并通过标准输入交给初始化命令，不打印到日志。 + +## 生产边界 + +这是一条功能和发布前置回归，不是生产容量结论。生产环境还必须按[交付核对清单](../release-checklist.md)验证 TLS 终止、备份恢复、复制切换、容量、RPO/RTO 和告警通知；Worker 仍然由业务方独立部署。 diff --git a/scripts/test-console-e2e.sh b/scripts/test-console-e2e.sh new file mode 100755 index 0000000..97b3c16 --- /dev/null +++ b/scripts/test-console-e2e.sh @@ -0,0 +1,223 @@ +#!/bin/sh +# 在临时 Docker 网络和一次性 MySQL 中验证首次初始化、控制台登录、任务执行、重试和定时任务。 +set -eu + +E2E_DOCKER=${DOCKER:-docker} +E2E_GO=${GO:-go} +E2E_IMAGE=${ORBIT_E2E_IMAGE:-orbit:e2e} +E2E_SUFFIX=$$ +E2E_MYSQL="orbit-e2e-mysql-$E2E_SUFFIX" +E2E_ORBIT="orbit-e2e-server-$E2E_SUFFIX" +E2E_DIR=$(mktemp -d "${TMPDIR:-/tmp}/orbit-console-e2e.XXXXXX") +E2E_PASSWORD_FILE="$E2E_DIR/console-password" +E2E_CREDENTIALS_FILE="$E2E_DIR/credentials.json" + +cleanup() { + "$E2E_DOCKER" stop "$E2E_ORBIT" "$E2E_MYSQL" >/dev/null 2>&1 || true + "$E2E_DOCKER" container rm "$E2E_ORBIT" "$E2E_MYSQL" >/dev/null 2>&1 || true + for e2e_artifact in "$E2E_DIR"/*; do + if [ -f "$e2e_artifact" ]; then + unlink "$e2e_artifact" + fi + done + rmdir "$E2E_DIR" >/dev/null 2>&1 || true +} +trap cleanup EXIT HUP INT TERM + +umask 077 +E2E_DB_PASSWORD=$(openssl rand -hex 24) +E2E_CONSOLE_PASSWORD=$(openssl rand -hex 24) +E2E_APPLICATION="console-e2e-$E2E_SUFFIX" +E2E_ADMIN="orbit-admin-$E2E_SUFFIX" +printf '%s\n' "$E2E_CONSOLE_PASSWORD" > "$E2E_PASSWORD_FILE" + +printf '%s\n' "[1/8] build image" +"$E2E_DOCKER" build --tag "$E2E_IMAGE" . >/dev/null +"$E2E_DOCKER" run -d --name "$E2E_MYSQL" --network bridge \ + --env MYSQL_ROOT_PASSWORD="$E2E_DB_PASSWORD" \ + --env MYSQL_DATABASE=orbit_test mysql:8.4 >/dev/null + +E2E_READY=0 +for e2e_attempt in $(seq 1 60); do + if "$E2E_DOCKER" exec "$E2E_MYSQL" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=tcp -e "SELECT 1" >/dev/null 2>&1'; then + E2E_READY=1 + break + fi + sleep 1 +done +if [ "$E2E_READY" -ne 1 ]; then + printf '%s\n' 'temporary MySQL failed to become ready' >&2 + "$E2E_DOCKER" logs --tail 40 "$E2E_MYSQL" >&2 + exit 1 +fi + +E2E_DSN="root:$E2E_DB_PASSWORD@tcp(mysql:3306)/orbit_test" +printf '%s\n' "[2/8] migrate empty database" +"$E2E_DOCKER" run --rm --network bridge --link "$E2E_MYSQL:mysql" \ + --env ORBIT_MYSQL_DSN="$E2E_DSN" \ + --entrypoint /usr/local/bin/orbit-migrate "$E2E_IMAGE" up >/dev/null + +printf '%s\n' "[3/8] bootstrap resources and console administrator" +"$E2E_DOCKER" run --rm -i --network bridge --link "$E2E_MYSQL:mysql" \ + --env ORBIT_MYSQL_DSN="$E2E_DSN" \ + --volume "$E2E_DIR:/run/orbit" \ + --entrypoint /usr/local/bin/orbit-admin "$E2E_IMAGE" bootstrap \ + --application "$E2E_APPLICATION" \ + --namespace default \ + --environment test \ + --queue files \ + --worker-access \ + --schedule-access \ + --console-username "$E2E_ADMIN" \ + --console-display-name 'Orbit E2E 管理员' \ + --console-password-stdin \ + --output /run/orbit/credentials.json < "$E2E_PASSWORD_FILE" >/dev/null +E2E_QUEUE=$(python3 - "$E2E_CREDENTIALS_FILE" <<'PY' +import json +import sys + +value = json.load(open(sys.argv[1], encoding="utf-8")) +queue = value.get("queue_id", "") +if not queue: + raise SystemExit("bootstrap output has no queue_id") +print(queue) +PY +) + +printf '%s\n' "[4/8] start Orbit container" +"$E2E_DOCKER" run -d --name "$E2E_ORBIT" --network bridge --link "$E2E_MYSQL:mysql" \ + --env ORBIT_MYSQL_DSN="$E2E_DSN" \ + --env ORBIT_HTTP_ADDR=0.0.0.0:8080 \ + --env ORBIT_CONSOLE_QUEUE_ID="$E2E_QUEUE" \ + --publish 127.0.0.1::8080 "$E2E_IMAGE" >/dev/null +E2E_PORT= +for e2e_attempt in $(seq 1 30); do + E2E_PORT=$("$E2E_DOCKER" port "$E2E_ORBIT" 8080/tcp 2>/dev/null | sed -n 's/.*:\([0-9][0-9]*\)$/\1/p' | head -n 1) + if [ -n "$E2E_PORT" ]; then + break + fi + sleep 1 +done +if [ -z "$E2E_PORT" ]; then + printf '%s\n' 'Orbit container did not publish HTTP port' >&2 + "$E2E_DOCKER" logs --tail 60 "$E2E_ORBIT" >&2 + exit 1 +fi +E2E_URL="http://127.0.0.1:$E2E_PORT" +E2E_READY=0 +for e2e_attempt in $(seq 1 30); do + if curl -fsS "$E2E_URL/health/ready" >/dev/null 2>&1; then + E2E_READY=1 + break + fi + sleep 1 +done +if [ "$E2E_READY" -ne 1 ]; then + printf '%s\n' 'Orbit readiness check failed' >&2 + "$E2E_DOCKER" logs --tail 60 "$E2E_ORBIT" >&2 + exit 1 +fi + +printf '%s\n' "[5/8] console login and protected-page check" +ORBIT_E2E_URL="$E2E_URL" ORBIT_E2E_QUEUE="$E2E_QUEUE" ORBIT_E2E_USER="$E2E_ADMIN" \ + ORBIT_E2E_PASSWORD_FILE="$E2E_PASSWORD_FILE" python3 - <<'PY' +import http.cookiejar +import json +import os +import pathlib +import urllib.error +import urllib.parse +import urllib.request + +base = os.environ["ORBIT_E2E_URL"] +queue = os.environ["ORBIT_E2E_QUEUE"] +username = os.environ["ORBIT_E2E_USER"] +password = pathlib.Path(os.environ["ORBIT_E2E_PASSWORD_FILE"]).read_text(encoding="utf-8").strip() +jar = http.cookiejar.CookieJar() +client = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar)) + +def request(path, method="GET", body=None): + request_body = None if body is None else json.dumps(body).encode("utf-8") + request = urllib.request.Request(base + path, method=method, data=request_body) + request.add_header("Accept", "application/json") + if body is not None: + request.add_header("Content-Type", "application/json") + try: + with client.open(request, timeout=8) as response: + return response.status, response.read() + except urllib.error.HTTPError as error: + return error.code, error.read() + +status, _ = request("/console/auth/session") +assert status == 200, status +status, _ = request("/console/auth/login", "POST", {"username": username, "password": password}) +assert status == 200, status +status, body = request("/console/auth/session") +assert status == 200 and json.loads(body).get("authenticated") is True +status, body = request("/console/") +assert status == 200 and "运行统计".encode("utf-8") in body, status +status, body = request("/console/config") +assert status == 200 and json.loads(body).get("default_queue_id") == queue +status, _ = request("/v1/tasks?" + urllib.parse.urlencode({"queue_id": queue, "limit": 20})) +assert status == 200, status +print("console_login=ok protected_page=ok") +PY + +printf '%s\n' "[6/8] WebSocket Worker task execution" +(cd sdk/go && ORBIT_URL="$E2E_URL" ORBIT_CREDENTIALS_FILE="$E2E_CREDENTIALS_FILE" \ + "$E2E_GO" run ./examples/worker) | tee "$E2E_DIR/worker.log" +grep -q 'worker_drained=true started=3 confirmed=3 running=0' "$E2E_DIR/worker.log" + +printf '%s\n' "[7/8] retry and schedule execution" +(cd sdk/go && ORBIT_URL="$E2E_URL" ORBIT_CREDENTIALS_FILE="$E2E_CREDENTIALS_FILE" \ + "$E2E_GO" run ./examples/retry) | tee "$E2E_DIR/retry.log" +grep -q 'attempts=2' "$E2E_DIR/retry.log" +(cd sdk/go && ORBIT_URL="$E2E_URL" ORBIT_CREDENTIALS_FILE="$E2E_CREDENTIALS_FILE" \ + "$E2E_GO" run ./examples/schedules) | tee "$E2E_DIR/schedule.log" +grep -q 'source_kind=SCHEDULE' "$E2E_DIR/schedule.log" + +printf '%s\n' "[8/8] final console data and logout check" +ORBIT_E2E_URL="$E2E_URL" ORBIT_E2E_QUEUE="$E2E_QUEUE" ORBIT_E2E_PASSWORD_FILE="$E2E_PASSWORD_FILE" \ + ORBIT_E2E_USER="$E2E_ADMIN" python3 - <<'PY' +import http.cookiejar +import json +import os +import pathlib +import urllib.error +import urllib.parse +import urllib.request + +base = os.environ["ORBIT_E2E_URL"] +queue = os.environ["ORBIT_E2E_QUEUE"] +username = os.environ["ORBIT_E2E_USER"] +password = pathlib.Path(os.environ["ORBIT_E2E_PASSWORD_FILE"]).read_text(encoding="utf-8").strip() +jar = http.cookiejar.CookieJar() +client = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar)) + +def request(path, method="GET", body=None): + data = None if body is None else json.dumps(body).encode("utf-8") + req = urllib.request.Request(base + path, method=method, data=data) + req.add_header("Accept", "application/json") + if body is not None: + req.add_header("Content-Type", "application/json") + try: + with client.open(req, timeout=8) as response: + return response.status, response.read() + except urllib.error.HTTPError as error: + return error.code, error.read() + +status, _ = request("/console/auth/login", "POST", {"username": username, "password": password}) +assert status == 200, status +status, body = request("/v1/tasks?" + urllib.parse.urlencode({"queue_id": queue, "limit": 100})) +assert status == 200, status +items = json.loads(body).get("data", {}).get("items", []) +assert any(item.get("state") == "SUCCEEDED" for item in items), "no succeeded task visible in console scope" +assert any(item.get("handler", "").startswith("text.normalize.runtime.") for item in items) +status, _ = request("/console/auth/logout", "POST", {}) +assert status == 200, status +status, _ = request("/v1/tasks?" + urllib.parse.urlencode({"queue_id": queue, "limit": 1})) +assert status == 401, status +print("console_task_data=ok succeeded_tasks=%d logout=ok unauthorized_after_logout=%d" % (sum(item.get("state") == "SUCCEEDED" for item in items), status)) +PY + +printf '%s\n' "console-e2e=passed image=$E2E_IMAGE url=$E2E_URL"