test: add reverse proxy deployment check
This commit is contained in:
@@ -17,6 +17,7 @@
|
|||||||
- 补充 `MaxConnections` 满载拒绝与槽位释放验收,确认请求完成后连接容量可再次使用。
|
- 补充 `MaxConnections` 满载拒绝与槽位释放验收,确认请求完成后连接容量可再次使用。
|
||||||
- 补充 `MaxConcurrentRequests` 过载恢复验收,确认 503 拒绝后请求槽位释放并可继续处理新请求。
|
- 补充 `MaxConcurrentRequests` 过载恢复验收,确认 503 拒绝后请求槽位释放并可继续处理新请求。
|
||||||
- 增加高频建连/断连的状态池 churn 验收,覆盖连接对象回收与容量复用。
|
- 增加高频建连/断连的状态池 churn 验收,覆盖连接对象回收与容量复用。
|
||||||
|
- 新增 Linux Nginx 反向代理验收脚本,覆盖 native TLS、Keep-Alive、上游超时和优雅关闭。
|
||||||
|
|
||||||
## v0.6.0-preview
|
## v0.6.0-preview
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,9 @@ python3 scripts/verifyLinux.py --load --tls --duration 30s \
|
|||||||
python3 scripts/verifyLinux.py --load --tls --duration 30m \
|
python3 scripts/verifyLinux.py --load --tls --duration 30m \
|
||||||
--concurrency 128 --idle 512 --count 1 \
|
--concurrency 128 --idle 512 --count 1 \
|
||||||
--output performance/linuxSoakTLS-30m.json
|
--output performance/linuxSoakTLS-30m.json
|
||||||
|
|
||||||
|
# Linux 上用 Nginx 反向代理接入 native TLS 服务,检查 Keep-Alive、超时和优雅关闭
|
||||||
|
python3 scripts/verifyProxy.py
|
||||||
```
|
```
|
||||||
|
|
||||||
可通过 `--concurrency 1,32,128` / `--idle 128,1024` 调整矩阵;传空字符串可跳过对应场景。工具只监听本机随机端口,每个场景启动独立服务进程,并在结束后关闭客户端、检查连接排空、关闭服务。响应状态、完整响应体、requestId 或传输错误均计入失败,不能把快速返回的错误页算成成功吞吐。
|
可通过 `--concurrency 1,32,128` / `--idle 128,1024` 调整矩阵;传空字符串可跳过对应场景。工具只监听本机随机端口,每个场景启动独立服务进程,并在结束后关闭客户端、检查连接排空、关闭服务。响应状态、完整响应体、requestId 或传输错误均计入失败,不能把快速返回的错误页算成成功吞吐。
|
||||||
@@ -75,6 +78,8 @@ Linux arm64 容器使用 Go 1.25.14、GOMAXPROCS=4 运行 3 秒矩阵,见 [tls
|
|||||||
|
|
||||||
Linux arm64 的 30 秒高并发回归见 [linuxSoak.json](performance/linuxSoak.json) 和 [linuxSoakTLS.json](performance/linuxSoakTLS.json):两份报告均使用 128 并发流量和 512 条空闲连接,所有场景错误为 0;native 非 TLS 约 146,614 req/s、TLS 约 149,720 req/s,空闲连接堆增量分别约 2.57 MiB 和 4.47 MiB,FD 均由 7 增至 519 并回到 7,goroutine 均回到启动附近。
|
Linux arm64 的 30 秒高并发回归见 [linuxSoak.json](performance/linuxSoak.json) 和 [linuxSoakTLS.json](performance/linuxSoakTLS.json):两份报告均使用 128 并发流量和 512 条空闲连接,所有场景错误为 0;native 非 TLS 约 146,614 req/s、TLS 约 149,720 req/s,空闲连接堆增量分别约 2.57 MiB 和 4.47 MiB,FD 均由 7 增至 519 并回到 7,goroutine 均回到启动附近。
|
||||||
|
|
||||||
|
目标 Linux amd64 服务器的 30 分钟 TLS 验收使用 1 个 CPU、1 GiB 内存的隔离 Docker 容器完成:net/http 约 11,865 req/s,native 约 13,098 req/s,两个流量场景均为 0 错误;native 的 P95/P99 为 44.040/52.429 ms。512 条空闲 TLS 连接时,native 服务端堆增量约 4.50 MiB(约 9,215 B/连接),文件描述符由 7 增至 519 后回到 7,goroutine 和连接均正常回收。原始报告保存在服务器的 `performance/linuxServerSoakTLS.json`。
|
||||||
|
|
||||||
`BenchmarkTransportTCPApplicationRequest/{nethttp,native}/concurrency{1,32,128}` 使用同一测试入口、客户端、应用与响应检查。原先独立的 `BenchmarkNativeApplicationRequest` 已合并,避免两个入口配置漂移。`verifyPerformance.py` 会一并运行。
|
`BenchmarkTransportTCPApplicationRequest/{nethttp,native}/concurrency{1,32,128}` 使用同一测试入口、客户端、应用与响应检查。原先独立的 `BenchmarkNativeApplicationRequest` 已合并,避免两个入口配置漂移。`verifyPerformance.py` 会一并运行。
|
||||||
|
|
||||||
并行 benchmark 的 **`ns/op` 是总经过时间除以请求数,反映吞吐成本,不是单请求延迟**。`B/op` / `allocs/op` 统计同一进程中的客户端与服务端总分配。此前报告的 native 15.4–16.7 µs/op、约 7.8 KB/op、100 allocs/op 应按这个含义理解,不能作为服务端延迟或独立内存指标。
|
并行 benchmark 的 **`ns/op` 是总经过时间除以请求数,反映吞吐成本,不是单请求延迟**。`B/op` / `allocs/op` 统计同一进程中的客户端与服务端总分配。此前报告的 native 15.4–16.7 µs/op、约 7.8 KB/op、100 allocs/op 应按这个含义理解,不能作为服务端延迟或独立内存指标。
|
||||||
|
|||||||
@@ -51,6 +51,8 @@
|
|||||||
- 已用 16 个并发 worker 完成 3,200 次建连/断连 race 验收;MaxConnections 满载拒绝、槽位释放和 TLS 连接上限均已覆盖。
|
- 已用 16 个并发 worker 完成 3,200 次建连/断连 race 验收;MaxConnections 满载拒绝、槽位释放和 TLS 连接上限均已覆盖。
|
||||||
- 新增 `scripts/verifyLinux.py`,固定在 Go 1.25 Linux 容器内运行 race、vet、build 和可选 TLS/非 TLS perfcheck;perfcheck 记录堆、goroutine、连接和 Linux 文件描述符。
|
- 新增 `scripts/verifyLinux.py`,固定在 Go 1.25 Linux 容器内运行 race、vet、build 和可选 TLS/非 TLS perfcheck;perfcheck 记录堆、goroutine、连接和 Linux 文件描述符。
|
||||||
- Linux/arm64 已完成 30 秒、128 并发、512 空闲连接的 TLS 与非 TLS 持续回归,报告见 `performance/linuxSoak.json` 和 `performance/linuxSoakTLS.json`,均为 0 错误且连接、goroutine、FD 回收正常。
|
- Linux/arm64 已完成 30 秒、128 并发、512 空闲连接的 TLS 与非 TLS 持续回归,报告见 `performance/linuxSoak.json` 和 `performance/linuxSoakTLS.json`,均为 0 错误且连接、goroutine、FD 回收正常。
|
||||||
- 自动验收入口:`python3 scripts/verifyPerformance.py`;Linux 固定验收入口:`python3 scripts/verifyLinux.py --load --tls --duration 30m`。真实部署仍需按目标代理和内核参数执行 30 分钟发布前压测。
|
- 目标 Linux amd64 服务器已完成 30 分钟、128 并发、512 空闲连接的 TLS 验收:net/http 约 11,865 req/s、native 约 13,098 req/s,均为 0 错误;native P95/P99 为 44.040/52.429 ms,空闲连接堆增量约 4.50 MiB,FD 由 7 增至 519 后回到 7。
|
||||||
|
- 新增 `scripts/verifyProxy.py`,在 Linux Docker 中用 Nginx 反向代理 native TLS 服务,检查真实 TLS、Keep-Alive、上游超时和 native 优雅关闭。
|
||||||
|
- 自动验收入口:`python3 scripts/verifyPerformance.py`;Linux 固定验收入口:`python3 scripts/verifyLinux.py --load --tls --duration 30m`;代理联调入口:`python3 scripts/verifyProxy.py`。
|
||||||
|
|
||||||
以上阶段以验收结果判断完成。当前仍是基础框架试用阶段;MySQL、适配层契约和一次业务模块试用已完成,自研网络层已有可运行预览,稳定性与生产规模验证尚未完成。
|
以上阶段以验收结果判断完成。当前仍是基础框架试用阶段;MySQL、适配层契约和一次业务模块试用已完成,自研网络层已有可运行预览,稳定性与生产规模验证尚未完成。
|
||||||
|
|||||||
@@ -181,6 +181,16 @@ func serve(adapter string, tlsEnabled bool) error {
|
|||||||
application.GET("/health", func(c *omni.Context) error {
|
application.GET("/health", func(c *omni.Context) error {
|
||||||
return c.OK(map[string]string{"status": "ok"})
|
return c.OK(map[string]string{"status": "ok"})
|
||||||
})
|
})
|
||||||
|
application.GET("/slow", func(c *omni.Context) error {
|
||||||
|
timer := time.NewTimer(2 * time.Second)
|
||||||
|
defer timer.Stop()
|
||||||
|
select {
|
||||||
|
case <-timer.C:
|
||||||
|
return c.OK(map[string]string{"status": "slow"})
|
||||||
|
case <-c.Context().Done():
|
||||||
|
return c.Context().Err()
|
||||||
|
}
|
||||||
|
})
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
var shutdown func(context.Context) error
|
var shutdown func(context.Context) error
|
||||||
switch adapter {
|
switch adapter {
|
||||||
|
|||||||
227
scripts/verifyProxy.py
Normal file
227
scripts/verifyProxy.py
Normal file
@@ -0,0 +1,227 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Verify an isolated Nginx -> native TLS reverse-proxy deployment on Linux."""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import http.client
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import selectors
|
||||||
|
import shutil
|
||||||
|
import socket
|
||||||
|
import ssl
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
def free_port() -> int:
|
||||||
|
with socket.socket() as listener:
|
||||||
|
listener.bind(("127.0.0.1", 0))
|
||||||
|
return listener.getsockname()[1]
|
||||||
|
|
||||||
|
|
||||||
|
def wait_for_native(process: subprocess.Popen[str], timeout: float) -> int:
|
||||||
|
selector = selectors.DefaultSelector()
|
||||||
|
assert process.stdout is not None
|
||||||
|
selector.register(process.stdout, selectors.EVENT_READ)
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
try:
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
if process.poll() is not None:
|
||||||
|
raise RuntimeError(f"native server exited with code {process.returncode}")
|
||||||
|
for _, _ in selector.select(timeout=0.5):
|
||||||
|
line = process.stdout.readline()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
value = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
address = value.get("address", "")
|
||||||
|
if address:
|
||||||
|
return int(address.rsplit(":", 1)[1])
|
||||||
|
raise RuntimeError("native TLS server did not announce a listener")
|
||||||
|
finally:
|
||||||
|
selector.close()
|
||||||
|
|
||||||
|
|
||||||
|
def write_edge_certificate(directory: Path) -> None:
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
"openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes",
|
||||||
|
"-keyout", str(directory / "key.pem"),
|
||||||
|
"-out", str(directory / "cert.pem"),
|
||||||
|
"-days", "1", "-subj", "/CN=localhost",
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def write_nginx_config(path: Path, backend_port: int, edge_port: int) -> None:
|
||||||
|
path.write_text(
|
||||||
|
f"""events {{
|
||||||
|
worker_connections 512;
|
||||||
|
}}
|
||||||
|
|
||||||
|
http {{
|
||||||
|
upstream go_omni_native {{
|
||||||
|
server 127.0.0.1:{backend_port};
|
||||||
|
keepalive 64;
|
||||||
|
}}
|
||||||
|
|
||||||
|
server {{
|
||||||
|
listen 127.0.0.1:{edge_port} ssl;
|
||||||
|
ssl_certificate /etc/nginx/certs/cert.pem;
|
||||||
|
ssl_certificate_key /etc/nginx/certs/key.pem;
|
||||||
|
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Request-ID $http_x_request_id;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_connect_timeout 1s;
|
||||||
|
proxy_read_timeout 300ms;
|
||||||
|
proxy_send_timeout 1s;
|
||||||
|
|
||||||
|
location / {{
|
||||||
|
proxy_pass https://go_omni_native;
|
||||||
|
proxy_ssl_verify off;
|
||||||
|
proxy_ssl_server_name off;
|
||||||
|
}}
|
||||||
|
}}
|
||||||
|
}}
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def request(connection: http.client.HTTPSConnection, path: str) -> tuple[int, dict[str, str], bytes]:
|
||||||
|
connection.request("GET", path, headers={"X-Request-ID": "proxyCheck"})
|
||||||
|
response = connection.getresponse()
|
||||||
|
body = response.read()
|
||||||
|
return response.status, dict(response.headers), body
|
||||||
|
|
||||||
|
|
||||||
|
def verify_keep_alive(edge_port: int) -> None:
|
||||||
|
context = ssl._create_unverified_context()
|
||||||
|
connection = http.client.HTTPSConnection("127.0.0.1", edge_port, context=context, timeout=3)
|
||||||
|
expected = b'{"code":"OK","message":"success","data":{"status":"ok"},"requestId":"proxyCheck"}'
|
||||||
|
try:
|
||||||
|
for attempt in range(5):
|
||||||
|
status, headers, body = request(connection, "/health")
|
||||||
|
if status != 200 or headers.get("X-Request-ID") != "proxyCheck" or body != expected:
|
||||||
|
raise AssertionError(f"unexpected proxied response: status={status} body={body!r}")
|
||||||
|
if attempt < 4 and getattr(connection, "sock", None) is None:
|
||||||
|
raise AssertionError("Nginx did not keep the TLS client connection alive")
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def verify_timeout(edge_port: int) -> None:
|
||||||
|
context = ssl._create_unverified_context()
|
||||||
|
connection = http.client.HTTPSConnection("127.0.0.1", edge_port, context=context, timeout=3)
|
||||||
|
try:
|
||||||
|
status, _, _ = request(connection, "/slow")
|
||||||
|
if status != 504:
|
||||||
|
raise AssertionError(f"expected Nginx upstream timeout 504, got {status}")
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--image", default="golang:1.25")
|
||||||
|
parser.add_argument("--nginx-image", default="nginx:1.27-alpine")
|
||||||
|
args = parser.parse_args()
|
||||||
|
if shutil.which("docker") is None:
|
||||||
|
raise SystemExit("docker is required for proxy verification")
|
||||||
|
if shutil.which("openssl") is None:
|
||||||
|
raise SystemExit("openssl is required for proxy verification")
|
||||||
|
if not Path("/proc").exists():
|
||||||
|
raise SystemExit("proxy verification must run on Linux")
|
||||||
|
|
||||||
|
root = Path(__file__).resolve().parent.parent
|
||||||
|
backend_name = "go-omni-proxy-native"
|
||||||
|
nginx_name = "go-omni-proxy-nginx"
|
||||||
|
backend = None
|
||||||
|
nginx = None
|
||||||
|
with tempfile.TemporaryDirectory(prefix="go-omni-proxy-") as temporary:
|
||||||
|
directory = Path(temporary)
|
||||||
|
edge_port = free_port()
|
||||||
|
write_edge_certificate(directory)
|
||||||
|
backend = subprocess.Popen(
|
||||||
|
[
|
||||||
|
"docker", "run", "--rm", "--pull=missing", "--name", backend_name,
|
||||||
|
"--network", "host", "--cpus=1", "--memory=512m", "--pids-limit=512",
|
||||||
|
"-v", f"{root}:/workspace", "-w", "/workspace", "-e", "GOWORK=off",
|
||||||
|
args.image, "sh", "-c", "go run ./internal/perfcheck --server native --tls",
|
||||||
|
],
|
||||||
|
stdin=subprocess.PIPE,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
bufsize=1,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
backend_port = wait_for_native(backend, 120)
|
||||||
|
write_nginx_config(directory / "nginx.conf", backend_port, edge_port)
|
||||||
|
nginx = subprocess.Popen(
|
||||||
|
[
|
||||||
|
"docker", "run", "--rm", "--pull=missing", "--name", nginx_name,
|
||||||
|
"--network", "host",
|
||||||
|
"-v", f"{directory / 'nginx.conf'}:/etc/nginx/nginx.conf:ro",
|
||||||
|
"-v", f"{directory}:/etc/nginx/certs:ro",
|
||||||
|
args.nginx_image, "nginx", "-g", "daemon off;",
|
||||||
|
],
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
deadline = time.monotonic() + 30
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
if nginx.poll() is not None:
|
||||||
|
raise RuntimeError(f"Nginx exited with code {nginx.returncode}")
|
||||||
|
try:
|
||||||
|
with socket.create_connection(("127.0.0.1", edge_port), timeout=0.5):
|
||||||
|
break
|
||||||
|
except OSError:
|
||||||
|
time.sleep(0.2)
|
||||||
|
else:
|
||||||
|
raise RuntimeError("Nginx did not start listening")
|
||||||
|
|
||||||
|
verify_keep_alive(edge_port)
|
||||||
|
verify_timeout(edge_port)
|
||||||
|
print("PASS Nginx TLS proxy, Keep-Alive and upstream timeout", flush=True)
|
||||||
|
|
||||||
|
assert backend.stdin is not None
|
||||||
|
backend.stdin.write('"stop"\n')
|
||||||
|
backend.stdin.flush()
|
||||||
|
if backend.wait(timeout=5) != 0:
|
||||||
|
raise AssertionError("native server did not shut down cleanly behind Nginx")
|
||||||
|
print("PASS native graceful shutdown behind proxy", flush=True)
|
||||||
|
finally:
|
||||||
|
if nginx is not None and nginx.poll() is None:
|
||||||
|
nginx.terminate()
|
||||||
|
try:
|
||||||
|
nginx.wait(timeout=5)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
nginx.kill()
|
||||||
|
nginx.wait(timeout=5)
|
||||||
|
if backend is not None and backend.poll() is None:
|
||||||
|
if backend.stdin is not None:
|
||||||
|
try:
|
||||||
|
backend.stdin.write('"stop"\n')
|
||||||
|
backend.stdin.flush()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
backend.terminate()
|
||||||
|
try:
|
||||||
|
backend.wait(timeout=5)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
backend.kill()
|
||||||
|
backend.wait(timeout=5)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user