From d94df4fc549d12397cf20a4e179ae0bdc8bcfc04 Mon Sep 17 00:00:00 2001 From: k2safe Date: Wed, 23 Sep 2026 01:23:20 +0800 Subject: [PATCH] fix: bound HTTP2 soak resources and cleanup --- CHANGELOG.md | 3 +- README.md | 4 +- ROADMAP.md | 3 +- performance/http2SmokeLinux.json | 84 ++++++++++++++++++++++++++++++++ scripts/verifyHTTP2.py | 62 ++++++++++++++++++++--- scripts/verifyHTTP2_test.py | 65 ++++++++++++++++++++++++ transport/http2/soak_test.go | 81 +++++++++++++++++++++++++----- 7 files changed, 279 insertions(+), 23 deletions(-) create mode 100644 performance/http2SmokeLinux.json create mode 100644 scripts/verifyHTTP2_test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 55f7ffd..d33425e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,8 @@ - 新增独立 `transport/http2` 实验 Runner：支持 h2c/TLS h2、连接上限、全局请求并发闸门、GOAWAY 排空、RST_STREAM 单流取消和生命周期统计；不改变 native HTTP/1.1 默认路径。 - HTTP/2 实验 Runner 增加标准 1 MiB 头部上限、默认 8 MiB 请求体上限、128 路同连接多路复用、2 MiB 慢读取流控验收、协议错误、RST_STREAM 和 GOAWAY 帧计数；已声明超限请求在进入业务 handler 前返回 413。 - 新增可配置时长的 HTTP/2 TLS soak：默认 3 秒，Linux 验收可通过 `GO_OMNI_HTTP2_SOAK_DURATION=30m` 覆盖，混合验证 128 路 stream、慢 handler、请求体、Keep-Alive 和退出回收。 -- 新增 `scripts/verifyHTTP2.py`，固定在 Go 1.25 Linux 容器中运行 HTTP/2 TLS race 长测。 +- HTTP/2 soak 每 15 秒输出请求/失败数、Go 堆、Linux RSS、goroutine、FD 和协议统计；校验响应协议、内容和独立请求 ID，不再忽略压测期限附近已发出请求的失败。 +- 新增 `scripts/verifyHTTP2.py`，固定在 Go 1.25 Linux 容器中运行 HTTP/2 TLS 长测；默认使用普通模式，`--race` 默认运行 60 秒。容器默认限制为 2 核、2 GiB 内存、256 个进程/线程，带总时限、中断清理和日志大小限制。 - native TLS 明确限制 ALPN 为 `http/1.1`：缺省自动宣告该协议，误配置 `h2` 等未实现协议时在启动阶段报错。 - native `Server.Stats()` 新增连接和请求过载拒绝计数；新增可信代理配置，访问日志只在 socket 对端属于配置网段时解析 `X-Forwarded-For`。 - 新增 transport-neutral `metrics` 包，支持请求数、错误数、in-flight、累计耗时和状态分布，并可直接注册 `/metrics` 接口。 diff --git a/README.md b/README.md index 7222658..4fbfa5b 100644 --- a/README.md +++ b/README.md @@ -228,7 +228,9 @@ python3 scripts/verifyBusiness.py --version v0.6.0-preview 标准库与 native 的并发、延迟和长连接内存对照见 [PERFORMANCE.md](PERFORMANCE.md)。`python3 scripts/verifyPerformance.py` 运行检查与 benchmark；增加 `--load --output /tmp/goOmniPerf.json` 可测量独立服务进程并保存报告。 -HTTP/2 仍是显式 opt-in 实验适配器；Linux TLS 长测使用 `python3 scripts/verifyHTTP2.py --duration 30m`，不会改变 native 的 HTTP/1.1 默认路径。 +HTTP/2 仍是显式 opt-in 实验适配器，不会改变 native 的 HTTP/1.1 默认路径。Linux TLS 长测使用 `python3 scripts/verifyHTTP2.py --duration 30m`，默认使用普通 Go 运行模式；`--race` 默认改为 60 秒，用于并发检查，其额外内存开销不能当成生产运行基线。 + +该脚本默认把测试容器限制为 2 核、2 GiB 内存（禁用 swap）、256 个进程/线程，整个命令最多运行 40 分钟，中断和超时会清理本次容器。这些限制可用 `--cpus`、`--memory-mb`、`--pids-limit`、`--wall-timeout` 调整；共享服务器应先用 `--duration 60s` 短测，再逐步延长。每 15 秒输出的堆、RSS 和 FD 统计包含同一进程内的客户端与服务端；当前验证范围见 [Linux HTTP/2 短测记录](performance/http2SmokeLinux.json)。 在框架根目录运行： diff --git a/ROADMAP.md b/ROADMAP.md index c5a315e..6ee0918 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -51,7 +51,8 @@ - native `Stats()` 已记录连接和请求过载拒绝数；访问日志已支持可信代理网段配置，默认拒绝伪造的 `X-Forwarded-For`。 - native TLS 已明确只宣告 HTTP/1.1 ALPN；HTTP/2 暂由边缘代理终止或使用 `transport/nethttp`，已整理独立协议设计、流控和压测门槛，见 [native HTTP/2 进入条件](transport/native/HTTP2.md)，完成后再接入 native。 - 已加入独立 `transport/http2` 实验 Runner，验证 h2c/TLS h2、连接上限、全局请求并发拒绝和恢复，以及 GOAWAY 排空、RST_STREAM 单流取消和服务关闭超时；它使用 `x/net/http2` 作为临时帧/流控实现，尚未达到 native 生产协议的流控和长测门槛。 -- HTTP/2 实验已补齐头部/请求体上限、128 路同连接多路复用、慢读取流控、协议错误、RST_STREAM 与 GOAWAY 统计，并提供 `scripts/verifyHTTP2.py --duration 30m` 的 Linux TLS 长测入口；代理联调和长测结果仍是进入 native 前的门槛。 +- HTTP/2 实验已补齐头部/请求体上限、128 路同连接多路复用、慢读取流控、协议错误、RST_STREAM 与 GOAWAY 统计，并提供 `scripts/verifyHTTP2.py --duration 30m` 的 Linux TLS 长测入口；长测默认不启用 race，短时并发检查使用 `--race`（默认 60 秒）。测试容器默认限制为 2 核、2 GiB 内存、256 个进程/线程，并设总时限和退出清理；代理联调和长测结果仍是进入 native 前的门槛。 +- 新 Linux amd64 环境在上述硬上限内完成 HTTP/2 普通模式 2 分钟与 race 模式 60 秒短测，分别为 3,885,273 和 377,788 次请求，均为 0 失败；连接/请求归零、FD 回到基线。SIGTERM、SIGHUP 和总时限退出清理已实测。报告见 [HTTP/2 短测记录](performance/http2SmokeLinux.json)；旧的无资源上限 race 长测因 RSS 增长被停止，30 分钟 HTTP/2 验收仍未完成。 - 已用 16 个并发 worker 完成 3,200 次建连/断连 race 验收；MaxConnections 满载拒绝、槽位释放和 TLS 连接上限均已覆盖。 - 新增 `scripts/verifyLinux.py`，固定在 Go 1.25 Linux 容器内运行 race、vet、build 和可选 TLS/非 TLS perfcheck；perfcheck 记录堆、goroutine、连接和 Linux 文件描述符。 - Linux/arm64 已完成 30 秒、128 并发、512 空闲连接的 TLS 与非 TLS 持续回归，报告见 `performance/linuxSoak.json` 和 `performance/linuxSoakTLS.json`，均为 0 错误且连接、goroutine、FD 回收正常。 diff --git a/performance/http2SmokeLinux.json b/performance/http2SmokeLinux.json new file mode 100644 index 0000000..ecae615 --- /dev/null +++ b/performance/http2SmokeLinux.json @@ -0,0 +1,84 @@ +{ + "createdAt": "2026-09-22T17:22:44.674797+00:00", + "platform": "linux/amd64", + "image": "golang:1.25", + "imageId": "sha256:699337d620559a59b4a2bb298ad59611e535d2ee755a34cf2d2a98f37578dc80", + "sourceSha256": { + "scripts/verifyHTTP2.py": "2c9f4ca095e16e9b30b80b5d812b2644577222274e5a8041e0fb178e305197a4", + "transport/http2/soak_test.go": "9964a1670c8ffe30b27e96375e0ae72d00996b59beede748756013e8fd52eacf" + }, + "containerLimits": { + "cpus": 2, + "memoryBytes": 2147483648, + "memoryAndSwapBytes": 2147483648, + "pids": 256 + }, + "gomaxprocs": 2, + "workers": 128, + "notes": [ + "Bounded smoke validation, not completion of the 30-minute HTTP/2 acceptance gate.", + "TLS HTTP/2 over container loopback, with the client and server in one process; memory is not server-only.", + "One TCP connection multiplexes the 128 workers; each response checks HTTP/2, status, body and a unique request ID.", + "MiB uses 1024*1024 bytes. RSS is /proc/self/statm; heap is runtime.ReadMemStats. Samples every 15 seconds.", + "The after sample includes the one monitoring goroutine; it is stopped and joined before the test returns.", + "All seven pre-existing containers remained running, and no soak containers remained after cleanup.", + "The prior unbounded 30-minute race run was stopped after RSS growth; its result is not counted as a pass.", + "The short A/B comparison does not establish the complete cause of the old server incident." + ], + "results": [ + { + "mode": "regular", + "duration": "2m", + "elapsedSeconds": 120.04, + "requests": 3885273, + "failures": 0, + "protocolErrors": 0, + "sampledPeakRssBytes": 22437888, + "before": { + "heapBytes": 168040, + "rssBytes": 8491008, + "goroutines": 2, + "fileDescriptors": 6 + }, + "after": { + "heapBytes": 3925848, + "rssBytes": 21364736, + "goroutines": 3, + "fileDescriptors": 6, + "activeConnections": 0, + "activeRequests": 0 + }, + "exitCode": 0 + }, + { + "mode": "race", + "duration": "60s", + "elapsedSeconds": 60.127, + "requests": 377788, + "failures": 0, + "protocolErrors": 0, + "sampledPeakRssBytes": 492707840, + "before": { + "heapBytes": 169256, + "rssBytes": 23732224, + "goroutines": 2, + "fileDescriptors": 6 + }, + "after": { + "heapBytes": 2722632, + "rssBytes": 487239680, + "goroutines": 3, + "fileDescriptors": 6, + "activeConnections": 0, + "activeRequests": 0 + }, + "exitCode": 0 + } + ], + "cleanupValidation": { + "sigterm": "passed", + "sighup": "passed", + "wallTimeout": "passed" + }, + "thirtyMinuteAcceptance": "pending" +} diff --git a/scripts/verifyHTTP2.py b/scripts/verifyHTTP2.py index 36335e3..078d9cb 100755 --- a/scripts/verifyHTTP2.py +++ b/scripts/verifyHTTP2.py @@ -2,42 +2,90 @@ """Run the opt-in HTTP/2 TLS soak in a pinned Linux Go image.""" import argparse +import math from pathlib import Path import shutil +import signal import subprocess -import sys +import uuid def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--image", default="golang:1.25") parser.add_argument("--goproxy", default="", help="optional Go module proxy passed to the container") - parser.add_argument("--duration", default="30m") + parser.add_argument("--duration", help="soak duration (default: 30m, or 60s with --race)") parser.add_argument("--timeout", default="35m", help="go test timeout (default: 35m)") parser.add_argument("--gomaxprocs", type=int, default=2) + parser.add_argument("--cpus", type=float, default=2, help="container CPU quota (default: 2)") + parser.add_argument("--memory-mb", type=int, default=2048, help="container memory cap, with no swap (default: 2048 MiB)") + parser.add_argument("--pids-limit", type=int, default=256, help="container process/thread cap (default: 256)") + parser.add_argument("--wall-timeout", type=int, default=2400, help="total time limit including build/download, in seconds (default: 2400)") + parser.add_argument( + "--race", + action="store_true", + help="enable the race detector for short concurrency checks; its memory overhead is not a production baseline", + ) args = parser.parse_args() + if args.duration is None: + args.duration = "60s" if args.race else "30m" if not args.duration or not args.timeout or args.gomaxprocs < 1: parser.error("--duration and --timeout must be non-empty and --gomaxprocs must be positive") + if not math.isfinite(args.cpus) or args.cpus <= 0 or args.memory_mb < 128 or args.pids_limit < 16 or args.wall_timeout < 1: + parser.error("--cpus and --wall-timeout must be positive; --memory-mb >= 128 and --pids-limit >= 16 are required") docker = shutil.which("docker") if docker is None: raise SystemExit("docker is required for HTTP/2 Linux soak") root = Path(__file__).resolve().parent.parent + name = "go-omni-http2-soak-" + uuid.uuid4().hex[:12] command = [ - docker, "run", "--rm", "--pull=missing", - "-v", f"{root}:/workspace", "-w", "/workspace", + docker, "run", "--rm", "--pull=missing", "--name", name, + "--cpus", str(args.cpus), "--memory", f"{args.memory_mb}m", + "--memory-swap", f"{args.memory_mb}m", "--pids-limit", str(args.pids_limit), + "--log-opt", "max-size=10m", "--log-opt", "max-file=1", + "-v", f"{root}:/workspace:ro", "-w", "/workspace", "-e", "GOWORK=off", "-e", f"GOMAXPROCS={args.gomaxprocs}", "-e", f"GO_OMNI_HTTP2_SOAK_DURATION={args.duration}", ] if args.goproxy: command += ["-e", f"GOPROXY={args.goproxy}"] + command += [args.image, "go", "test"] + if args.race: + command.append("-race") command += [ - args.image, "go", "test", "-race", "-timeout", args.timeout, "./transport/http2", - "-run", "^TestTLSSoak$", "-count=1", + "-v", "-timeout", args.timeout, "./transport/http2", "-run", "^TestTLSSoak$", "-count=1", ] print(f"HTTP/2 Linux soak image: {args.image}", flush=True) print(f"HTTP/2 TLS soak duration: {args.duration}", flush=True) - result = subprocess.run(command, cwd=root) + print(f"HTTP/2 TLS soak mode: {'race' if args.race else 'regular'}", flush=True) + print(f"HTTP/2 soak container: {name}; limits: {args.cpus:g} CPUs, {args.memory_mb} MiB, {args.pids_limit} tasks", flush=True) + + def interrupt(signum, frame): + raise KeyboardInterrupt + + previous_handlers = { + sig: signal.signal(sig, interrupt) for sig in (signal.SIGTERM, signal.SIGHUP) + } + try: + result = subprocess.run(command, cwd=root, timeout=args.wall_timeout) + except subprocess.TimeoutExpired: + raise SystemExit("HTTP/2 Linux soak exceeded --wall-timeout; stopping its container") + except KeyboardInterrupt: + raise SystemExit("HTTP/2 Linux soak interrupted; stopping its container") + finally: + for sig, handler in previous_handlers.items(): + signal.signal(sig, handler) + # Only this run's unique container is targeted, including on timeout or + # a disconnected SSH session. Do not prune other jobs. + try: + cleanup = subprocess.run( + [docker, "rm", "--force", name], capture_output=True, text=True, timeout=15, + ) + if cleanup.returncode != 0 and "No such container" not in cleanup.stderr: + raise SystemExit(f"Could not clean up {name}: {cleanup.stderr.strip()}") + except subprocess.TimeoutExpired: + raise SystemExit(f"Cleanup timed out; inspect container {name} before another run") if result.returncode != 0: raise SystemExit(f"HTTP/2 Linux soak failed with exit code {result.returncode}") print("PASS HTTP/2 TLS soak", flush=True) diff --git a/scripts/verifyHTTP2_test.py b/scripts/verifyHTTP2_test.py new file mode 100644 index 0000000..dc06c7a --- /dev/null +++ b/scripts/verifyHTTP2_test.py @@ -0,0 +1,65 @@ +"""Regression checks for stress-container cleanup on failure and interruption.""" + +import contextlib +import importlib.util +import io +from pathlib import Path +import signal +import subprocess +import sys +import unittest +from unittest.mock import patch + + +spec = importlib.util.spec_from_file_location("http2Runner", Path(__file__).with_name("verifyHTTP2.py")) +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) + + +class CleanupTests(unittest.TestCase): + def testFailedRunsCleanUpOnlyTheirOwnContainer(self): + failures = ( + subprocess.TimeoutExpired("docker", 1), + KeyboardInterrupt(), + subprocess.CompletedProcess([], 7), + ) + for failure in failures: + with self.subTest(failure=type(failure).__name__): + calls = [] + original_handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGTERM, signal.SIGHUP)} + + def execute(command, **kwargs): + calls.append(command) + if command[1] == "run": + if isinstance(failure, BaseException): + raise failure + return failure + return subprocess.CompletedProcess(command, 0, "", "") + + with patch.object(sys, "argv", ["verifyHTTP2.py", "--duration", "1s"]), \ + patch.object(runner.shutil, "which", return_value="/fake/docker"), \ + patch.object(runner.subprocess, "run", side_effect=execute), \ + contextlib.redirect_stdout(io.StringIO()): + with self.assertRaises(SystemExit) as raised: + runner.main() + self.assertNotEqual(raised.exception.code, 0) + name = calls[0][calls[0].index("--name") + 1] + self.assertEqual(calls[1:], [["/fake/docker", "rm", "--force", name]]) + self.assertEqual(original_handlers, {sig: signal.getsignal(sig) for sig in original_handlers}) + + def testCleanupFailureIsNotReportedAsPass(self): + results = [ + subprocess.CompletedProcess([], 0), + subprocess.CompletedProcess([], 1, "", "Cannot connect to the Docker daemon"), + ] + with patch.object(sys, "argv", ["verifyHTTP2.py", "--duration", "1s"]), \ + patch.object(runner.shutil, "which", return_value="/fake/docker"), \ + patch.object(runner.subprocess, "run", side_effect=results), \ + contextlib.redirect_stdout(io.StringIO()) as output: + with self.assertRaisesRegex(SystemExit, "Could not clean up"): + runner.main() + self.assertNotIn("PASS", output.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/transport/http2/soak_test.go b/transport/http2/soak_test.go index 6c1e1a4..b11ef0d 100644 --- a/transport/http2/soak_test.go +++ b/transport/http2/soak_test.go @@ -7,6 +7,9 @@ import ( "net" "net/http" "os" + "runtime" + "strconv" + "strings" "sync" "sync/atomic" "testing" @@ -18,6 +21,7 @@ import ( // TestTLSSoak is short by default so it remains part of normal package // verification. Linux acceptance runs set GO_OMNI_HTTP2_SOAK_DURATION=30m. func TestTLSSoak(t *testing.T) { + started := time.Now() duration := 3 * time.Second if value := os.Getenv("GO_OMNI_HTTP2_SOAK_DURATION"); value != "" { parsed, err := time.ParseDuration(value) @@ -26,6 +30,7 @@ func TestTLSSoak(t *testing.T) { } duration = parsed } + logSoakResources(t, started, "before", 0, 0, Stats{}) certificate, roots := testCertificate(t) server := New(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { @@ -44,6 +49,7 @@ func TestTLSSoak(t *testing.T) { return } } + writer.Header().Set("X-Soak-Request-Id", request.Header.Get("X-Soak-Request-Id")) _, _ = io.WriteString(writer, request.URL.Path) })) server.MaxConnections = 8 @@ -73,13 +79,37 @@ func TestTLSSoak(t *testing.T) { client := &http.Client{Transport: transport, Timeout: 3 * time.Second} var requests, failures atomic.Uint64 + var firstFailure string + var failureOnce sync.Once + recordFailure := func(message string) { + failures.Add(1) + failureOnce.Do(func() { firstFailure = message }) + } + monitorStop, monitorDone := make(chan struct{}), make(chan struct{}) + go func() { + defer close(monitorDone) + ticker := time.NewTicker(15 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + logSoakResources(t, started, "running", requests.Load(), failures.Load(), server.Stats()) + case <-monitorStop: + return + } + } + }() + defer func() { + close(monitorStop) + <-monitorDone + }() deadline := time.Now().Add(duration) var workers sync.WaitGroup for worker := 0; worker < 128; worker++ { workers.Add(1) go func(worker int) { defer workers.Done() - for time.Now().Before(deadline) { + for sequence := uint64(0); time.Now().Before(deadline); sequence++ { path := "/health" method := http.MethodGet var body io.Reader @@ -95,24 +125,26 @@ func TestTLSSoak(t *testing.T) { request, err := http.NewRequestWithContext(requestContext, method, "https://"+listener.Addr().String()+path, body) if err != nil { cancelRequest() - failures.Add(1) + recordFailure(err.Error()) continue } + requestID := strconv.Itoa(worker) + ":" + strconv.FormatUint(sequence, 10) + request.Header.Set("X-Soak-Request-Id", requestID) response, err := client.Do(request) if err != nil { cancelRequest() - if time.Now().Before(deadline) { - failures.Add(1) - } + recordFailure(err.Error()) continue } - _, readErr := io.Copy(io.Discard, response.Body) + responseBody, readErr := io.ReadAll(io.LimitReader(response.Body, 1024)) response.Body.Close() cancelRequest() - if readErr != nil || response.StatusCode != http.StatusOK { - if time.Now().Before(deadline) { - failures.Add(1) - } + if readErr != nil { + recordFailure(readErr.Error()) + continue + } + if response.StatusCode != http.StatusOK || response.ProtoMajor != 2 || string(responseBody) != path || response.Header.Get("X-Soak-Request-Id") != requestID { + recordFailure("unexpected status, protocol, body or request ID: " + response.Status) continue } requests.Add(1) @@ -131,14 +163,37 @@ func TestTLSSoak(t *testing.T) { if err := <-done; err != nil { t.Fatalf("TLS soak serve: %v", err) } + logSoakResources(t, started, "after", requests.Load(), failures.Load(), server.Stats()) if failures.Load() != 0 || requests.Load() == 0 { - t.Fatalf("TLS soak requests=%d failures=%d stats=%+v", requests.Load(), failures.Load(), server.Stats()) + t.Fatalf("TLS soak requests=%d failures=%d firstFailure=%q stats=%+v", requests.Load(), failures.Load(), firstFailure, server.Stats()) } - if stats := server.Stats(); stats.ActiveConnections != 0 || stats.ActiveRequests != 0 { - t.Fatalf("TLS soak did not reclaim resources: %+v", stats) + if stats := server.Stats(); stats.ActiveConnections != 0 || stats.ActiveRequests != 0 || stats.ProtocolErrors != 0 || stats.RejectedRequests != 0 { + t.Fatalf("TLS soak protocol errors, unexpected rejection or unreclaimed resources: %+v", stats) } } +// These measurements cover the client and server in the same test process. +// RSS includes race runtime allocations that do not appear in Go heap stats. +func logSoakResources(t *testing.T, started time.Time, phase string, requests, failures uint64, stats Stats) { + t.Helper() + var memory runtime.MemStats + runtime.ReadMemStats(&memory) + rssBytes, fileDescriptors := int64(-1), -1 + if data, err := os.ReadFile("/proc/self/statm"); err == nil { + if fields := strings.Fields(string(data)); len(fields) >= 2 { + if pages, err := strconv.ParseInt(fields[1], 10, 64); err == nil { + rssBytes = pages * int64(os.Getpagesize()) + } + } + } + if entries, err := os.ReadDir("/proc/self/fd"); err == nil { + fileDescriptors = len(entries) + } + t.Logf("phase=%s elapsed=%s requests=%d failures=%d heapBytes=%d rssBytes=%d goroutines=%d fileDescriptors=%d stats=%+v", + phase, time.Since(started).Round(time.Millisecond), requests, failures, + memory.HeapAlloc, rssBytes, runtime.NumGoroutine(), fileDescriptors, stats) +} + type repeatingReader struct { remaining int }