196 lines
13 KiB
Python
196 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Verify Connector TLS/isolation and HTTP limits from a local or remote module.
|
|
|
|
Uses a disposable local MySQL container (1 CPU, 768 MiB), private test credentials,
|
|
and generated test certificates. Never connects to a project database.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import secrets
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
MODULE = "git.yuntuo.hk/k2safe/go-omni"
|
|
SOURCES = ("database/connector_test.go", "database/mysqlConnector_test.go",
|
|
"app/httpConfig_test.go", "config/http_test.go", "database/mysql/integration_test.go")
|
|
|
|
|
|
def run(args, cwd, env, timeout=300):
|
|
result = subprocess.run([str(a) for a in args], cwd=cwd, env=env,
|
|
text=True, capture_output=True, timeout=timeout)
|
|
output = re.sub(r"https://[^/@\s]+:[^/@\s]+@", "https://", result.stdout + result.stderr)
|
|
# Never print generated database credentials, including from a failed command.
|
|
for key in ("MYSQL_ROOT_PASSWORD", "MYSQL_PASSWORD", "GO_OMNI_CONNECTOR_PASSWORD", "GO_OMNI_DB_PASSWORD"):
|
|
if env.get(key):
|
|
output = output.replace(env[key], "[redacted]")
|
|
if result.returncode:
|
|
raise RuntimeError(f"{args[0]} failed ({result.returncode}): {output}")
|
|
return output
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
source = parser.add_mutually_exclusive_group(required=True)
|
|
source.add_argument("--local", action="store_true")
|
|
source.add_argument("--version")
|
|
parser.add_argument("--report", type=Path, required=True)
|
|
parser.add_argument("--image", default="mysql:8.4")
|
|
parser.add_argument("--gorm", action="store_true", help="also verify the optional GORM adapter and generated svc extension")
|
|
args = parser.parse_args()
|
|
if args.version and not re.fullmatch(r"v\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?", args.version):
|
|
parser.error("a fixed release version is required")
|
|
env = {k: v for k, v in os.environ.items() if not k.startswith("GO_OMNI_")}
|
|
env.update(GOWORK="off", GOFLAGS="")
|
|
for key in ("GOPRIVATE", "GONOSUMDB"):
|
|
env[key] = ",".join(filter(None, [env.get(key), "git.yuntuo.hk"]))
|
|
began = time.monotonic()
|
|
with tempfile.TemporaryDirectory(prefix="goOmniConnector-") as directory:
|
|
temp = Path(directory)
|
|
endpoint = env.get("DOCKER_HOST") or run(["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], temp, env).strip()
|
|
if not endpoint.startswith(("unix://", "npipe://")):
|
|
raise RuntimeError("requires local Docker; remote servers are not used")
|
|
certs = temp / "certs"
|
|
certs.mkdir(mode=0o755)
|
|
run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout", certs / "ca.key", "-out", certs / "ca.pem", "-days", "2", "-subj", "/CN=GoOmni Test CA"], temp, env)
|
|
run(["openssl", "req", "-newkey", "rsa:2048", "-nodes", "-keyout", certs / "server.key", "-out", temp / "server.csr", "-subj", "/CN=localhost"], temp, env)
|
|
(temp / "extensions.cnf").write_text("subjectAltName=DNS:localhost,IP:127.0.0.1\nbasicConstraints=CA:FALSE\nextendedKeyUsage=serverAuth\n")
|
|
run(["openssl", "x509", "-req", "-in", temp / "server.csr", "-CA", certs / "ca.pem", "-CAkey", certs / "ca.key", "-CAcreateserial", "-out", certs / "server.pem", "-days", "2", "-extfile", temp / "extensions.cnf"], temp, env)
|
|
run(["openssl", "req", "-newkey", "rsa:2048", "-nodes", "-keyout", certs / "client.key", "-out", temp / "client.csr", "-subj", "/CN=GoOmni Test Client"], temp, env)
|
|
(temp / "clientExtensions.cnf").write_text("basicConstraints=CA:FALSE\nextendedKeyUsage=clientAuth\n")
|
|
run(["openssl", "x509", "-req", "-in", temp / "client.csr", "-CA", certs / "ca.pem", "-CAkey", certs / "ca.key", "-CAcreateserial", "-out", certs / "client.pem", "-days", "2", "-extfile", temp / "clientExtensions.cnf"], temp, env)
|
|
(certs / "ca.key").unlink()
|
|
for path in certs.iterdir():
|
|
path.chmod(0o644) # Disposable fixtures, readable by the container's mysql uid.
|
|
project = temp / "consumer"
|
|
project.mkdir()
|
|
version = args.version or "v0.14.1-preview"
|
|
(project / "go.mod").write_text(f"module example.com/connectorAcceptance\n\ngo 1.22.0\n\nrequire {MODULE} {version}\n")
|
|
if args.local:
|
|
run(["go", "mod", "edit", f"-replace={MODULE}={ROOT}"], project, env)
|
|
else:
|
|
env["GOMODCACHE"] = str(temp / "modcache")
|
|
sources = list(SOURCES)
|
|
if args.gorm:
|
|
sources.append("database/gormx/integration_test.go")
|
|
for name in sources:
|
|
target = project / name
|
|
target.parent.mkdir(exist_ok=True, parents=True)
|
|
shutil.copyfile(ROOT / name, target)
|
|
print("Resolving consumer dependency", flush=True)
|
|
run(["go", "mod", "tidy"], project, env)
|
|
module = json.loads(run(["go", "list", "-m", "-json", MODULE], project, env))
|
|
if not args.local:
|
|
assert module["Version"] == args.version and module.get("Sum") and not module.get("Replace"), module
|
|
cli = temp / "bin/go-omni"
|
|
cli.parent.mkdir()
|
|
if args.local:
|
|
run(["go", "build", "-o", cli, "./cmd/go-omni"], ROOT, env)
|
|
else:
|
|
env["GOBIN"] = str(cli.parent)
|
|
run(["go", "install", MODULE + "/cmd/go-omni@" + args.version], temp, env)
|
|
generated = temp / "generated"
|
|
command = [cli, "new", "--database", "mysql", "--module", "example.com/mysqlAcceptance"]
|
|
if args.local:
|
|
command += ["--framework-dir", ROOT]
|
|
run([*command, generated], temp, env)
|
|
shutil.copyfile(ROOT / "scripts/testdata/mysqlGenerated_test.go", generated / "mysql_test.go")
|
|
if args.gorm:
|
|
# Only the documented project-owned extension is customized.
|
|
service = generated / "internal/svc/serviceContext.go"
|
|
code = service.read_text()
|
|
assert '"context"' in code and 'type ServiceContext struct {' in code
|
|
code = code.replace('"context"', '"context"\n "gorm.io/gorm"\n "' + MODULE + '/database/gormx"', 1)
|
|
code = code.replace('type ServiceContext struct {', 'type ServiceContext struct {\n ORM *gorm.DB', 1)
|
|
assert 'return nil' in code
|
|
code = code.replace('return nil', 'var err error\n c.ORM, err = gormx.MySQL(ctx, c.DB, gormx.WithLogger(c.Logger))\n return err', 1)
|
|
service.write_text(code)
|
|
run(["gofmt", "-w", service], generated, env)
|
|
shutil.copyfile(ROOT / "scripts/testdata/gormGenerated_test.go", generated / "gorm_test.go")
|
|
run(["go", "mod", "tidy"], generated, env)
|
|
generated_module = json.loads(run(["go", "list", "-m", "-json", MODULE], generated, env))
|
|
if not args.local:
|
|
assert generated_module["Version"] == args.version and generated_module.get("Sum") == module["Sum"] and not generated_module.get("Replace")
|
|
env.update(MYSQL_ROOT_PASSWORD=secrets.token_hex(24), MYSQL_PASSWORD=secrets.token_hex(24))
|
|
name = "go-omni-connector-" + uuid.uuid4().hex[:12]
|
|
created = False
|
|
try:
|
|
print("Starting isolated MySQL TLS fixture (1 CPU, 768 MiB)", flush=True)
|
|
run(["docker", "create", "--name", name, "--label", "go-omni.test=connector", "--cpus", "1", "--memory", "768m", "--pids-limit", "128", "--publish", "127.0.0.1::3306", "--mount", f"type=bind,src={certs},dst=/certs,readonly", "--env", "MYSQL_ROOT_PASSWORD", "--env", "MYSQL_PASSWORD", "--env", "MYSQL_DATABASE=omni_test", "--env", "MYSQL_USER=omni_test", args.image, "--ssl-ca=/certs/ca.pem", "--ssl-cert=/certs/server.pem", "--ssl-key=/certs/server.key", "--require-secure-transport=ON", "--innodb-buffer-pool-size=64M", "--max-connections=20"], temp, env)
|
|
created = True
|
|
run(["docker", "start", name], temp, env)
|
|
ready = False
|
|
for _ in range(60):
|
|
try:
|
|
check_env = dict(env, MYSQL_PWD=env["MYSQL_PASSWORD"])
|
|
result = run(["docker", "exec", "--env", "MYSQL_PWD", name, "mysql", "--protocol=tcp", "-h127.0.0.1", "-uomni_test", "--batch", "--skip-column-names", "-e", "SELECT 1"], temp, check_env, timeout=5)
|
|
if result.strip() == "1":
|
|
ready = True
|
|
break
|
|
except (RuntimeError, subprocess.TimeoutExpired):
|
|
pass
|
|
time.sleep(1)
|
|
if not ready:
|
|
raise RuntimeError("isolated MySQL fixture did not become ready")
|
|
admin_env = dict(env, MYSQL_PWD=env["MYSQL_ROOT_PASSWORD"])
|
|
run(["docker", "exec", "--env", "MYSQL_PWD", name, "mysql", "-uroot", "-e", "CREATE USER 'omni_mtls'@'%' REQUIRE X509"], temp, admin_env)
|
|
addr = run(["docker", "port", name, "3306/tcp"], temp, env).strip()
|
|
assert addr.startswith("127.0.0.1:"), addr
|
|
env.update(GO_OMNI_CONNECTOR_MYSQL_TEST="1", GO_OMNI_CONNECTOR_ADDR=addr,
|
|
GO_OMNI_CONNECTOR_USER="omni_test", GO_OMNI_CONNECTOR_PASSWORD=env["MYSQL_PASSWORD"],
|
|
GO_OMNI_CONNECTOR_DATABASE="omni_test", GO_OMNI_CONNECTOR_CA_FILE=str(certs / "ca.pem"),
|
|
GO_OMNI_CONNECTOR_CLIENT_CERT=str(certs / "client.pem"), GO_OMNI_CONNECTOR_CLIENT_KEY=str(certs / "client.key"))
|
|
print("Testing TLS trust, per-connector options, driver logging, lifecycle and HTTP limits", flush=True)
|
|
output = run(["go", "test", "-race", "-count=1", "-json", "-timeout=60s", "./..."], project, env)
|
|
events = [json.loads(line) for line in output.splitlines() if line.startswith("{")]
|
|
passed = [e["Test"] for e in events if e.get("Action") == "pass" and "Test" in e]
|
|
assert "TestMySQLConnectorTLSAndPoolIsolation" in passed
|
|
assert "TestStructuredMySQLAdapterTLS" in passed
|
|
if args.gorm:
|
|
assert "TestGORMMySQLTransactionsCancellationAndLogs" in passed
|
|
assert not any(e.get("Action") in ("fail", "skip") for e in events), output
|
|
env.update(GO_OMNI_DB_HOST="127.0.0.1", GO_OMNI_DB_PORT=addr.rsplit(":", 1)[1],
|
|
GO_OMNI_DB_USER="omni_test", GO_OMNI_DB_PASSWORD=env["MYSQL_PASSWORD"],
|
|
GO_OMNI_DB_DATABASE="omni_test", GO_OMNI_DB_TLS_CA_FILE=str(certs / "ca.pem"),
|
|
GO_OMNI_DB_TLS_SERVER_NAME="localhost", GO_OMNI_DB_POOL_MAX_OPEN_CONNS="2",
|
|
GO_OMNI_DB_POOL_MAX_IDLE_CONNS="2")
|
|
print("Testing generated MySQL service config, independent pools and cleanup", flush=True)
|
|
check_output = run([cli, "api", "generate", "--check"], generated, env)
|
|
if not args.local:
|
|
assert "frameworkVersion" not in check_output, check_output
|
|
output = run(["go", "test", "-race", "-count=1", "-json", "-timeout=60s", "./..."], generated, env)
|
|
generated_events = [json.loads(line) for line in output.splitlines() if line.startswith("{")]
|
|
generated_passed = [e["Test"] for e in generated_events if e.get("Action") == "pass" and "Test" in e]
|
|
assert "TestGeneratedMySQLConfigurationAndLifecycle" in generated_passed, output
|
|
if args.gorm:
|
|
assert "TestGeneratedOptionalGORM" in generated_passed, output
|
|
assert not any(e.get("Action") == "fail" or (e.get("Action") == "skip" and "Test" in e) for e in generated_events), output
|
|
passed += generated_passed
|
|
image = run(["docker", "inspect", "--format", "{{.Image}}", name], temp, env).strip()
|
|
finally:
|
|
if created:
|
|
run(["docker", "rm", "--force", "--volumes", name], temp, env)
|
|
report_sources = sources + ["scripts/verifyConnector.py", "scripts/testdata/mysqlGenerated_test.go"]
|
|
if args.gorm:
|
|
report_sources += ["scripts/testdata/gormGenerated_test.go", "database/gormx/gorm.go", "database/gormx/pool.go", "database/gormx/logger.go"]
|
|
report = {"passed": True, "frameworkVersion": version, "localSource": args.local, "gormAdapter": args.gorm,
|
|
"frameworkSum": module.get("Sum"), "freshModuleCache": not args.local,
|
|
"noLocalReplace": not bool(module.get("Replace")), "mysqlImage": image,
|
|
"mysqlLimits": {"cpus": 1, "memoryMiB": 768, "pids": 128},
|
|
"durationSeconds": round(time.monotonic() - began, 2), "tests": passed,
|
|
"sourceHashes": {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in report_sources}}
|
|
args.report.parent.mkdir(parents=True, exist_ok=True)
|
|
args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
|
print("PASS; isolated container, credentials, certificates and consumer removed", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|