feat: support database connectors and HTTP server configuration
This commit is contained in:
133
scripts/verifyConnector.py
Normal file
133
scripts/verifyConnector.py
Normal file
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify Connector TLS/isolation and HTTP limits from a local or remote module.
|
||||
|
||||
Uses a disposable local MySQL container (1 CPU, 768 MiB), private test credentials,
|
||||
and generated test certificates. Never connects to a project database.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MODULE = "git.yuntuo.hk/k2safe/go-omni"
|
||||
SOURCES = ("database/connector_test.go", "database/mysqlConnector_test.go",
|
||||
"app/httpConfig_test.go", "config/http_test.go")
|
||||
|
||||
|
||||
def run(args, cwd, env, timeout=300):
|
||||
result = subprocess.run([str(a) for a in args], cwd=cwd, env=env,
|
||||
text=True, capture_output=True, timeout=timeout)
|
||||
output = re.sub(r"https://[^/@\s]+:[^/@\s]+@", "https://", result.stdout + result.stderr)
|
||||
# Never print generated database credentials, including from a failed command.
|
||||
for key in ("MYSQL_ROOT_PASSWORD", "MYSQL_PASSWORD", "GO_OMNI_CONNECTOR_PASSWORD"):
|
||||
if env.get(key):
|
||||
output = output.replace(env[key], "[redacted]")
|
||||
if result.returncode:
|
||||
raise RuntimeError(f"{args[0]} failed ({result.returncode}): {output}")
|
||||
return output
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
source = parser.add_mutually_exclusive_group(required=True)
|
||||
source.add_argument("--local", action="store_true")
|
||||
source.add_argument("--version")
|
||||
parser.add_argument("--report", type=Path, required=True)
|
||||
parser.add_argument("--image", default="mysql:8.4")
|
||||
args = parser.parse_args()
|
||||
if args.version and not re.fullmatch(r"v\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?", args.version):
|
||||
parser.error("a fixed release version is required")
|
||||
env = {k: v for k, v in os.environ.items() if not k.startswith("GO_OMNI_")}
|
||||
env.update(GOWORK="off", GOFLAGS="")
|
||||
for key in ("GOPRIVATE", "GONOSUMDB"):
|
||||
env[key] = ",".join(filter(None, [env.get(key), "git.yuntuo.hk"]))
|
||||
began = time.monotonic()
|
||||
with tempfile.TemporaryDirectory(prefix="goOmniConnector-") as directory:
|
||||
temp = Path(directory)
|
||||
endpoint = env.get("DOCKER_HOST") or run(["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], temp, env).strip()
|
||||
if not endpoint.startswith(("unix://", "npipe://")):
|
||||
raise RuntimeError("requires local Docker; remote servers are not used")
|
||||
certs = temp / "certs"
|
||||
certs.mkdir(mode=0o755)
|
||||
run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout", certs / "ca.key", "-out", certs / "ca.pem", "-days", "2", "-subj", "/CN=GoOmni Test CA"], temp, env)
|
||||
run(["openssl", "req", "-newkey", "rsa:2048", "-nodes", "-keyout", certs / "server.key", "-out", temp / "server.csr", "-subj", "/CN=localhost"], temp, env)
|
||||
(temp / "extensions.cnf").write_text("subjectAltName=DNS:localhost,IP:127.0.0.1\nbasicConstraints=CA:FALSE\nextendedKeyUsage=serverAuth\n")
|
||||
run(["openssl", "x509", "-req", "-in", temp / "server.csr", "-CA", certs / "ca.pem", "-CAkey", certs / "ca.key", "-CAcreateserial", "-out", certs / "server.pem", "-days", "2", "-extfile", temp / "extensions.cnf"], temp, env)
|
||||
(certs / "ca.key").unlink()
|
||||
for path in certs.iterdir():
|
||||
path.chmod(0o644) # Disposable fixtures, readable by the container's mysql uid.
|
||||
project = temp / "consumer"
|
||||
project.mkdir()
|
||||
version = args.version or "v0.9.1-preview"
|
||||
(project / "go.mod").write_text(f"module example.com/connectorAcceptance\n\ngo 1.22.0\n\nrequire {MODULE} {version}\n")
|
||||
if args.local:
|
||||
run(["go", "mod", "edit", f"-replace={MODULE}={ROOT}"], project, env)
|
||||
else:
|
||||
env["GOMODCACHE"] = str(temp / "modcache")
|
||||
for name in SOURCES:
|
||||
target = project / name
|
||||
target.parent.mkdir(exist_ok=True)
|
||||
shutil.copyfile(ROOT / name, target)
|
||||
print("Resolving consumer dependency", flush=True)
|
||||
run(["go", "mod", "tidy"], project, env)
|
||||
module = json.loads(run(["go", "list", "-m", "-json", MODULE], project, env))
|
||||
if not args.local:
|
||||
assert module["Version"] == args.version and module.get("Sum") and not module.get("Replace"), module
|
||||
env.update(MYSQL_ROOT_PASSWORD=secrets.token_hex(24), MYSQL_PASSWORD=secrets.token_hex(24))
|
||||
name = "go-omni-connector-" + uuid.uuid4().hex[:12]
|
||||
created = False
|
||||
try:
|
||||
print("Starting isolated MySQL TLS fixture (1 CPU, 768 MiB)", flush=True)
|
||||
run(["docker", "create", "--name", name, "--label", "go-omni.test=connector", "--cpus", "1", "--memory", "768m", "--pids-limit", "128", "--publish", "127.0.0.1::3306", "--mount", f"type=bind,src={certs},dst=/certs,readonly", "--env", "MYSQL_ROOT_PASSWORD", "--env", "MYSQL_PASSWORD", "--env", "MYSQL_DATABASE=omni_test", "--env", "MYSQL_USER=omni_test", args.image, "--ssl-ca=/certs/ca.pem", "--ssl-cert=/certs/server.pem", "--ssl-key=/certs/server.key", "--require-secure-transport=ON", "--innodb-buffer-pool-size=64M", "--max-connections=20"], temp, env)
|
||||
created = True
|
||||
run(["docker", "start", name], temp, env)
|
||||
ready = False
|
||||
for _ in range(60):
|
||||
try:
|
||||
check_env = dict(env, MYSQL_PWD=env["MYSQL_PASSWORD"])
|
||||
result = run(["docker", "exec", "--env", "MYSQL_PWD", name, "mysql", "--protocol=tcp", "-h127.0.0.1", "-uomni_test", "--batch", "--skip-column-names", "-e", "SELECT 1"], temp, check_env, timeout=5)
|
||||
if result.strip() == "1":
|
||||
ready = True
|
||||
break
|
||||
except (RuntimeError, subprocess.TimeoutExpired):
|
||||
pass
|
||||
time.sleep(1)
|
||||
if not ready:
|
||||
raise RuntimeError("isolated MySQL fixture did not become ready")
|
||||
addr = run(["docker", "port", name, "3306/tcp"], temp, env).strip()
|
||||
assert addr.startswith("127.0.0.1:"), addr
|
||||
env.update(GO_OMNI_CONNECTOR_MYSQL_TEST="1", GO_OMNI_CONNECTOR_ADDR=addr,
|
||||
GO_OMNI_CONNECTOR_USER="omni_test", GO_OMNI_CONNECTOR_PASSWORD=env["MYSQL_PASSWORD"],
|
||||
GO_OMNI_CONNECTOR_DATABASE="omni_test", GO_OMNI_CONNECTOR_CA_FILE=str(certs / "ca.pem"))
|
||||
print("Testing TLS trust, per-connector options, driver logging, lifecycle and HTTP limits", flush=True)
|
||||
output = run(["go", "test", "-race", "-count=1", "-json", "-timeout=60s", "./..."], project, env)
|
||||
events = [json.loads(line) for line in output.splitlines() if line.startswith("{")]
|
||||
passed = [e["Test"] for e in events if e.get("Action") == "pass" and "Test" in e]
|
||||
assert "TestMySQLConnectorTLSAndPoolIsolation" in passed
|
||||
assert not any(e.get("Action") in ("fail", "skip") for e in events), output
|
||||
image = run(["docker", "inspect", "--format", "{{.Image}}", name], temp, env).strip()
|
||||
finally:
|
||||
if created:
|
||||
run(["docker", "rm", "--force", "--volumes", name], temp, env)
|
||||
report = {"passed": True, "frameworkVersion": version, "localSource": args.local,
|
||||
"frameworkSum": module.get("Sum"), "freshModuleCache": not args.local,
|
||||
"noLocalReplace": not bool(module.get("Replace")), "mysqlImage": image,
|
||||
"mysqlLimits": {"cpus": 1, "memoryMiB": 768, "pids": 128},
|
||||
"durationSeconds": round(time.monotonic() - began, 2), "tests": passed,
|
||||
"sourceHashes": {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in (*SOURCES, "scripts/verifyConnector.py")}}
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
print("PASS; isolated container, credentials, certificates and consumer removed", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user