k2safe/go-omni
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add independent platform modules and multi-provider storage

Browse Source
This commit is contained in:
k2safe
2026-09-23 14:51:14 +08:00
parent f91034477b
commit 208a9c3fbd
57 changed files with 4890 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
scripts/verifyModules.py
View File

@@ -22,6 +22,9 @@ def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--image", default="golang:1.25")
parser.add_argument("--redis-image", default="redis:7.4-alpine")
parser.add_argument("--advanced", action="store_true", help="also verify shared state, durable queue, S3 and MySQL migrations")
parser.add_argument("--s3-image", default="quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z")
parser.add_argument("--mysql-image", default="m.daocloud.io/docker.io/library/mysql:8.4")
parser.add_argument("--goproxy", default="")
parser.add_argument("--wall-timeout", type=int, default=900)
parser.add_argument("--output", type=Path, default=Path("performance/modulesLinux.json"))
@@ -40,9 +43,13 @@ def main():
output_path.parent.mkdir(parents=True, exist_ok=True)
log_path = output_path.with_suffix(".txt")
prefix = "go-omni-modules-" + uuid.uuid4().hex[:12]
names = [prefix + "-" + stage for stage in ("build", "redis")]
stages = ("build", "redis", "s3", "mysql") if args.advanced else ("build", "redis")
names = [prefix + "-" + stage for stage in stages]
paths = [root / "go.mod", root / "go.sum", root / "httpx/context.go", Path(__file__).resolve()]
for directory in ("cache", "httpc", "task", "middleware", "cmd/go-omni", "examples/modules"):
directories = ["cache", "httpc", "task", "middleware", "cmd/go-omni", "examples/modules"]
if args.advanced:
directories += ["auth", "config", "logx", "openapi", "telemetry", "resilience", "storage", "queue", "audit", "pagination", "database/migrate", "internal/storagefixture", "examples/platform"]
for directory in directories:
paths.extend(p for p in (root / directory).rglob("*") if p.is_file() and p.suffix in (".go", ".tmpl"))
report = {
"status": "running", "startedAt": datetime.now(timezone.utc).isoformat(),
@@ -86,7 +93,10 @@ def main():
report["cleanedContainers"].append(name)
try:
for image in (args.image, args.redis_image):
images = [args.image, args.redis_image]
if args.advanced:
images += [args.s3_image, args.mysql_image]
for image in images:
result = subprocess.run([docker, "image", "inspect", image], capture_output=True, timeout=15)
if result.returncode:
print("Pulling " + image, flush=True)
@@ -101,9 +111,11 @@ def main():
"-e", "GOWORK=off", "-e", "GOMAXPROCS=2", "-e", "GO_OMNI_SCAFFOLD_INTEGRATION=1"]
if args.goproxy:
arguments += ["-e", "GOPROXY=" + args.goproxy]
arguments += [args.image, "sh", "-c",
"go version && go test -race -count=1 -timeout=240s ./... && go vet ./... && go build ./... && "
"CGO_ENABLED=0 go test -c -o /output/redis.test ./cache/redis"]
checks = "go version && go test -race -count=1 -timeout=240s ./... && go vet ./... && go build ./... && CGO_ENABLED=0 go test -c -o /output/redis.test ./cache/redis"
if args.advanced:
for binary, package in (("shared", "middleware/redis"), ("queue", "queue/redis"), ("s3", "storage/s3"), ("mysql", "database/migrate")):
checks += f" && CGO_ENABLED=0 go test -c -o /output/{binary}.test ./{package}"
arguments += [args.image, "sh", "-c", checks]
container(names[0], arguments)
print("Real Redis TTL/read/write/delete/failure checks; external network disabled", flush=True)
redis_check = '''set -eu
@@ -118,15 +130,54 @@ until redis-cli -h 127.0.0.1 ping >/dev/null 2>&1; do
done
/test/redis.test -test.v -test.run '^TestRedisIntegration$' -test.count=1 -test.timeout=30s
'''
if args.advanced:
redis_check += "/test/shared.test -test.v -test.count=1 -test.timeout=45s\n/test/queue.test -test.v -test.count=1 -test.timeout=45s\n"
container(names[1], ["--network", "none", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges",
"--user", "999:999", "--tmpfs", "/tmp:rw,nosuid,noexec,size=64m,mode=1777",
"-v", f"{temporary}:/test:ro", "--entrypoint", "sh",
"-e", "GOMAXPROCS=2", "-e", "GO_OMNI_REDIS_TEST_ADDR=127.0.0.1:6379",
args.redis_image, "-c", redis_check])
if args.advanced:
isolated = ["--network", "none", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges",
"-v", f"{temporary}:/test:ro", "--entrypoint", "sh", "-e", "GOMAXPROCS=2"]
print("Real S3 object operations and signature verification in isolated MinIO", flush=True)
s3_check = '''set -eu
minio --version
minio server /data --address 127.0.0.1:9000 --console-address 127.0.0.1:9001 >/tmp/minio.log 2>&1 &
server=$!
trap 'kill "$server" 2>/dev/null || true; wait "$server" 2>/dev/null || true' EXIT
/test/s3.test -test.v -test.run '^TestS3Integration$' -test.count=1 -test.timeout=45s || { cat /tmp/minio.log; exit 1; }
'''
container(names[2], isolated + ["--user", "1000:1000", "--tmpfs", "/tmp:rw,nosuid,noexec,size=64m,mode=1777",
"--tmpfs", "/data:rw,nosuid,noexec,size=128m,mode=1777", "-e", "MINIO_ROOT_USER=goOmniTest", "-e", "MINIO_ROOT_PASSWORD=goOmniTestSecret",
"-e", "GO_OMNI_S3_TEST_ADDR=127.0.0.1:9000", args.s3_image, "-c", s3_check])
print("Real MySQL migrations, repeat apply, dirty-state blocking and SQL spans", flush=True)
mysql_check = '''set -eu
mysqld --version
mkdir /tmp/mysql
mysqld --no-defaults --initialize-insecure --datadir=/tmp/mysql --innodb-redo-log-capacity=67108864 >/tmp/init.log 2>&1 || { cat /tmp/init.log; exit 1; }
mysqld --no-defaults --datadir=/tmp/mysql --socket=/tmp/mysql.sock --pid-file=/tmp/mysql.pid --bind-address=127.0.0.1 --port=3306 --mysqlx=OFF --skip-log-bin --skip-name-resolve --innodb-buffer-pool-size=134217728 --innodb-redo-log-capacity=67108864 --max-connections=8 --log-error=/tmp/mysql.log &
server=$!
trap 'kill "$server" 2>/dev/null || true; wait "$server" 2>/dev/null || true' EXIT
attempt=0
until mysql --no-defaults --socket=/tmp/mysql.sock -uroot -e 'SELECT 1' >/dev/null 2>&1; do
attempt=$((attempt + 1)); if [ "$attempt" -gt 200 ]; then cat /tmp/mysql.log; exit 1; fi; sleep 0.1
done
mysql --no-defaults --socket=/tmp/mysql.sock -uroot -e 'CREATE DATABASE omni_test'
/test/mysql.test -test.v -test.run '^TestMySQLIntegration$' -test.count=1 -test.timeout=60s
'''
container(names[3], isolated + ["--user", "999:999", "--tmpfs", "/tmp:rw,nosuid,noexec,size=512m,mode=1777",
"--tmpfs", "/var/lib/mysql:rw,nosuid,noexec,size=1m,mode=1777",
"-e", "GO_OMNI_MIGRATE_TEST_DSN=root@unix(/tmp/mysql.sock)/omni_test?parseTime=true&timeout=2s&readTimeout=5s&writeTimeout=5s",
args.mysql_image, "-c", mysql_check])
log.flush()
content = log_path.read_text()
if "--- PASS: TestRedisIntegration" not in content:
raise RuntimeError("missing real Redis pass evidence")
if args.advanced:
for test in ("TestSharedRedisIntegration", "TestExpiredTargetBeyondCleanupBatch", "TestQueueRedisIntegration", "TestS3Integration", "TestMySQLIntegration"):
if "--- PASS: " + test not in content:
raise RuntimeError("missing integration pass evidence: " + test)
report["status"] = "passed"
except (Exception, KeyboardInterrupt) as error:
failure = str(error) or type(error).__name__