96 lines
5.3 KiB
Python
96 lines
5.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Run real S3 uploads in one bounded, network-isolated local Docker container."""
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import tempfile
|
|
import uuid
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--image', default='quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z')
|
|
parser.add_argument('--output', type=Path, default=Path('performance/uploadsLocal.json'))
|
|
args = parser.parse_args()
|
|
root = Path(__file__).resolve().parent.parent
|
|
output = args.output.resolve()
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
log_path = output.with_suffix('.txt')
|
|
name = 'go-omni-upload-' + uuid.uuid4().hex[:12]
|
|
|
|
def capture(command):
|
|
return subprocess.check_output(command, text=True, timeout=15).strip()
|
|
|
|
arch = capture(['docker', 'info', '--format', '{{.Architecture}}'])
|
|
arch = {'aarch64': 'arm64', 'x86_64': 'amd64'}.get(arch, arch)
|
|
if arch not in ('arm64', 'amd64'):
|
|
raise SystemExit('unsupported Docker architecture: ' + arch)
|
|
before = set(capture(['docker', 'ps', '-q']).split())
|
|
report = {
|
|
'status': 'running', 'startedAt': datetime.now(timezone.utc).isoformat(),
|
|
'architecture': arch, 'goVersion': capture(['go', 'version']),
|
|
'image': args.image, 'imageId': capture(['docker', 'image', 'inspect', args.image, '--format', '{{.Id}}']),
|
|
'sourceSha256': {}, 'commands': [],
|
|
'containerLimits': {'cpus': 2, 'memoryBytes': 2147483648, 'pids': 256, 'network': 'none'},
|
|
'notes': ['MinIO verifies S3 signatures; OSS/R2 real cloud accounts are not exercised.'],
|
|
}
|
|
for p in sorted(list((root / 'storage').rglob('*.go')) + list((root / 'internal/storagefixture').glob('*.go')) + [root / 'go.mod', root / 'go.sum']):
|
|
report['sourceSha256'][str(p.relative_to(root))] = hashlib.sha256(p.read_bytes()).hexdigest()
|
|
failure = None
|
|
with log_path.open('w') as log:
|
|
try:
|
|
with tempfile.TemporaryDirectory(prefix='goOmniUpload') as temporary:
|
|
Path(temporary).chmod(0o755)
|
|
env = dict(os.environ, GOOS='linux', GOARCH=arch, CGO_ENABLED='0', GOMAXPROCS='2', GOWORK='off')
|
|
build = ['go', 'test', '-p=2', '-c', '-o', temporary + '/s3.test', './storage/s3']
|
|
report['commands'].append(build)
|
|
subprocess.run(build, cwd=root, env=env, stdout=log, stderr=subprocess.STDOUT, check=True, timeout=240)
|
|
check = '''set -eu
|
|
minio --version
|
|
minio server /data --address 127.0.0.1:9000 --console-address 127.0.0.1:9001 >/tmp/minio.log 2>&1 &
|
|
server=$!
|
|
trap 'kill "$server" 2>/dev/null || true; wait "$server" 2>/dev/null || true' EXIT
|
|
/test/s3.test -test.v -test.run '^TestS3Integration$' -test.count=1 -test.timeout=60s || { cat /tmp/minio.log; exit 1; }
|
|
'''
|
|
command = ['docker', 'run', '--rm', '--name', name, '--platform', 'linux/' + arch,
|
|
'--cpus', '2', '--memory', '2048m', '--memory-swap', '2048m', '--pids-limit', '256',
|
|
'--network', 'none', '--read-only', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges',
|
|
'--user', '1000:1000', '--tmpfs', '/tmp:rw,nosuid,noexec,size=64m,mode=1777',
|
|
'--tmpfs', '/data:rw,nosuid,noexec,size=128m,mode=1777', '-v', temporary + ':/test:ro',
|
|
'-e', 'GOMAXPROCS=2', '-e', 'MINIO_ROOT_USER=goOmniTest', '-e', 'MINIO_ROOT_PASSWORD=goOmniTestSecret',
|
|
'-e', 'GO_OMNI_S3_TEST_ADDR=127.0.0.1:9000', '--entrypoint', 'sh', args.image, '-c', check]
|
|
report['commands'].append(command)
|
|
subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, check=True, timeout=90)
|
|
log.flush()
|
|
if '--- PASS: TestS3Integration' not in log_path.read_text():
|
|
raise RuntimeError('missing integration pass evidence')
|
|
report['status'] = 'passed'
|
|
except Exception as error:
|
|
failure = str(error)
|
|
report['status'], report['error'] = 'failed', failure
|
|
finally:
|
|
remaining = subprocess.run(['docker', 'inspect', name], capture_output=True, timeout=15)
|
|
if remaining.returncode == 0:
|
|
subprocess.run(['docker', 'stop', '-t', '3', name], capture_output=True, timeout=15, check=True)
|
|
after = set(capture(['docker', 'ps', '-q']).split())
|
|
report['preExistingContainersStillRunning'] = before.issubset(after)
|
|
if not before.issubset(after):
|
|
failure = 'a pre-existing container is no longer running'
|
|
report['status'], report['error'] = 'failed', failure
|
|
report['cleanedContainer'] = name
|
|
report['finishedAt'] = datetime.now(timezone.utc).isoformat()
|
|
log.flush()
|
|
report['logSha256'] = hashlib.sha256(log_path.read_bytes()).hexdigest()
|
|
output.write_text(json.dumps(report, indent=2) + '\n')
|
|
if failure:
|
|
raise SystemExit('FAIL: ' + failure + '; see ' + str(log_path))
|
|
print('PASS real S3 direct/multipart uploads, pagination, abort, signed constraints; report: ' + str(output))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|