链接到期时间：{{.Preview.ExpiresAt}}
只读演示：无数据库、业务身份或公开磁盘目录。
`)) + mux := http.NewServeMux() + mux.Handle("/file", signer) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" || r.Method != http.MethodGet { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("Cache-Control", "private, no-store") + w.Header().Set("Referrer-Policy", "no-referrer") + _ = page.Execute(w, struct{ Preview, Download storage.ReadLink }{preview, download}) + }) + server := &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second, IdleTimeout: 30 * time.Second} + done := make(chan error, 1) + go func() { done <- server.Serve(listener) }() + defer server.Close() + if check { + if err := checkHTTP(preview, download, content); err != nil { + return err + } + fmt.Println("signed read: PDF preview, Chinese filename, download, HEAD and Range passed") + return nil + } + // Never print signed URLs or enable request logging containing query strings. + fmt.Println("Open " + baseURL + " (Ctrl+C to stop)") + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + select { + case err := <-done: + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err + case <-ctx.Done(): + shutdown, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return server.Shutdown(shutdown) + } +} + +func checkHTTP(preview, download storage.ReadLink, content []byte) error { + client := &http.Client{Timeout: 5 * time.Second} + for _, tc := range []struct { + link storage.ReadLink + method, span, disposition string + status int + body []byte + }{ + {preview, "GET", "", "inline", 200, content}, + {preview, "HEAD", "", "inline", 200, nil}, + {preview, "GET", "bytes=0-7", "inline", 206, content[:8]}, + {download, "GET", "", "attachment", 200, content}, + } { + req, err := http.NewRequest(tc.method, tc.link.URL, nil) + if err != nil { + return errors.New("invalid example link") + } + if tc.span != "" { + req.Header.Set("Range", tc.span) + } + response, err := client.Do(req) + if err != nil { + return errors.New("example request failed") + } + body, err := io.ReadAll(response.Body) + response.Body.Close() + kind, params, headerErr := mime.ParseMediaType(response.Header.Get("Content-Disposition")) + if err != nil || headerErr != nil || response.StatusCode != tc.status || !bytes.Equal(body, tc.body) || kind != tc.disposition || params["filename"] != "示例报告.pdf" || response.Header.Get("Content-Type") != "application/pdf" { + return errors.New("example response did not match signed options") + } + size := len(tc.body) + if tc.method == "HEAD" { + size = len(content) + } + if response.Header.Get("Content-Length") != strconv.Itoa(size) || response.Header.Get("Cache-Control") != "private, no-store" { + return errors.New("example size or cache policy mismatch") + } + } + return nil +} + +// A complete, single-page PDF with a real cross-reference table. No generator +// dependency or business document is needed for this example. +func samplePDF() []byte { + stream := "BT /F1 24 Tf 72 720 Td (go-omni signed read) Tj ET\n" + objects := []string{ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>", + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + fmt.Sprintf("<< /Length %d >>\nstream\n%sendstream", len(stream), stream), + } + var pdf bytes.Buffer + pdf.WriteString("%PDF-1.4\n") + offsets := []int{0} + for i, object := range objects { + offsets = append(offsets, pdf.Len()) + fmt.Fprintf(&pdf, "%d 0 obj\n%s\nendobj\n", i+1, object) + } + xref := pdf.Len() + fmt.Fprintf(&pdf, "xref\n0 %d\n0000000000 65535 f \n", len(offsets)) + for _, offset := range offsets[1:] { + fmt.Fprintf(&pdf, "%010d 00000 n \n", offset) + } + fmt.Fprintf(&pdf, "trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(offsets), xref) + return pdf.Bytes() +} diff --git a/examples/signedRead/main_test.go b/examples/signedRead/main_test.go new file mode 100644 index 0000000..91a8f28 --- /dev/null +++ b/examples/signedRead/main_test.go @@ -0,0 +1,9 @@ +package main + +import "testing" + +func TestSignedReadHTTP(t *testing.T) { + if err := run(true); err != nil { + t.Fatal(err) + } +} diff --git a/reports/signedReadAcceptance.md b/reports/signedReadAcceptance.md new file mode 100644 index 0000000..929b5d9 --- /dev/null +++ b/reports/signedReadAcceptance.md @@ -0,0 +1,33 @@ +# v0.29.0 通用签名读取验收 + +范围：Local、S3/R2、OSS 的签名读取选项和本地范围读取。接口不包含业务身份、租户、分享记录、数据库、撤销或固定有效期。旧 Presigner、旧本地链接和默认下载行为兼容。 + +## 本地回归 + +2026-09-30，macOS arm64，Go 1.27.1： + +- `GO_OMNI_SCAFFOLD_INTEGRATION=1 go test -race -count=1 ./...`：57 个测试包通过，apigen 集成包 123.143 秒。 +- `GOTOOLCHAIN=go1.22.12 GO_OMNI_SCAFFOLD_INTEGRATION=1 go test -count=1 ./...`：57 个测试包通过，apigen 集成包 113.212 秒。 +- `go vet ./...`、`go build ./...`、`git diff --check`：通过。 +- `go run ./examples/signedRead -check`：真实回环 HTTP 预览、中文文件名、下载、HEAD、Range 通过。 +- `python3 scripts/verifyUploads.py --output reports/signedReadMinio.json`：真实隔离 MinIO 的新读取选项、原上传/分片与存储契约通过，见 [报告](signedReadMinio.json)和[日志](signedReadMinio.txt)。独立容器无外部网络、不映射端口，已清理，原有容器保持运行。 + +首次 MinIO 回归中，原有契约故意发送错误长度触发 SDK 重试退避，耗尽契约的 10 秒预算。测试专用 client 改为 MaxRetries=1，与现有协议测试一致；生产 client 配置未变，修正后通过。 + +## 覆盖内容 + +- 默认 attachment、inline、可选 MIME、ASCII/中文/特殊字符文件名，拒绝 CR/LF、控制字符、路径、过长参数、非法 MIME 和 disposition。 +- 本地签名绑定版本、key、有效期、全部响应选项，重复参数、未知参数、签名篡改、版本剥离和旧下载 URL 注入 inline 均拒绝，拒绝前不读存储。 +- 30 分钟实际签名到期时间、TTL 上下限、精确到期边界、取消 context；过期 Range/HEAD/条件请求不得读取文件或返回缓存命中。 +- 本地真实文件 GET/HEAD 内容长度、前缀/后缀/开放区间/截断/多段范围、206/416，Seek 不暴露元数据，按范围读取而非全文件缓冲；顺序读取 Store 回退完整 200。 +- 错误及条件响应保持 private/no-store，HEAD 成功与失败均无响应体。 +- OSS SDK V1/V4 独立重新计算签名，验证响应头在签名前加入，中文文件名、配置域名、到期时间及选项篡改；拒绝未签名配置。S3/R2 验证原生 SDK 查询参数与实际到期时间。 +- MinIO 实际 GET/Range 的响应头、字节和状态正确，篡改 disposition/type/cache/expiry 被服务端拒绝。 + +## 验证边界 + +OSS/R2 未使用云账号实测；SDK/MinIO 验证不能代替实际桶和域名验收。OSS 默认域名可能强制下载，其响应头覆盖受平台规则限制，需部署方检查实际 200/206。 + +浏览器控制工具不可用，隔离 Chrome/Chromium 截图尝试未获得成功渲染证据，因此不计为浏览器预览通过。真实 PDF 的 HTTP 响应已检查，浏览器视觉预览与实际云域名接入仍需项目侧回归。 + +远程固定版本空缓存验收结果在发布后记录；不移动已发布标签。 diff --git a/reports/signedReadMinio.json b/reports/signedReadMinio.json new file mode 100644 index 0000000..48c1fc4 --- /dev/null +++ b/reports/signedReadMinio.json @@ -0,0 +1,111 @@ +{ + "status": "passed", + "startedAt": "2026-09-30T14:35:32.113432+00:00", + "architecture": "arm64", + "goVersion": "go version go1.27.1 darwin/arm64", + "image": "quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z", + "imageId": "sha256:8f08aee614800a237906bd48114d733e5ac5bfac4ccdf731f141b0e880d7a253", + "sourceSha256": { + "go.mod": "e3877788b18b605a007530f80b684f06a7ae308844c2959a8750ed55ae8037a3", + "go.sum": "53c136b153ec7c91731f7c927dfda540fbd683037f3fbd1f3d067b1ec7aabc15", + "internal/storagefixture/multipart.go": "da9ec938b3bdb872e10e4f874b6a0224121c14a631dcf4e86d7ebbc1cd34a770", + "internal/storagefixture/server.go": "defacc905246a643faf5371e437f9ce458f05461e0a922766034d1f9ffdaf003", + "storage/local/local.go": "5d1ca75d0d2ca87981eef84a60bf98d0feefa2a9594940d6bd231877f4f8ec02", + "storage/local/local_test.go": "3a2241840968df8e37fd67169c3ef708228449e81f65aff908c8b60d49434dc0", + "storage/local/read_test.go": "ef1f7feb0887ea4b1f30304dfaf542fe9788b6e17856c942f1cd98448656194f", + "storage/multipart.go": "3b718ebe6da681c89a3f797892c88e7acb15c295df2376d64ac35da0976ed69d", + "storage/multipart_test.go": "4053f680f7795db2a6b4aa23590b3669eec7a609dd92c4a1e20891a5299274ee", + "storage/oss/oss.go": "4216ce9f3fe3ac5418a5b809db3cef7e99dcca72a25a2a12301f5897cda85dd5", + "storage/oss/oss_test.go": "555a954010b1218804dc7951fc75e42eae24eba9a99c0871f5ba787f3fc87d45", + "storage/oss/read.go": "429a9a9f6c586a885d7cccb76c9d572d53a79df637ff128caacb2eac35b8d862", + "storage/oss/read_test.go": "a1c273cfedd2f3ee31ce407a7ddd6fd5ad02c65ad632a91bc7f3ead3712b5bbd", + "storage/oss/upload.go": "556129611f9e5549cac8a36b4026bfb19c112a295b55192579f9ac116e20626f", + "storage/oss/upload_test.go": "08eb0ed157f3743288d781f61d4d2b7bfe77852a8259c5e598c719d35889409e", + "storage/read.go": "92787c74ae024822f82b5b84bffc9f8a355f1f2565b99ca7fbefe49157ef4851", + "storage/read_test.go": "9b11ab9640e98c1c9fa7ce1e2587df217e874e3a6a6e27a0dd0ade6fbf3d5e1a", + "storage/s3/config.go": "626ec6a2c0b06c95ee7eb7aa0832b190cfcdeb8a127b6b89e99204040bc3b041", + "storage/s3/read.go": "319b57f2ecf435b71085c7cdcafd50a1e30caa5f357f83d4db3cdcef085792b6", + "storage/s3/read_test.go": "9b64fda5c83ba41508ec23cf41aeeb9826b677caaa851dcbeb04abe1f43eed65", + "storage/s3/s3.go": "016bef5948e97f3d043d2e858716f1c02d9f27c4e02802bdc1be3860116f4b6c", + "storage/s3/s3_test.go": "9ef284926ec9936ddfec8c52a9bf04fc7b4be55c43a535f6c5a428d1535f6fec", + "storage/s3/upload.go": "3ce0b3626249ffe7a041606d6a9481743af8a0d4868fadea20ed3ed44ee0c765", + "storage/s3/upload_test.go": "a41fdbd100d9c79a3606c5942f98017afe26d841b251d96e979df87d0af93867", + "storage/signed.go": "ae4b144137627be35718c40743ebc2f063ec667c24b6d8984ccfcca034a73787", + "storage/signedUpload.go": "a8d31bcbbad64796f04611e69271e94d917c9bcd5aeabd7a97ea155b0093c814", + "storage/signed_test.go": "eaffd40894018814e976477f96cf0fbaecfa1987caad78830d9fc8e96995882f", + "storage/storage.go": "cb670872f01cf1d7b4ebf21cec9fc6f52ad93bfdf523b2f28bbb8d9c7fe39a6c", + "storage/storagetest/contract.go": "e7ad7242102437882269d4066462059d7974586b36a82d18639eca10fd932be9", + "storage/storagetest/uploads.go": "9b66f86dd94ccdf30c7d603d432bc2a55956608898e46c1274fa27d16fa1a969", + "storage/upload.go": "9ff2930ba389163acc8063b51107c09a60290609ba384de8004c498ce09f20a1", + "storage/upload_test.go": "12b60419807ed64b89e30861674e19abee7b0b7bb0e8ec9ac5b3aa2e0710a1da" + }, + "commands": [ + [ + "go", + "test", + "-p=2", + "-c", + "-o", + "/var/folders/wm/1sg73x8n7879sp3t6m4jpnrr0000gn/T/goOmniUpload3cba6ibp/s3.test", + "./storage/s3" + ], + [ + "docker", + "run", + "--rm", + "--name", + "go-omni-upload-c388755df4ec", + "--platform", + "linux/arm64", + "--cpus", + "2", + "--memory", + "2048m", + "--memory-swap", + "2048m", + "--pids-limit", + "256", + "--network", + "none", + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--user", + "1000:1000", + "--tmpfs", + "/tmp:rw,nosuid,noexec,size=64m,mode=1777", + "--tmpfs", + "/data:rw,nosuid,noexec,size=128m,mode=1777", + "-v", + "/var/folders/wm/1sg73x8n7879sp3t6m4jpnrr0000gn/T/goOmniUpload3cba6ibp:/test:ro", + "-e", + "GOMAXPROCS=2", + "-e", + "MINIO_ROOT_USER=goOmniTest", + "-e", + "MINIO_ROOT_PASSWORD=goOmniTestSecret", + "-e", + "GO_OMNI_S3_TEST_ADDR=127.0.0.1:9000", + "--entrypoint", + "sh", + "quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z", + "-c", + "set -eu\nminio --version\nminio server /data --address 127.0.0.1:9000 --console-address 127.0.0.1:9001 >/tmp/minio.log 2>&1 &\nserver=$!\ntrap 'kill \"$server\" 2>/dev/null || true; wait \"$server\" 2>/dev/null || true' EXIT\n/test/s3.test -test.v -test.run '^TestS3Integration$' -test.count=1 -test.timeout=60s || { cat /tmp/minio.log; exit 1; }\n" + ] + ], + "containerLimits": { + "cpus": 2, + "memoryBytes": 2147483648, + "pids": 256, + "network": "none" + }, + "notes": [ + "MinIO verifies S3 signatures; OSS/R2 real cloud accounts are not exercised." + ], + "preExistingContainersStillRunning": true, + "cleanedContainer": "go-omni-upload-c388755df4ec", + "finishedAt": "2026-09-30T14:35:34.723540+00:00", + "logSha256": "c25db6c44baee149c1e3d6d3b7f81faac9483eb79709bed25f61b03d7b9b3133" +} diff --git a/reports/signedReadMinio.txt b/reports/signedReadMinio.txt new file mode 100644 index 0000000..e5f13a1 --- /dev/null +++ b/reports/signedReadMinio.txt @@ -0,0 +1,7 @@ +minio version RELEASE.2025-09-07T16-13-09Z (commit-id=07c3a429bfed433e49018cb0f78a52145d4bedeb) +Runtime: go1.24.6 linux/arm64 +License: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html +Copyright: 2015-2025 MinIO, Inc. +=== RUN TestS3Integration +--- PASS: TestS3Integration (0.63s) +PASS diff --git a/scripts/verifyManagedRelease.py b/scripts/verifyManagedRelease.py index 692d3fa..b1bae68 100644 --- a/scripts/verifyManagedRelease.py +++ b/scripts/verifyManagedRelease.py @@ -162,6 +162,11 @@ def main(): "remoteCheckConfigBeforeDependencies", "removedImplicitEnvironmentRules"] for package, expected in { + "storage": {"TestReadOptions", "TestReadSignatureAndExpiry", "TestReadRangeStreamingAndCache"}, + "storage/local": {"TestSignedPayloadRanges"}, + "storage/oss": {"TestReadOptionsNativeSignatures"}, + "storage/s3": {"TestReadOptionsS3AndR2"}, + "examples/signedRead": {"TestSignedReadHTTP"}, "copyx": {"TestCheckTypesAndEmptyCollections", "TestCheckRulesAndConverterBoundaries", "TestCopierCheckConcurrentAndOptionsIsolation"}, "logx": {"TestLoggerTimeZoneIsolation", "TestLoggerTimeZoneValidationAndDST", "TestLoggerTimeZoneConcurrentChildren"}, "cors": {"TestPolicyPreflightAndActual", "TestPolicyValidation", "TestPolicyCredentialsWildcardAndDuplicates", "TestPolicyPreservesHTTPAndSnapshot"}, @@ -181,6 +186,11 @@ def main(): events = [json.loads(line) for line in output.splitlines() if line.startswith("{")] passed = {event.get("Test") for event in events if event.get("Action") == "pass"} assert expected <= passed + signed_read_output = run(["go", "run", MODULE + "/examples/signedRead", "-check"], starter, env) + assert "PDF preview, Chinese filename, download, HEAD and Range passed" in signed_read_output + checks += ["remoteSignedReadOptionsAndLegacyLinks", "remoteReadSignatureTamperingAndExpiry", + "remoteLocalPayloadRangesAndHEAD", "remoteOSSNativeV1V4Signatures", + "remoteS3R2ReadOptions", "remoteSignedReadHTTPExample"] checks += ["remotePreciseModuleDiagnostics", "remoteJSONSuccessStatuses", "remoteRejectedStatusBodyCleanup", "remoteCopyTypePreflight", "remoteEmptyCollectionTypeRejection", "remoteLoggerTimeZoneIsolation", "remoteCORSPolicyRace", "remoteCORSErrorResponses", "remoteCORSTransports", "generatedCORSConfigOnly"] @@ -247,6 +257,8 @@ def main(): sources += ["app/body.go", "app/router.go", "app/requests.go", "app/bodyPolicy_test.go", "app/requests_test.go", "app/transport_test.go", "httpx/context.go", "internal/httplimits/limits.go", "lifecycle/runtime_test.go", "transport/http2/server.go", "transport/http2/lifecycle_test.go", "transport/native/server.go", "transport/native/earlyReject_test.go", "examples/httpSafety/main.go"] + sources += [str(p.relative_to(ROOT)) for p in (ROOT / "storage").rglob("*.go")] + sources += [str(p.relative_to(ROOT)) for p in (ROOT / "examples/signedRead").glob("*.go")] report = {"passed": True, "frameworkVersion": args.version, "frameworkSum": dependency["Sum"], "freshModuleCache": True, "noLocalReplace": True, "durationSeconds": round(time.monotonic() - started, 2), "checks": checks, diff --git a/storage/README.md b/storage/README.md index 6267702..8fb594a 100644 --- a/storage/README.md +++ b/storage/README.md @@ -6,12 +6,42 @@ | --- | --- | --- | --- | --- | | 读写删除 | Store | 支持 | 支持 | 支持 | | 临时下载 | Presigner | NewSigner | 支持 | 支持 | +| 带选项的预览/下载 | ReadPresigner | NewSigner | 支持 | 支持 | | 限定大小/类型的直传 | UploadPresigner | NewUploadSigner | 支持 | 支持，需要配置签名 Header | | 创建、上传、查询、完成、取消分片 | MultipartStore | 当前不提供 | 支持 | 支持 | | 分片直传签名 | PartPresigner | 当前不提供 | 支持 | 支持，需要配置签名 Header | 没有固定业务字段、目录规则或身份模型。鉴权、对象归属、业务配额由应用定义。SDK 客户端和连接池由应用持有；适配器不会在一次调用后关闭客户端。 +## 签名预览与下载 + +`v0.29.0-preview` 新增独立的可选接口 `ReadPresigner`，不扩充原 Store/Presigner，不要求第三方适配器修改代码。原 `PresignGet(ctx, key, ttl)` 和既有本地下载链接继续有效、默认 attachment。 + +```go +// signer 可以是 storage.NewSigner 返回的本地 Handler，或 S3/R2、OSS Store。 +link, err := signer.PresignGetWithOptions(ctx, key, 30*time.Minute, storage.ReadOptions{ + Disposition: storage.DispositionInline, + FileName: "示例报告.pdf", + ContentType: "application/pdf", +}) +if err != nil { return err } +// 把 link.URL 和 link.ExpiresAt 返回给已授权的调用方。 +``` + +TTL 仍为 1 秒到 1 小时，30 分钟无需改配置。`ReadLink` 包含 `url` 和 `expiresAt`，到期时间取自实际签名，精确到秒（不足一秒的部分舍弃）；临时云凭据可能更早失效。无需额外请求 Header，客户端直接打开 URL。可通过 `storage.ReadPresigner` 做能力判断；不支持时由调用方明确处理，不应静默降级成公开地址。 + +`Disposition` 只接受 inline/attachment，空值为 attachment。`FileName` 是显示名，与存储 key 无关；中文使用标准 `filename*` 编码，拒绝控制字符（包括 CR/LF）、路径分隔符及超长值。`ContentType` 为空时沿用对象元数据，可显式指定合法 MIME 类型；它只控制响应，不转换或验证文件内容。PDF 应在上传时就保存 `application/pdf` 元数据。inline 请求内联显示，能否展示还取决于浏览器的格式支持与用户设置。 + +本地签名绑定 key、到期时间、版本及全部选项，拒绝重复/未知参数和篡改；先验签、验有效期，再读取对象或处理条件请求。Local 使用受文件内容边界约束的流，GET/HEAD 的长度不包含内部元数据；支持单段/多段 Range、206/416，以及 Last-Modified 条件请求。HEAD 返回完整对象头、无响应体，忽略 Range。自定义 Store 的 Body 实现 `io.ReadSeeker` 时复用同一行为；仅支持顺序读取时返回完整 200，不宣称支持范围读取，也不把整个对象加载到内存。 + +云端适配器在 OSS/S3 原生 SDK 签名前传入响应参数，禁止先签名后拼参数或替换域名。Range 由云存储处理；云端 GET 签名不能直接改成 HEAD 使用。新接口请求 `Cache-Control: private, no-store`，本地成功、错误及条件响应均保持该策略；云端响应头覆盖受平台规则约束，需检查实际 200/206 响应与代理缓存配置。 + +OSS 默认域名或加速域名可能强制下载。需要自定义域名时，在应用装配 SDK client 时设置已绑定桶的域名和 `WithUseCName(true)`，再传给适配器；SDK 的区域、签名版本、凭据由应用配置。例如在现有 SDK config 上调用 `WithEndpoint("https://files.example").WithUseCName(true)`。参见 [OSS 预览规则](https://help.aliyun.com/zh/oss/how-to-ensure-an-object-is-previewed-when-you-access-the-object)与 [GetObject 响应参数规则](https://www.alibabacloud.com/help/en/oss/developer-reference/getobject)。 + +签发权限、对象是否不可变、审计和前端链接复用由项目维护。框架不添加分享表、撤销、租户或数据库依赖；重新签发不会撤销旧链接，到期不会收回已下载内容，也不会中断已开始的传输。本地入口使用显式配置的公开 baseURL，不猜测 Host/转发头，不公开磁盘目录。应用与代理日志应隐藏查询参数。 + +完整离线 [预览/下载示例](../examples/signedRead/README.md)，包含真实 HTTP 检查与一页 PDF。OSS V1/V4、S3/R2 SDK 签名和隔离 MinIO 有回归；OSS/R2 云账号尚未实测，部署方仍需对实际桶、域名及浏览器进行验收。 + ## 服务端上传限制 ```go diff --git a/storage/local/local.go b/storage/local/local.go index 20c8a2b..d0d55ba 100644 --- a/storage/local/local.go +++ b/storage/local/local.go @@ -151,14 +151,31 @@ func (s *Store) Get(ctx context.Context, key string) (storage.Object, error) { } info.ModifiedAt = stat.ModTime() ok = true - return storage.Object{Info: info, Body: &body{Reader: storage.ContextReader{Context: ctx, Reader: io.LimitReader(f, info.Size)}, Closer: f}}, nil + // Expose only the payload as a seekable section. Seeks are relative to the + // object, never to our private metadata header or other disk bytes. + return storage.Object{Info: info, Body: &body{section: io.NewSectionReader(f, int64(length)+4, info.Size), file: f, ctx: ctx}}, nil } type body struct { - io.Reader - io.Closer + section *io.SectionReader + file *os.File + ctx context.Context } +func (b *body) Read(p []byte) (int, error) { + if err := b.ctx.Err(); err != nil { + return 0, err + } + return b.section.Read(p) +} +func (b *body) Seek(offset int64, whence int) (int64, error) { + if err := b.ctx.Err(); err != nil { + return 0, err + } + return b.section.Seek(offset, whence) +} +func (b *body) Close() error { return b.file.Close() } + func (s *Store) Delete(ctx context.Context, key string) error { if err := storage.Check(ctx, key); err != nil { return err diff --git a/storage/local/read_test.go b/storage/local/read_test.go new file mode 100644 index 0000000..6fb589f --- /dev/null +++ b/storage/local/read_test.go @@ -0,0 +1,125 @@ +package local + +import ( + "context" + "io" + "mime" + "mime/multipart" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "git.yuntuo.hk/k2safe/go-omni/storage" +) + +func TestSignedPayloadRanges(t *testing.T) { + store, err := New(t.TempDir(), 1<<20) + if err != nil { + t.Fatal(err) + } + data := "%PDF-1.4\n" + strings.Repeat("0123456789", 100) + "\n%%EOF\n" + ctx := context.Background() + if _, err = store.Put(ctx, "private/document", strings.NewReader(data), int64(len(data)), "application/pdf"); err != nil { + t.Fatal(err) + } + signer, _ := storage.NewSigner(store, "https://files.example/read", []byte(strings.Repeat("s", 32))) + link, err := signer.PresignGetWithOptions(ctx, "private/document", 30*time.Minute, storage.ReadOptions{Disposition: storage.DispositionInline, FileName: "示例.pdf"}) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + method, span string + status int + want string + }{ + {"GET", "", 200, data}, {"HEAD", "", 200, ""}, {"HEAD", "bytes=0-3", 200, ""}, + {"GET", "bytes=0-7", 206, data[:8]}, {"GET", "bytes=20-", 206, data[20:]}, {"GET", "bytes=-6", 206, data[len(data)-6:]}, + {"GET", "bytes=1000-999999", 206, data[1000:]}, {"GET", "bytes=999999-", 416, ""}, {"GET", "bytes=bad", 416, ""}, + } { + t.Run(tc.method+tc.span, func(t *testing.T) { + r := httptest.NewRequest(tc.method, link.URL, nil) + if tc.span != "" { + r.Header.Set("Range", tc.span) + } + w := httptest.NewRecorder() + signer.ServeHTTP(w, r) + if w.Code != tc.status { + t.Fatal(w.Code) + } + if tc.status != 416 && w.Body.String() != tc.want { + t.Fatal("wrong payload slice") + } + if tc.status == 206 && (w.Header().Get("Content-Length") != strconv.Itoa(len(tc.want)) || !strings.HasSuffix(w.Header().Get("Content-Range"), "/"+strconv.Itoa(len(data)))) { + t.Fatal(w.Header()) + } + if tc.status == 200 && w.Header().Get("Content-Length") != strconv.Itoa(len(data)) { + t.Fatal("metadata included in length") + } + if tc.span == "bytes=999999-" && w.Header().Get("Content-Range") != "bytes */"+strconv.Itoa(len(data)) { + t.Fatal(w.Header()) + } + if w.Header().Get("Cache-Control") != "private, no-store" { + t.Fatal("cache policy") + } + if tc.status != 416 { + kind, params, err := mime.ParseMediaType(w.Header().Get("Content-Disposition")) + if err != nil || kind != "inline" || params["filename"] != "示例.pdf" || w.Header().Get("Content-Type") != "application/pdf" { + t.Fatal(w.Header(), err) + } + } + }) + } + r := httptest.NewRequest("GET", link.URL, nil) + r.Header.Set("Range", "bytes=0-3,10-13") + w := httptest.NewRecorder() + signer.ServeHTTP(w, r) + kind, params, err := mime.ParseMediaType(w.Header().Get("Content-Type")) + if err != nil || w.Code != 206 || kind != "multipart/byteranges" { + t.Fatal(w.Header(), err) + } + reader := multipart.NewReader(w.Body, params["boundary"]) + for _, want := range []string{data[:4], data[10:14]} { + p, err := reader.NextPart() + if err != nil { + t.Fatal(err) + } + got, err := io.ReadAll(p) + if err != nil || string(got) != want { + t.Fatal("multipart bytes", err) + } + } + if _, err := reader.NextPart(); err != io.EOF { + t.Fatal(err) + } + // Random access cannot expose the managed header and still observes context. + ctx, cancel := context.WithCancel(ctx) + defer cancel() + object, err := store.Get(ctx, "private/document") + if err != nil { + t.Fatal(err) + } + defer object.Body.Close() + seeker := object.Body.(io.ReadSeeker) + if size, err := seeker.Seek(0, io.SeekEnd); err != nil || size != int64(len(data)) { + t.Fatal(size, err) + } + if _, err := seeker.Seek(-1, io.SeekStart); err == nil { + t.Fatal("seek into header") + } + if _, err := seeker.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } + prefix := make([]byte, 8) + if _, err := io.ReadFull(seeker, prefix); err != nil || string(prefix) != data[:8] { + t.Fatal("metadata exposed", err) + } + cancel() + if _, err := seeker.Seek(0, io.SeekStart); err != context.Canceled { + t.Fatal(err) + } + if _, err := seeker.Read(prefix); err != context.Canceled { + t.Fatal(err) + } +} diff --git a/storage/oss/read.go b/storage/oss/read.go new file mode 100644 index 0000000..3b15d3f --- /dev/null +++ b/storage/oss/read.go @@ -0,0 +1,59 @@ +package oss + +import ( + "context" + "errors" + "net/url" + "strconv" + "time" + + "git.yuntuo.hk/k2safe/go-omni/storage" + sdk "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss" +) + +func (s *Store) PresignGetWithOptions(ctx context.Context, key string, ttl time.Duration, options storage.ReadOptions) (storage.ReadLink, error) { + if err := storage.Check(ctx, key); err != nil { + return storage.ReadLink{}, err + } + if ttl < time.Second || ttl > time.Hour { + return storage.ReadLink{}, errors.New("storage/oss: TTL must be 1s..1h") + } + headers, err := options.ResponseHeaders() + if err != nil { + return storage.ReadLink{}, err + } + request := &sdk.GetObjectRequest{Bucket: sdk.Ptr(s.bucket), Key: sdk.Ptr(key), ResponseContentDisposition: sdk.Ptr(headers.Get("Content-Disposition")), ResponseCacheControl: sdk.Ptr(headers.Get("Cache-Control"))} + if contentType := headers.Get("Content-Type"); contentType != "" { + request.ResponseContentType = sdk.Ptr(contentType) + } + r, err := s.client.Presign(ctx, request, sdk.PresignExpires(ttl)) + if err != nil { + return storage.ReadLink{}, err + } + if len(r.SignedHeaders) != 0 { + return storage.ReadLink{}, errors.New("storage/oss: direct GET requires a signature without additional request headers") + } + u, err := url.Parse(r.URL) + if err != nil { + return storage.ReadLink{}, errors.New("storage/oss: invalid signed URL") + } + q := u.Query() + var expires time.Time + if q.Get("x-oss-signature") != "" { + start, e := time.Parse("20060102T150405Z", q.Get("x-oss-date")) + seconds, e2 := strconv.ParseInt(q.Get("x-oss-expires"), 10, 64) + if e != nil || e2 != nil || seconds < 1 || seconds > 3600 { + return storage.ReadLink{}, errors.New("storage/oss: invalid signed expiry") + } + expires = start.Add(time.Duration(seconds) * time.Second) + } else { + seconds, e := strconv.ParseInt(q.Get("Expires"), 10, 64) + if e != nil || q.Get("Signature") == "" { + return storage.ReadLink{}, errors.New("storage/oss: missing signed expiry") + } + expires = time.Unix(seconds, 0).UTC() + } + return storage.ReadLink{URL: r.URL, ExpiresAt: expires}, nil +} + +var _ storage.ReadPresigner = (*Store)(nil) diff --git a/storage/oss/read_test.go b/storage/oss/read_test.go new file mode 100644 index 0000000..3980f1d --- /dev/null +++ b/storage/oss/read_test.go @@ -0,0 +1,140 @@ +package oss + +import ( + "context" + "crypto/hmac" + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "mime" + "net/url" + "sort" + "strconv" + "strings" + "testing" + "time" + + "git.yuntuo.hk/k2safe/go-omni/storage" + sdk "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss" + "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials" + "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/signer" +) + +// Recompute signatures independently from the SDK using fixed test credentials. +// This catches response parameters appended after signing, not just their presence. +func validReadSignature(u *url.URL, version sdk.SignatureVersionType) bool { + q := u.Query() + if version == sdk.SignatureVersionV1 { + var keys []string + for key := range q { + if strings.HasPrefix(key, "response-") { + keys = append(keys, key) + } + } + sort.Strings(keys) + var params []string + for _, key := range keys { + params = append(params, key+"="+q.Get(key)) + } + message := "GET\n\n\n" + q.Get("Expires") + "\n/testbucket/folder/file.pdf?" + strings.Join(params, "&") + mac := hmac.New(sha1.New, []byte("fixtureSecret")) + mac.Write([]byte(message)) + return hmac.Equal([]byte(q.Get("Signature")), []byte(base64.StdEncoding.EncodeToString(mac.Sum(nil)))) + } + signature := q.Get("x-oss-signature") + q.Del("x-oss-signature") + canonical := "GET\n/testbucket/folder/file.pdf\n" + strings.ReplaceAll(q.Encode(), "+", "%20") + "\n\n\nUNSIGNED-PAYLOAD" + hash := sha256.Sum256([]byte(canonical)) + _, scope, _ := strings.Cut(q.Get("x-oss-credential"), "/") + message := "OSS4-HMAC-SHA256\n" + q.Get("x-oss-date") + "\n" + scope + "\n" + hex.EncodeToString(hash[:]) + key := []byte("aliyun_v4fixtureSecret") + for _, part := range strings.Split(scope, "/") { + mac := hmac.New(sha256.New, key) + mac.Write([]byte(part)) + key = mac.Sum(nil) + } + mac := hmac.New(sha256.New, key) + mac.Write([]byte(message)) + return hmac.Equal([]byte(signature), []byte(hex.EncodeToString(mac.Sum(nil)))) +} + +func TestReadOptionsNativeSignatures(t *testing.T) { + for name, version := range map[string]sdk.SignatureVersionType{"v1": sdk.SignatureVersionV1, "v4": sdk.SignatureVersionV4} { + t.Run(name, func(t *testing.T) { + cfg := sdk.LoadDefaultConfig().WithRegion("cn-hangzhou").WithEndpoint("https://files.example").WithUseCName(true).WithSignatureVersion(version).WithCredentialsProvider(credentials.NewStaticCredentialsProvider("fixtureAccess", "fixtureSecret")) + store, err := New(sdk.NewClient(cfg), "testbucket", 64) + if err != nil { + t.Fatal(err) + } + for _, options := range []storage.ReadOptions{{}, {Disposition: storage.DispositionInline, FileName: "报告 2026.pdf", ContentType: "application/pdf"}, {Disposition: storage.DispositionAttachment, FileName: "download.pdf"}} { + before := time.Now().UTC().Truncate(time.Second) + link, err := store.PresignGetWithOptions(context.Background(), "folder/file.pdf", 30*time.Minute, options) + if err != nil { + t.Fatal(err) + } + u, err := url.Parse(link.URL) + if err != nil { + t.Fatal(err) + } + q := u.Query() + if u.Host != "files.example" || u.Path != "/folder/file.pdf" || !validReadSignature(u, version) { + t.Fatal("SDK response overrides were not signed") + } + disposition, params, err := mime.ParseMediaType(q.Get("response-content-disposition")) + want := options.Disposition + if want == "" { + want = "attachment" + } + if err != nil || disposition != want || params["filename"] != options.FileName || q.Get("response-content-type") != options.ContentType || q.Get("response-cache-control") != "private, no-store" { + t.Fatal("wrong signed headers", err) + } + if link.ExpiresAt.Before(before.Add(30*time.Minute)) || link.ExpiresAt.After(time.Now().Add(30*time.Minute)) { + t.Fatal("expiry drift") + } + var expires time.Time + if version == sdk.SignatureVersionV4 { + start, _ := time.Parse("20060102T150405Z", q.Get("x-oss-date")) + seconds, _ := strconv.Atoi(q.Get("x-oss-expires")) + expires = start.Add(time.Duration(seconds) * time.Second) + } else { + seconds, _ := strconv.ParseInt(q.Get("Expires"), 10, 64) + expires = time.Unix(seconds, 0) + } + if !expires.Equal(link.ExpiresAt) { + t.Fatal("result does not match signature expiry") + } + for _, name := range []string{"response-content-disposition", "response-content-type", "response-cache-control"} { + changed := *u + query := changed.Query() + query.Set(name, "changed") + changed.RawQuery = query.Encode() + if validReadSignature(&changed, version) { + t.Fatal("unsigned override", name) + } + } + } + for _, ttl := range []time.Duration{0, time.Second - time.Nanosecond, time.Hour + time.Nanosecond} { + if _, err := store.PresignGetWithOptions(context.Background(), "file", ttl, storage.ReadOptions{}); err == nil { + t.Fatal("TTL accepted") + } + } + for _, options := range []storage.ReadOptions{{FileName: "x\r\nY: z"}, {ContentType: "text/plain\r\nY: z"}, {Disposition: "bad"}} { + if _, err := store.PresignGetWithOptions(context.Background(), "file", time.Minute, options); err == nil { + t.Fatal("invalid header accepted") + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := store.PresignGetWithOptions(ctx, "file", time.Minute, storage.ReadOptions{}); err != context.Canceled { + t.Fatal(err) + } + }) + } + // A caller-supplied unsigned SDK configuration cannot masquerade as a link. + client := sdk.NewClient(sdk.LoadDefaultConfig().WithRegion("cn-hangzhou").WithEndpoint("https://files.example").WithCredentialsProvider(credentials.NewStaticCredentialsProvider("fixtureAccess", "fixtureSecret")), func(o *sdk.Options) { o.Signer = &signer.NopSigner{} }) + store, _ := New(client, "testbucket", 64) + if _, err := store.PresignGetWithOptions(context.Background(), "file", time.Minute, storage.ReadOptions{}); err == nil { + t.Fatal("unsigned URL accepted") + } +} diff --git a/storage/read.go b/storage/read.go new file mode 100644 index 0000000..b5db2a4 --- /dev/null +++ b/storage/read.go @@ -0,0 +1,72 @@ +package storage + +import ( + "context" + "errors" + "mime" + "net/http" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +const ( + DispositionAttachment = "attachment" + DispositionInline = "inline" +) + +// ReadOptions controls the response to one signed GET. Empty Disposition means +// attachment; empty ContentType preserves object metadata. FileName is a display +// name, not an object key or filesystem path. All options are signed. +type ReadOptions struct { + Disposition string + FileName string + ContentType string +} + +// ReadLink grants GET access without additional request headers. ExpiresAt is +// the expiry encoded in the signature, at second precision. Provider credentials +// may expire earlier. Do not log the URL. Issuing another link does not revoke it. +type ReadLink struct { + URL string `json:"url"` + ExpiresAt time.Time `json:"expiresAt"` +} + +// ReadPresigner is optional; existing Store and Presigner implementations do not +// need to change. Cloud GET signatures do not also authorize HEAD requests. +type ReadPresigner interface { + PresignGetWithOptions(context.Context, string, time.Duration, ReadOptions) (ReadLink, error) +} + +// ResponseHeaders validates options and returns a fresh set of response headers +// for provider adapters to include BEFORE signing, requesting private, no-store. +func (o ReadOptions) ResponseHeaders() (http.Header, error) { + disposition := o.Disposition + if disposition == "" { + disposition = DispositionAttachment + } + if disposition != DispositionAttachment && disposition != DispositionInline { + return nil, errors.New("storage: disposition must be inline or attachment") + } + if len(o.FileName) > 1024 || !utf8.ValidString(o.FileName) || strings.IndexFunc(o.FileName, unicode.IsControl) >= 0 || strings.ContainsAny(o.FileName, `/\`) || o.FileName == "." || o.FileName == ".." { + return nil, errors.New("storage: invalid display filename") + } + params := map[string]string{} + if o.FileName != "" { + params["filename"] = o.FileName + } + h := http.Header{"Content-Disposition": {mime.FormatMediaType(disposition, params)}, "Cache-Control": {"private, no-store"}} + if o.ContentType != "" { + if !utf8.ValidString(o.ContentType) || strings.IndexFunc(o.ContentType, unicode.IsControl) >= 0 { + return nil, ErrContentType + } + kind, err := MediaType(o.ContentType) + if err != nil { + return nil, err + } + _, params, _ := mime.ParseMediaType(o.ContentType) + h.Set("Content-Type", mime.FormatMediaType(kind, params)) + } + return h, nil +} diff --git a/storage/read_test.go b/storage/read_test.go new file mode 100644 index 0000000..647fc49 --- /dev/null +++ b/storage/read_test.go @@ -0,0 +1,219 @@ +package storage + +import ( + "context" + "io" + "mime" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" +) + +func TestReadOptions(t *testing.T) { + for _, name := range []string{"", "report.pdf", "报告 2026.pdf", `a";b.pdf`, "100%.pdf"} { + h, err := (ReadOptions{Disposition: DispositionInline, FileName: name, ContentType: "Application/PDF"}).ResponseHeaders() + if err != nil { + t.Fatal(err) + } + disposition, params, err := mime.ParseMediaType(h.Get("Content-Disposition")) + if err != nil || disposition != "inline" || params["filename"] != name || h.Get("Content-Type") != "application/pdf" || h.Get("Cache-Control") != "private, no-store" { + t.Fatal("invalid read headers", h, err) + } + } + h, err := (ReadOptions{}).ResponseHeaders() + if err != nil || h.Get("Content-Disposition") != "attachment" || h.Get("Content-Type") != "" { + t.Fatal(h, err) + } + for _, options := range []ReadOptions{ + {Disposition: "form-data"}, {Disposition: "Inline"}, {Disposition: "inline\r\nX: y"}, + {FileName: "x\r\nX: y"}, {FileName: "a\x00b"}, {FileName: "a\tb"}, {FileName: "bad\xff"}, {FileName: "a/b"}, {FileName: `a\b`}, {FileName: ".."}, {FileName: strings.Repeat("a", 1025)}, + {ContentType: "bad"}, {ContentType: "image/*"}, {ContentType: "application/pdf\r\nX: y"}, {ContentType: "application/pdf\x00"}, {ContentType: strings.Repeat("x", 257)}, + } { + if _, err := options.ResponseHeaders(); err == nil { + t.Fatal("invalid options accepted", options) + } + } +} + +type readTestStore struct { + calls, bytes int + streaming bool +} + +func (*readTestStore) Put(context.Context, string, io.Reader, int64, string) (Info, error) { + panic("unused") +} +func (*readTestStore) Delete(context.Context, string) error { panic("unused") } +func (s *readTestStore) Get(context.Context, string) (Object, error) { + s.calls++ + body := &readTestBody{Reader: strings.NewReader("0123456789"), store: s} + var stream io.ReadCloser = body + if s.streaming { + stream = struct { + io.Reader + io.Closer + }{body, body} + } + return Object{Info: Info{Size: 10, ContentType: "application/pdf", ModifiedAt: time.Unix(1000, 0)}, Body: stream}, nil +} + +type readTestBody struct { + *strings.Reader + store *readTestStore +} + +func (b *readTestBody) Read(p []byte) (int, error) { + n, err := b.Reader.Read(p) + b.store.bytes += n + return n, err +} +func (*readTestBody) Close() error { return nil } + +func TestReadSignatureAndExpiry(t *testing.T) { + store := &readTestStore{} + s, _ := NewSigner(store, "https://files.example/read", []byte(strings.Repeat("s", 32))) + now := time.Unix(1800000000, 0).UTC() + s.now = func() time.Time { return now } + link, err := s.PresignGetWithOptions(context.Background(), "folder/file", 30*time.Minute, ReadOptions{Disposition: DispositionInline, FileName: "报告.pdf", ContentType: "application/pdf"}) + if err != nil || !link.ExpiresAt.Equal(now.Add(30*time.Minute)) { + t.Fatal("incorrect expiry", err) + } + serve := func(method, address string, headers http.Header) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, address, nil) + if headers != nil { + r.Header = headers + } + w := httptest.NewRecorder() + s.ServeHTTP(w, r) + return w + } + for _, method := range []string{"GET", "HEAD"} { + w := serve(method, link.URL, nil) + if w.Code != 200 || w.Header().Get("Content-Type") != "application/pdf" || w.Header().Get("Cache-Control") != "private, no-store" { + t.Fatal(w) + } + if method == "HEAD" && w.Body.Len() != 0 { + t.Fatal("HEAD body") + } + } + old, err := s.PresignGet(context.Background(), "folder/file", 30*time.Minute) + if err != nil { + t.Fatal(err) + } + if w := serve("GET", old.String(), nil); w.Code != 200 || w.Header().Get("Content-Disposition") != "attachment" { + t.Fatal("legacy download changed") + } + mutations := []func(url.Values){ + func(q url.Values) { q.Set("key", "other") }, func(q url.Values) { q.Set("expires", "1800001900") }, func(q url.Values) { q.Set("signature", "bad") }, + func(q url.Values) { q.Set("filename", "other.pdf") }, func(q url.Values) { q.Set("disposition", "attachment") }, func(q url.Values) { q.Set("contentType", "text/html") }, + func(q url.Values) { q.Set("v", "3") }, func(q url.Values) { q.Del("v") }, func(q url.Values) { + for _, k := range []string{"v", "filename", "disposition", "contentType"} { + q.Del(k) + } + }, + func(q url.Values) { q.Set("extra", "x") }, func(q url.Values) { q.Set("filename", "evil\r\nX: yes") }, + } + for _, key := range []string{"key", "expires", "signature", "v", "disposition", "filename", "contentType"} { + key := key + mutations = append(mutations, func(q url.Values) { q.Add(key, q.Get(key)) }) + } + for _, mutate := range mutations { + u, _ := url.Parse(link.URL) + q := u.Query() + mutate(q) + u.RawQuery = q.Encode() + before := store.calls + if w := serve("GET", u.String(), http.Header{"Range": {"bytes=0-1"}}); w.Code != 403 || store.calls != before { + t.Fatal("tampered link reached storage") + } + } + u := *old + q := u.Query() + q.Set("disposition", "inline") + u.RawQuery = q.Encode() + if serve("GET", u.String(), nil).Code != 403 { + t.Fatal("legacy download upgraded by query") + } + if serve("GET", link.URL+"&bad=%ZZ", nil).Code != 403 { + t.Fatal("malformed query accepted") + } + before := store.calls + now = link.ExpiresAt.Add(-time.Nanosecond) + if serve("GET", link.URL, nil).Code != 200 { + t.Fatal("early expiration") + } + before = store.calls + now = link.ExpiresAt + for _, method := range []string{"GET", "HEAD"} { + w := serve(method, link.URL, http.Header{"Range": {"bytes=0-1"}, "If-Modified-Since": {"Fri, 01 Jan 2100 00:00:00 GMT"}}) + if w.Code != 403 || store.calls != before || w.Header().Get("Cache-Control") != "private, no-store" { + t.Fatal("expired link reached storage or cache") + } + if method == "HEAD" && w.Body.Len() != 0 { + t.Fatal("HEAD error has body") + } + } + for _, ttl := range []time.Duration{0, time.Second - time.Nanosecond, time.Hour + time.Nanosecond} { + if _, err := s.PresignGetWithOptions(context.Background(), "key", ttl, ReadOptions{}); err == nil { + t.Fatal("invalid TTL") + } + } + for _, ttl := range []time.Duration{time.Second, time.Hour} { + if _, err := s.PresignGetWithOptions(context.Background(), "key", ttl, ReadOptions{}); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := s.PresignGetWithOptions(ctx, "key", time.Minute, ReadOptions{}); err != context.Canceled { + t.Fatal(err) + } +} + +func TestReadRangeStreamingAndCache(t *testing.T) { + for _, streaming := range []bool{false, true} { + store := &readTestStore{streaming: streaming} + s, _ := NewSigner(store, "https://files.example/read", []byte(strings.Repeat("s", 32))) + link, _ := s.PresignGetWithOptions(context.Background(), "key", 30*time.Minute, ReadOptions{}) + for _, method := range []string{"GET", "HEAD"} { + before := store.bytes + r := httptest.NewRequest(method, link.URL, nil) + r.Header.Set("Range", "bytes=2-4") + w := httptest.NewRecorder() + s.ServeHTTP(w, r) + if method == "HEAD" { + if w.Code != 200 || w.Body.Len() != 0 || store.bytes != before { + t.Fatal("HEAD read body", w) + } + } else if streaming { + if w.Code != 200 || w.Body.String() != "0123456789" || w.Header().Get("Accept-Ranges") != "" { + t.Fatal("stream fallback", w) + } + } else { + if w.Code != 206 || w.Body.String() != "234" || store.bytes-before != 3 { + t.Fatal("range buffered full body", w) + } + } + if w.Header().Get("Cache-Control") != "private, no-store" { + t.Fatal("missing cache policy") + } + } + if !streaming { + for _, h := range []http.Header{{"Range": {"bytes=99-100"}}, {"If-Modified-Since": {"Fri, 01 Jan 2100 00:00:00 GMT"}}} { + r := httptest.NewRequest("GET", link.URL, nil) + r.Header = h + w := httptest.NewRecorder() + s.ServeHTTP(w, r) + if w.Code != 416 && w.Code != 304 { + t.Fatal(w.Code) + } + if w.Header().Get("Cache-Control") != "private, no-store" { + t.Fatal("cache policy lost on error/conditional") + } + } + } + } +} diff --git a/storage/s3/read.go b/storage/s3/read.go new file mode 100644 index 0000000..580db9e --- /dev/null +++ b/storage/s3/read.go @@ -0,0 +1,53 @@ +package s3 + +import ( + "context" + "errors" + "net/url" + "strconv" + "time" + + "git.yuntuo.hk/k2safe/go-omni/storage" +) + +func (s *Store) PresignGetWithOptions(ctx context.Context, key string, ttl time.Duration, options storage.ReadOptions) (storage.ReadLink, error) { + if err := storage.Check(ctx, key); err != nil { + return storage.ReadLink{}, err + } + if ttl < time.Second || ttl > time.Hour { + return storage.ReadLink{}, errors.New("storage/s3: TTL must be 1s..1h") + } + headers, err := options.ResponseHeaders() + if err != nil { + return storage.ReadLink{}, err + } + query := url.Values{"response-content-disposition": {headers.Get("Content-Disposition")}, "response-cache-control": {headers.Get("Cache-Control")}} + if contentType := headers.Get("Content-Type"); contentType != "" { + query.Set("response-content-type", contentType) + } + u, err := s.client.PresignedGetObject(ctx, s.bucket, key, ttl, query) + if err != nil { + return storage.ReadLink{}, err + } + // Derive the exact expiry from the SDK's signed URL, not a second clock read + // around credential discovery or bucket-region lookup. + q := u.Query() + var expires time.Time + if q.Get("X-Amz-Signature") != "" { + start, e := time.Parse("20060102T150405Z", q.Get("X-Amz-Date")) + seconds, e2 := strconv.ParseInt(q.Get("X-Amz-Expires"), 10, 64) + if e != nil || e2 != nil || seconds < 1 || seconds > 3600 { + return storage.ReadLink{}, errors.New("storage/s3: invalid signed expiry") + } + expires = start.Add(time.Duration(seconds) * time.Second) + } else { + seconds, e := strconv.ParseInt(q.Get("Expires"), 10, 64) + if e != nil || q.Get("Signature") == "" { + return storage.ReadLink{}, errors.New("storage/s3: missing signed expiry") + } + expires = time.Unix(seconds, 0).UTC() + } + return storage.ReadLink{URL: u.String(), ExpiresAt: expires}, nil +} + +var _ storage.ReadPresigner = (*Store)(nil) diff --git a/storage/s3/read_test.go b/storage/s3/read_test.go new file mode 100644 index 0000000..eb3a05d --- /dev/null +++ b/storage/s3/read_test.go @@ -0,0 +1,145 @@ +package s3 + +import ( + "context" + "io" + "mime" + "net/http" + "net/url" + "strconv" + "strings" + "testing" + "time" + + "git.yuntuo.hk/k2safe/go-omni/storage" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +func TestReadOptionsS3AndR2(t *testing.T) { + for _, r2 := range []bool{false, true} { + cfg := Config{Endpoint: "files.example", Bucket: "testbucket", Region: "us-east-1", Secure: true, Credentials: credentials.NewStaticV4("fixtureAccess", "fixtureSecret", "")} + if r2 { + var err error + cfg, err = R2Config(strings.Repeat("a", 32), "testbucket", cfg.Credentials) + if err != nil { + t.Fatal(err) + } + } + store, err := Open(cfg) + if err != nil { + t.Fatal(err) + } + for _, options := range []storage.ReadOptions{{}, {Disposition: storage.DispositionInline, FileName: "报告.pdf", ContentType: "application/pdf"}} { + link, err := store.PresignGetWithOptions(context.Background(), "folder/file.pdf", 30*time.Minute, options) + if err != nil { + t.Fatal(err) + } + u, _ := url.Parse(link.URL) + q := u.Query() + kind, params, err := mime.ParseMediaType(q.Get("response-content-disposition")) + want := options.Disposition + if want == "" { + want = "attachment" + } + if err != nil || kind != want || params["filename"] != options.FileName || q.Get("response-content-type") != options.ContentType || q.Get("response-cache-control") != "private, no-store" || q.Get("X-Amz-Signature") == "" { + t.Fatal("missing response parameters", err) + } + if u.Host != cfg.Endpoint || q.Get("X-Amz-Expires") != "1800" { + t.Fatal("wrong endpoint/TTL") + } + start, err := time.Parse("20060102T150405Z", q.Get("X-Amz-Date")) + if err != nil || !link.ExpiresAt.Equal(start.Add(30*time.Minute)) { + t.Fatal("expiry drift", err) + } + } + for _, ttl := range []time.Duration{0, time.Hour + time.Nanosecond} { + if _, err := store.PresignGetWithOptions(context.Background(), "file", ttl, storage.ReadOptions{}); err == nil { + t.Fatal("invalid TTL") + } + } + if _, err := store.PresignGetWithOptions(context.Background(), "file", time.Minute, storage.ReadOptions{FileName: "x\r\nY: z"}); err == nil { + t.Fatal("invalid filename") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := store.PresignGetWithOptions(ctx, "file", time.Minute, storage.ReadOptions{}); err != context.Canceled { + t.Fatal(err) + } + } +} + +// Called inside the existing isolated MinIO integration. The actual provider +// verifies signatures and rejects tampered response parameters. +func checkReadOptionsIntegration(t *testing.T, store *Store) { + t.Helper() + ctx := context.Background() + data := "%PDF-1.4\nread-test\n%%EOF" + _, err := store.Put(ctx, "preview.pdf", strings.NewReader(data), int64(len(data)), "application/pdf") + if err != nil { + t.Fatal(err) + } + defer store.Delete(ctx, "preview.pdf") + for _, kind := range []string{storage.DispositionInline, storage.DispositionAttachment} { + link, err := store.PresignGetWithOptions(ctx, "preview.pdf", 30*time.Minute, storage.ReadOptions{Disposition: kind, FileName: "示例.pdf", ContentType: "application/pdf"}) + if err != nil { + t.Fatal(err) + } + client := &http.Client{Timeout: 5 * time.Second} + for _, span := range []string{"", "bytes=0-7", "bytes=999-"} { + req, _ := http.NewRequest("GET", link.URL, nil) + if span != "" { + req.Header.Set("Range", span) + } + r, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(r.Body) + r.Body.Close() + if err != nil { + t.Fatal(err) + } + want := 200 + if span == "bytes=0-7" { + want = 206 + } + if span == "bytes=999-" { + want = 416 + } + if r.StatusCode != want { + t.Fatal("status", r.StatusCode) + } + if want == 416 { + continue + } + disposition, params, err := mime.ParseMediaType(r.Header.Get("Content-Disposition")) + if err != nil || disposition != kind || params["filename"] != "示例.pdf" || r.Header.Get("Content-Type") != "application/pdf" || r.Header.Get("Cache-Control") != "private, no-store" { + t.Fatal("provider headers", r.Header, err) + } + value := data + if want == 206 { + value = data[:8] + } + if string(body) != value || r.Header.Get("Content-Length") != strconv.Itoa(len(value)) { + t.Fatal("provider body") + } + } + for _, name := range []string{"response-content-disposition", "response-content-type", "response-cache-control", "X-Amz-Expires"} { + u, _ := url.Parse(link.URL) + q := u.Query() + q.Set(name, "changed") + if name == "X-Amz-Expires" { + q.Set(name, "1801") + } + u.RawQuery = q.Encode() + r, err := client.Get(u.String()) + if err != nil { + t.Fatal(err) + } + r.Body.Close() + if r.StatusCode != 403 { + t.Fatal("tampered parameter", name, r.StatusCode) + } + } + } +} diff --git a/storage/s3/s3_test.go b/storage/s3/s3_test.go index a896682..dd4f609 100644 --- a/storage/s3/s3_test.go +++ b/storage/s3/s3_test.go @@ -84,7 +84,9 @@ func TestS3Integration(t *testing.T) { if !ready { t.Fatal("isolated S3 server did not start") } - client, err := minio.New(address, &minio.Options{Creds: credentials.NewStaticV4("goOmniTest", "goOmniTestSecret", ""), Region: "us-east-1"}) + // The contract deliberately sends malformed bodies. Avoid retry backoff + // exhausting its deadline; these are expected failures, not network retries. + client, err := minio.New(address, &minio.Options{Creds: credentials.NewStaticV4("goOmniTest", "goOmniTestSecret", ""), Region: "us-east-1", MaxRetries: 1}) if err != nil { t.Fatal(err) } @@ -114,6 +116,7 @@ func TestS3Integration(t *testing.T) { } } s, _ := New(client, bucket, 64) + checkReadOptionsIntegration(t, s) storagetest.Run(t, s) s.Put(ctx, "signed", strings.NewReader("hello"), 5, "text/plain") defer s.Delete(context.Background(), "signed") diff --git a/storage/signed.go b/storage/signed.go index a3f02e7..9f7d709 100644 --- a/storage/signed.go +++ b/storage/signed.go @@ -49,30 +49,88 @@ func (s *Signer) PresignGet(ctx context.Context, key string, ttl time.Duration) return &u, nil } -// ServeHTTP serves signed downloads as attachments. Issue links only after +func (s *Signer) readSignature(q url.Values) string { + values := make(url.Values, len(q)) + for key, value := range q { + if key != "signature" { + values[key] = value + } + } + mac := hmac.New(sha256.New, s.secret) + io.WriteString(mac, "GET-OPTIONS-v2\n"+values.Encode()) + return base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) +} + +func (s *Signer) PresignGetWithOptions(ctx context.Context, key string, ttl time.Duration, options ReadOptions) (ReadLink, error) { + if err := Check(ctx, key); err != nil { + return ReadLink{}, err + } + if ttl < time.Second || ttl > time.Hour { + return ReadLink{}, errors.New("storage: TTL must be 1s..1h") + } + headers, err := options.ResponseHeaders() + if err != nil { + return ReadLink{}, err + } + if options.Disposition == "" { + options.Disposition = DispositionAttachment + } + expires := s.now().Add(ttl).UTC().Truncate(time.Second) + q := url.Values{"v": {"2"}, "key": {key}, "expires": {strconv.FormatInt(expires.Unix(), 10)}, "disposition": {options.Disposition}, "filename": {options.FileName}, "contentType": {headers.Get("Content-Type")}} + q.Set("signature", s.readSignature(q)) + u := *s.base + u.RawQuery = q.Encode() + return ReadLink{URL: u.String(), ExpiresAt: expires}, nil +} + +// ServeHTTP serves signed GET/HEAD requests, with optional preview headers. +// Seekable object bodies support Range and conditional reads; other stores keep +// streaming full responses. No full-object buffer is used. Issue links only after // authorization; possession of an unexpired URL grants access to its object. func (s *Signer) ServeHTTP(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodPut && s.uploadPolicy != nil { s.serveUpload(w, r) return } + w = &readResponseWriter{ResponseWriter: w, head: r.Method == http.MethodHead} if r.Method != "GET" && r.Method != "HEAD" { w.Header().Set("Allow", "GET, HEAD") http.Error(w, "method not allowed", 405) return } q, err := url.ParseQuery(r.URL.RawQuery) - if err != nil || len(q) != 3 || len(q["key"]) != 1 || len(q["expires"]) != 1 || len(q["signature"]) != 1 { + keys := []string{"key", "expires", "signature"} + if q.Get("v") == "2" { + keys = append(keys, "v", "disposition", "filename", "contentType") + } + valid := err == nil && len(q) == len(keys) + for _, key := range keys { + valid = valid && len(q[key]) == 1 + } + if !valid { http.Error(w, "invalid link", 403) return } key, expiry := q.Get("key"), q.Get("expires") expires, err := strconv.ParseInt(expiry, 10, 64) now := s.now().Unix() - if err != nil || expires <= now || expires > now+3600 || Check(r.Context(), key) != nil || !hmac.Equal([]byte(q.Get("signature")), []byte(s.signature(key, expiry))) { + expected := s.signature(key, expiry) + if q.Get("v") == "2" { + expected = s.readSignature(q) + } + if err != nil || expires <= now || expires > now+3600 || Check(r.Context(), key) != nil || !hmac.Equal([]byte(q.Get("signature")), []byte(expected)) { http.Error(w, "invalid or expired link", 403) return } + options := ReadOptions{} + if q.Get("v") == "2" { + options = ReadOptions{Disposition: q.Get("disposition"), FileName: q.Get("filename"), ContentType: q.Get("contentType")} + } + headers, err := options.ResponseHeaders() + if err != nil { + http.Error(w, "invalid link", 403) + return + } object, err := s.store.Get(r.Context(), key) if err != nil { status := 500 @@ -82,17 +140,66 @@ func (s *Signer) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, "object unavailable", status) return } + if object.Body == nil { + http.Error(w, "object unavailable", 500) + return + } defer object.Body.Close() + if object.Size < 0 { + http.Error(w, "object unavailable", 500) + return + } contentType := object.ContentType + if options.ContentType != "" { + contentType = headers.Get("Content-Type") + } if contentType == "" { contentType = "application/octet-stream" } w.Header().Set("Content-Type", contentType) - w.Header().Set("Content-Disposition", "attachment") - w.Header().Set("X-Content-Type-Options", "nosniff") - w.Header().Set("Cache-Control", "private, no-store") + w.Header().Set("Content-Disposition", headers.Get("Content-Disposition")) + if body, ok := object.Body.(io.ReadSeeker); ok { + // Range describes a GET representation. HEAD reports full-object headers. + if r.Method == http.MethodHead { + r = r.Clone(r.Context()) + r.Header.Del("Range") + } + http.ServeContent(w, r, "", object.ModifiedAt, body) + return + } w.Header().Set("Content-Length", strconv.FormatInt(object.Size, 10)) + w.WriteHeader(http.StatusOK) if r.Method == "GET" { _, _ = io.Copy(w, object.Body) } } + +// ServeContent may strip cache headers on errors in newer Go versions. Restore +// the signed-link policy at the actual write, including 304/416 and HEAD errors. +type readResponseWriter struct { + http.ResponseWriter + head, wrote bool +} + +func (w *readResponseWriter) WriteHeader(status int) { + if w.wrote { + return + } + w.wrote = true + w.Header().Set("Cache-Control", "private, no-store") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.ResponseWriter.WriteHeader(status) +} +func (w *readResponseWriter) Write(p []byte) (int, error) { + if !w.wrote { + w.WriteHeader(http.StatusOK) + } + if w.head { + return len(p), nil + } + return w.ResponseWriter.Write(p) +} +func (w *readResponseWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter } + +var _ Presigner = (*Signer)(nil) +var _ ReadPresigner = (*Signer)(nil)