feat(storage): support signed read options and local byte ranges
This commit is contained in:
@@ -6,12 +6,42 @@
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 读写删除 | Store | 支持 | 支持 | 支持 |
|
||||
| 临时下载 | Presigner | NewSigner | 支持 | 支持 |
|
||||
| 带选项的预览/下载 | ReadPresigner | NewSigner | 支持 | 支持 |
|
||||
| 限定大小/类型的直传 | UploadPresigner | NewUploadSigner | 支持 | 支持,需要配置签名 Header |
|
||||
| 创建、上传、查询、完成、取消分片 | MultipartStore | 当前不提供 | 支持 | 支持 |
|
||||
| 分片直传签名 | PartPresigner | 当前不提供 | 支持 | 支持,需要配置签名 Header |
|
||||
|
||||
没有固定业务字段、目录规则或身份模型。鉴权、对象归属、业务配额由应用定义。SDK 客户端和连接池由应用持有;适配器不会在一次调用后关闭客户端。
|
||||
|
||||
## 签名预览与下载
|
||||
|
||||
`v0.29.0-preview` 新增独立的可选接口 `ReadPresigner`,不扩充原 Store/Presigner,不要求第三方适配器修改代码。原 `PresignGet(ctx, key, ttl)` 和既有本地下载链接继续有效、默认 attachment。
|
||||
|
||||
```go
|
||||
// signer 可以是 storage.NewSigner 返回的本地 Handler,或 S3/R2、OSS Store。
|
||||
link, err := signer.PresignGetWithOptions(ctx, key, 30*time.Minute, storage.ReadOptions{
|
||||
Disposition: storage.DispositionInline,
|
||||
FileName: "示例报告.pdf",
|
||||
ContentType: "application/pdf",
|
||||
})
|
||||
if err != nil { return err }
|
||||
// 把 link.URL 和 link.ExpiresAt 返回给已授权的调用方。
|
||||
```
|
||||
|
||||
TTL 仍为 1 秒到 1 小时,30 分钟无需改配置。`ReadLink` 包含 `url` 和 `expiresAt`,到期时间取自实际签名,精确到秒(不足一秒的部分舍弃);临时云凭据可能更早失效。无需额外请求 Header,客户端直接打开 URL。可通过 `storage.ReadPresigner` 做能力判断;不支持时由调用方明确处理,不应静默降级成公开地址。
|
||||
|
||||
`Disposition` 只接受 inline/attachment,空值为 attachment。`FileName` 是显示名,与存储 key 无关;中文使用标准 `filename*` 编码,拒绝控制字符(包括 CR/LF)、路径分隔符及超长值。`ContentType` 为空时沿用对象元数据,可显式指定合法 MIME 类型;它只控制响应,不转换或验证文件内容。PDF 应在上传时就保存 `application/pdf` 元数据。inline 请求内联显示,能否展示还取决于浏览器的格式支持与用户设置。
|
||||
|
||||
本地签名绑定 key、到期时间、版本及全部选项,拒绝重复/未知参数和篡改;先验签、验有效期,再读取对象或处理条件请求。Local 使用受文件内容边界约束的流,GET/HEAD 的长度不包含内部元数据;支持单段/多段 Range、206/416,以及 Last-Modified 条件请求。HEAD 返回完整对象头、无响应体,忽略 Range。自定义 Store 的 Body 实现 `io.ReadSeeker` 时复用同一行为;仅支持顺序读取时返回完整 200,不宣称支持范围读取,也不把整个对象加载到内存。
|
||||
|
||||
云端适配器在 OSS/S3 原生 SDK 签名前传入响应参数,禁止先签名后拼参数或替换域名。Range 由云存储处理;云端 GET 签名不能直接改成 HEAD 使用。新接口请求 `Cache-Control: private, no-store`,本地成功、错误及条件响应均保持该策略;云端响应头覆盖受平台规则约束,需检查实际 200/206 响应与代理缓存配置。
|
||||
|
||||
OSS 默认域名或加速域名可能强制下载。需要自定义域名时,在应用装配 SDK client 时设置已绑定桶的域名和 `WithUseCName(true)`,再传给适配器;SDK 的区域、签名版本、凭据由应用配置。例如在现有 SDK config 上调用 `WithEndpoint("https://files.example").WithUseCName(true)`。参见 [OSS 预览规则](https://help.aliyun.com/zh/oss/how-to-ensure-an-object-is-previewed-when-you-access-the-object)与 [GetObject 响应参数规则](https://www.alibabacloud.com/help/en/oss/developer-reference/getobject)。
|
||||
|
||||
签发权限、对象是否不可变、审计和前端链接复用由项目维护。框架不添加分享表、撤销、租户或数据库依赖;重新签发不会撤销旧链接,到期不会收回已下载内容,也不会中断已开始的传输。本地入口使用显式配置的公开 baseURL,不猜测 Host/转发头,不公开磁盘目录。应用与代理日志应隐藏查询参数。
|
||||
|
||||
完整离线 [预览/下载示例](../examples/signedRead/README.md),包含真实 HTTP 检查与一页 PDF。OSS V1/V4、S3/R2 SDK 签名和隔离 MinIO 有回归;OSS/R2 云账号尚未实测,部署方仍需对实际桶、域名及浏览器进行验收。
|
||||
|
||||
## 服务端上传限制
|
||||
|
||||
```go
|
||||
|
||||
@@ -151,14 +151,31 @@ func (s *Store) Get(ctx context.Context, key string) (storage.Object, error) {
|
||||
}
|
||||
info.ModifiedAt = stat.ModTime()
|
||||
ok = true
|
||||
return storage.Object{Info: info, Body: &body{Reader: storage.ContextReader{Context: ctx, Reader: io.LimitReader(f, info.Size)}, Closer: f}}, nil
|
||||
// Expose only the payload as a seekable section. Seeks are relative to the
|
||||
// object, never to our private metadata header or other disk bytes.
|
||||
return storage.Object{Info: info, Body: &body{section: io.NewSectionReader(f, int64(length)+4, info.Size), file: f, ctx: ctx}}, nil
|
||||
}
|
||||
|
||||
type body struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
section *io.SectionReader
|
||||
file *os.File
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func (b *body) Read(p []byte) (int, error) {
|
||||
if err := b.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return b.section.Read(p)
|
||||
}
|
||||
func (b *body) Seek(offset int64, whence int) (int64, error) {
|
||||
if err := b.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return b.section.Seek(offset, whence)
|
||||
}
|
||||
func (b *body) Close() error { return b.file.Close() }
|
||||
|
||||
func (s *Store) Delete(ctx context.Context, key string) error {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return err
|
||||
|
||||
125
storage/local/read_test.go
Normal file
125
storage/local/read_test.go
Normal file
@@ -0,0 +1,125 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage"
|
||||
)
|
||||
|
||||
func TestSignedPayloadRanges(t *testing.T) {
|
||||
store, err := New(t.TempDir(), 1<<20)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data := "%PDF-1.4\n" + strings.Repeat("0123456789", 100) + "\n%%EOF\n"
|
||||
ctx := context.Background()
|
||||
if _, err = store.Put(ctx, "private/document", strings.NewReader(data), int64(len(data)), "application/pdf"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
signer, _ := storage.NewSigner(store, "https://files.example/read", []byte(strings.Repeat("s", 32)))
|
||||
link, err := signer.PresignGetWithOptions(ctx, "private/document", 30*time.Minute, storage.ReadOptions{Disposition: storage.DispositionInline, FileName: "示例.pdf"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
method, span string
|
||||
status int
|
||||
want string
|
||||
}{
|
||||
{"GET", "", 200, data}, {"HEAD", "", 200, ""}, {"HEAD", "bytes=0-3", 200, ""},
|
||||
{"GET", "bytes=0-7", 206, data[:8]}, {"GET", "bytes=20-", 206, data[20:]}, {"GET", "bytes=-6", 206, data[len(data)-6:]},
|
||||
{"GET", "bytes=1000-999999", 206, data[1000:]}, {"GET", "bytes=999999-", 416, ""}, {"GET", "bytes=bad", 416, ""},
|
||||
} {
|
||||
t.Run(tc.method+tc.span, func(t *testing.T) {
|
||||
r := httptest.NewRequest(tc.method, link.URL, nil)
|
||||
if tc.span != "" {
|
||||
r.Header.Set("Range", tc.span)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
signer.ServeHTTP(w, r)
|
||||
if w.Code != tc.status {
|
||||
t.Fatal(w.Code)
|
||||
}
|
||||
if tc.status != 416 && w.Body.String() != tc.want {
|
||||
t.Fatal("wrong payload slice")
|
||||
}
|
||||
if tc.status == 206 && (w.Header().Get("Content-Length") != strconv.Itoa(len(tc.want)) || !strings.HasSuffix(w.Header().Get("Content-Range"), "/"+strconv.Itoa(len(data)))) {
|
||||
t.Fatal(w.Header())
|
||||
}
|
||||
if tc.status == 200 && w.Header().Get("Content-Length") != strconv.Itoa(len(data)) {
|
||||
t.Fatal("metadata included in length")
|
||||
}
|
||||
if tc.span == "bytes=999999-" && w.Header().Get("Content-Range") != "bytes */"+strconv.Itoa(len(data)) {
|
||||
t.Fatal(w.Header())
|
||||
}
|
||||
if w.Header().Get("Cache-Control") != "private, no-store" {
|
||||
t.Fatal("cache policy")
|
||||
}
|
||||
if tc.status != 416 {
|
||||
kind, params, err := mime.ParseMediaType(w.Header().Get("Content-Disposition"))
|
||||
if err != nil || kind != "inline" || params["filename"] != "示例.pdf" || w.Header().Get("Content-Type") != "application/pdf" {
|
||||
t.Fatal(w.Header(), err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
r := httptest.NewRequest("GET", link.URL, nil)
|
||||
r.Header.Set("Range", "bytes=0-3,10-13")
|
||||
w := httptest.NewRecorder()
|
||||
signer.ServeHTTP(w, r)
|
||||
kind, params, err := mime.ParseMediaType(w.Header().Get("Content-Type"))
|
||||
if err != nil || w.Code != 206 || kind != "multipart/byteranges" {
|
||||
t.Fatal(w.Header(), err)
|
||||
}
|
||||
reader := multipart.NewReader(w.Body, params["boundary"])
|
||||
for _, want := range []string{data[:4], data[10:14]} {
|
||||
p, err := reader.NextPart()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := io.ReadAll(p)
|
||||
if err != nil || string(got) != want {
|
||||
t.Fatal("multipart bytes", err)
|
||||
}
|
||||
}
|
||||
if _, err := reader.NextPart(); err != io.EOF {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Random access cannot expose the managed header and still observes context.
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
object, err := store.Get(ctx, "private/document")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer object.Body.Close()
|
||||
seeker := object.Body.(io.ReadSeeker)
|
||||
if size, err := seeker.Seek(0, io.SeekEnd); err != nil || size != int64(len(data)) {
|
||||
t.Fatal(size, err)
|
||||
}
|
||||
if _, err := seeker.Seek(-1, io.SeekStart); err == nil {
|
||||
t.Fatal("seek into header")
|
||||
}
|
||||
if _, err := seeker.Seek(0, io.SeekStart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prefix := make([]byte, 8)
|
||||
if _, err := io.ReadFull(seeker, prefix); err != nil || string(prefix) != data[:8] {
|
||||
t.Fatal("metadata exposed", err)
|
||||
}
|
||||
cancel()
|
||||
if _, err := seeker.Seek(0, io.SeekStart); err != context.Canceled {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := seeker.Read(prefix); err != context.Canceled {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
59
storage/oss/read.go
Normal file
59
storage/oss/read.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage"
|
||||
sdk "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
|
||||
)
|
||||
|
||||
func (s *Store) PresignGetWithOptions(ctx context.Context, key string, ttl time.Duration, options storage.ReadOptions) (storage.ReadLink, error) {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return storage.ReadLink{}, err
|
||||
}
|
||||
if ttl < time.Second || ttl > time.Hour {
|
||||
return storage.ReadLink{}, errors.New("storage/oss: TTL must be 1s..1h")
|
||||
}
|
||||
headers, err := options.ResponseHeaders()
|
||||
if err != nil {
|
||||
return storage.ReadLink{}, err
|
||||
}
|
||||
request := &sdk.GetObjectRequest{Bucket: sdk.Ptr(s.bucket), Key: sdk.Ptr(key), ResponseContentDisposition: sdk.Ptr(headers.Get("Content-Disposition")), ResponseCacheControl: sdk.Ptr(headers.Get("Cache-Control"))}
|
||||
if contentType := headers.Get("Content-Type"); contentType != "" {
|
||||
request.ResponseContentType = sdk.Ptr(contentType)
|
||||
}
|
||||
r, err := s.client.Presign(ctx, request, sdk.PresignExpires(ttl))
|
||||
if err != nil {
|
||||
return storage.ReadLink{}, err
|
||||
}
|
||||
if len(r.SignedHeaders) != 0 {
|
||||
return storage.ReadLink{}, errors.New("storage/oss: direct GET requires a signature without additional request headers")
|
||||
}
|
||||
u, err := url.Parse(r.URL)
|
||||
if err != nil {
|
||||
return storage.ReadLink{}, errors.New("storage/oss: invalid signed URL")
|
||||
}
|
||||
q := u.Query()
|
||||
var expires time.Time
|
||||
if q.Get("x-oss-signature") != "" {
|
||||
start, e := time.Parse("20060102T150405Z", q.Get("x-oss-date"))
|
||||
seconds, e2 := strconv.ParseInt(q.Get("x-oss-expires"), 10, 64)
|
||||
if e != nil || e2 != nil || seconds < 1 || seconds > 3600 {
|
||||
return storage.ReadLink{}, errors.New("storage/oss: invalid signed expiry")
|
||||
}
|
||||
expires = start.Add(time.Duration(seconds) * time.Second)
|
||||
} else {
|
||||
seconds, e := strconv.ParseInt(q.Get("Expires"), 10, 64)
|
||||
if e != nil || q.Get("Signature") == "" {
|
||||
return storage.ReadLink{}, errors.New("storage/oss: missing signed expiry")
|
||||
}
|
||||
expires = time.Unix(seconds, 0).UTC()
|
||||
}
|
||||
return storage.ReadLink{URL: r.URL, ExpiresAt: expires}, nil
|
||||
}
|
||||
|
||||
var _ storage.ReadPresigner = (*Store)(nil)
|
||||
140
storage/oss/read_test.go
Normal file
140
storage/oss/read_test.go
Normal file
@@ -0,0 +1,140 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"mime"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage"
|
||||
sdk "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
|
||||
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
|
||||
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/signer"
|
||||
)
|
||||
|
||||
// Recompute signatures independently from the SDK using fixed test credentials.
|
||||
// This catches response parameters appended after signing, not just their presence.
|
||||
func validReadSignature(u *url.URL, version sdk.SignatureVersionType) bool {
|
||||
q := u.Query()
|
||||
if version == sdk.SignatureVersionV1 {
|
||||
var keys []string
|
||||
for key := range q {
|
||||
if strings.HasPrefix(key, "response-") {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var params []string
|
||||
for _, key := range keys {
|
||||
params = append(params, key+"="+q.Get(key))
|
||||
}
|
||||
message := "GET\n\n\n" + q.Get("Expires") + "\n/testbucket/folder/file.pdf?" + strings.Join(params, "&")
|
||||
mac := hmac.New(sha1.New, []byte("fixtureSecret"))
|
||||
mac.Write([]byte(message))
|
||||
return hmac.Equal([]byte(q.Get("Signature")), []byte(base64.StdEncoding.EncodeToString(mac.Sum(nil))))
|
||||
}
|
||||
signature := q.Get("x-oss-signature")
|
||||
q.Del("x-oss-signature")
|
||||
canonical := "GET\n/testbucket/folder/file.pdf\n" + strings.ReplaceAll(q.Encode(), "+", "%20") + "\n\n\nUNSIGNED-PAYLOAD"
|
||||
hash := sha256.Sum256([]byte(canonical))
|
||||
_, scope, _ := strings.Cut(q.Get("x-oss-credential"), "/")
|
||||
message := "OSS4-HMAC-SHA256\n" + q.Get("x-oss-date") + "\n" + scope + "\n" + hex.EncodeToString(hash[:])
|
||||
key := []byte("aliyun_v4fixtureSecret")
|
||||
for _, part := range strings.Split(scope, "/") {
|
||||
mac := hmac.New(sha256.New, key)
|
||||
mac.Write([]byte(part))
|
||||
key = mac.Sum(nil)
|
||||
}
|
||||
mac := hmac.New(sha256.New, key)
|
||||
mac.Write([]byte(message))
|
||||
return hmac.Equal([]byte(signature), []byte(hex.EncodeToString(mac.Sum(nil))))
|
||||
}
|
||||
|
||||
func TestReadOptionsNativeSignatures(t *testing.T) {
|
||||
for name, version := range map[string]sdk.SignatureVersionType{"v1": sdk.SignatureVersionV1, "v4": sdk.SignatureVersionV4} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cfg := sdk.LoadDefaultConfig().WithRegion("cn-hangzhou").WithEndpoint("https://files.example").WithUseCName(true).WithSignatureVersion(version).WithCredentialsProvider(credentials.NewStaticCredentialsProvider("fixtureAccess", "fixtureSecret"))
|
||||
store, err := New(sdk.NewClient(cfg), "testbucket", 64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, options := range []storage.ReadOptions{{}, {Disposition: storage.DispositionInline, FileName: "报告 2026.pdf", ContentType: "application/pdf"}, {Disposition: storage.DispositionAttachment, FileName: "download.pdf"}} {
|
||||
before := time.Now().UTC().Truncate(time.Second)
|
||||
link, err := store.PresignGetWithOptions(context.Background(), "folder/file.pdf", 30*time.Minute, options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u, err := url.Parse(link.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q := u.Query()
|
||||
if u.Host != "files.example" || u.Path != "/folder/file.pdf" || !validReadSignature(u, version) {
|
||||
t.Fatal("SDK response overrides were not signed")
|
||||
}
|
||||
disposition, params, err := mime.ParseMediaType(q.Get("response-content-disposition"))
|
||||
want := options.Disposition
|
||||
if want == "" {
|
||||
want = "attachment"
|
||||
}
|
||||
if err != nil || disposition != want || params["filename"] != options.FileName || q.Get("response-content-type") != options.ContentType || q.Get("response-cache-control") != "private, no-store" {
|
||||
t.Fatal("wrong signed headers", err)
|
||||
}
|
||||
if link.ExpiresAt.Before(before.Add(30*time.Minute)) || link.ExpiresAt.After(time.Now().Add(30*time.Minute)) {
|
||||
t.Fatal("expiry drift")
|
||||
}
|
||||
var expires time.Time
|
||||
if version == sdk.SignatureVersionV4 {
|
||||
start, _ := time.Parse("20060102T150405Z", q.Get("x-oss-date"))
|
||||
seconds, _ := strconv.Atoi(q.Get("x-oss-expires"))
|
||||
expires = start.Add(time.Duration(seconds) * time.Second)
|
||||
} else {
|
||||
seconds, _ := strconv.ParseInt(q.Get("Expires"), 10, 64)
|
||||
expires = time.Unix(seconds, 0)
|
||||
}
|
||||
if !expires.Equal(link.ExpiresAt) {
|
||||
t.Fatal("result does not match signature expiry")
|
||||
}
|
||||
for _, name := range []string{"response-content-disposition", "response-content-type", "response-cache-control"} {
|
||||
changed := *u
|
||||
query := changed.Query()
|
||||
query.Set(name, "changed")
|
||||
changed.RawQuery = query.Encode()
|
||||
if validReadSignature(&changed, version) {
|
||||
t.Fatal("unsigned override", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, ttl := range []time.Duration{0, time.Second - time.Nanosecond, time.Hour + time.Nanosecond} {
|
||||
if _, err := store.PresignGetWithOptions(context.Background(), "file", ttl, storage.ReadOptions{}); err == nil {
|
||||
t.Fatal("TTL accepted")
|
||||
}
|
||||
}
|
||||
for _, options := range []storage.ReadOptions{{FileName: "x\r\nY: z"}, {ContentType: "text/plain\r\nY: z"}, {Disposition: "bad"}} {
|
||||
if _, err := store.PresignGetWithOptions(context.Background(), "file", time.Minute, options); err == nil {
|
||||
t.Fatal("invalid header accepted")
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := store.PresignGetWithOptions(ctx, "file", time.Minute, storage.ReadOptions{}); err != context.Canceled {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
// A caller-supplied unsigned SDK configuration cannot masquerade as a link.
|
||||
client := sdk.NewClient(sdk.LoadDefaultConfig().WithRegion("cn-hangzhou").WithEndpoint("https://files.example").WithCredentialsProvider(credentials.NewStaticCredentialsProvider("fixtureAccess", "fixtureSecret")), func(o *sdk.Options) { o.Signer = &signer.NopSigner{} })
|
||||
store, _ := New(client, "testbucket", 64)
|
||||
if _, err := store.PresignGetWithOptions(context.Background(), "file", time.Minute, storage.ReadOptions{}); err == nil {
|
||||
t.Fatal("unsigned URL accepted")
|
||||
}
|
||||
}
|
||||
72
storage/read.go
Normal file
72
storage/read.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
DispositionAttachment = "attachment"
|
||||
DispositionInline = "inline"
|
||||
)
|
||||
|
||||
// ReadOptions controls the response to one signed GET. Empty Disposition means
|
||||
// attachment; empty ContentType preserves object metadata. FileName is a display
|
||||
// name, not an object key or filesystem path. All options are signed.
|
||||
type ReadOptions struct {
|
||||
Disposition string
|
||||
FileName string
|
||||
ContentType string
|
||||
}
|
||||
|
||||
// ReadLink grants GET access without additional request headers. ExpiresAt is
|
||||
// the expiry encoded in the signature, at second precision. Provider credentials
|
||||
// may expire earlier. Do not log the URL. Issuing another link does not revoke it.
|
||||
type ReadLink struct {
|
||||
URL string `json:"url"`
|
||||
ExpiresAt time.Time `json:"expiresAt"`
|
||||
}
|
||||
|
||||
// ReadPresigner is optional; existing Store and Presigner implementations do not
|
||||
// need to change. Cloud GET signatures do not also authorize HEAD requests.
|
||||
type ReadPresigner interface {
|
||||
PresignGetWithOptions(context.Context, string, time.Duration, ReadOptions) (ReadLink, error)
|
||||
}
|
||||
|
||||
// ResponseHeaders validates options and returns a fresh set of response headers
|
||||
// for provider adapters to include BEFORE signing, requesting private, no-store.
|
||||
func (o ReadOptions) ResponseHeaders() (http.Header, error) {
|
||||
disposition := o.Disposition
|
||||
if disposition == "" {
|
||||
disposition = DispositionAttachment
|
||||
}
|
||||
if disposition != DispositionAttachment && disposition != DispositionInline {
|
||||
return nil, errors.New("storage: disposition must be inline or attachment")
|
||||
}
|
||||
if len(o.FileName) > 1024 || !utf8.ValidString(o.FileName) || strings.IndexFunc(o.FileName, unicode.IsControl) >= 0 || strings.ContainsAny(o.FileName, `/\`) || o.FileName == "." || o.FileName == ".." {
|
||||
return nil, errors.New("storage: invalid display filename")
|
||||
}
|
||||
params := map[string]string{}
|
||||
if o.FileName != "" {
|
||||
params["filename"] = o.FileName
|
||||
}
|
||||
h := http.Header{"Content-Disposition": {mime.FormatMediaType(disposition, params)}, "Cache-Control": {"private, no-store"}}
|
||||
if o.ContentType != "" {
|
||||
if !utf8.ValidString(o.ContentType) || strings.IndexFunc(o.ContentType, unicode.IsControl) >= 0 {
|
||||
return nil, ErrContentType
|
||||
}
|
||||
kind, err := MediaType(o.ContentType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_, params, _ := mime.ParseMediaType(o.ContentType)
|
||||
h.Set("Content-Type", mime.FormatMediaType(kind, params))
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
219
storage/read_test.go
Normal file
219
storage/read_test.go
Normal file
@@ -0,0 +1,219 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestReadOptions(t *testing.T) {
|
||||
for _, name := range []string{"", "report.pdf", "报告 2026.pdf", `a";b.pdf`, "100%.pdf"} {
|
||||
h, err := (ReadOptions{Disposition: DispositionInline, FileName: name, ContentType: "Application/PDF"}).ResponseHeaders()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
disposition, params, err := mime.ParseMediaType(h.Get("Content-Disposition"))
|
||||
if err != nil || disposition != "inline" || params["filename"] != name || h.Get("Content-Type") != "application/pdf" || h.Get("Cache-Control") != "private, no-store" {
|
||||
t.Fatal("invalid read headers", h, err)
|
||||
}
|
||||
}
|
||||
h, err := (ReadOptions{}).ResponseHeaders()
|
||||
if err != nil || h.Get("Content-Disposition") != "attachment" || h.Get("Content-Type") != "" {
|
||||
t.Fatal(h, err)
|
||||
}
|
||||
for _, options := range []ReadOptions{
|
||||
{Disposition: "form-data"}, {Disposition: "Inline"}, {Disposition: "inline\r\nX: y"},
|
||||
{FileName: "x\r\nX: y"}, {FileName: "a\x00b"}, {FileName: "a\tb"}, {FileName: "bad\xff"}, {FileName: "a/b"}, {FileName: `a\b`}, {FileName: ".."}, {FileName: strings.Repeat("a", 1025)},
|
||||
{ContentType: "bad"}, {ContentType: "image/*"}, {ContentType: "application/pdf\r\nX: y"}, {ContentType: "application/pdf\x00"}, {ContentType: strings.Repeat("x", 257)},
|
||||
} {
|
||||
if _, err := options.ResponseHeaders(); err == nil {
|
||||
t.Fatal("invalid options accepted", options)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type readTestStore struct {
|
||||
calls, bytes int
|
||||
streaming bool
|
||||
}
|
||||
|
||||
func (*readTestStore) Put(context.Context, string, io.Reader, int64, string) (Info, error) {
|
||||
panic("unused")
|
||||
}
|
||||
func (*readTestStore) Delete(context.Context, string) error { panic("unused") }
|
||||
func (s *readTestStore) Get(context.Context, string) (Object, error) {
|
||||
s.calls++
|
||||
body := &readTestBody{Reader: strings.NewReader("0123456789"), store: s}
|
||||
var stream io.ReadCloser = body
|
||||
if s.streaming {
|
||||
stream = struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{body, body}
|
||||
}
|
||||
return Object{Info: Info{Size: 10, ContentType: "application/pdf", ModifiedAt: time.Unix(1000, 0)}, Body: stream}, nil
|
||||
}
|
||||
|
||||
type readTestBody struct {
|
||||
*strings.Reader
|
||||
store *readTestStore
|
||||
}
|
||||
|
||||
func (b *readTestBody) Read(p []byte) (int, error) {
|
||||
n, err := b.Reader.Read(p)
|
||||
b.store.bytes += n
|
||||
return n, err
|
||||
}
|
||||
func (*readTestBody) Close() error { return nil }
|
||||
|
||||
func TestReadSignatureAndExpiry(t *testing.T) {
|
||||
store := &readTestStore{}
|
||||
s, _ := NewSigner(store, "https://files.example/read", []byte(strings.Repeat("s", 32)))
|
||||
now := time.Unix(1800000000, 0).UTC()
|
||||
s.now = func() time.Time { return now }
|
||||
link, err := s.PresignGetWithOptions(context.Background(), "folder/file", 30*time.Minute, ReadOptions{Disposition: DispositionInline, FileName: "报告.pdf", ContentType: "application/pdf"})
|
||||
if err != nil || !link.ExpiresAt.Equal(now.Add(30*time.Minute)) {
|
||||
t.Fatal("incorrect expiry", err)
|
||||
}
|
||||
serve := func(method, address string, headers http.Header) *httptest.ResponseRecorder {
|
||||
r := httptest.NewRequest(method, address, nil)
|
||||
if headers != nil {
|
||||
r.Header = headers
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
s.ServeHTTP(w, r)
|
||||
return w
|
||||
}
|
||||
for _, method := range []string{"GET", "HEAD"} {
|
||||
w := serve(method, link.URL, nil)
|
||||
if w.Code != 200 || w.Header().Get("Content-Type") != "application/pdf" || w.Header().Get("Cache-Control") != "private, no-store" {
|
||||
t.Fatal(w)
|
||||
}
|
||||
if method == "HEAD" && w.Body.Len() != 0 {
|
||||
t.Fatal("HEAD body")
|
||||
}
|
||||
}
|
||||
old, err := s.PresignGet(context.Background(), "folder/file", 30*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w := serve("GET", old.String(), nil); w.Code != 200 || w.Header().Get("Content-Disposition") != "attachment" {
|
||||
t.Fatal("legacy download changed")
|
||||
}
|
||||
mutations := []func(url.Values){
|
||||
func(q url.Values) { q.Set("key", "other") }, func(q url.Values) { q.Set("expires", "1800001900") }, func(q url.Values) { q.Set("signature", "bad") },
|
||||
func(q url.Values) { q.Set("filename", "other.pdf") }, func(q url.Values) { q.Set("disposition", "attachment") }, func(q url.Values) { q.Set("contentType", "text/html") },
|
||||
func(q url.Values) { q.Set("v", "3") }, func(q url.Values) { q.Del("v") }, func(q url.Values) {
|
||||
for _, k := range []string{"v", "filename", "disposition", "contentType"} {
|
||||
q.Del(k)
|
||||
}
|
||||
},
|
||||
func(q url.Values) { q.Set("extra", "x") }, func(q url.Values) { q.Set("filename", "evil\r\nX: yes") },
|
||||
}
|
||||
for _, key := range []string{"key", "expires", "signature", "v", "disposition", "filename", "contentType"} {
|
||||
key := key
|
||||
mutations = append(mutations, func(q url.Values) { q.Add(key, q.Get(key)) })
|
||||
}
|
||||
for _, mutate := range mutations {
|
||||
u, _ := url.Parse(link.URL)
|
||||
q := u.Query()
|
||||
mutate(q)
|
||||
u.RawQuery = q.Encode()
|
||||
before := store.calls
|
||||
if w := serve("GET", u.String(), http.Header{"Range": {"bytes=0-1"}}); w.Code != 403 || store.calls != before {
|
||||
t.Fatal("tampered link reached storage")
|
||||
}
|
||||
}
|
||||
u := *old
|
||||
q := u.Query()
|
||||
q.Set("disposition", "inline")
|
||||
u.RawQuery = q.Encode()
|
||||
if serve("GET", u.String(), nil).Code != 403 {
|
||||
t.Fatal("legacy download upgraded by query")
|
||||
}
|
||||
if serve("GET", link.URL+"&bad=%ZZ", nil).Code != 403 {
|
||||
t.Fatal("malformed query accepted")
|
||||
}
|
||||
before := store.calls
|
||||
now = link.ExpiresAt.Add(-time.Nanosecond)
|
||||
if serve("GET", link.URL, nil).Code != 200 {
|
||||
t.Fatal("early expiration")
|
||||
}
|
||||
before = store.calls
|
||||
now = link.ExpiresAt
|
||||
for _, method := range []string{"GET", "HEAD"} {
|
||||
w := serve(method, link.URL, http.Header{"Range": {"bytes=0-1"}, "If-Modified-Since": {"Fri, 01 Jan 2100 00:00:00 GMT"}})
|
||||
if w.Code != 403 || store.calls != before || w.Header().Get("Cache-Control") != "private, no-store" {
|
||||
t.Fatal("expired link reached storage or cache")
|
||||
}
|
||||
if method == "HEAD" && w.Body.Len() != 0 {
|
||||
t.Fatal("HEAD error has body")
|
||||
}
|
||||
}
|
||||
for _, ttl := range []time.Duration{0, time.Second - time.Nanosecond, time.Hour + time.Nanosecond} {
|
||||
if _, err := s.PresignGetWithOptions(context.Background(), "key", ttl, ReadOptions{}); err == nil {
|
||||
t.Fatal("invalid TTL")
|
||||
}
|
||||
}
|
||||
for _, ttl := range []time.Duration{time.Second, time.Hour} {
|
||||
if _, err := s.PresignGetWithOptions(context.Background(), "key", ttl, ReadOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := s.PresignGetWithOptions(ctx, "key", time.Minute, ReadOptions{}); err != context.Canceled {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadRangeStreamingAndCache(t *testing.T) {
|
||||
for _, streaming := range []bool{false, true} {
|
||||
store := &readTestStore{streaming: streaming}
|
||||
s, _ := NewSigner(store, "https://files.example/read", []byte(strings.Repeat("s", 32)))
|
||||
link, _ := s.PresignGetWithOptions(context.Background(), "key", 30*time.Minute, ReadOptions{})
|
||||
for _, method := range []string{"GET", "HEAD"} {
|
||||
before := store.bytes
|
||||
r := httptest.NewRequest(method, link.URL, nil)
|
||||
r.Header.Set("Range", "bytes=2-4")
|
||||
w := httptest.NewRecorder()
|
||||
s.ServeHTTP(w, r)
|
||||
if method == "HEAD" {
|
||||
if w.Code != 200 || w.Body.Len() != 0 || store.bytes != before {
|
||||
t.Fatal("HEAD read body", w)
|
||||
}
|
||||
} else if streaming {
|
||||
if w.Code != 200 || w.Body.String() != "0123456789" || w.Header().Get("Accept-Ranges") != "" {
|
||||
t.Fatal("stream fallback", w)
|
||||
}
|
||||
} else {
|
||||
if w.Code != 206 || w.Body.String() != "234" || store.bytes-before != 3 {
|
||||
t.Fatal("range buffered full body", w)
|
||||
}
|
||||
}
|
||||
if w.Header().Get("Cache-Control") != "private, no-store" {
|
||||
t.Fatal("missing cache policy")
|
||||
}
|
||||
}
|
||||
if !streaming {
|
||||
for _, h := range []http.Header{{"Range": {"bytes=99-100"}}, {"If-Modified-Since": {"Fri, 01 Jan 2100 00:00:00 GMT"}}} {
|
||||
r := httptest.NewRequest("GET", link.URL, nil)
|
||||
r.Header = h
|
||||
w := httptest.NewRecorder()
|
||||
s.ServeHTTP(w, r)
|
||||
if w.Code != 416 && w.Code != 304 {
|
||||
t.Fatal(w.Code)
|
||||
}
|
||||
if w.Header().Get("Cache-Control") != "private, no-store" {
|
||||
t.Fatal("cache policy lost on error/conditional")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
53
storage/s3/read.go
Normal file
53
storage/s3/read.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage"
|
||||
)
|
||||
|
||||
func (s *Store) PresignGetWithOptions(ctx context.Context, key string, ttl time.Duration, options storage.ReadOptions) (storage.ReadLink, error) {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return storage.ReadLink{}, err
|
||||
}
|
||||
if ttl < time.Second || ttl > time.Hour {
|
||||
return storage.ReadLink{}, errors.New("storage/s3: TTL must be 1s..1h")
|
||||
}
|
||||
headers, err := options.ResponseHeaders()
|
||||
if err != nil {
|
||||
return storage.ReadLink{}, err
|
||||
}
|
||||
query := url.Values{"response-content-disposition": {headers.Get("Content-Disposition")}, "response-cache-control": {headers.Get("Cache-Control")}}
|
||||
if contentType := headers.Get("Content-Type"); contentType != "" {
|
||||
query.Set("response-content-type", contentType)
|
||||
}
|
||||
u, err := s.client.PresignedGetObject(ctx, s.bucket, key, ttl, query)
|
||||
if err != nil {
|
||||
return storage.ReadLink{}, err
|
||||
}
|
||||
// Derive the exact expiry from the SDK's signed URL, not a second clock read
|
||||
// around credential discovery or bucket-region lookup.
|
||||
q := u.Query()
|
||||
var expires time.Time
|
||||
if q.Get("X-Amz-Signature") != "" {
|
||||
start, e := time.Parse("20060102T150405Z", q.Get("X-Amz-Date"))
|
||||
seconds, e2 := strconv.ParseInt(q.Get("X-Amz-Expires"), 10, 64)
|
||||
if e != nil || e2 != nil || seconds < 1 || seconds > 3600 {
|
||||
return storage.ReadLink{}, errors.New("storage/s3: invalid signed expiry")
|
||||
}
|
||||
expires = start.Add(time.Duration(seconds) * time.Second)
|
||||
} else {
|
||||
seconds, e := strconv.ParseInt(q.Get("Expires"), 10, 64)
|
||||
if e != nil || q.Get("Signature") == "" {
|
||||
return storage.ReadLink{}, errors.New("storage/s3: missing signed expiry")
|
||||
}
|
||||
expires = time.Unix(seconds, 0).UTC()
|
||||
}
|
||||
return storage.ReadLink{URL: u.String(), ExpiresAt: expires}, nil
|
||||
}
|
||||
|
||||
var _ storage.ReadPresigner = (*Store)(nil)
|
||||
145
storage/s3/read_test.go
Normal file
145
storage/s3/read_test.go
Normal file
@@ -0,0 +1,145 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
func TestReadOptionsS3AndR2(t *testing.T) {
|
||||
for _, r2 := range []bool{false, true} {
|
||||
cfg := Config{Endpoint: "files.example", Bucket: "testbucket", Region: "us-east-1", Secure: true, Credentials: credentials.NewStaticV4("fixtureAccess", "fixtureSecret", "")}
|
||||
if r2 {
|
||||
var err error
|
||||
cfg, err = R2Config(strings.Repeat("a", 32), "testbucket", cfg.Credentials)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
store, err := Open(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, options := range []storage.ReadOptions{{}, {Disposition: storage.DispositionInline, FileName: "报告.pdf", ContentType: "application/pdf"}} {
|
||||
link, err := store.PresignGetWithOptions(context.Background(), "folder/file.pdf", 30*time.Minute, options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u, _ := url.Parse(link.URL)
|
||||
q := u.Query()
|
||||
kind, params, err := mime.ParseMediaType(q.Get("response-content-disposition"))
|
||||
want := options.Disposition
|
||||
if want == "" {
|
||||
want = "attachment"
|
||||
}
|
||||
if err != nil || kind != want || params["filename"] != options.FileName || q.Get("response-content-type") != options.ContentType || q.Get("response-cache-control") != "private, no-store" || q.Get("X-Amz-Signature") == "" {
|
||||
t.Fatal("missing response parameters", err)
|
||||
}
|
||||
if u.Host != cfg.Endpoint || q.Get("X-Amz-Expires") != "1800" {
|
||||
t.Fatal("wrong endpoint/TTL")
|
||||
}
|
||||
start, err := time.Parse("20060102T150405Z", q.Get("X-Amz-Date"))
|
||||
if err != nil || !link.ExpiresAt.Equal(start.Add(30*time.Minute)) {
|
||||
t.Fatal("expiry drift", err)
|
||||
}
|
||||
}
|
||||
for _, ttl := range []time.Duration{0, time.Hour + time.Nanosecond} {
|
||||
if _, err := store.PresignGetWithOptions(context.Background(), "file", ttl, storage.ReadOptions{}); err == nil {
|
||||
t.Fatal("invalid TTL")
|
||||
}
|
||||
}
|
||||
if _, err := store.PresignGetWithOptions(context.Background(), "file", time.Minute, storage.ReadOptions{FileName: "x\r\nY: z"}); err == nil {
|
||||
t.Fatal("invalid filename")
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := store.PresignGetWithOptions(ctx, "file", time.Minute, storage.ReadOptions{}); err != context.Canceled {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Called inside the existing isolated MinIO integration. The actual provider
|
||||
// verifies signatures and rejects tampered response parameters.
|
||||
func checkReadOptionsIntegration(t *testing.T, store *Store) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
data := "%PDF-1.4\nread-test\n%%EOF"
|
||||
_, err := store.Put(ctx, "preview.pdf", strings.NewReader(data), int64(len(data)), "application/pdf")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Delete(ctx, "preview.pdf")
|
||||
for _, kind := range []string{storage.DispositionInline, storage.DispositionAttachment} {
|
||||
link, err := store.PresignGetWithOptions(ctx, "preview.pdf", 30*time.Minute, storage.ReadOptions{Disposition: kind, FileName: "示例.pdf", ContentType: "application/pdf"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
for _, span := range []string{"", "bytes=0-7", "bytes=999-"} {
|
||||
req, _ := http.NewRequest("GET", link.URL, nil)
|
||||
if span != "" {
|
||||
req.Header.Set("Range", span)
|
||||
}
|
||||
r, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := io.ReadAll(r.Body)
|
||||
r.Body.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := 200
|
||||
if span == "bytes=0-7" {
|
||||
want = 206
|
||||
}
|
||||
if span == "bytes=999-" {
|
||||
want = 416
|
||||
}
|
||||
if r.StatusCode != want {
|
||||
t.Fatal("status", r.StatusCode)
|
||||
}
|
||||
if want == 416 {
|
||||
continue
|
||||
}
|
||||
disposition, params, err := mime.ParseMediaType(r.Header.Get("Content-Disposition"))
|
||||
if err != nil || disposition != kind || params["filename"] != "示例.pdf" || r.Header.Get("Content-Type") != "application/pdf" || r.Header.Get("Cache-Control") != "private, no-store" {
|
||||
t.Fatal("provider headers", r.Header, err)
|
||||
}
|
||||
value := data
|
||||
if want == 206 {
|
||||
value = data[:8]
|
||||
}
|
||||
if string(body) != value || r.Header.Get("Content-Length") != strconv.Itoa(len(value)) {
|
||||
t.Fatal("provider body")
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"response-content-disposition", "response-content-type", "response-cache-control", "X-Amz-Expires"} {
|
||||
u, _ := url.Parse(link.URL)
|
||||
q := u.Query()
|
||||
q.Set(name, "changed")
|
||||
if name == "X-Amz-Expires" {
|
||||
q.Set(name, "1801")
|
||||
}
|
||||
u.RawQuery = q.Encode()
|
||||
r, err := client.Get(u.String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.Body.Close()
|
||||
if r.StatusCode != 403 {
|
||||
t.Fatal("tampered parameter", name, r.StatusCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -84,7 +84,9 @@ func TestS3Integration(t *testing.T) {
|
||||
if !ready {
|
||||
t.Fatal("isolated S3 server did not start")
|
||||
}
|
||||
client, err := minio.New(address, &minio.Options{Creds: credentials.NewStaticV4("goOmniTest", "goOmniTestSecret", ""), Region: "us-east-1"})
|
||||
// The contract deliberately sends malformed bodies. Avoid retry backoff
|
||||
// exhausting its deadline; these are expected failures, not network retries.
|
||||
client, err := minio.New(address, &minio.Options{Creds: credentials.NewStaticV4("goOmniTest", "goOmniTestSecret", ""), Region: "us-east-1", MaxRetries: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -114,6 +116,7 @@ func TestS3Integration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
s, _ := New(client, bucket, 64)
|
||||
checkReadOptionsIntegration(t, s)
|
||||
storagetest.Run(t, s)
|
||||
s.Put(ctx, "signed", strings.NewReader("hello"), 5, "text/plain")
|
||||
defer s.Delete(context.Background(), "signed")
|
||||
|
||||
@@ -49,30 +49,88 @@ func (s *Signer) PresignGet(ctx context.Context, key string, ttl time.Duration)
|
||||
return &u, nil
|
||||
}
|
||||
|
||||
// ServeHTTP serves signed downloads as attachments. Issue links only after
|
||||
func (s *Signer) readSignature(q url.Values) string {
|
||||
values := make(url.Values, len(q))
|
||||
for key, value := range q {
|
||||
if key != "signature" {
|
||||
values[key] = value
|
||||
}
|
||||
}
|
||||
mac := hmac.New(sha256.New, s.secret)
|
||||
io.WriteString(mac, "GET-OPTIONS-v2\n"+values.Encode())
|
||||
return base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
func (s *Signer) PresignGetWithOptions(ctx context.Context, key string, ttl time.Duration, options ReadOptions) (ReadLink, error) {
|
||||
if err := Check(ctx, key); err != nil {
|
||||
return ReadLink{}, err
|
||||
}
|
||||
if ttl < time.Second || ttl > time.Hour {
|
||||
return ReadLink{}, errors.New("storage: TTL must be 1s..1h")
|
||||
}
|
||||
headers, err := options.ResponseHeaders()
|
||||
if err != nil {
|
||||
return ReadLink{}, err
|
||||
}
|
||||
if options.Disposition == "" {
|
||||
options.Disposition = DispositionAttachment
|
||||
}
|
||||
expires := s.now().Add(ttl).UTC().Truncate(time.Second)
|
||||
q := url.Values{"v": {"2"}, "key": {key}, "expires": {strconv.FormatInt(expires.Unix(), 10)}, "disposition": {options.Disposition}, "filename": {options.FileName}, "contentType": {headers.Get("Content-Type")}}
|
||||
q.Set("signature", s.readSignature(q))
|
||||
u := *s.base
|
||||
u.RawQuery = q.Encode()
|
||||
return ReadLink{URL: u.String(), ExpiresAt: expires}, nil
|
||||
}
|
||||
|
||||
// ServeHTTP serves signed GET/HEAD requests, with optional preview headers.
|
||||
// Seekable object bodies support Range and conditional reads; other stores keep
|
||||
// streaming full responses. No full-object buffer is used. Issue links only after
|
||||
// authorization; possession of an unexpired URL grants access to its object.
|
||||
func (s *Signer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPut && s.uploadPolicy != nil {
|
||||
s.serveUpload(w, r)
|
||||
return
|
||||
}
|
||||
w = &readResponseWriter{ResponseWriter: w, head: r.Method == http.MethodHead}
|
||||
if r.Method != "GET" && r.Method != "HEAD" {
|
||||
w.Header().Set("Allow", "GET, HEAD")
|
||||
http.Error(w, "method not allowed", 405)
|
||||
return
|
||||
}
|
||||
q, err := url.ParseQuery(r.URL.RawQuery)
|
||||
if err != nil || len(q) != 3 || len(q["key"]) != 1 || len(q["expires"]) != 1 || len(q["signature"]) != 1 {
|
||||
keys := []string{"key", "expires", "signature"}
|
||||
if q.Get("v") == "2" {
|
||||
keys = append(keys, "v", "disposition", "filename", "contentType")
|
||||
}
|
||||
valid := err == nil && len(q) == len(keys)
|
||||
for _, key := range keys {
|
||||
valid = valid && len(q[key]) == 1
|
||||
}
|
||||
if !valid {
|
||||
http.Error(w, "invalid link", 403)
|
||||
return
|
||||
}
|
||||
key, expiry := q.Get("key"), q.Get("expires")
|
||||
expires, err := strconv.ParseInt(expiry, 10, 64)
|
||||
now := s.now().Unix()
|
||||
if err != nil || expires <= now || expires > now+3600 || Check(r.Context(), key) != nil || !hmac.Equal([]byte(q.Get("signature")), []byte(s.signature(key, expiry))) {
|
||||
expected := s.signature(key, expiry)
|
||||
if q.Get("v") == "2" {
|
||||
expected = s.readSignature(q)
|
||||
}
|
||||
if err != nil || expires <= now || expires > now+3600 || Check(r.Context(), key) != nil || !hmac.Equal([]byte(q.Get("signature")), []byte(expected)) {
|
||||
http.Error(w, "invalid or expired link", 403)
|
||||
return
|
||||
}
|
||||
options := ReadOptions{}
|
||||
if q.Get("v") == "2" {
|
||||
options = ReadOptions{Disposition: q.Get("disposition"), FileName: q.Get("filename"), ContentType: q.Get("contentType")}
|
||||
}
|
||||
headers, err := options.ResponseHeaders()
|
||||
if err != nil {
|
||||
http.Error(w, "invalid link", 403)
|
||||
return
|
||||
}
|
||||
object, err := s.store.Get(r.Context(), key)
|
||||
if err != nil {
|
||||
status := 500
|
||||
@@ -82,17 +140,66 @@ func (s *Signer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "object unavailable", status)
|
||||
return
|
||||
}
|
||||
if object.Body == nil {
|
||||
http.Error(w, "object unavailable", 500)
|
||||
return
|
||||
}
|
||||
defer object.Body.Close()
|
||||
if object.Size < 0 {
|
||||
http.Error(w, "object unavailable", 500)
|
||||
return
|
||||
}
|
||||
contentType := object.ContentType
|
||||
if options.ContentType != "" {
|
||||
contentType = headers.Get("Content-Type")
|
||||
}
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Disposition", "attachment")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.Header().Set("Cache-Control", "private, no-store")
|
||||
w.Header().Set("Content-Disposition", headers.Get("Content-Disposition"))
|
||||
if body, ok := object.Body.(io.ReadSeeker); ok {
|
||||
// Range describes a GET representation. HEAD reports full-object headers.
|
||||
if r.Method == http.MethodHead {
|
||||
r = r.Clone(r.Context())
|
||||
r.Header.Del("Range")
|
||||
}
|
||||
http.ServeContent(w, r, "", object.ModifiedAt, body)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(object.Size, 10))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if r.Method == "GET" {
|
||||
_, _ = io.Copy(w, object.Body)
|
||||
}
|
||||
}
|
||||
|
||||
// ServeContent may strip cache headers on errors in newer Go versions. Restore
|
||||
// the signed-link policy at the actual write, including 304/416 and HEAD errors.
|
||||
type readResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
head, wrote bool
|
||||
}
|
||||
|
||||
func (w *readResponseWriter) WriteHeader(status int) {
|
||||
if w.wrote {
|
||||
return
|
||||
}
|
||||
w.wrote = true
|
||||
w.Header().Set("Cache-Control", "private, no-store")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.ResponseWriter.WriteHeader(status)
|
||||
}
|
||||
func (w *readResponseWriter) Write(p []byte) (int, error) {
|
||||
if !w.wrote {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
if w.head {
|
||||
return len(p), nil
|
||||
}
|
||||
return w.ResponseWriter.Write(p)
|
||||
}
|
||||
func (w *readResponseWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter }
|
||||
|
||||
var _ Presigner = (*Signer)(nil)
|
||||
var _ ReadPresigner = (*Signer)(nil)
|
||||
|
||||
Reference in New Issue
Block a user