feat(storage): support signed read options and local byte ranges

This commit is contained in:
2026-09-30 22:44:54 +08:00
parent 01a03149f9
commit f0c62152d7
29 changed files with 1395 additions and 30 deletions

View File

@@ -6,12 +6,42 @@
| --- | --- | --- | --- | --- |
| 读写删除 | Store | 支持 | 支持 | 支持 |
| 临时下载 | Presigner | NewSigner | 支持 | 支持 |
| 带选项的预览/下载 | ReadPresigner | NewSigner | 支持 | 支持 |
| 限定大小/类型的直传 | UploadPresigner | NewUploadSigner | 支持 | 支持,需要配置签名 Header |
| 创建、上传、查询、完成、取消分片 | MultipartStore | 当前不提供 | 支持 | 支持 |
| 分片直传签名 | PartPresigner | 当前不提供 | 支持 | 支持,需要配置签名 Header |
没有固定业务字段、目录规则或身份模型。鉴权、对象归属、业务配额由应用定义。SDK 客户端和连接池由应用持有;适配器不会在一次调用后关闭客户端。
## 签名预览与下载
`v0.29.0-preview` 新增独立的可选接口 `ReadPresigner`,不扩充原 Store/Presigner,不要求第三方适配器修改代码。原 `PresignGet(ctx, key, ttl)` 和既有本地下载链接继续有效、默认 attachment。
```go
// signer 可以是 storage.NewSigner 返回的本地 Handler,或 S3/R2、OSS Store。
link, err := signer.PresignGetWithOptions(ctx, key, 30*time.Minute, storage.ReadOptions{
Disposition: storage.DispositionInline,
FileName: "示例报告.pdf",
ContentType: "application/pdf",
})
if err != nil { return err }
// 把 link.URL 和 link.ExpiresAt 返回给已授权的调用方。
```
TTL 仍为 1 秒到 1 小时,30 分钟无需改配置。`ReadLink` 包含 `url` 和 `expiresAt`,到期时间取自实际签名,精确到秒(不足一秒的部分舍弃);临时云凭据可能更早失效。无需额外请求 Header,客户端直接打开 URL。可通过 `storage.ReadPresigner` 做能力判断;不支持时由调用方明确处理,不应静默降级成公开地址。
`Disposition` 只接受 inline/attachment,空值为 attachment。`FileName` 是显示名,与存储 key 无关;中文使用标准 `filename*` 编码,拒绝控制字符(包括 CR/LF)、路径分隔符及超长值。`ContentType` 为空时沿用对象元数据,可显式指定合法 MIME 类型;它只控制响应,不转换或验证文件内容。PDF 应在上传时就保存 `application/pdf` 元数据。inline 请求内联显示,能否展示还取决于浏览器的格式支持与用户设置。
本地签名绑定 key、到期时间、版本及全部选项,拒绝重复/未知参数和篡改;先验签、验有效期,再读取对象或处理条件请求。Local 使用受文件内容边界约束的流,GET/HEAD 的长度不包含内部元数据;支持单段/多段 Range、206/416,以及 Last-Modified 条件请求。HEAD 返回完整对象头、无响应体,忽略 Range。自定义 Store 的 Body 实现 `io.ReadSeeker` 时复用同一行为;仅支持顺序读取时返回完整 200,不宣称支持范围读取,也不把整个对象加载到内存。
云端适配器在 OSS/S3 原生 SDK 签名前传入响应参数,禁止先签名后拼参数或替换域名。Range 由云存储处理;云端 GET 签名不能直接改成 HEAD 使用。新接口请求 `Cache-Control: private, no-store`,本地成功、错误及条件响应均保持该策略;云端响应头覆盖受平台规则约束,需检查实际 200/206 响应与代理缓存配置。
OSS 默认域名或加速域名可能强制下载。需要自定义域名时,在应用装配 SDK client 时设置已绑定桶的域名和 `WithUseCName(true)`,再传给适配器;SDK 的区域、签名版本、凭据由应用配置。例如在现有 SDK config 上调用 `WithEndpoint("https://files.example").WithUseCName(true)`。参见 [OSS 预览规则](https://help.aliyun.com/zh/oss/how-to-ensure-an-object-is-previewed-when-you-access-the-object)与 [GetObject 响应参数规则](https://www.alibabacloud.com/help/en/oss/developer-reference/getobject)。
签发权限、对象是否不可变、审计和前端链接复用由项目维护。框架不添加分享表、撤销、租户或数据库依赖;重新签发不会撤销旧链接,到期不会收回已下载内容,也不会中断已开始的传输。本地入口使用显式配置的公开 baseURL,不猜测 Host/转发头,不公开磁盘目录。应用与代理日志应隐藏查询参数。
完整离线 [预览/下载示例](../examples/signedRead/README.md),包含真实 HTTP 检查与一页 PDF。OSS V1/V4、S3/R2 SDK 签名和隔离 MinIO 有回归;OSS/R2 云账号尚未实测,部署方仍需对实际桶、域名及浏览器进行验收。
## 服务端上传限制
```go

View File

@@ -151,14 +151,31 @@ func (s *Store) Get(ctx context.Context, key string) (storage.Object, error) {
}
info.ModifiedAt = stat.ModTime()
ok = true
return storage.Object{Info: info, Body: &body{Reader: storage.ContextReader{Context: ctx, Reader: io.LimitReader(f, info.Size)}, Closer: f}}, nil
// Expose only the payload as a seekable section. Seeks are relative to the
// object, never to our private metadata header or other disk bytes.
return storage.Object{Info: info, Body: &body{section: io.NewSectionReader(f, int64(length)+4, info.Size), file: f, ctx: ctx}}, nil
}
type body struct {
io.Reader
io.Closer
section *io.SectionReader
file *os.File
ctx context.Context
}
func (b *body) Read(p []byte) (int, error) {
if err := b.ctx.Err(); err != nil {
return 0, err
}
return b.section.Read(p)
}
func (b *body) Seek(offset int64, whence int) (int64, error) {
if err := b.ctx.Err(); err != nil {
return 0, err
}
return b.section.Seek(offset, whence)
}
func (b *body) Close() error { return b.file.Close() }
func (s *Store) Delete(ctx context.Context, key string) error {
if err := storage.Check(ctx, key); err != nil {
return err

125
storage/local/read_test.go Normal file
View File

@@ -0,0 +1,125 @@
package local
import (
"context"
"io"
"mime"
"mime/multipart"
"net/http/httptest"
"strconv"
"strings"
"testing"
"time"
"git.yuntuo.hk/k2safe/go-omni/storage"
)
func TestSignedPayloadRanges(t *testing.T) {
store, err := New(t.TempDir(), 1<<20)
if err != nil {
t.Fatal(err)
}
data := "%PDF-1.4\n" + strings.Repeat("0123456789", 100) + "\n%%EOF\n"
ctx := context.Background()
if _, err = store.Put(ctx, "private/document", strings.NewReader(data), int64(len(data)), "application/pdf"); err != nil {
t.Fatal(err)
}
signer, _ := storage.NewSigner(store, "https://files.example/read", []byte(strings.Repeat("s", 32)))
link, err := signer.PresignGetWithOptions(ctx, "private/document", 30*time.Minute, storage.ReadOptions{Disposition: storage.DispositionInline, FileName: "示例.pdf"})
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
method, span string
status int
want string
}{
{"GET", "", 200, data}, {"HEAD", "", 200, ""}, {"HEAD", "bytes=0-3", 200, ""},
{"GET", "bytes=0-7", 206, data[:8]}, {"GET", "bytes=20-", 206, data[20:]}, {"GET", "bytes=-6", 206, data[len(data)-6:]},
{"GET", "bytes=1000-999999", 206, data[1000:]}, {"GET", "bytes=999999-", 416, ""}, {"GET", "bytes=bad", 416, ""},
} {
t.Run(tc.method+tc.span, func(t *testing.T) {
r := httptest.NewRequest(tc.method, link.URL, nil)
if tc.span != "" {
r.Header.Set("Range", tc.span)
}
w := httptest.NewRecorder()
signer.ServeHTTP(w, r)
if w.Code != tc.status {
t.Fatal(w.Code)
}
if tc.status != 416 && w.Body.String() != tc.want {
t.Fatal("wrong payload slice")
}
if tc.status == 206 && (w.Header().Get("Content-Length") != strconv.Itoa(len(tc.want)) || !strings.HasSuffix(w.Header().Get("Content-Range"), "/"+strconv.Itoa(len(data)))) {
t.Fatal(w.Header())
}
if tc.status == 200 && w.Header().Get("Content-Length") != strconv.Itoa(len(data)) {
t.Fatal("metadata included in length")
}
if tc.span == "bytes=999999-" && w.Header().Get("Content-Range") != "bytes */"+strconv.Itoa(len(data)) {
t.Fatal(w.Header())
}
if w.Header().Get("Cache-Control") != "private, no-store" {
t.Fatal("cache policy")
}
if tc.status != 416 {
kind, params, err := mime.ParseMediaType(w.Header().Get("Content-Disposition"))
if err != nil || kind != "inline" || params["filename"] != "示例.pdf" || w.Header().Get("Content-Type") != "application/pdf" {
t.Fatal(w.Header(), err)
}
}
})
}
r := httptest.NewRequest("GET", link.URL, nil)
r.Header.Set("Range", "bytes=0-3,10-13")
w := httptest.NewRecorder()
signer.ServeHTTP(w, r)
kind, params, err := mime.ParseMediaType(w.Header().Get("Content-Type"))
if err != nil || w.Code != 206 || kind != "multipart/byteranges" {
t.Fatal(w.Header(), err)
}
reader := multipart.NewReader(w.Body, params["boundary"])
for _, want := range []string{data[:4], data[10:14]} {
p, err := reader.NextPart()
if err != nil {
t.Fatal(err)
}
got, err := io.ReadAll(p)
if err != nil || string(got) != want {
t.Fatal("multipart bytes", err)
}
}
if _, err := reader.NextPart(); err != io.EOF {
t.Fatal(err)
}
// Random access cannot expose the managed header and still observes context.
ctx, cancel := context.WithCancel(ctx)
defer cancel()
object, err := store.Get(ctx, "private/document")
if err != nil {
t.Fatal(err)
}
defer object.Body.Close()
seeker := object.Body.(io.ReadSeeker)
if size, err := seeker.Seek(0, io.SeekEnd); err != nil || size != int64(len(data)) {
t.Fatal(size, err)
}
if _, err := seeker.Seek(-1, io.SeekStart); err == nil {
t.Fatal("seek into header")
}
if _, err := seeker.Seek(0, io.SeekStart); err != nil {
t.Fatal(err)
}
prefix := make([]byte, 8)
if _, err := io.ReadFull(seeker, prefix); err != nil || string(prefix) != data[:8] {
t.Fatal("metadata exposed", err)
}
cancel()
if _, err := seeker.Seek(0, io.SeekStart); err != context.Canceled {
t.Fatal(err)
}
if _, err := seeker.Read(prefix); err != context.Canceled {
t.Fatal(err)
}
}

59
storage/oss/read.go Normal file
View File

@@ -0,0 +1,59 @@
package oss
import (
"context"
"errors"
"net/url"
"strconv"
"time"
"git.yuntuo.hk/k2safe/go-omni/storage"
sdk "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
)
func (s *Store) PresignGetWithOptions(ctx context.Context, key string, ttl time.Duration, options storage.ReadOptions) (storage.ReadLink, error) {
if err := storage.Check(ctx, key); err != nil {
return storage.ReadLink{}, err
}
if ttl < time.Second || ttl > time.Hour {
return storage.ReadLink{}, errors.New("storage/oss: TTL must be 1s..1h")
}
headers, err := options.ResponseHeaders()
if err != nil {
return storage.ReadLink{}, err
}
request := &sdk.GetObjectRequest{Bucket: sdk.Ptr(s.bucket), Key: sdk.Ptr(key), ResponseContentDisposition: sdk.Ptr(headers.Get("Content-Disposition")), ResponseCacheControl: sdk.Ptr(headers.Get("Cache-Control"))}
if contentType := headers.Get("Content-Type"); contentType != "" {
request.ResponseContentType = sdk.Ptr(contentType)
}
r, err := s.client.Presign(ctx, request, sdk.PresignExpires(ttl))
if err != nil {
return storage.ReadLink{}, err
}
if len(r.SignedHeaders) != 0 {
return storage.ReadLink{}, errors.New("storage/oss: direct GET requires a signature without additional request headers")
}
u, err := url.Parse(r.URL)
if err != nil {
return storage.ReadLink{}, errors.New("storage/oss: invalid signed URL")
}
q := u.Query()
var expires time.Time
if q.Get("x-oss-signature") != "" {
start, e := time.Parse("20060102T150405Z", q.Get("x-oss-date"))
seconds, e2 := strconv.ParseInt(q.Get("x-oss-expires"), 10, 64)
if e != nil || e2 != nil || seconds < 1 || seconds > 3600 {
return storage.ReadLink{}, errors.New("storage/oss: invalid signed expiry")
}
expires = start.Add(time.Duration(seconds) * time.Second)
} else {
seconds, e := strconv.ParseInt(q.Get("Expires"), 10, 64)
if e != nil || q.Get("Signature") == "" {
return storage.ReadLink{}, errors.New("storage/oss: missing signed expiry")
}
expires = time.Unix(seconds, 0).UTC()
}
return storage.ReadLink{URL: r.URL, ExpiresAt: expires}, nil
}
var _ storage.ReadPresigner = (*Store)(nil)

140
storage/oss/read_test.go Normal file
View File

@@ -0,0 +1,140 @@
package oss
import (
"context"
"crypto/hmac"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"mime"
"net/url"
"sort"
"strconv"
"strings"
"testing"
"time"
"git.yuntuo.hk/k2safe/go-omni/storage"
sdk "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/signer"
)
// Recompute signatures independently from the SDK using fixed test credentials.
// This catches response parameters appended after signing, not just their presence.
func validReadSignature(u *url.URL, version sdk.SignatureVersionType) bool {
q := u.Query()
if version == sdk.SignatureVersionV1 {
var keys []string
for key := range q {
if strings.HasPrefix(key, "response-") {
keys = append(keys, key)
}
}
sort.Strings(keys)
var params []string
for _, key := range keys {
params = append(params, key+"="+q.Get(key))
}
message := "GET\n\n\n" + q.Get("Expires") + "\n/testbucket/folder/file.pdf?" + strings.Join(params, "&")
mac := hmac.New(sha1.New, []byte("fixtureSecret"))
mac.Write([]byte(message))
return hmac.Equal([]byte(q.Get("Signature")), []byte(base64.StdEncoding.EncodeToString(mac.Sum(nil))))
}
signature := q.Get("x-oss-signature")
q.Del("x-oss-signature")
canonical := "GET\n/testbucket/folder/file.pdf\n" + strings.ReplaceAll(q.Encode(), "+", "%20") + "\n\n\nUNSIGNED-PAYLOAD"
hash := sha256.Sum256([]byte(canonical))
_, scope, _ := strings.Cut(q.Get("x-oss-credential"), "/")
message := "OSS4-HMAC-SHA256\n" + q.Get("x-oss-date") + "\n" + scope + "\n" + hex.EncodeToString(hash[:])
key := []byte("aliyun_v4fixtureSecret")
for _, part := range strings.Split(scope, "/") {
mac := hmac.New(sha256.New, key)
mac.Write([]byte(part))
key = mac.Sum(nil)
}
mac := hmac.New(sha256.New, key)
mac.Write([]byte(message))
return hmac.Equal([]byte(signature), []byte(hex.EncodeToString(mac.Sum(nil))))
}
func TestReadOptionsNativeSignatures(t *testing.T) {
for name, version := range map[string]sdk.SignatureVersionType{"v1": sdk.SignatureVersionV1, "v4": sdk.SignatureVersionV4} {
t.Run(name, func(t *testing.T) {
cfg := sdk.LoadDefaultConfig().WithRegion("cn-hangzhou").WithEndpoint("https://files.example").WithUseCName(true).WithSignatureVersion(version).WithCredentialsProvider(credentials.NewStaticCredentialsProvider("fixtureAccess", "fixtureSecret"))
store, err := New(sdk.NewClient(cfg), "testbucket", 64)
if err != nil {
t.Fatal(err)
}
for _, options := range []storage.ReadOptions{{}, {Disposition: storage.DispositionInline, FileName: "报告 2026.pdf", ContentType: "application/pdf"}, {Disposition: storage.DispositionAttachment, FileName: "download.pdf"}} {
before := time.Now().UTC().Truncate(time.Second)
link, err := store.PresignGetWithOptions(context.Background(), "folder/file.pdf", 30*time.Minute, options)
if err != nil {
t.Fatal(err)
}
u, err := url.Parse(link.URL)
if err != nil {
t.Fatal(err)
}
q := u.Query()
if u.Host != "files.example" || u.Path != "/folder/file.pdf" || !validReadSignature(u, version) {
t.Fatal("SDK response overrides were not signed")
}
disposition, params, err := mime.ParseMediaType(q.Get("response-content-disposition"))
want := options.Disposition
if want == "" {
want = "attachment"
}
if err != nil || disposition != want || params["filename"] != options.FileName || q.Get("response-content-type") != options.ContentType || q.Get("response-cache-control") != "private, no-store" {
t.Fatal("wrong signed headers", err)
}
if link.ExpiresAt.Before(before.Add(30*time.Minute)) || link.ExpiresAt.After(time.Now().Add(30*time.Minute)) {
t.Fatal("expiry drift")
}
var expires time.Time
if version == sdk.SignatureVersionV4 {
start, _ := time.Parse("20060102T150405Z", q.Get("x-oss-date"))
seconds, _ := strconv.Atoi(q.Get("x-oss-expires"))
expires = start.Add(time.Duration(seconds) * time.Second)
} else {
seconds, _ := strconv.ParseInt(q.Get("Expires"), 10, 64)
expires = time.Unix(seconds, 0)
}
if !expires.Equal(link.ExpiresAt) {
t.Fatal("result does not match signature expiry")
}
for _, name := range []string{"response-content-disposition", "response-content-type", "response-cache-control"} {
changed := *u
query := changed.Query()
query.Set(name, "changed")
changed.RawQuery = query.Encode()
if validReadSignature(&changed, version) {
t.Fatal("unsigned override", name)
}
}
}
for _, ttl := range []time.Duration{0, time.Second - time.Nanosecond, time.Hour + time.Nanosecond} {
if _, err := store.PresignGetWithOptions(context.Background(), "file", ttl, storage.ReadOptions{}); err == nil {
t.Fatal("TTL accepted")
}
}
for _, options := range []storage.ReadOptions{{FileName: "x\r\nY: z"}, {ContentType: "text/plain\r\nY: z"}, {Disposition: "bad"}} {
if _, err := store.PresignGetWithOptions(context.Background(), "file", time.Minute, options); err == nil {
t.Fatal("invalid header accepted")
}
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := store.PresignGetWithOptions(ctx, "file", time.Minute, storage.ReadOptions{}); err != context.Canceled {
t.Fatal(err)
}
})
}
// A caller-supplied unsigned SDK configuration cannot masquerade as a link.
client := sdk.NewClient(sdk.LoadDefaultConfig().WithRegion("cn-hangzhou").WithEndpoint("https://files.example").WithCredentialsProvider(credentials.NewStaticCredentialsProvider("fixtureAccess", "fixtureSecret")), func(o *sdk.Options) { o.Signer = &signer.NopSigner{} })
store, _ := New(client, "testbucket", 64)
if _, err := store.PresignGetWithOptions(context.Background(), "file", time.Minute, storage.ReadOptions{}); err == nil {
t.Fatal("unsigned URL accepted")
}
}

72
storage/read.go Normal file
View File

@@ -0,0 +1,72 @@
package storage
import (
"context"
"errors"
"mime"
"net/http"
"strings"
"time"
"unicode"
"unicode/utf8"
)
const (
DispositionAttachment = "attachment"
DispositionInline = "inline"
)
// ReadOptions controls the response to one signed GET. Empty Disposition means
// attachment; empty ContentType preserves object metadata. FileName is a display
// name, not an object key or filesystem path. All options are signed.
type ReadOptions struct {
Disposition string
FileName string
ContentType string
}
// ReadLink grants GET access without additional request headers. ExpiresAt is
// the expiry encoded in the signature, at second precision. Provider credentials
// may expire earlier. Do not log the URL. Issuing another link does not revoke it.
type ReadLink struct {
URL string `json:"url"`
ExpiresAt time.Time `json:"expiresAt"`
}
// ReadPresigner is optional; existing Store and Presigner implementations do not
// need to change. Cloud GET signatures do not also authorize HEAD requests.
type ReadPresigner interface {
PresignGetWithOptions(context.Context, string, time.Duration, ReadOptions) (ReadLink, error)
}
// ResponseHeaders validates options and returns a fresh set of response headers
// for provider adapters to include BEFORE signing, requesting private, no-store.
func (o ReadOptions) ResponseHeaders() (http.Header, error) {
disposition := o.Disposition
if disposition == "" {
disposition = DispositionAttachment
}
if disposition != DispositionAttachment && disposition != DispositionInline {
return nil, errors.New("storage: disposition must be inline or attachment")
}
if len(o.FileName) > 1024 || !utf8.ValidString(o.FileName) || strings.IndexFunc(o.FileName, unicode.IsControl) >= 0 || strings.ContainsAny(o.FileName, `/\`) || o.FileName == "." || o.FileName == ".." {
return nil, errors.New("storage: invalid display filename")
}
params := map[string]string{}
if o.FileName != "" {
params["filename"] = o.FileName
}
h := http.Header{"Content-Disposition": {mime.FormatMediaType(disposition, params)}, "Cache-Control": {"private, no-store"}}
if o.ContentType != "" {
if !utf8.ValidString(o.ContentType) || strings.IndexFunc(o.ContentType, unicode.IsControl) >= 0 {
return nil, ErrContentType
}
kind, err := MediaType(o.ContentType)
if err != nil {
return nil, err
}
_, params, _ := mime.ParseMediaType(o.ContentType)
h.Set("Content-Type", mime.FormatMediaType(kind, params))
}
return h, nil
}

219
storage/read_test.go Normal file
View File

@@ -0,0 +1,219 @@
package storage
import (
"context"
"io"
"mime"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
)
func TestReadOptions(t *testing.T) {
for _, name := range []string{"", "report.pdf", "报告 2026.pdf", `a";b.pdf`, "100%.pdf"} {
h, err := (ReadOptions{Disposition: DispositionInline, FileName: name, ContentType: "Application/PDF"}).ResponseHeaders()
if err != nil {
t.Fatal(err)
}
disposition, params, err := mime.ParseMediaType(h.Get("Content-Disposition"))
if err != nil || disposition != "inline" || params["filename"] != name || h.Get("Content-Type") != "application/pdf" || h.Get("Cache-Control") != "private, no-store" {
t.Fatal("invalid read headers", h, err)
}
}
h, err := (ReadOptions{}).ResponseHeaders()
if err != nil || h.Get("Content-Disposition") != "attachment" || h.Get("Content-Type") != "" {
t.Fatal(h, err)
}
for _, options := range []ReadOptions{
{Disposition: "form-data"}, {Disposition: "Inline"}, {Disposition: "inline\r\nX: y"},
{FileName: "x\r\nX: y"}, {FileName: "a\x00b"}, {FileName: "a\tb"}, {FileName: "bad\xff"}, {FileName: "a/b"}, {FileName: `a\b`}, {FileName: ".."}, {FileName: strings.Repeat("a", 1025)},
{ContentType: "bad"}, {ContentType: "image/*"}, {ContentType: "application/pdf\r\nX: y"}, {ContentType: "application/pdf\x00"}, {ContentType: strings.Repeat("x", 257)},
} {
if _, err := options.ResponseHeaders(); err == nil {
t.Fatal("invalid options accepted", options)
}
}
}
type readTestStore struct {
calls, bytes int
streaming bool
}
func (*readTestStore) Put(context.Context, string, io.Reader, int64, string) (Info, error) {
panic("unused")
}
func (*readTestStore) Delete(context.Context, string) error { panic("unused") }
func (s *readTestStore) Get(context.Context, string) (Object, error) {
s.calls++
body := &readTestBody{Reader: strings.NewReader("0123456789"), store: s}
var stream io.ReadCloser = body
if s.streaming {
stream = struct {
io.Reader
io.Closer
}{body, body}
}
return Object{Info: Info{Size: 10, ContentType: "application/pdf", ModifiedAt: time.Unix(1000, 0)}, Body: stream}, nil
}
type readTestBody struct {
*strings.Reader
store *readTestStore
}
func (b *readTestBody) Read(p []byte) (int, error) {
n, err := b.Reader.Read(p)
b.store.bytes += n
return n, err
}
func (*readTestBody) Close() error { return nil }
func TestReadSignatureAndExpiry(t *testing.T) {
store := &readTestStore{}
s, _ := NewSigner(store, "https://files.example/read", []byte(strings.Repeat("s", 32)))
now := time.Unix(1800000000, 0).UTC()
s.now = func() time.Time { return now }
link, err := s.PresignGetWithOptions(context.Background(), "folder/file", 30*time.Minute, ReadOptions{Disposition: DispositionInline, FileName: "报告.pdf", ContentType: "application/pdf"})
if err != nil || !link.ExpiresAt.Equal(now.Add(30*time.Minute)) {
t.Fatal("incorrect expiry", err)
}
serve := func(method, address string, headers http.Header) *httptest.ResponseRecorder {
r := httptest.NewRequest(method, address, nil)
if headers != nil {
r.Header = headers
}
w := httptest.NewRecorder()
s.ServeHTTP(w, r)
return w
}
for _, method := range []string{"GET", "HEAD"} {
w := serve(method, link.URL, nil)
if w.Code != 200 || w.Header().Get("Content-Type") != "application/pdf" || w.Header().Get("Cache-Control") != "private, no-store" {
t.Fatal(w)
}
if method == "HEAD" && w.Body.Len() != 0 {
t.Fatal("HEAD body")
}
}
old, err := s.PresignGet(context.Background(), "folder/file", 30*time.Minute)
if err != nil {
t.Fatal(err)
}
if w := serve("GET", old.String(), nil); w.Code != 200 || w.Header().Get("Content-Disposition") != "attachment" {
t.Fatal("legacy download changed")
}
mutations := []func(url.Values){
func(q url.Values) { q.Set("key", "other") }, func(q url.Values) { q.Set("expires", "1800001900") }, func(q url.Values) { q.Set("signature", "bad") },
func(q url.Values) { q.Set("filename", "other.pdf") }, func(q url.Values) { q.Set("disposition", "attachment") }, func(q url.Values) { q.Set("contentType", "text/html") },
func(q url.Values) { q.Set("v", "3") }, func(q url.Values) { q.Del("v") }, func(q url.Values) {
for _, k := range []string{"v", "filename", "disposition", "contentType"} {
q.Del(k)
}
},
func(q url.Values) { q.Set("extra", "x") }, func(q url.Values) { q.Set("filename", "evil\r\nX: yes") },
}
for _, key := range []string{"key", "expires", "signature", "v", "disposition", "filename", "contentType"} {
key := key
mutations = append(mutations, func(q url.Values) { q.Add(key, q.Get(key)) })
}
for _, mutate := range mutations {
u, _ := url.Parse(link.URL)
q := u.Query()
mutate(q)
u.RawQuery = q.Encode()
before := store.calls
if w := serve("GET", u.String(), http.Header{"Range": {"bytes=0-1"}}); w.Code != 403 || store.calls != before {
t.Fatal("tampered link reached storage")
}
}
u := *old
q := u.Query()
q.Set("disposition", "inline")
u.RawQuery = q.Encode()
if serve("GET", u.String(), nil).Code != 403 {
t.Fatal("legacy download upgraded by query")
}
if serve("GET", link.URL+"&bad=%ZZ", nil).Code != 403 {
t.Fatal("malformed query accepted")
}
before := store.calls
now = link.ExpiresAt.Add(-time.Nanosecond)
if serve("GET", link.URL, nil).Code != 200 {
t.Fatal("early expiration")
}
before = store.calls
now = link.ExpiresAt
for _, method := range []string{"GET", "HEAD"} {
w := serve(method, link.URL, http.Header{"Range": {"bytes=0-1"}, "If-Modified-Since": {"Fri, 01 Jan 2100 00:00:00 GMT"}})
if w.Code != 403 || store.calls != before || w.Header().Get("Cache-Control") != "private, no-store" {
t.Fatal("expired link reached storage or cache")
}
if method == "HEAD" && w.Body.Len() != 0 {
t.Fatal("HEAD error has body")
}
}
for _, ttl := range []time.Duration{0, time.Second - time.Nanosecond, time.Hour + time.Nanosecond} {
if _, err := s.PresignGetWithOptions(context.Background(), "key", ttl, ReadOptions{}); err == nil {
t.Fatal("invalid TTL")
}
}
for _, ttl := range []time.Duration{time.Second, time.Hour} {
if _, err := s.PresignGetWithOptions(context.Background(), "key", ttl, ReadOptions{}); err != nil {
t.Fatal(err)
}
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := s.PresignGetWithOptions(ctx, "key", time.Minute, ReadOptions{}); err != context.Canceled {
t.Fatal(err)
}
}
func TestReadRangeStreamingAndCache(t *testing.T) {
for _, streaming := range []bool{false, true} {
store := &readTestStore{streaming: streaming}
s, _ := NewSigner(store, "https://files.example/read", []byte(strings.Repeat("s", 32)))
link, _ := s.PresignGetWithOptions(context.Background(), "key", 30*time.Minute, ReadOptions{})
for _, method := range []string{"GET", "HEAD"} {
before := store.bytes
r := httptest.NewRequest(method, link.URL, nil)
r.Header.Set("Range", "bytes=2-4")
w := httptest.NewRecorder()
s.ServeHTTP(w, r)
if method == "HEAD" {
if w.Code != 200 || w.Body.Len() != 0 || store.bytes != before {
t.Fatal("HEAD read body", w)
}
} else if streaming {
if w.Code != 200 || w.Body.String() != "0123456789" || w.Header().Get("Accept-Ranges") != "" {
t.Fatal("stream fallback", w)
}
} else {
if w.Code != 206 || w.Body.String() != "234" || store.bytes-before != 3 {
t.Fatal("range buffered full body", w)
}
}
if w.Header().Get("Cache-Control") != "private, no-store" {
t.Fatal("missing cache policy")
}
}
if !streaming {
for _, h := range []http.Header{{"Range": {"bytes=99-100"}}, {"If-Modified-Since": {"Fri, 01 Jan 2100 00:00:00 GMT"}}} {
r := httptest.NewRequest("GET", link.URL, nil)
r.Header = h
w := httptest.NewRecorder()
s.ServeHTTP(w, r)
if w.Code != 416 && w.Code != 304 {
t.Fatal(w.Code)
}
if w.Header().Get("Cache-Control") != "private, no-store" {
t.Fatal("cache policy lost on error/conditional")
}
}
}
}
}

53
storage/s3/read.go Normal file
View File

@@ -0,0 +1,53 @@
package s3
import (
"context"
"errors"
"net/url"
"strconv"
"time"
"git.yuntuo.hk/k2safe/go-omni/storage"
)
func (s *Store) PresignGetWithOptions(ctx context.Context, key string, ttl time.Duration, options storage.ReadOptions) (storage.ReadLink, error) {
if err := storage.Check(ctx, key); err != nil {
return storage.ReadLink{}, err
}
if ttl < time.Second || ttl > time.Hour {
return storage.ReadLink{}, errors.New("storage/s3: TTL must be 1s..1h")
}
headers, err := options.ResponseHeaders()
if err != nil {
return storage.ReadLink{}, err
}
query := url.Values{"response-content-disposition": {headers.Get("Content-Disposition")}, "response-cache-control": {headers.Get("Cache-Control")}}
if contentType := headers.Get("Content-Type"); contentType != "" {
query.Set("response-content-type", contentType)
}
u, err := s.client.PresignedGetObject(ctx, s.bucket, key, ttl, query)
if err != nil {
return storage.ReadLink{}, err
}
// Derive the exact expiry from the SDK's signed URL, not a second clock read
// around credential discovery or bucket-region lookup.
q := u.Query()
var expires time.Time
if q.Get("X-Amz-Signature") != "" {
start, e := time.Parse("20060102T150405Z", q.Get("X-Amz-Date"))
seconds, e2 := strconv.ParseInt(q.Get("X-Amz-Expires"), 10, 64)
if e != nil || e2 != nil || seconds < 1 || seconds > 3600 {
return storage.ReadLink{}, errors.New("storage/s3: invalid signed expiry")
}
expires = start.Add(time.Duration(seconds) * time.Second)
} else {
seconds, e := strconv.ParseInt(q.Get("Expires"), 10, 64)
if e != nil || q.Get("Signature") == "" {
return storage.ReadLink{}, errors.New("storage/s3: missing signed expiry")
}
expires = time.Unix(seconds, 0).UTC()
}
return storage.ReadLink{URL: u.String(), ExpiresAt: expires}, nil
}
var _ storage.ReadPresigner = (*Store)(nil)

145
storage/s3/read_test.go Normal file
View File

@@ -0,0 +1,145 @@
package s3
import (
"context"
"io"
"mime"
"net/http"
"net/url"
"strconv"
"strings"
"testing"
"time"
"git.yuntuo.hk/k2safe/go-omni/storage"
"github.com/minio/minio-go/v7/pkg/credentials"
)
func TestReadOptionsS3AndR2(t *testing.T) {
for _, r2 := range []bool{false, true} {
cfg := Config{Endpoint: "files.example", Bucket: "testbucket", Region: "us-east-1", Secure: true, Credentials: credentials.NewStaticV4("fixtureAccess", "fixtureSecret", "")}
if r2 {
var err error
cfg, err = R2Config(strings.Repeat("a", 32), "testbucket", cfg.Credentials)
if err != nil {
t.Fatal(err)
}
}
store, err := Open(cfg)
if err != nil {
t.Fatal(err)
}
for _, options := range []storage.ReadOptions{{}, {Disposition: storage.DispositionInline, FileName: "报告.pdf", ContentType: "application/pdf"}} {
link, err := store.PresignGetWithOptions(context.Background(), "folder/file.pdf", 30*time.Minute, options)
if err != nil {
t.Fatal(err)
}
u, _ := url.Parse(link.URL)
q := u.Query()
kind, params, err := mime.ParseMediaType(q.Get("response-content-disposition"))
want := options.Disposition
if want == "" {
want = "attachment"
}
if err != nil || kind != want || params["filename"] != options.FileName || q.Get("response-content-type") != options.ContentType || q.Get("response-cache-control") != "private, no-store" || q.Get("X-Amz-Signature") == "" {
t.Fatal("missing response parameters", err)
}
if u.Host != cfg.Endpoint || q.Get("X-Amz-Expires") != "1800" {
t.Fatal("wrong endpoint/TTL")
}
start, err := time.Parse("20060102T150405Z", q.Get("X-Amz-Date"))
if err != nil || !link.ExpiresAt.Equal(start.Add(30*time.Minute)) {
t.Fatal("expiry drift", err)
}
}
for _, ttl := range []time.Duration{0, time.Hour + time.Nanosecond} {
if _, err := store.PresignGetWithOptions(context.Background(), "file", ttl, storage.ReadOptions{}); err == nil {
t.Fatal("invalid TTL")
}
}
if _, err := store.PresignGetWithOptions(context.Background(), "file", time.Minute, storage.ReadOptions{FileName: "x\r\nY: z"}); err == nil {
t.Fatal("invalid filename")
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := store.PresignGetWithOptions(ctx, "file", time.Minute, storage.ReadOptions{}); err != context.Canceled {
t.Fatal(err)
}
}
}
// Called inside the existing isolated MinIO integration. The actual provider
// verifies signatures and rejects tampered response parameters.
func checkReadOptionsIntegration(t *testing.T, store *Store) {
t.Helper()
ctx := context.Background()
data := "%PDF-1.4\nread-test\n%%EOF"
_, err := store.Put(ctx, "preview.pdf", strings.NewReader(data), int64(len(data)), "application/pdf")
if err != nil {
t.Fatal(err)
}
defer store.Delete(ctx, "preview.pdf")
for _, kind := range []string{storage.DispositionInline, storage.DispositionAttachment} {
link, err := store.PresignGetWithOptions(ctx, "preview.pdf", 30*time.Minute, storage.ReadOptions{Disposition: kind, FileName: "示例.pdf", ContentType: "application/pdf"})
if err != nil {
t.Fatal(err)
}
client := &http.Client{Timeout: 5 * time.Second}
for _, span := range []string{"", "bytes=0-7", "bytes=999-"} {
req, _ := http.NewRequest("GET", link.URL, nil)
if span != "" {
req.Header.Set("Range", span)
}
r, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
body, err := io.ReadAll(r.Body)
r.Body.Close()
if err != nil {
t.Fatal(err)
}
want := 200
if span == "bytes=0-7" {
want = 206
}
if span == "bytes=999-" {
want = 416
}
if r.StatusCode != want {
t.Fatal("status", r.StatusCode)
}
if want == 416 {
continue
}
disposition, params, err := mime.ParseMediaType(r.Header.Get("Content-Disposition"))
if err != nil || disposition != kind || params["filename"] != "示例.pdf" || r.Header.Get("Content-Type") != "application/pdf" || r.Header.Get("Cache-Control") != "private, no-store" {
t.Fatal("provider headers", r.Header, err)
}
value := data
if want == 206 {
value = data[:8]
}
if string(body) != value || r.Header.Get("Content-Length") != strconv.Itoa(len(value)) {
t.Fatal("provider body")
}
}
for _, name := range []string{"response-content-disposition", "response-content-type", "response-cache-control", "X-Amz-Expires"} {
u, _ := url.Parse(link.URL)
q := u.Query()
q.Set(name, "changed")
if name == "X-Amz-Expires" {
q.Set(name, "1801")
}
u.RawQuery = q.Encode()
r, err := client.Get(u.String())
if err != nil {
t.Fatal(err)
}
r.Body.Close()
if r.StatusCode != 403 {
t.Fatal("tampered parameter", name, r.StatusCode)
}
}
}
}

View File

@@ -84,7 +84,9 @@ func TestS3Integration(t *testing.T) {
if !ready {
t.Fatal("isolated S3 server did not start")
}
client, err := minio.New(address, &minio.Options{Creds: credentials.NewStaticV4("goOmniTest", "goOmniTestSecret", ""), Region: "us-east-1"})
// The contract deliberately sends malformed bodies. Avoid retry backoff
// exhausting its deadline; these are expected failures, not network retries.
client, err := minio.New(address, &minio.Options{Creds: credentials.NewStaticV4("goOmniTest", "goOmniTestSecret", ""), Region: "us-east-1", MaxRetries: 1})
if err != nil {
t.Fatal(err)
}
@@ -114,6 +116,7 @@ func TestS3Integration(t *testing.T) {
}
}
s, _ := New(client, bucket, 64)
checkReadOptionsIntegration(t, s)
storagetest.Run(t, s)
s.Put(ctx, "signed", strings.NewReader("hello"), 5, "text/plain")
defer s.Delete(context.Background(), "signed")

View File

@@ -49,30 +49,88 @@ func (s *Signer) PresignGet(ctx context.Context, key string, ttl time.Duration)
return &u, nil
}
// ServeHTTP serves signed downloads as attachments. Issue links only after
func (s *Signer) readSignature(q url.Values) string {
values := make(url.Values, len(q))
for key, value := range q {
if key != "signature" {
values[key] = value
}
}
mac := hmac.New(sha256.New, s.secret)
io.WriteString(mac, "GET-OPTIONS-v2\n"+values.Encode())
return base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
}
func (s *Signer) PresignGetWithOptions(ctx context.Context, key string, ttl time.Duration, options ReadOptions) (ReadLink, error) {
if err := Check(ctx, key); err != nil {
return ReadLink{}, err
}
if ttl < time.Second || ttl > time.Hour {
return ReadLink{}, errors.New("storage: TTL must be 1s..1h")
}
headers, err := options.ResponseHeaders()
if err != nil {
return ReadLink{}, err
}
if options.Disposition == "" {
options.Disposition = DispositionAttachment
}
expires := s.now().Add(ttl).UTC().Truncate(time.Second)
q := url.Values{"v": {"2"}, "key": {key}, "expires": {strconv.FormatInt(expires.Unix(), 10)}, "disposition": {options.Disposition}, "filename": {options.FileName}, "contentType": {headers.Get("Content-Type")}}
q.Set("signature", s.readSignature(q))
u := *s.base
u.RawQuery = q.Encode()
return ReadLink{URL: u.String(), ExpiresAt: expires}, nil
}
// ServeHTTP serves signed GET/HEAD requests, with optional preview headers.
// Seekable object bodies support Range and conditional reads; other stores keep
// streaming full responses. No full-object buffer is used. Issue links only after
// authorization; possession of an unexpired URL grants access to its object.
func (s *Signer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPut && s.uploadPolicy != nil {
s.serveUpload(w, r)
return
}
w = &readResponseWriter{ResponseWriter: w, head: r.Method == http.MethodHead}
if r.Method != "GET" && r.Method != "HEAD" {
w.Header().Set("Allow", "GET, HEAD")
http.Error(w, "method not allowed", 405)
return
}
q, err := url.ParseQuery(r.URL.RawQuery)
if err != nil || len(q) != 3 || len(q["key"]) != 1 || len(q["expires"]) != 1 || len(q["signature"]) != 1 {
keys := []string{"key", "expires", "signature"}
if q.Get("v") == "2" {
keys = append(keys, "v", "disposition", "filename", "contentType")
}
valid := err == nil && len(q) == len(keys)
for _, key := range keys {
valid = valid && len(q[key]) == 1
}
if !valid {
http.Error(w, "invalid link", 403)
return
}
key, expiry := q.Get("key"), q.Get("expires")
expires, err := strconv.ParseInt(expiry, 10, 64)
now := s.now().Unix()
if err != nil || expires <= now || expires > now+3600 || Check(r.Context(), key) != nil || !hmac.Equal([]byte(q.Get("signature")), []byte(s.signature(key, expiry))) {
expected := s.signature(key, expiry)
if q.Get("v") == "2" {
expected = s.readSignature(q)
}
if err != nil || expires <= now || expires > now+3600 || Check(r.Context(), key) != nil || !hmac.Equal([]byte(q.Get("signature")), []byte(expected)) {
http.Error(w, "invalid or expired link", 403)
return
}
options := ReadOptions{}
if q.Get("v") == "2" {
options = ReadOptions{Disposition: q.Get("disposition"), FileName: q.Get("filename"), ContentType: q.Get("contentType")}
}
headers, err := options.ResponseHeaders()
if err != nil {
http.Error(w, "invalid link", 403)
return
}
object, err := s.store.Get(r.Context(), key)
if err != nil {
status := 500
@@ -82,17 +140,66 @@ func (s *Signer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
http.Error(w, "object unavailable", status)
return
}
if object.Body == nil {
http.Error(w, "object unavailable", 500)
return
}
defer object.Body.Close()
if object.Size < 0 {
http.Error(w, "object unavailable", 500)
return
}
contentType := object.ContentType
if options.ContentType != "" {
contentType = headers.Get("Content-Type")
}
if contentType == "" {
contentType = "application/octet-stream"
}
w.Header().Set("Content-Type", contentType)
w.Header().Set("Content-Disposition", "attachment")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("Cache-Control", "private, no-store")
w.Header().Set("Content-Disposition", headers.Get("Content-Disposition"))
if body, ok := object.Body.(io.ReadSeeker); ok {
// Range describes a GET representation. HEAD reports full-object headers.
if r.Method == http.MethodHead {
r = r.Clone(r.Context())
r.Header.Del("Range")
}
http.ServeContent(w, r, "", object.ModifiedAt, body)
return
}
w.Header().Set("Content-Length", strconv.FormatInt(object.Size, 10))
w.WriteHeader(http.StatusOK)
if r.Method == "GET" {
_, _ = io.Copy(w, object.Body)
}
}
// ServeContent may strip cache headers on errors in newer Go versions. Restore
// the signed-link policy at the actual write, including 304/416 and HEAD errors.
type readResponseWriter struct {
http.ResponseWriter
head, wrote bool
}
func (w *readResponseWriter) WriteHeader(status int) {
if w.wrote {
return
}
w.wrote = true
w.Header().Set("Cache-Control", "private, no-store")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.ResponseWriter.WriteHeader(status)
}
func (w *readResponseWriter) Write(p []byte) (int, error) {
if !w.wrote {
w.WriteHeader(http.StatusOK)
}
if w.head {
return len(p), nil
}
return w.ResponseWriter.Write(p)
}
func (w *readResponseWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter }
var _ Presigner = (*Signer)(nil)
var _ ReadPresigner = (*Signer)(nil)