feat(storage): support signed read options and local byte ranges
This commit is contained in:
53
storage/s3/read.go
Normal file
53
storage/s3/read.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage"
|
||||
)
|
||||
|
||||
func (s *Store) PresignGetWithOptions(ctx context.Context, key string, ttl time.Duration, options storage.ReadOptions) (storage.ReadLink, error) {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return storage.ReadLink{}, err
|
||||
}
|
||||
if ttl < time.Second || ttl > time.Hour {
|
||||
return storage.ReadLink{}, errors.New("storage/s3: TTL must be 1s..1h")
|
||||
}
|
||||
headers, err := options.ResponseHeaders()
|
||||
if err != nil {
|
||||
return storage.ReadLink{}, err
|
||||
}
|
||||
query := url.Values{"response-content-disposition": {headers.Get("Content-Disposition")}, "response-cache-control": {headers.Get("Cache-Control")}}
|
||||
if contentType := headers.Get("Content-Type"); contentType != "" {
|
||||
query.Set("response-content-type", contentType)
|
||||
}
|
||||
u, err := s.client.PresignedGetObject(ctx, s.bucket, key, ttl, query)
|
||||
if err != nil {
|
||||
return storage.ReadLink{}, err
|
||||
}
|
||||
// Derive the exact expiry from the SDK's signed URL, not a second clock read
|
||||
// around credential discovery or bucket-region lookup.
|
||||
q := u.Query()
|
||||
var expires time.Time
|
||||
if q.Get("X-Amz-Signature") != "" {
|
||||
start, e := time.Parse("20060102T150405Z", q.Get("X-Amz-Date"))
|
||||
seconds, e2 := strconv.ParseInt(q.Get("X-Amz-Expires"), 10, 64)
|
||||
if e != nil || e2 != nil || seconds < 1 || seconds > 3600 {
|
||||
return storage.ReadLink{}, errors.New("storage/s3: invalid signed expiry")
|
||||
}
|
||||
expires = start.Add(time.Duration(seconds) * time.Second)
|
||||
} else {
|
||||
seconds, e := strconv.ParseInt(q.Get("Expires"), 10, 64)
|
||||
if e != nil || q.Get("Signature") == "" {
|
||||
return storage.ReadLink{}, errors.New("storage/s3: missing signed expiry")
|
||||
}
|
||||
expires = time.Unix(seconds, 0).UTC()
|
||||
}
|
||||
return storage.ReadLink{URL: u.String(), ExpiresAt: expires}, nil
|
||||
}
|
||||
|
||||
var _ storage.ReadPresigner = (*Store)(nil)
|
||||
145
storage/s3/read_test.go
Normal file
145
storage/s3/read_test.go
Normal file
@@ -0,0 +1,145 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
func TestReadOptionsS3AndR2(t *testing.T) {
|
||||
for _, r2 := range []bool{false, true} {
|
||||
cfg := Config{Endpoint: "files.example", Bucket: "testbucket", Region: "us-east-1", Secure: true, Credentials: credentials.NewStaticV4("fixtureAccess", "fixtureSecret", "")}
|
||||
if r2 {
|
||||
var err error
|
||||
cfg, err = R2Config(strings.Repeat("a", 32), "testbucket", cfg.Credentials)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
store, err := Open(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, options := range []storage.ReadOptions{{}, {Disposition: storage.DispositionInline, FileName: "报告.pdf", ContentType: "application/pdf"}} {
|
||||
link, err := store.PresignGetWithOptions(context.Background(), "folder/file.pdf", 30*time.Minute, options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u, _ := url.Parse(link.URL)
|
||||
q := u.Query()
|
||||
kind, params, err := mime.ParseMediaType(q.Get("response-content-disposition"))
|
||||
want := options.Disposition
|
||||
if want == "" {
|
||||
want = "attachment"
|
||||
}
|
||||
if err != nil || kind != want || params["filename"] != options.FileName || q.Get("response-content-type") != options.ContentType || q.Get("response-cache-control") != "private, no-store" || q.Get("X-Amz-Signature") == "" {
|
||||
t.Fatal("missing response parameters", err)
|
||||
}
|
||||
if u.Host != cfg.Endpoint || q.Get("X-Amz-Expires") != "1800" {
|
||||
t.Fatal("wrong endpoint/TTL")
|
||||
}
|
||||
start, err := time.Parse("20060102T150405Z", q.Get("X-Amz-Date"))
|
||||
if err != nil || !link.ExpiresAt.Equal(start.Add(30*time.Minute)) {
|
||||
t.Fatal("expiry drift", err)
|
||||
}
|
||||
}
|
||||
for _, ttl := range []time.Duration{0, time.Hour + time.Nanosecond} {
|
||||
if _, err := store.PresignGetWithOptions(context.Background(), "file", ttl, storage.ReadOptions{}); err == nil {
|
||||
t.Fatal("invalid TTL")
|
||||
}
|
||||
}
|
||||
if _, err := store.PresignGetWithOptions(context.Background(), "file", time.Minute, storage.ReadOptions{FileName: "x\r\nY: z"}); err == nil {
|
||||
t.Fatal("invalid filename")
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := store.PresignGetWithOptions(ctx, "file", time.Minute, storage.ReadOptions{}); err != context.Canceled {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Called inside the existing isolated MinIO integration. The actual provider
|
||||
// verifies signatures and rejects tampered response parameters.
|
||||
func checkReadOptionsIntegration(t *testing.T, store *Store) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
data := "%PDF-1.4\nread-test\n%%EOF"
|
||||
_, err := store.Put(ctx, "preview.pdf", strings.NewReader(data), int64(len(data)), "application/pdf")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Delete(ctx, "preview.pdf")
|
||||
for _, kind := range []string{storage.DispositionInline, storage.DispositionAttachment} {
|
||||
link, err := store.PresignGetWithOptions(ctx, "preview.pdf", 30*time.Minute, storage.ReadOptions{Disposition: kind, FileName: "示例.pdf", ContentType: "application/pdf"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
for _, span := range []string{"", "bytes=0-7", "bytes=999-"} {
|
||||
req, _ := http.NewRequest("GET", link.URL, nil)
|
||||
if span != "" {
|
||||
req.Header.Set("Range", span)
|
||||
}
|
||||
r, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := io.ReadAll(r.Body)
|
||||
r.Body.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := 200
|
||||
if span == "bytes=0-7" {
|
||||
want = 206
|
||||
}
|
||||
if span == "bytes=999-" {
|
||||
want = 416
|
||||
}
|
||||
if r.StatusCode != want {
|
||||
t.Fatal("status", r.StatusCode)
|
||||
}
|
||||
if want == 416 {
|
||||
continue
|
||||
}
|
||||
disposition, params, err := mime.ParseMediaType(r.Header.Get("Content-Disposition"))
|
||||
if err != nil || disposition != kind || params["filename"] != "示例.pdf" || r.Header.Get("Content-Type") != "application/pdf" || r.Header.Get("Cache-Control") != "private, no-store" {
|
||||
t.Fatal("provider headers", r.Header, err)
|
||||
}
|
||||
value := data
|
||||
if want == 206 {
|
||||
value = data[:8]
|
||||
}
|
||||
if string(body) != value || r.Header.Get("Content-Length") != strconv.Itoa(len(value)) {
|
||||
t.Fatal("provider body")
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"response-content-disposition", "response-content-type", "response-cache-control", "X-Amz-Expires"} {
|
||||
u, _ := url.Parse(link.URL)
|
||||
q := u.Query()
|
||||
q.Set(name, "changed")
|
||||
if name == "X-Amz-Expires" {
|
||||
q.Set(name, "1801")
|
||||
}
|
||||
u.RawQuery = q.Encode()
|
||||
r, err := client.Get(u.String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.Body.Close()
|
||||
if r.StatusCode != 403 {
|
||||
t.Fatal("tampered parameter", name, r.StatusCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -84,7 +84,9 @@ func TestS3Integration(t *testing.T) {
|
||||
if !ready {
|
||||
t.Fatal("isolated S3 server did not start")
|
||||
}
|
||||
client, err := minio.New(address, &minio.Options{Creds: credentials.NewStaticV4("goOmniTest", "goOmniTestSecret", ""), Region: "us-east-1"})
|
||||
// The contract deliberately sends malformed bodies. Avoid retry backoff
|
||||
// exhausting its deadline; these are expected failures, not network retries.
|
||||
client, err := minio.New(address, &minio.Options{Creds: credentials.NewStaticV4("goOmniTest", "goOmniTestSecret", ""), Region: "us-east-1", MaxRetries: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -114,6 +116,7 @@ func TestS3Integration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
s, _ := New(client, bucket, 64)
|
||||
checkReadOptionsIntegration(t, s)
|
||||
storagetest.Run(t, s)
|
||||
s.Put(ctx, "signed", strings.NewReader("hello"), 5, "text/plain")
|
||||
defer s.Delete(context.Background(), "signed")
|
||||
|
||||
Reference in New Issue
Block a user