feat(storage): support signed read options and local byte ranges
This commit is contained in:
59
storage/oss/read.go
Normal file
59
storage/oss/read.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage"
|
||||
sdk "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
|
||||
)
|
||||
|
||||
func (s *Store) PresignGetWithOptions(ctx context.Context, key string, ttl time.Duration, options storage.ReadOptions) (storage.ReadLink, error) {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return storage.ReadLink{}, err
|
||||
}
|
||||
if ttl < time.Second || ttl > time.Hour {
|
||||
return storage.ReadLink{}, errors.New("storage/oss: TTL must be 1s..1h")
|
||||
}
|
||||
headers, err := options.ResponseHeaders()
|
||||
if err != nil {
|
||||
return storage.ReadLink{}, err
|
||||
}
|
||||
request := &sdk.GetObjectRequest{Bucket: sdk.Ptr(s.bucket), Key: sdk.Ptr(key), ResponseContentDisposition: sdk.Ptr(headers.Get("Content-Disposition")), ResponseCacheControl: sdk.Ptr(headers.Get("Cache-Control"))}
|
||||
if contentType := headers.Get("Content-Type"); contentType != "" {
|
||||
request.ResponseContentType = sdk.Ptr(contentType)
|
||||
}
|
||||
r, err := s.client.Presign(ctx, request, sdk.PresignExpires(ttl))
|
||||
if err != nil {
|
||||
return storage.ReadLink{}, err
|
||||
}
|
||||
if len(r.SignedHeaders) != 0 {
|
||||
return storage.ReadLink{}, errors.New("storage/oss: direct GET requires a signature without additional request headers")
|
||||
}
|
||||
u, err := url.Parse(r.URL)
|
||||
if err != nil {
|
||||
return storage.ReadLink{}, errors.New("storage/oss: invalid signed URL")
|
||||
}
|
||||
q := u.Query()
|
||||
var expires time.Time
|
||||
if q.Get("x-oss-signature") != "" {
|
||||
start, e := time.Parse("20060102T150405Z", q.Get("x-oss-date"))
|
||||
seconds, e2 := strconv.ParseInt(q.Get("x-oss-expires"), 10, 64)
|
||||
if e != nil || e2 != nil || seconds < 1 || seconds > 3600 {
|
||||
return storage.ReadLink{}, errors.New("storage/oss: invalid signed expiry")
|
||||
}
|
||||
expires = start.Add(time.Duration(seconds) * time.Second)
|
||||
} else {
|
||||
seconds, e := strconv.ParseInt(q.Get("Expires"), 10, 64)
|
||||
if e != nil || q.Get("Signature") == "" {
|
||||
return storage.ReadLink{}, errors.New("storage/oss: missing signed expiry")
|
||||
}
|
||||
expires = time.Unix(seconds, 0).UTC()
|
||||
}
|
||||
return storage.ReadLink{URL: r.URL, ExpiresAt: expires}, nil
|
||||
}
|
||||
|
||||
var _ storage.ReadPresigner = (*Store)(nil)
|
||||
140
storage/oss/read_test.go
Normal file
140
storage/oss/read_test.go
Normal file
@@ -0,0 +1,140 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"mime"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage"
|
||||
sdk "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
|
||||
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
|
||||
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/signer"
|
||||
)
|
||||
|
||||
// Recompute signatures independently from the SDK using fixed test credentials.
|
||||
// This catches response parameters appended after signing, not just their presence.
|
||||
func validReadSignature(u *url.URL, version sdk.SignatureVersionType) bool {
|
||||
q := u.Query()
|
||||
if version == sdk.SignatureVersionV1 {
|
||||
var keys []string
|
||||
for key := range q {
|
||||
if strings.HasPrefix(key, "response-") {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var params []string
|
||||
for _, key := range keys {
|
||||
params = append(params, key+"="+q.Get(key))
|
||||
}
|
||||
message := "GET\n\n\n" + q.Get("Expires") + "\n/testbucket/folder/file.pdf?" + strings.Join(params, "&")
|
||||
mac := hmac.New(sha1.New, []byte("fixtureSecret"))
|
||||
mac.Write([]byte(message))
|
||||
return hmac.Equal([]byte(q.Get("Signature")), []byte(base64.StdEncoding.EncodeToString(mac.Sum(nil))))
|
||||
}
|
||||
signature := q.Get("x-oss-signature")
|
||||
q.Del("x-oss-signature")
|
||||
canonical := "GET\n/testbucket/folder/file.pdf\n" + strings.ReplaceAll(q.Encode(), "+", "%20") + "\n\n\nUNSIGNED-PAYLOAD"
|
||||
hash := sha256.Sum256([]byte(canonical))
|
||||
_, scope, _ := strings.Cut(q.Get("x-oss-credential"), "/")
|
||||
message := "OSS4-HMAC-SHA256\n" + q.Get("x-oss-date") + "\n" + scope + "\n" + hex.EncodeToString(hash[:])
|
||||
key := []byte("aliyun_v4fixtureSecret")
|
||||
for _, part := range strings.Split(scope, "/") {
|
||||
mac := hmac.New(sha256.New, key)
|
||||
mac.Write([]byte(part))
|
||||
key = mac.Sum(nil)
|
||||
}
|
||||
mac := hmac.New(sha256.New, key)
|
||||
mac.Write([]byte(message))
|
||||
return hmac.Equal([]byte(signature), []byte(hex.EncodeToString(mac.Sum(nil))))
|
||||
}
|
||||
|
||||
func TestReadOptionsNativeSignatures(t *testing.T) {
|
||||
for name, version := range map[string]sdk.SignatureVersionType{"v1": sdk.SignatureVersionV1, "v4": sdk.SignatureVersionV4} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cfg := sdk.LoadDefaultConfig().WithRegion("cn-hangzhou").WithEndpoint("https://files.example").WithUseCName(true).WithSignatureVersion(version).WithCredentialsProvider(credentials.NewStaticCredentialsProvider("fixtureAccess", "fixtureSecret"))
|
||||
store, err := New(sdk.NewClient(cfg), "testbucket", 64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, options := range []storage.ReadOptions{{}, {Disposition: storage.DispositionInline, FileName: "报告 2026.pdf", ContentType: "application/pdf"}, {Disposition: storage.DispositionAttachment, FileName: "download.pdf"}} {
|
||||
before := time.Now().UTC().Truncate(time.Second)
|
||||
link, err := store.PresignGetWithOptions(context.Background(), "folder/file.pdf", 30*time.Minute, options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u, err := url.Parse(link.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q := u.Query()
|
||||
if u.Host != "files.example" || u.Path != "/folder/file.pdf" || !validReadSignature(u, version) {
|
||||
t.Fatal("SDK response overrides were not signed")
|
||||
}
|
||||
disposition, params, err := mime.ParseMediaType(q.Get("response-content-disposition"))
|
||||
want := options.Disposition
|
||||
if want == "" {
|
||||
want = "attachment"
|
||||
}
|
||||
if err != nil || disposition != want || params["filename"] != options.FileName || q.Get("response-content-type") != options.ContentType || q.Get("response-cache-control") != "private, no-store" {
|
||||
t.Fatal("wrong signed headers", err)
|
||||
}
|
||||
if link.ExpiresAt.Before(before.Add(30*time.Minute)) || link.ExpiresAt.After(time.Now().Add(30*time.Minute)) {
|
||||
t.Fatal("expiry drift")
|
||||
}
|
||||
var expires time.Time
|
||||
if version == sdk.SignatureVersionV4 {
|
||||
start, _ := time.Parse("20060102T150405Z", q.Get("x-oss-date"))
|
||||
seconds, _ := strconv.Atoi(q.Get("x-oss-expires"))
|
||||
expires = start.Add(time.Duration(seconds) * time.Second)
|
||||
} else {
|
||||
seconds, _ := strconv.ParseInt(q.Get("Expires"), 10, 64)
|
||||
expires = time.Unix(seconds, 0)
|
||||
}
|
||||
if !expires.Equal(link.ExpiresAt) {
|
||||
t.Fatal("result does not match signature expiry")
|
||||
}
|
||||
for _, name := range []string{"response-content-disposition", "response-content-type", "response-cache-control"} {
|
||||
changed := *u
|
||||
query := changed.Query()
|
||||
query.Set(name, "changed")
|
||||
changed.RawQuery = query.Encode()
|
||||
if validReadSignature(&changed, version) {
|
||||
t.Fatal("unsigned override", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, ttl := range []time.Duration{0, time.Second - time.Nanosecond, time.Hour + time.Nanosecond} {
|
||||
if _, err := store.PresignGetWithOptions(context.Background(), "file", ttl, storage.ReadOptions{}); err == nil {
|
||||
t.Fatal("TTL accepted")
|
||||
}
|
||||
}
|
||||
for _, options := range []storage.ReadOptions{{FileName: "x\r\nY: z"}, {ContentType: "text/plain\r\nY: z"}, {Disposition: "bad"}} {
|
||||
if _, err := store.PresignGetWithOptions(context.Background(), "file", time.Minute, options); err == nil {
|
||||
t.Fatal("invalid header accepted")
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := store.PresignGetWithOptions(ctx, "file", time.Minute, storage.ReadOptions{}); err != context.Canceled {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
// A caller-supplied unsigned SDK configuration cannot masquerade as a link.
|
||||
client := sdk.NewClient(sdk.LoadDefaultConfig().WithRegion("cn-hangzhou").WithEndpoint("https://files.example").WithCredentialsProvider(credentials.NewStaticCredentialsProvider("fixtureAccess", "fixtureSecret")), func(o *sdk.Options) { o.Signer = &signer.NopSigner{} })
|
||||
store, _ := New(client, "testbucket", 64)
|
||||
if _, err := store.PresignGetWithOptions(context.Background(), "file", time.Minute, storage.ReadOptions{}); err == nil {
|
||||
t.Fatal("unsigned URL accepted")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user