feat: add independent platform modules and multi-provider storage
This commit is contained in:
171
storage/local/local.go
Normal file
171
storage/local/local.go
Normal file
@@ -0,0 +1,171 @@
|
||||
// Package local stores objects atomically in a private, flat managed directory.
|
||||
// This directory must not be writable by untrusted local users/processes.
|
||||
package local
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
root string
|
||||
maxBytes int64
|
||||
}
|
||||
|
||||
func New(root string, maxBytes int64) (*Store, error) {
|
||||
if root == "" || maxBytes < 0 || maxBytes > 1<<40 {
|
||||
return nil, errors.New("storage/local: invalid configuration")
|
||||
}
|
||||
if maxBytes == 0 {
|
||||
maxBytes = 32 << 20
|
||||
}
|
||||
if err := os.MkdirAll(root, 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root, err := filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root, err = filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !info.IsDir() || info.Mode().Perm()&0022 != 0 {
|
||||
return nil, errors.New("storage/local: root must be a private writable directory")
|
||||
}
|
||||
return &Store{root: root, maxBytes: maxBytes}, nil
|
||||
}
|
||||
func (s *Store) path(key string) string {
|
||||
sum := sha256.Sum256([]byte(key))
|
||||
return filepath.Join(s.root, hex.EncodeToString(sum[:])+".object")
|
||||
}
|
||||
func (s *Store) Put(ctx context.Context, key string, reader io.Reader, size int64, contentType string) (storage.Info, error) {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return storage.Info{}, err
|
||||
}
|
||||
if reader == nil || size < 0 || len(contentType) > 256 || strings.ContainsAny(contentType, "\r\n") {
|
||||
return storage.Info{}, errors.New("storage/local: invalid upload")
|
||||
}
|
||||
if size > s.maxBytes {
|
||||
return storage.Info{}, storage.ErrTooLarge
|
||||
}
|
||||
info := storage.Info{Key: key, Size: size, ContentType: contentType}
|
||||
metadata, _ := json.Marshal(info)
|
||||
f, err := os.CreateTemp(s.root, ".upload-")
|
||||
if err != nil {
|
||||
return storage.Info{}, err
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
defer f.Close()
|
||||
if err = binary.Write(f, binary.BigEndian, uint32(len(metadata))); err != nil {
|
||||
return storage.Info{}, err
|
||||
}
|
||||
if _, err = f.Write(metadata); err != nil {
|
||||
return storage.Info{}, err
|
||||
}
|
||||
n, err := io.Copy(f, io.LimitReader(storage.ContextReader{Context: ctx, Reader: reader}, size+1))
|
||||
if err != nil {
|
||||
return storage.Info{}, err
|
||||
}
|
||||
if n != size {
|
||||
return storage.Info{}, errors.New("storage/local: declared size mismatch")
|
||||
}
|
||||
if err = ctx.Err(); err != nil {
|
||||
return storage.Info{}, err
|
||||
}
|
||||
if err = f.Sync(); err != nil {
|
||||
return storage.Info{}, err
|
||||
}
|
||||
stat, err := f.Stat()
|
||||
if err != nil {
|
||||
return storage.Info{}, err
|
||||
}
|
||||
if err = f.Close(); err != nil {
|
||||
return storage.Info{}, err
|
||||
}
|
||||
if err = os.Rename(f.Name(), s.path(key)); err != nil {
|
||||
return storage.Info{}, err
|
||||
}
|
||||
info.ModifiedAt = stat.ModTime()
|
||||
return info, nil
|
||||
}
|
||||
func (s *Store) Get(ctx context.Context, key string) (storage.Object, error) {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return storage.Object{}, err
|
||||
}
|
||||
path := s.path(key)
|
||||
stat, err := os.Lstat(path)
|
||||
if os.IsNotExist(err) {
|
||||
return storage.Object{}, storage.ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return storage.Object{}, err
|
||||
}
|
||||
if !stat.Mode().IsRegular() {
|
||||
return storage.Object{}, errors.New("storage/local: not a regular managed object")
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return storage.Object{}, err
|
||||
}
|
||||
// A concurrent Put can atomically replace the path between Lstat/Open.
|
||||
// Validate size against the descriptor whose bytes we actually read.
|
||||
stat, err = f.Stat()
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return storage.Object{}, err
|
||||
}
|
||||
ok := false
|
||||
defer func() {
|
||||
if !ok {
|
||||
f.Close()
|
||||
}
|
||||
}()
|
||||
var length uint32
|
||||
if err = binary.Read(f, binary.BigEndian, &length); err != nil {
|
||||
return storage.Object{}, err
|
||||
}
|
||||
if length > 4096 {
|
||||
return storage.Object{}, errors.New("storage/local: invalid object header")
|
||||
}
|
||||
data := make([]byte, length)
|
||||
if _, err = io.ReadFull(f, data); err != nil {
|
||||
return storage.Object{}, err
|
||||
}
|
||||
var info storage.Info
|
||||
if json.Unmarshal(data, &info) != nil || info.Key != key || info.Size < 0 || info.Size > s.maxBytes || int64(length)+4+info.Size != stat.Size() {
|
||||
return storage.Object{}, errors.New("storage/local: corrupt object")
|
||||
}
|
||||
info.ModifiedAt = stat.ModTime()
|
||||
ok = true
|
||||
return storage.Object{Info: info, Body: &body{Reader: storage.ContextReader{Context: ctx, Reader: io.LimitReader(f, info.Size)}, Closer: f}}, nil
|
||||
}
|
||||
|
||||
type body struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}
|
||||
|
||||
func (s *Store) Delete(ctx context.Context, key string) error {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return err
|
||||
}
|
||||
err := os.Remove(s.path(key))
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
42
storage/local/local_test.go
Normal file
42
storage/local/local_test.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"context"
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage/storagetest"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestContractAndAtomicFailure(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
s, err := New(root, 64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
storagetest.Run(t, s)
|
||||
ctx := context.Background()
|
||||
if _, err = s.Put(ctx, "saved", strings.NewReader("old"), 3, "text/plain"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = s.Put(ctx, "saved", strings.NewReader("short"), 20, "text/plain"); err == nil {
|
||||
t.Fatal("short upload committed")
|
||||
}
|
||||
again, _ := New(root, 64)
|
||||
object, err := again.Get(ctx, "saved")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := io.ReadAll(object.Body)
|
||||
object.Body.Close()
|
||||
if string(raw) != "old" {
|
||||
t.Fatal(string(raw))
|
||||
}
|
||||
if err = os.Symlink(s.path("saved"), s.path("linked")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = s.Get(ctx, "linked"); err == nil {
|
||||
t.Fatal("symlink followed")
|
||||
}
|
||||
}
|
||||
101
storage/oss/oss.go
Normal file
101
storage/oss/oss.go
Normal file
@@ -0,0 +1,101 @@
|
||||
// Package oss adapts the caller-owned official Alibaba Cloud OSS v2 client.
|
||||
package oss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage"
|
||||
sdk "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
client *sdk.Client
|
||||
bucket string
|
||||
maxBytes int64
|
||||
}
|
||||
|
||||
// New borrows a configured SDK client. This adapter uses simple uploads; large
|
||||
// multipart uploads are a separate provider capability, not buffered in memory.
|
||||
func New(client *sdk.Client, bucket string, maxBytes int64) (*Store, error) {
|
||||
if client == nil || bucket == "" || maxBytes < 0 || maxBytes > 5<<30 {
|
||||
return nil, errors.New("storage/oss: invalid client, bucket or size limit")
|
||||
}
|
||||
if maxBytes == 0 {
|
||||
maxBytes = 32 << 20
|
||||
}
|
||||
return &Store{client: client, bucket: bucket, maxBytes: maxBytes}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Put(ctx context.Context, key string, reader io.Reader, size int64, contentType string) (storage.Info, error) {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return storage.Info{}, err
|
||||
}
|
||||
if reader == nil || size < 0 || len(contentType) > 256 || strings.ContainsAny(contentType, "\r\n") {
|
||||
return storage.Info{}, errors.New("storage/oss: invalid upload")
|
||||
}
|
||||
if size > s.maxBytes {
|
||||
return storage.Info{}, storage.ErrTooLarge
|
||||
}
|
||||
_, err := s.client.PutObject(ctx, &sdk.PutObjectRequest{Bucket: sdk.Ptr(s.bucket), Key: sdk.Ptr(key), ContentLength: sdk.Ptr(size), ContentType: sdk.Ptr(contentType), Body: storage.ContextReader{Context: ctx, Reader: reader}})
|
||||
if err != nil {
|
||||
return storage.Info{}, mapError(err)
|
||||
}
|
||||
return storage.Info{Key: key, Size: size, ContentType: contentType, ModifiedAt: time.Now().UTC()}, nil
|
||||
}
|
||||
func (s *Store) Get(ctx context.Context, key string) (storage.Object, error) {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return storage.Object{}, err
|
||||
}
|
||||
r, err := s.client.GetObject(ctx, &sdk.GetObjectRequest{Bucket: sdk.Ptr(s.bucket), Key: sdk.Ptr(key)})
|
||||
if err != nil {
|
||||
return storage.Object{}, mapError(err)
|
||||
}
|
||||
if r.ContentLength < 0 || r.ContentLength > s.maxBytes {
|
||||
r.Body.Close()
|
||||
return storage.Object{}, storage.ErrTooLarge
|
||||
}
|
||||
info := storage.Info{Key: key, Size: r.ContentLength, ContentType: sdk.ToString(r.ContentType)}
|
||||
if r.LastModified != nil {
|
||||
info.ModifiedAt = *r.LastModified
|
||||
}
|
||||
return storage.Object{Info: info, Body: r.Body}, nil
|
||||
}
|
||||
func (s *Store) Delete(ctx context.Context, key string) error {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := s.client.DeleteObject(ctx, &sdk.DeleteObjectRequest{Bucket: sdk.Ptr(s.bucket), Key: sdk.Ptr(key)})
|
||||
err = mapError(err)
|
||||
if errors.Is(err, storage.ErrNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
func (s *Store) PresignGet(ctx context.Context, key string, ttl time.Duration) (*url.URL, error) {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ttl < time.Second || ttl > time.Hour {
|
||||
return nil, errors.New("storage/oss: TTL must be 1s..1h")
|
||||
}
|
||||
r, err := s.client.Presign(ctx, &sdk.GetObjectRequest{Bucket: sdk.Ptr(s.bucket), Key: sdk.Ptr(key), ResponseContentDisposition: sdk.Ptr("attachment")}, sdk.PresignExpires(ttl))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return url.Parse(r.URL)
|
||||
}
|
||||
func mapError(err error) error {
|
||||
var serviceError *sdk.ServiceError
|
||||
if errors.As(err, &serviceError) && serviceError.Code == "NoSuchKey" {
|
||||
return storage.ErrNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
var _ storage.Store = (*Store)(nil)
|
||||
var _ storage.Presigner = (*Store)(nil)
|
||||
31
storage/oss/oss_test.go
Normal file
31
storage/oss/oss_test.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"git.yuntuo.hk/k2safe/go-omni/internal/storagefixture"
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage/storagetest"
|
||||
sdk "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
|
||||
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSDKProtocolContract(t *testing.T) {
|
||||
server := httptest.NewTLSServer(storagefixture.Handler())
|
||||
defer server.Close()
|
||||
cfg := sdk.LoadDefaultConfig().WithRegion("cn-hangzhou").WithEndpoint(server.URL).WithUsePathStyle(true).WithCredentialsProvider(credentials.NewStaticCredentialsProvider("fixtureAccess", "fixtureSecret")).WithHttpClient(server.Client())
|
||||
store, err := New(sdk.NewClient(cfg), "testbucket", 64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
storagetest.Run(t, store)
|
||||
u, err := store.PresignGet(context.Background(), "a/file.txt", time.Minute)
|
||||
if err != nil || u.Query().Get("x-oss-signature") == "" || !strings.Contains(u.Path, "a/file.txt") {
|
||||
t.Fatal("invalid signed URL", err)
|
||||
}
|
||||
if _, err = store.PresignGet(context.Background(), "a", 2*time.Hour); err == nil {
|
||||
t.Fatal("TTL accepted")
|
||||
}
|
||||
}
|
||||
42
storage/s3/config.go
Normal file
42
storage/s3/config.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"regexp"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// Open creates a store using an injected credential provider and transport.
|
||||
// The application owns transport shutdown; no global HTTP defaults are changed.
|
||||
type Config struct {
|
||||
Endpoint string `json:"endpoint"` // Host[:port], without scheme or path.
|
||||
Region string `json:"region"`
|
||||
Bucket string `json:"bucket"`
|
||||
Secure bool `json:"secure"`
|
||||
MaxBytes int64 `json:"maxBytes"`
|
||||
Credentials *credentials.Credentials `json:"-"`
|
||||
Transport http.RoundTripper `json:"-"`
|
||||
}
|
||||
|
||||
func Open(cfg Config) (*Store, error) {
|
||||
if cfg.Credentials == nil {
|
||||
return nil, errors.New("storage/s3: credentials provider required")
|
||||
}
|
||||
client, err := minio.New(cfg.Endpoint, &minio.Options{Creds: cfg.Credentials, Secure: cfg.Secure, Region: cfg.Region, Transport: cfg.Transport, BucketLookup: minio.BucketLookupPath})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(client, cfg.Bucket, cfg.MaxBytes)
|
||||
}
|
||||
|
||||
// R2Config selects Cloudflare's standard S3 API endpoint. Jurisdiction-specific
|
||||
// endpoints can be set explicitly with Config. R2 implements an S3 subset.
|
||||
func R2Config(accountID, bucket string, creds *credentials.Credentials) (Config, error) {
|
||||
if !regexp.MustCompile(`^[a-fA-F0-9]{32}$`).MatchString(accountID) {
|
||||
return Config{}, errors.New("storage/s3: invalid R2 account ID")
|
||||
}
|
||||
return Config{Endpoint: accountID + ".r2.cloudflarestorage.com", Region: "auto", Bucket: bucket, Secure: true, Credentials: creds}, nil
|
||||
}
|
||||
93
storage/s3/s3.go
Normal file
93
storage/s3/s3.go
Normal file
@@ -0,0 +1,93 @@
|
||||
// Package s3 adapts a caller-owned MinIO client to S3-compatible object storage.
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
maxBytes int64
|
||||
}
|
||||
|
||||
func New(client *minio.Client, bucket string, maxBytes int64) (*Store, error) {
|
||||
if client == nil || bucket == "" || maxBytes < 0 {
|
||||
return nil, errors.New("storage/s3: client, bucket and nonnegative limit required")
|
||||
}
|
||||
if maxBytes == 0 {
|
||||
maxBytes = 32 << 20
|
||||
}
|
||||
return &Store{client: client, bucket: bucket, maxBytes: maxBytes}, nil
|
||||
}
|
||||
func (s *Store) Put(ctx context.Context, key string, reader io.Reader, size int64, contentType string) (storage.Info, error) {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return storage.Info{}, err
|
||||
}
|
||||
if reader == nil || size < 0 || len(contentType) > 256 || strings.ContainsAny(contentType, "\r\n") {
|
||||
return storage.Info{}, errors.New("storage/s3: invalid upload")
|
||||
}
|
||||
if size > s.maxBytes {
|
||||
return storage.Info{}, storage.ErrTooLarge
|
||||
}
|
||||
result, err := s.client.PutObject(ctx, s.bucket, key, storage.ContextReader{Context: ctx, Reader: reader}, size, minio.PutObjectOptions{ContentType: contentType})
|
||||
if err != nil {
|
||||
return storage.Info{}, err
|
||||
}
|
||||
return storage.Info{Key: key, Size: result.Size, ContentType: contentType, ModifiedAt: time.Now().UTC()}, nil
|
||||
}
|
||||
func (s *Store) Get(ctx context.Context, key string) (storage.Object, error) {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return storage.Object{}, err
|
||||
}
|
||||
object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return storage.Object{}, mapError(err)
|
||||
}
|
||||
stat, err := object.Stat()
|
||||
if err != nil {
|
||||
object.Close()
|
||||
return storage.Object{}, mapError(err)
|
||||
}
|
||||
if stat.Size > s.maxBytes {
|
||||
object.Close()
|
||||
return storage.Object{}, storage.ErrTooLarge
|
||||
}
|
||||
return storage.Object{Info: storage.Info{Key: key, Size: stat.Size, ContentType: stat.ContentType, ModifiedAt: stat.LastModified}, Body: object}, nil
|
||||
}
|
||||
func (s *Store) Delete(ctx context.Context, key string) error {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return err
|
||||
}
|
||||
err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
|
||||
if errors.Is(mapError(err), storage.ErrNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
func (s *Store) PresignGet(ctx context.Context, key string, ttl time.Duration) (*url.URL, error) {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ttl < time.Second || ttl > time.Hour {
|
||||
return nil, errors.New("storage/s3: TTL must be 1s..1h")
|
||||
}
|
||||
return s.client.PresignedGetObject(ctx, s.bucket, key, ttl, url.Values{"response-content-disposition": {"attachment"}})
|
||||
}
|
||||
func mapError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
switch minio.ToErrorResponse(err).Code {
|
||||
case "NoSuchKey", "NoSuchObject":
|
||||
return storage.ErrNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
104
storage/s3/s3_test.go
Normal file
104
storage/s3/s3_test.go
Normal file
@@ -0,0 +1,104 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"git.yuntuo.hk/k2safe/go-omni/internal/storagefixture"
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage/storagetest"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestS3ProtocolAndR2Config(t *testing.T) {
|
||||
server := httptest.NewTLSServer(storagefixture.Handler())
|
||||
defer server.Close()
|
||||
creds := credentials.NewStaticV4("fixtureAccess", "fixtureSecret", "")
|
||||
s, err := Open(Config{Endpoint: strings.TrimPrefix(server.URL, "https://"), Region: "us-east-1", Bucket: "testbucket", Secure: true, MaxBytes: 64, Credentials: creds, Transport: server.Client().Transport})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
storagetest.Run(t, s)
|
||||
cfg, err := R2Config(strings.Repeat("a", 32), "bucket", creds)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r2, err := Open(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u, err := r2.PresignGet(context.Background(), "path/file", time.Minute)
|
||||
if err != nil || u.Host != strings.Repeat("a", 32)+".r2.cloudflarestorage.com" || u.Query().Get("X-Amz-Signature") == "" || !strings.Contains(u.Query().Get("X-Amz-Credential"), "/auto/s3/") {
|
||||
t.Fatal("R2 presign settings", err)
|
||||
}
|
||||
}
|
||||
func TestS3Integration(t *testing.T) {
|
||||
address := os.Getenv("GO_OMNI_S3_TEST_ADDR")
|
||||
if address == "" {
|
||||
t.Skip("requires isolated MinIO")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
|
||||
defer cancel()
|
||||
ready := false
|
||||
for n := 0; n < 100; n++ {
|
||||
req, _ := http.NewRequestWithContext(ctx, "GET", "http://"+address+"/minio/health/ready", nil)
|
||||
r, err := http.DefaultClient.Do(req)
|
||||
if err == nil {
|
||||
r.Body.Close()
|
||||
if r.StatusCode == 200 {
|
||||
ready = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
if !ready {
|
||||
t.Fatal("isolated S3 server did not start")
|
||||
}
|
||||
client, err := minio.New(address, &minio.Options{Creds: credentials.NewStaticV4("goOmniTest", "goOmniTestSecret", ""), Region: "us-east-1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bucket := fmt.Sprintf("test-%d", time.Now().UnixNano())
|
||||
if err = client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.RemoveBucket(context.Background(), bucket)
|
||||
s, _ := New(client, bucket, 64)
|
||||
storagetest.Run(t, s)
|
||||
s.Put(ctx, "signed", strings.NewReader("hello"), 5, "text/plain")
|
||||
defer s.Delete(context.Background(), "signed")
|
||||
u, err := s.PresignGet(ctx, "signed", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response, err := http.Get(u.String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := io.ReadAll(response.Body)
|
||||
response.Body.Close()
|
||||
if response.StatusCode != 200 || string(raw) != "hello" {
|
||||
t.Fatal(response.StatusCode, string(raw))
|
||||
}
|
||||
q := u.Query()
|
||||
q.Set("X-Amz-Signature", strings.Repeat("0", 64))
|
||||
u.RawQuery = q.Encode()
|
||||
response, err = http.Get(u.String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response.Body.Close()
|
||||
if response.StatusCode != 403 {
|
||||
t.Fatal("tampered signature", response.StatusCode)
|
||||
}
|
||||
}
|
||||
93
storage/signed.go
Normal file
93
storage/signed.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Signer struct {
|
||||
store Store
|
||||
base *url.URL
|
||||
secret []byte
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewSigner(store Store, baseURL string, secret []byte) (*Signer, error) {
|
||||
base, err := url.Parse(baseURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if store == nil || len(secret) < 32 || base.Host == "" || (base.Scheme != "https" && base.Scheme != "http") || base.RawQuery != "" || base.Fragment != "" || base.User != nil {
|
||||
return nil, errors.New("storage: store, HTTP base URL and >=32-byte secret required")
|
||||
}
|
||||
return &Signer{store: store, base: base, secret: append([]byte(nil), secret...), now: time.Now}, nil
|
||||
}
|
||||
func (s *Signer) signature(key, expiry string) string {
|
||||
mac := hmac.New(sha256.New, s.secret)
|
||||
io.WriteString(mac, "GET\n"+key+"\n"+expiry)
|
||||
return base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
func (s *Signer) PresignGet(ctx context.Context, key string, ttl time.Duration) (*url.URL, error) {
|
||||
if err := Check(ctx, key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ttl < time.Second || ttl > time.Hour {
|
||||
return nil, errors.New("storage: TTL must be 1s..1h")
|
||||
}
|
||||
expiry := strconv.FormatInt(s.now().Add(ttl).Unix(), 10)
|
||||
u := *s.base
|
||||
u.RawQuery = url.Values{"key": {key}, "expires": {expiry}, "signature": {s.signature(key, expiry)}}.Encode()
|
||||
return &u, nil
|
||||
}
|
||||
|
||||
// ServeHTTP serves signed downloads as attachments. Issue links only after
|
||||
// authorization; possession of an unexpired URL grants access to its object.
|
||||
func (s *Signer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "GET" && r.Method != "HEAD" {
|
||||
w.Header().Set("Allow", "GET, HEAD")
|
||||
http.Error(w, "method not allowed", 405)
|
||||
return
|
||||
}
|
||||
q, err := url.ParseQuery(r.URL.RawQuery)
|
||||
if err != nil || len(q) != 3 || len(q["key"]) != 1 || len(q["expires"]) != 1 || len(q["signature"]) != 1 {
|
||||
http.Error(w, "invalid link", 403)
|
||||
return
|
||||
}
|
||||
key, expiry := q.Get("key"), q.Get("expires")
|
||||
expires, err := strconv.ParseInt(expiry, 10, 64)
|
||||
now := s.now().Unix()
|
||||
if err != nil || expires <= now || expires > now+3600 || Check(r.Context(), key) != nil || !hmac.Equal([]byte(q.Get("signature")), []byte(s.signature(key, expiry))) {
|
||||
http.Error(w, "invalid or expired link", 403)
|
||||
return
|
||||
}
|
||||
object, err := s.store.Get(r.Context(), key)
|
||||
if err != nil {
|
||||
status := 500
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
status = 404
|
||||
}
|
||||
http.Error(w, "object unavailable", status)
|
||||
return
|
||||
}
|
||||
defer object.Body.Close()
|
||||
contentType := object.ContentType
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Disposition", "attachment")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.Header().Set("Cache-Control", "private, no-store")
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(object.Size, 10))
|
||||
if r.Method == "GET" {
|
||||
_, _ = io.Copy(w, object.Body)
|
||||
}
|
||||
}
|
||||
35
storage/signed_test.go
Normal file
35
storage/signed_test.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package storage_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage"
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage/local"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLocalSignedDownload(t *testing.T) {
|
||||
s, _ := local.New(t.TempDir(), 64)
|
||||
s.Put(context.Background(), "file", strings.NewReader("hello"), 5, "text/plain")
|
||||
signer, _ := storage.NewSigner(s, "https://example.test/download", []byte(strings.Repeat("x", 32)))
|
||||
u, err := signer.PresignGet(context.Background(), "file", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request := httptest.NewRequest("GET", u.String(), nil)
|
||||
response := httptest.NewRecorder()
|
||||
signer.ServeHTTP(response, request)
|
||||
if response.Code != 200 || response.Body.String() != "hello" || response.Header().Get("Content-Disposition") != "attachment" {
|
||||
t.Fatal(response)
|
||||
}
|
||||
q := u.Query()
|
||||
q.Set("key", "another")
|
||||
u.RawQuery = q.Encode()
|
||||
response = httptest.NewRecorder()
|
||||
signer.ServeHTTP(response, httptest.NewRequest("GET", u.String(), nil))
|
||||
if response.Code != 403 {
|
||||
t.Fatal(response.Code)
|
||||
}
|
||||
}
|
||||
68
storage/storage.go
Normal file
68
storage/storage.go
Normal file
@@ -0,0 +1,68 @@
|
||||
// Package storage defines streaming object storage; callers own returned bodies.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
var ErrNotFound = errors.New("storage: object not found")
|
||||
var ErrTooLarge = errors.New("storage: object exceeds limit")
|
||||
|
||||
type Info struct {
|
||||
Key string `json:"key"`
|
||||
Size int64 `json:"size"`
|
||||
ContentType string `json:"contentType"`
|
||||
ModifiedAt time.Time `json:"modifiedAt"`
|
||||
}
|
||||
type Object struct {
|
||||
Info
|
||||
Body io.ReadCloser
|
||||
}
|
||||
type Store interface {
|
||||
Put(context.Context, string, io.Reader, int64, string) (Info, error)
|
||||
Get(context.Context, string) (Object, error)
|
||||
Delete(context.Context, string) error
|
||||
}
|
||||
|
||||
// Presigner is an optional capability. URLs grant temporary read access and
|
||||
// must not be logged. Providers may expire them earlier when credentials expire.
|
||||
type Presigner interface {
|
||||
PresignGet(context.Context, string, time.Duration) (*url.URL, error)
|
||||
}
|
||||
|
||||
func Check(ctx context.Context, key string) error {
|
||||
if ctx == nil {
|
||||
return errors.New("storage: nil context")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if key == "" || len(key) > 1024 || !utf8.ValidString(key) || strings.Contains(key, "\\") || strings.IndexFunc(key, unicode.IsControl) >= 0 {
|
||||
return errors.New("storage: invalid key")
|
||||
}
|
||||
for _, part := range strings.Split(key, "/") {
|
||||
if part == "" || part == "." || part == ".." {
|
||||
return errors.New("storage: invalid key path")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ContextReader struct {
|
||||
Context context.Context
|
||||
Reader io.Reader
|
||||
}
|
||||
|
||||
func (r ContextReader) Read(p []byte) (int, error) {
|
||||
if err := r.Context.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.Reader.Read(p)
|
||||
}
|
||||
59
storage/storagetest/contract.go
Normal file
59
storage/storagetest/contract.go
Normal file
@@ -0,0 +1,59 @@
|
||||
// Package storagetest provides shared adapter checks. Use a dedicated test
|
||||
// bucket/root and a Store configured with a 64-byte object limit.
|
||||
package storagetest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func Run(t *testing.T, store storage.Store) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
key := "contract/文件 +%.txt"
|
||||
defer store.Delete(ctx, key)
|
||||
if _, err := store.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) {
|
||||
t.Fatal("missing object", err)
|
||||
}
|
||||
for _, body := range []string{"original", "replacement"} {
|
||||
info, err := store.Put(ctx, key, strings.NewReader(body), int64(len(body)), "text/plain")
|
||||
if err != nil || info.Size != int64(len(body)) {
|
||||
t.Fatal(info, err)
|
||||
}
|
||||
object, err := store.Get(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := io.ReadAll(object.Body)
|
||||
object.Body.Close()
|
||||
if err != nil || string(data) != body || object.ContentType != "text/plain" {
|
||||
t.Fatal(string(data), object.Info, err)
|
||||
}
|
||||
}
|
||||
if _, err := store.Put(ctx, "tooBig", strings.NewReader(strings.Repeat("a", 65)), 65, "text/plain"); !errors.Is(err, storage.ErrTooLarge) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, key := range []string{"../bad", "a/../b", "/absolute", "a//b", "a\\b"} {
|
||||
if _, err := store.Put(ctx, key, strings.NewReader("a"), 1, "text/plain"); err == nil {
|
||||
t.Fatal(key)
|
||||
}
|
||||
}
|
||||
canceled, cancel := context.WithCancel(ctx)
|
||||
cancel()
|
||||
if _, err := store.Get(canceled, key); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Delete(ctx, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Delete(ctx, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user