feat: add independent platform modules and multi-provider storage
This commit is contained in:
42
storage/s3/config.go
Normal file
42
storage/s3/config.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"regexp"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// Open creates a store using an injected credential provider and transport.
|
||||
// The application owns transport shutdown; no global HTTP defaults are changed.
|
||||
type Config struct {
|
||||
Endpoint string `json:"endpoint"` // Host[:port], without scheme or path.
|
||||
Region string `json:"region"`
|
||||
Bucket string `json:"bucket"`
|
||||
Secure bool `json:"secure"`
|
||||
MaxBytes int64 `json:"maxBytes"`
|
||||
Credentials *credentials.Credentials `json:"-"`
|
||||
Transport http.RoundTripper `json:"-"`
|
||||
}
|
||||
|
||||
func Open(cfg Config) (*Store, error) {
|
||||
if cfg.Credentials == nil {
|
||||
return nil, errors.New("storage/s3: credentials provider required")
|
||||
}
|
||||
client, err := minio.New(cfg.Endpoint, &minio.Options{Creds: cfg.Credentials, Secure: cfg.Secure, Region: cfg.Region, Transport: cfg.Transport, BucketLookup: minio.BucketLookupPath})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(client, cfg.Bucket, cfg.MaxBytes)
|
||||
}
|
||||
|
||||
// R2Config selects Cloudflare's standard S3 API endpoint. Jurisdiction-specific
|
||||
// endpoints can be set explicitly with Config. R2 implements an S3 subset.
|
||||
func R2Config(accountID, bucket string, creds *credentials.Credentials) (Config, error) {
|
||||
if !regexp.MustCompile(`^[a-fA-F0-9]{32}$`).MatchString(accountID) {
|
||||
return Config{}, errors.New("storage/s3: invalid R2 account ID")
|
||||
}
|
||||
return Config{Endpoint: accountID + ".r2.cloudflarestorage.com", Region: "auto", Bucket: bucket, Secure: true, Credentials: creds}, nil
|
||||
}
|
||||
93
storage/s3/s3.go
Normal file
93
storage/s3/s3.go
Normal file
@@ -0,0 +1,93 @@
|
||||
// Package s3 adapts a caller-owned MinIO client to S3-compatible object storage.
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
maxBytes int64
|
||||
}
|
||||
|
||||
func New(client *minio.Client, bucket string, maxBytes int64) (*Store, error) {
|
||||
if client == nil || bucket == "" || maxBytes < 0 {
|
||||
return nil, errors.New("storage/s3: client, bucket and nonnegative limit required")
|
||||
}
|
||||
if maxBytes == 0 {
|
||||
maxBytes = 32 << 20
|
||||
}
|
||||
return &Store{client: client, bucket: bucket, maxBytes: maxBytes}, nil
|
||||
}
|
||||
func (s *Store) Put(ctx context.Context, key string, reader io.Reader, size int64, contentType string) (storage.Info, error) {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return storage.Info{}, err
|
||||
}
|
||||
if reader == nil || size < 0 || len(contentType) > 256 || strings.ContainsAny(contentType, "\r\n") {
|
||||
return storage.Info{}, errors.New("storage/s3: invalid upload")
|
||||
}
|
||||
if size > s.maxBytes {
|
||||
return storage.Info{}, storage.ErrTooLarge
|
||||
}
|
||||
result, err := s.client.PutObject(ctx, s.bucket, key, storage.ContextReader{Context: ctx, Reader: reader}, size, minio.PutObjectOptions{ContentType: contentType})
|
||||
if err != nil {
|
||||
return storage.Info{}, err
|
||||
}
|
||||
return storage.Info{Key: key, Size: result.Size, ContentType: contentType, ModifiedAt: time.Now().UTC()}, nil
|
||||
}
|
||||
func (s *Store) Get(ctx context.Context, key string) (storage.Object, error) {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return storage.Object{}, err
|
||||
}
|
||||
object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return storage.Object{}, mapError(err)
|
||||
}
|
||||
stat, err := object.Stat()
|
||||
if err != nil {
|
||||
object.Close()
|
||||
return storage.Object{}, mapError(err)
|
||||
}
|
||||
if stat.Size > s.maxBytes {
|
||||
object.Close()
|
||||
return storage.Object{}, storage.ErrTooLarge
|
||||
}
|
||||
return storage.Object{Info: storage.Info{Key: key, Size: stat.Size, ContentType: stat.ContentType, ModifiedAt: stat.LastModified}, Body: object}, nil
|
||||
}
|
||||
func (s *Store) Delete(ctx context.Context, key string) error {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return err
|
||||
}
|
||||
err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
|
||||
if errors.Is(mapError(err), storage.ErrNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
func (s *Store) PresignGet(ctx context.Context, key string, ttl time.Duration) (*url.URL, error) {
|
||||
if err := storage.Check(ctx, key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ttl < time.Second || ttl > time.Hour {
|
||||
return nil, errors.New("storage/s3: TTL must be 1s..1h")
|
||||
}
|
||||
return s.client.PresignedGetObject(ctx, s.bucket, key, ttl, url.Values{"response-content-disposition": {"attachment"}})
|
||||
}
|
||||
func mapError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
switch minio.ToErrorResponse(err).Code {
|
||||
case "NoSuchKey", "NoSuchObject":
|
||||
return storage.ErrNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
104
storage/s3/s3_test.go
Normal file
104
storage/s3/s3_test.go
Normal file
@@ -0,0 +1,104 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"git.yuntuo.hk/k2safe/go-omni/internal/storagefixture"
|
||||
"git.yuntuo.hk/k2safe/go-omni/storage/storagetest"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestS3ProtocolAndR2Config(t *testing.T) {
|
||||
server := httptest.NewTLSServer(storagefixture.Handler())
|
||||
defer server.Close()
|
||||
creds := credentials.NewStaticV4("fixtureAccess", "fixtureSecret", "")
|
||||
s, err := Open(Config{Endpoint: strings.TrimPrefix(server.URL, "https://"), Region: "us-east-1", Bucket: "testbucket", Secure: true, MaxBytes: 64, Credentials: creds, Transport: server.Client().Transport})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
storagetest.Run(t, s)
|
||||
cfg, err := R2Config(strings.Repeat("a", 32), "bucket", creds)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r2, err := Open(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u, err := r2.PresignGet(context.Background(), "path/file", time.Minute)
|
||||
if err != nil || u.Host != strings.Repeat("a", 32)+".r2.cloudflarestorage.com" || u.Query().Get("X-Amz-Signature") == "" || !strings.Contains(u.Query().Get("X-Amz-Credential"), "/auto/s3/") {
|
||||
t.Fatal("R2 presign settings", err)
|
||||
}
|
||||
}
|
||||
func TestS3Integration(t *testing.T) {
|
||||
address := os.Getenv("GO_OMNI_S3_TEST_ADDR")
|
||||
if address == "" {
|
||||
t.Skip("requires isolated MinIO")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
|
||||
defer cancel()
|
||||
ready := false
|
||||
for n := 0; n < 100; n++ {
|
||||
req, _ := http.NewRequestWithContext(ctx, "GET", "http://"+address+"/minio/health/ready", nil)
|
||||
r, err := http.DefaultClient.Do(req)
|
||||
if err == nil {
|
||||
r.Body.Close()
|
||||
if r.StatusCode == 200 {
|
||||
ready = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
if !ready {
|
||||
t.Fatal("isolated S3 server did not start")
|
||||
}
|
||||
client, err := minio.New(address, &minio.Options{Creds: credentials.NewStaticV4("goOmniTest", "goOmniTestSecret", ""), Region: "us-east-1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bucket := fmt.Sprintf("test-%d", time.Now().UnixNano())
|
||||
if err = client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.RemoveBucket(context.Background(), bucket)
|
||||
s, _ := New(client, bucket, 64)
|
||||
storagetest.Run(t, s)
|
||||
s.Put(ctx, "signed", strings.NewReader("hello"), 5, "text/plain")
|
||||
defer s.Delete(context.Background(), "signed")
|
||||
u, err := s.PresignGet(ctx, "signed", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response, err := http.Get(u.String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := io.ReadAll(response.Body)
|
||||
response.Body.Close()
|
||||
if response.StatusCode != 200 || string(raw) != "hello" {
|
||||
t.Fatal(response.StatusCode, string(raw))
|
||||
}
|
||||
q := u.Query()
|
||||
q.Set("X-Amz-Signature", strings.Repeat("0", 64))
|
||||
u.RawQuery = q.Encode()
|
||||
response, err = http.Get(u.String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response.Body.Close()
|
||||
if response.StatusCode != 403 {
|
||||
t.Fatal("tampered signature", response.StatusCode)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user