feat: complete API-driven projects with infrastructure and business acceptance

2026-09-23 17:33:50 +08:00
parent 04a1f7999d
commit 2c74dc0859
137 changed files with 11926 additions and 105 deletions
--- a/storage/s3/s3.go
+++ b/storage/s3/s3.go
@@ -19,7 +19,7 @@ type Store struct {
 }

 func New(client *minio.Client, bucket string, maxBytes int64) (*Store, error) {
-	if client == nil || bucket == "" || maxBytes < 0 {
+	if client == nil || bucket == "" || maxBytes < 0 || maxBytes > 5<<40 {
 		return nil, errors.New("storage/s3: client, bucket and nonnegative limit required")
 	}
 	if maxBytes == 0 {
@@ -37,7 +37,11 @@ func (s *Store) Put(ctx context.Context, key string, reader io.Reader, size int6
 	if size > s.maxBytes {
 		return storage.Info{}, storage.ErrTooLarge
 	}
-	result, err := s.client.PutObject(ctx, s.bucket, key, storage.ContextReader{Context: ctx, Reader: reader}, size, minio.PutObjectOptions{ContentType: contentType})
+	checked, err := (storage.UploadPolicy{MaxBytes: s.maxBytes}).PrepareUpload(ctx, reader, size, contentType)
+	if err != nil {
+		return storage.Info{}, err
+	}
+	result, err := s.client.PutObject(ctx, s.bucket, key, checked, size, minio.PutObjectOptions{ContentType: contentType})
 	if err != nil {
 		return storage.Info{}, err
 	}
--- a/storage/s3/s3_test.go
+++ b/storage/s3/s3_test.go
@@ -73,6 +73,26 @@ func TestS3Integration(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer client.RemoveBucket(context.Background(), bucket)
+	uploads, _ := New(client, bucket, 6<<20)
+	httpClient := &http.Client{Timeout: 10 * time.Second}
+	storagetest.RunUploads(t, uploads, httpClient)
+	link, err := uploads.PresignPut(ctx, "constrained", 5, "text/plain", time.Minute)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, tc := range []struct{ body, contentType string }{{"longer", "text/plain"}, {"hello", "image/png"}} {
+		req, _ := http.NewRequestWithContext(ctx, link.Method, link.URL, strings.NewReader(tc.body))
+		req.Header = link.Headers.Clone()
+		req.Header.Set("Content-Type", tc.contentType)
+		response, requestErr := httpClient.Do(req)
+		if requestErr != nil {
+			t.Fatal(requestErr)
+		}
+		response.Body.Close()
+		if response.StatusCode != 403 {
+			t.Fatalf("tampered upload constraints accepted: %d", response.StatusCode)
+		}
+	}
 	s, _ := New(client, bucket, 64)
 	storagetest.Run(t, s)
 	s.Put(ctx, "signed", strings.NewReader("hello"), 5, "text/plain")
--- a/storage/s3/upload.go
+++ b/storage/s3/upload.go
@@ -0,0 +1,128 @@
+package s3
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"git.yuntuo.hk/k2safe/go-omni/storage"
+	"github.com/minio/minio-go/v7"
+)
+
+func (s *Store) PresignPut(ctx context.Context, key string, size int64, contentType string, ttl time.Duration) (storage.UploadRequest, error) {
+	if err := storage.CheckUpload(ctx, key, size, s.maxBytes, contentType, ttl); err != nil {
+		return storage.UploadRequest{}, err
+	}
+	if size > 5<<30 {
+		return storage.UploadRequest{}, storage.ErrTooLarge
+	}
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+	headers := http.Header{"Content-Length": {strconv.FormatInt(size, 10)}, "Content-Type": {contentType}}
+	return s.signUpload(ctx, key, nil, headers, ttl)
+}
+func (s *Store) signUpload(ctx context.Context, key string, query url.Values, headers http.Header, ttl time.Duration) (storage.UploadRequest, error) {
+	expires := time.Now().UTC().Add(ttl)
+	u, err := s.client.PresignHeader(ctx, http.MethodPut, s.bucket, key, ttl, query, headers)
+	if err != nil {
+		return storage.UploadRequest{}, err
+	}
+	signed := ";" + u.Query().Get("X-Amz-SignedHeaders") + ";"
+	for name := range headers {
+		if !strings.Contains(signed, ";"+strings.ToLower(name)+";") {
+			return storage.UploadRequest{}, errors.New("storage/s3: upload constraints were not signed (SigV4 required)")
+		}
+	}
+	return storage.UploadRequest{Method: http.MethodPut, URL: u.String(), Headers: headers, ExpiresAt: expires}, nil
+}
+func (s *Store) CreateMultipart(ctx context.Context, key, contentType string) (storage.MultipartUpload, error) {
+	if err := storage.Check(ctx, key); err != nil {
+		return storage.MultipartUpload{}, err
+	}
+	if _, err := storage.MediaType(contentType); err != nil {
+		return storage.MultipartUpload{}, err
+	}
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+	id, err := (minio.Core{Client: s.client}).NewMultipartUpload(ctx, s.bucket, key, minio.PutObjectOptions{ContentType: contentType})
+	if err != nil {
+		return storage.MultipartUpload{}, err
+	}
+	return storage.MultipartUpload{Key: key, UploadID: id}, nil
+}
+func (s *Store) UploadPart(ctx context.Context, upload storage.MultipartUpload, number int, body io.Reader, size int64) (storage.Part, error) {
+	if err := storage.CheckPart(ctx, upload, number, size, s.maxBytes); err != nil {
+		return storage.Part{}, err
+	}
+	reader, err := (storage.UploadPolicy{MaxBytes: s.maxBytes}).PrepareUpload(ctx, body, size, "")
+	if err != nil {
+		return storage.Part{}, err
+	}
+	part, err := (minio.Core{Client: s.client}).PutObjectPart(ctx, s.bucket, upload.Key, upload.UploadID, number, reader, size, minio.PutObjectPartOptions{})
+	if err != nil {
+		return storage.Part{}, err
+	}
+	return storage.Part{Number: number, ETag: part.ETag, Size: size}, nil
+}
+func (s *Store) PresignPart(ctx context.Context, upload storage.MultipartUpload, number int, size int64, ttl time.Duration) (storage.UploadRequest, error) {
+	if err := storage.CheckPart(ctx, upload, number, size, s.maxBytes); err != nil {
+		return storage.UploadRequest{}, err
+	}
+	if err := storage.CheckUpload(ctx, upload.Key, size, s.maxBytes, "", ttl); err != nil {
+		return storage.UploadRequest{}, err
+	}
+	return s.signUpload(ctx, upload.Key, url.Values{"uploadId": {upload.UploadID}, "partNumber": {strconv.Itoa(number)}}, http.Header{"Content-Length": {strconv.FormatInt(size, 10)}, "Content-Type": {"application/octet-stream"}}, ttl)
+}
+func (s *Store) ListParts(ctx context.Context, upload storage.MultipartUpload, after, limit int) (storage.PartsPage, error) {
+	if err := storage.CheckPartsPage(ctx, upload, after, limit); err != nil {
+		return storage.PartsPage{}, err
+	}
+	r, err := (minio.Core{Client: s.client}).ListObjectParts(ctx, s.bucket, upload.Key, upload.UploadID, after, limit)
+	if err != nil {
+		return storage.PartsPage{}, err
+	}
+	page := storage.PartsPage{Parts: make([]storage.Part, 0, len(r.ObjectParts))}
+	for _, p := range r.ObjectParts {
+		page.Parts = append(page.Parts, storage.Part{Number: p.PartNumber, ETag: p.ETag, Size: p.Size})
+	}
+	if r.IsTruncated {
+		page.NextMarker = r.NextPartNumberMarker
+	}
+	return page, nil
+}
+func (s *Store) CompleteMultipart(ctx context.Context, upload storage.MultipartUpload, parts []storage.Part) (storage.Info, error) {
+	size, err := storage.VerifyParts(ctx, s, upload, parts, s.maxBytes)
+	if err != nil {
+		return storage.Info{}, err
+	}
+	completed := make([]minio.CompletePart, len(parts))
+	for i, p := range parts {
+		completed[i] = minio.CompletePart{PartNumber: p.Number, ETag: strings.Trim(p.ETag, "\"")}
+	}
+	_, err = (minio.Core{Client: s.client}).CompleteMultipartUpload(ctx, s.bucket, upload.Key, upload.UploadID, completed, minio.PutObjectOptions{})
+	if err != nil {
+		return storage.Info{}, err
+	}
+	return storage.Info{Key: upload.Key, Size: size, ModifiedAt: time.Now().UTC()}, nil
+}
+func (s *Store) AbortMultipart(ctx context.Context, upload storage.MultipartUpload) error {
+	if err := storage.CheckMultipart(ctx, upload); err != nil {
+		return err
+	}
+	err := (minio.Core{Client: s.client}).AbortMultipartUpload(ctx, s.bucket, upload.Key, upload.UploadID)
+	if minio.ToErrorResponse(err).Code == "NoSuchUpload" {
+		return nil
+	}
+	return err
+}
+
+var _ storage.UploadPresigner = (*Store)(nil)
+var _ storage.MultipartStore = (*Store)(nil)
+var _ storage.PartPresigner = (*Store)(nil)
--- a/storage/s3/upload_test.go
+++ b/storage/s3/upload_test.go
@@ -0,0 +1,34 @@
+package s3
+
+import (
+	"context"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"git.yuntuo.hk/k2safe/go-omni/internal/storagefixture"
+	"git.yuntuo.hk/k2safe/go-omni/storage/storagetest"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+func TestUploadProtocol(t *testing.T) {
+	server := httptest.NewTLSServer(storagefixture.MultipartHandler())
+	defer server.Close()
+	s, err := Open(Config{Endpoint: strings.TrimPrefix(server.URL, "https://"), Region: "us-east-1", Bucket: "testbucket", Secure: true, MaxBytes: 6 << 20, Credentials: credentials.NewStaticV4("fixtureAccess", "fixtureSecret", ""), Transport: server.Client().Transport})
+	if err != nil {
+		t.Fatal(err)
+	}
+	storagetest.RunUploads(t, s, server.Client())
+}
+func TestR2UploadSigning(t *testing.T) {
+	cfg, _ := R2Config(strings.Repeat("a", 32), "testbucket", credentials.NewStaticV4("fixtureAccess", "fixtureSecret", ""))
+	s, err := Open(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r, err := s.PresignPut(context.Background(), "file", 5, "text/plain", time.Minute)
+	if err != nil || !strings.Contains(r.URL, "/auto/s3/") && !strings.Contains(r.URL, "%2Fauto%2Fs3%2F") {
+		t.Fatal("R2 upload signing", err)
+	}
+}