feat: add resumable file archive jobs and offline verification

cmd/orbit-admin/main.go

@@ -27,7 +27,7 @@ func main() {
 }
 func run() error {
 	if len(os.Args) < 2 {
-		return errors.New("usage: orbit-admin release manifest|bootstrap|console-user create|verify|observe snapshot|retention inventory|credential status|rotate|revoke|apply")
+		return errors.New("usage: orbit-admin release manifest|bootstrap|console-user create|verify|observe snapshot|retention inventory|archive|verify-archive|credential status|rotate|revoke|apply")
 	}
 	switch os.Args[1] {
 	case "release":
@@ -45,7 +45,7 @@ func run() error {
 	case "credential":
 		return runCredential(os.Args[2:])
 	}
-	return errors.New("usage: orbit-admin release manifest|bootstrap|console-user create|verify|observe snapshot|retention inventory|credential status|rotate|revoke|apply")
+	return errors.New("usage: orbit-admin release manifest|bootstrap|console-user create|verify|observe snapshot|retention inventory|archive|verify-archive|credential status|rotate|revoke|apply")
 }
 
 // releaseManifest 是发布物自身的静态身份清单，不读取数据库、不包含 DSN、凭据或业务数据。
@@ -208,8 +208,16 @@ func runObserve(args []string) error {
 }
 
 func runRetention(args []string) error {
+	if len(args) > 0 {
+		switch args[0] {
+		case "archive":
+			return runRetentionArchive(args[1:])
+		case "verify-archive":
+			return runRetentionVerifyArchive(args[1:])
+		}
+	}
 	if len(args) == 0 || args[0] != "inventory" {
-		return errors.New("usage: orbit-admin retention inventory --before RFC3339")
+		return errors.New("usage: orbit-admin retention inventory --before RFC3339 | archive | verify-archive")
 	}
 	flags := flag.NewFlagSet("retention inventory", flag.ContinueOnError)
 	var before string

cmd/orbit-admin/retention.go

@@ -0,0 +1,108 @@
+package main
+
+import (
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"git.yuntuo.hk/k2safe/orbit/internal/config"
+	"git.yuntuo.hk/k2safe/orbit/internal/fault"
+	"git.yuntuo.hk/k2safe/orbit/internal/retention"
+	"git.yuntuo.hk/k2safe/orbit/internal/storage/mysql"
+	core "git.yuntuo.hk/k2safe/orbit/model"
+)
+
+func runRetentionArchive(args []string) error {
+	flags := flag.NewFlagSet("retention archive", flag.ContinueOnError)
+	var spec retention.JobSpec
+	var directory, namespace string
+	var options retention.JobOptions
+	var timeout time.Duration
+	flags.StringVar(&directory, "directory", os.Getenv("ORBIT_ARCHIVE_DIRECTORY"), "private per-job output directory; same directory resumes saved batches")
+	flags.StringVar(&namespace, "namespace", "", "namespace ID to archive")
+	flags.StringVar(&spec.Kind, "kind", "", "tasks|workflows|events|audit|deliveries|inbox|idempotency")
+	flags.Uint64Var(&spec.PolicyRevision, "revision", 0, "confirmed database retention policy revision")
+	flags.IntVar(&spec.BatchSize, "batch-size", 20, "roots per batch, 1-100; immutable for this job")
+	flags.IntVar(&options.MaxBatches, "max-batches", 10, "maximum new batches per invocation, 1-1000")
+	flags.Int64Var(&options.MaxBytes, "max-bytes", 256<<20, "total saved manifest and batch bytes, 1 MiB-100 GiB")
+	flags.DurationVar(&timeout, "timeout", 5*time.Minute, "invocation deadline, 1s-1h")
+	if err := flags.Parse(args); err != nil {
+		return err
+	}
+	spec.NamespaceID = core.ID(namespace)
+	if flags.NArg() != 0 || directory == "" || !filepath.IsAbs(directory) || timeout < time.Second || timeout > time.Hour || options.MaxBatches < 1 || options.MaxBatches > 1000 || options.MaxBytes < 1<<20 || options.MaxBytes > 100<<30 {
+		return errors.New("invalid archive arguments; use --help for limits and an absolute --directory")
+	}
+	if err := spec.Validate(); err != nil {
+		return err
+	}
+	cfg, err := config.Load()
+	if err != nil {
+		return err
+	}
+	signalCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	ctx, cancel := context.WithTimeout(signalCtx, timeout)
+	defer cancel()
+	db, err := mysql.Open(ctx, cfg.MySQLDSN, 2, 1)
+	if err != nil {
+		return errors.New("database unavailable")
+	}
+	defer db.Close()
+	if err := mysql.CheckSchema(ctx, db); err != nil {
+		return errors.New("database schema is not ready")
+	}
+	store := mysql.NewRetentionPolicyStore(db, cfg.RetentionDefaults)
+	status, runErr := retention.RunArchive(ctx, directory, spec, options, func(ctx context.Context, job core.ID, input retention.ArchiveInput) (retention.ArchiveSnapshot, error) {
+		snapshot, err := store.ExportRetentionForJob(ctx, spec.NamespaceID, job, input)
+		if err != nil {
+			// 数据库原始错误可能含 SQL 参数，只输出固定错误码，业务正文不进入控制台日志。
+			if ctx.Err() != nil {
+				return retention.ArchiveSnapshot{}, ctx.Err()
+			}
+			return retention.ArchiveSnapshot{}, fmt.Errorf("archive database export failed (%s); keep original directory and verify policy before retrying", fault.Code(err))
+		}
+		return snapshot, nil
+	})
+	if err := json.NewEncoder(os.Stdout).Encode(status); err != nil {
+		return err
+	}
+	return runErr
+}
+
+func runRetentionVerifyArchive(args []string) error {
+	flags := flag.NewFlagSet("retention verify-archive", flag.ContinueOnError)
+	var directory, expectedSHA string
+	flags.StringVar(&directory, "directory", os.Getenv("ORBIT_ARCHIVE_DIRECTORY"), "saved archive job directory; no database connection required")
+	flags.StringVar(&expectedSHA, "expected-sha256", "", "optional independently saved last_sha256 to pin the entire chain")
+	if err := flags.Parse(args); err != nil {
+		return err
+	}
+	if flags.NArg() != 0 || directory == "" {
+		return errors.New("--directory is required")
+	}
+	if expectedSHA != "" {
+		raw, err := hex.DecodeString(expectedSHA)
+		if err != nil || len(raw) != 32 || hex.EncodeToString(raw) != expectedSHA {
+			return errors.New("--expected-sha256 must be 64 lowercase hexadecimal characters")
+		}
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+	status, err := retention.VerifyArchive(ctx, directory)
+	if encodeErr := json.NewEncoder(os.Stdout).Encode(status); encodeErr != nil {
+		return encodeErr
+	}
+	if err == nil && expectedSHA != "" && status.LastSHA256 != expectedSHA {
+		return errors.New("archive differs from independently saved digest")
+	}
+	return err
+}

cmd/orbit-admin/retention_test.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"git.yuntuo.hk/k2safe/orbit/internal/retention"
+	core "git.yuntuo.hk/k2safe/orbit/model"
+)
+
+func TestRetentionArchiveValidationBeforeDatabase(t *testing.T) {
+	t.Setenv("ORBIT_MYSQL_DSN", "")
+	for _, args := range [][]string{
+		{}, {"--directory", "relative"}, {"--directory", t.TempDir(), "--namespace", "invalid"},
+		{"--directory", t.TempDir(), "--namespace", "11111111111111111111111111111111", "--kind", "tasks", "--revision", "1", "--max-batches", "0"},
+	} {
+		if err := runRetentionArchive(args); err == nil {
+			t.Fatalf("invalid args accepted: %v", args)
+		}
+	}
+}
+
+func TestVerifyArchiveOfflinePinsDigest(t *testing.T) {
+	t.Setenv("ORBIT_MYSQL_DSN", "")
+	dir := filepath.Join(t.TempDir(), "job")
+	ns := core.ID("11111111111111111111111111111111")
+	spec := retention.JobSpec{NamespaceID: ns, Kind: "tasks", PolicyRevision: 1, BatchSize: 1}
+	status, err := retention.RunArchive(context.Background(), dir, spec, retention.JobOptions{MaxBatches: 1, MaxBytes: 1 << 20}, func(ctx context.Context, id core.ID, input retention.ArchiveInput) (retention.ArchiveSnapshot, error) {
+		policy := retention.Defaults(ns)
+		policy.Enabled = true
+		policy.Revision = 1
+		raw, err := json.Marshal(map[string]any{"policy": policy, "kind": "tasks", "cutoff": time.Now().UTC(), "root_count": 0, "source_retained": true, "next_cursor": ""})
+		digest := sha256.Sum256(raw)
+		return retention.ArchiveSnapshot{FormatVersion: 1, SHA256: hex.EncodeToString(digest[:]), Content: string(raw)}, err
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := runRetentionVerifyArchive([]string{"--directory", dir, "--expected-sha256", status.LastSHA256}); err != nil {
+		t.Fatal(err)
+	}
+	if err := runRetentionVerifyArchive([]string{"--directory", dir, "--expected-sha256", "0000000000000000000000000000000000000000000000000000000000000000"}); err == nil {
+		t.Fatal("trusted digest mismatch accepted")
+	}
+}