feat: add resumable file archive jobs and offline verification

2026-09-23 10:50:49 +08:00
parent 2d8467e50c
commit 6eb429e21f
19 changed files with 1160 additions and 15 deletions
--- a/internal/retention/job.go
+++ b/internal/retention/job.go
@@ -0,0 +1,413 @@
+package retention
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+
+	core "git.yuntuo.hk/k2safe/orbit/model"
+)
+
+const maxJobBatches = 10000
+const maxBatchFileBytes = 64 << 20
+
+// JobSpec 是首次落盘后不可变的作业范围，不包含凭据、DSN 或任意 SQL。
+type JobSpec struct {
+	NamespaceID    core.ID `json:"namespace_id"`           // NamespaceID 唯一命名空间。
+	Kind           string  `json:"kind"`                   // Kind 固定的历史类别。
+	PolicyRevision uint64  `json:"policy_revision,string"` // PolicyRevision 已在控制台确认的策略版本，必须非零。
+	BatchSize      int     `json:"batch_size"`             // BatchSize 每批根记录上限，1–100。
+}
+
+// JobManifest 为作业初始化清单；后续进度由连续且校验通过的批次文件重建。
+type JobManifest struct {
+	FormatVersion int       `json:"format_version"` // FormatVersion 当前为1。
+	JobID         core.ID   `json:"job_id"`         // JobID 随机且持久化的作业身份，也是数据库导出审计的执行者标识。
+	CreatedAt     time.Time `json:"created_at"`     // CreatedAt 清单创建时间，UTC，不作为数据库截止时间。
+	Spec          JobSpec   `json:"spec"`           // Spec 不可变范围，恢复时必须完全一致。
+}
+
+// JobBatch 是一个原子发布的归档批次，正文和摘要保持服务端原样。
+type JobBatch struct {
+	FormatVersion  int             `json:"format_version"`  // FormatVersion 当前为1。
+	JobID          core.ID         `json:"job_id"`          // JobID 与清单一致。
+	Sequence       int             `json:"sequence"`        // Sequence 从1开始的连续编号，缺批拒绝恢复。
+	PreviousSHA256 string          `json:"previous_sha256"` // PreviousSHA256 首批绑定清单文件摘要，后续绑定前一批文件摘要。
+	InputCursor    string          `json:"input_cursor"`    // InputCursor 此次导出使用的游标，首批为空。
+	Snapshot       ArchiveSnapshot `json:"snapshot"`        // Snapshot 包含原始正文与摘要。
+}
+
+// JobOptions 控制本次进程的工作量，重试时可显式提高上限，不改变作业身份。
+type JobOptions struct {
+	MaxBatches int   // MaxBatches 本次最多新增批次数，1–1000。
+	MaxBytes   int64 // MaxBytes 作业清单和全部批次文件总字节上限，1 MiB–100 GiB；不自动删除旧文件腾空间。
+}
+
+// JobStatus 是可输出到日志的摘要，不包含业务正文或游标。
+type JobStatus struct {
+	JobID      core.ID `json:"job_id"`      // JobID 原作业标识，恢复不会生成新标识。
+	Completed  bool    `json:"completed"`   // Completed 仅在末批成功持久化且游标为空时为真。
+	Batches    int     `json:"batches"`     // Batches 已校验的连续文件数。
+	Roots      int64   `json:"roots"`       // Roots 已归档根记录总数，不包含关联历史行。
+	Bytes      int64   `json:"bytes"`       // Bytes 清单及批次文件实际总字节，不包含临时文件和锁。
+	LastSHA256 string  `json:"last_sha256"` // LastSHA256 最新批次或初始清单的摘要，用于独立保存核对。
+}
+
+// ExportBatch 由运维调用方提供，数据库读取和授权仍由存储层完成。
+type ExportBatch func(context.Context, core.ID, ArchiveInput) (ArchiveSnapshot, error)
+
+func (s JobSpec) Validate() error {
+	if s.NamespaceID.Validate() != nil || s.PolicyRevision == 0 || s.BatchSize < 1 || s.BatchSize > 100 {
+		return errors.New("invalid archive namespace, revision or batch size")
+	}
+	switch s.Kind {
+	case "tasks", "workflows", "events", "audit", "deliveries", "inbox", "idempotency":
+		return nil
+	}
+	return errors.New("unsupported archive kind")
+}
+
+// RunArchive 自动处理有限批次，先验证全部已有文件再恢复。每个已发布文件就是检查点，
+// 不存在“文件成功但内存游标尚未落盘”的跨资源提交窗口；崩溃留下的临时文件不计为成功。
+func RunArchive(ctx context.Context, directory string, spec JobSpec, options JobOptions, export ExportBatch) (JobStatus, error) {
+	if err := spec.Validate(); err != nil {
+		return JobStatus{}, err
+	}
+	if options.MaxBatches < 1 || options.MaxBatches > 1000 || options.MaxBytes < 1<<20 || options.MaxBytes > 100<<30 || export == nil {
+		return JobStatus{}, errors.New("invalid archive execution limits")
+	}
+	root, lock, err := openJob(directory, true)
+	if err != nil {
+		return JobStatus{}, err
+	}
+	defer root.Close()
+	defer lock.Close()
+	if err := discardStaging(root); err != nil {
+		return JobStatus{}, err
+	}
+	if _, err := root.Lstat("manifest.json"); errors.Is(err, os.ErrNotExist) {
+		// 不在未知目录或缺少清单的残留批次上悄悄创建新身份。
+		entries, err := jobEntries(root)
+		if err != nil || len(entries) != 1 || entries[0].Name() != ".lock" {
+			return JobStatus{}, errors.New("archive directory is not empty or manifest is missing")
+		}
+		id, err := core.NewID()
+		if err != nil {
+			return JobStatus{}, err
+		}
+		manifest := JobManifest{1, id, time.Now().UTC(), spec}
+		raw, err := json.Marshal(manifest)
+		if err != nil {
+			return JobStatus{}, err
+		}
+		if err := publishFile(root, "manifest.json", raw); err != nil {
+			return JobStatus{}, err
+		}
+	} else if err != nil {
+		return JobStatus{}, err
+	}
+	manifest, status, cursor, cutoff, err := readJob(ctx, root, options.MaxBytes)
+	if err != nil {
+		return status, err
+	}
+	if manifest.Spec != spec {
+		return status, errors.New("archive scope differs from saved manifest; resume with original arguments")
+	}
+	if err := syncJobDirectory(root); err != nil {
+		return status, err
+	}
+	for i := 0; i < options.MaxBatches && !status.Completed; i++ {
+		if err := ctx.Err(); err != nil {
+			return status, err
+		}
+		if status.Batches >= maxJobBatches {
+			return status, errors.New("archive batch count limit reached")
+		}
+		snapshot, err := export(ctx, manifest.JobID, ArchiveInput{Kind: spec.Kind, Limit: spec.BatchSize, Cursor: cursor, ExpectedRevision: spec.PolicyRevision})
+		if err != nil {
+			return status, err
+		}
+		packet := JobBatch{1, manifest.JobID, status.Batches + 1, status.LastSHA256, cursor, snapshot}
+		next, err := verifyBatch(manifest, packet, status, cursor, cutoff)
+		if err != nil {
+			return status, err
+		}
+		raw, err := json.Marshal(packet)
+		if err != nil {
+			return status, err
+		}
+		if len(raw) > maxBatchFileBytes || int64(len(raw)) > options.MaxBytes-status.Bytes {
+			return status, errors.New("archive byte limit reached; source and saved progress retained")
+		}
+		if err := ctx.Err(); err != nil {
+			return status, err
+		}
+		if err := publishFile(root, batchName(packet.Sequence), raw); err != nil {
+			return status, err
+		}
+		// 重新读取原始字节核验，磁盘错误不能推进内存检查点或输出成功。
+		saved, err := readJobFile(root, batchName(packet.Sequence), maxBatchFileBytes)
+		if err != nil || !bytes.Equal(saved, raw) {
+			return status, errors.New("archive file readback failed; verify job before resuming")
+		}
+		advanceJob(&status, &cursor, &cutoff, next, raw)
+	}
+	return status, nil
+}
+
+// VerifyArchive 离线逐批校验整个作业，无需数据库或控制台会话；未到末批会返回 Completed=false。
+func VerifyArchive(ctx context.Context, directory string) (JobStatus, error) {
+	root, lock, err := openJob(directory, false)
+	if err != nil {
+		return JobStatus{}, err
+	}
+	defer root.Close()
+	defer lock.Close()
+	_, status, _, _, err := readJob(ctx, root, 100<<30)
+	return status, err
+}
+
+func openJob(directory string, create bool) (*os.Root, *os.File, error) {
+	if !filepath.IsAbs(directory) {
+		return nil, nil, errors.New("archive directory must be an absolute path")
+	}
+	if create {
+		if err := os.Mkdir(directory, 0700); err == nil {
+			parent, err := os.Open(filepath.Dir(directory))
+			if err != nil {
+				return nil, nil, err
+			}
+			err = parent.Sync()
+			parent.Close()
+			if err != nil {
+				return nil, nil, err
+			}
+		} else if !errors.Is(err, os.ErrExist) {
+			return nil, nil, err
+		}
+	}
+	info, err := os.Lstat(directory)
+	if err != nil {
+		return nil, nil, err
+	}
+	if !info.IsDir() || info.Mode().Perm()&0077 != 0 {
+		return nil, nil, errors.New("archive directory must be a private directory (0700), not a symlink")
+	}
+	root, err := os.OpenRoot(directory)
+	if err != nil {
+		return nil, nil, err
+	}
+	lock, err := lockJob(root)
+	if err != nil {
+		root.Close()
+		return nil, nil, err
+	}
+	return root, lock, nil
+}
+
+func jobEntries(root *os.Root) ([]os.DirEntry, error) {
+	dir, err := root.Open(".")
+	if err != nil {
+		return nil, err
+	}
+	defer dir.Close()
+	entries, err := dir.ReadDir(maxJobBatches + 4)
+	if err != nil && !errors.Is(err, io.EOF) {
+		return nil, err
+	}
+	if len(entries) >= maxJobBatches+4 {
+		return nil, errors.New("archive directory contains too many files")
+	}
+	return entries, nil
+}
+
+func readJob(ctx context.Context, root *os.Root, maxBytes int64) (JobManifest, JobStatus, string, time.Time, error) {
+	var manifest JobManifest
+	var status JobStatus
+	var cursor string
+	var cutoff time.Time
+	fail := func(err error) (JobManifest, JobStatus, string, time.Time, error) {
+		return manifest, status, cursor, cutoff, err
+	}
+	raw, err := readJobFile(root, "manifest.json", 65536)
+	if err != nil {
+		return fail(err)
+	}
+	if decodeJobJSON(raw, &manifest) != nil || manifest.FormatVersion != 1 || manifest.JobID.Validate() != nil || manifest.CreatedAt.IsZero() || manifest.Spec.Validate() != nil {
+		return fail(errors.New("invalid archive manifest"))
+	}
+	status.JobID, status.Bytes, status.LastSHA256 = manifest.JobID, int64(len(raw)), digest(raw)
+	entries, err := jobEntries(root)
+	if err != nil {
+		return fail(err)
+	}
+	files := make(map[string]bool, len(entries))
+	for _, e := range entries {
+		files[e.Name()] = true
+	}
+	delete(files, ".lock")
+	delete(files, "manifest.json")
+	// 临时文件不属于已发布链，离线校验不会删除它；恢复运行时再清理。
+	if files[".write.tmp"] {
+		if _, err := readJobFile(root, ".write.tmp", maxBatchFileBytes); err != nil {
+			return fail(err)
+		}
+		delete(files, ".write.tmp")
+	}
+	seen := map[string]bool{}
+	for files[batchName(status.Batches+1)] {
+		if err := ctx.Err(); err != nil {
+			return fail(err)
+		}
+		name := batchName(status.Batches + 1)
+		raw, err := readJobFile(root, name, maxBatchFileBytes)
+		if err != nil {
+			return fail(err)
+		}
+		if int64(len(raw)) > maxBytes-status.Bytes {
+			return fail(errors.New("archive byte limit reached"))
+		}
+		var packet JobBatch
+		if decodeJobJSON(raw, &packet) != nil {
+			return fail(errors.New("invalid archive batch JSON"))
+		}
+		next, err := verifyBatch(manifest, packet, status, cursor, cutoff)
+		if err != nil {
+			return fail(err)
+		}
+		if next.NextCursor != "" && seen[next.NextCursor] {
+			return fail(errors.New("archive cursor cycle detected"))
+		}
+		seen[next.NextCursor] = true
+		advanceJob(&status, &cursor, &cutoff, next, raw)
+		delete(files, name)
+	}
+	if len(files) != 0 {
+		return fail(errors.New("archive has missing batches or unexpected files"))
+	}
+	return manifest, status, cursor, cutoff, nil
+}
+
+type archivePage struct {
+	Policy         Policy    `json:"policy"`          // Policy 本批数据库策略。
+	Kind           string    `json:"kind"`            // Kind 本批根类别。
+	Cutoff         time.Time `json:"cutoff"`          // Cutoff 所有批次冻结的UTC边界。
+	RootCount      int       `json:"root_count"`      // RootCount 本批根数量。
+	SourceRetained bool      `json:"source_retained"` // SourceRetained 必须为真。
+	NextCursor     string    `json:"next_cursor"`     // NextCursor 空表示末批。
+}
+
+func verifyBatch(m JobManifest, b JobBatch, status JobStatus, cursor string, cutoff time.Time) (archivePage, error) {
+	var page archivePage
+	if status.Completed || b.FormatVersion != 1 || b.JobID != m.JobID || b.Sequence != status.Batches+1 || b.PreviousSHA256 != status.LastSHA256 || b.InputCursor != cursor || b.Snapshot.FormatVersion != 1 || len(b.Snapshot.Content) > 8<<20 || digest([]byte(b.Snapshot.Content)) != b.Snapshot.SHA256 {
+		return page, errors.New("archive batch identity or digest mismatch")
+	}
+	if json.Unmarshal([]byte(b.Snapshot.Content), &page) != nil || page.Policy.NamespaceID != m.Spec.NamespaceID || page.Policy.Revision != m.Spec.PolicyRevision || !page.Policy.Enabled || page.Kind != m.Spec.Kind || !page.SourceRetained || page.Cutoff.IsZero() || (!cutoff.IsZero() && !page.Cutoff.Equal(cutoff)) || page.RootCount < 0 || page.RootCount > m.Spec.BatchSize || len(page.NextCursor) > 2048 || (page.NextCursor != "" && (page.NextCursor == cursor || page.RootCount == 0)) {
+		return page, errors.New("archive batch scope or continuation mismatch")
+	}
+	return page, nil
+}
+
+func advanceJob(status *JobStatus, cursor *string, cutoff *time.Time, page archivePage, raw []byte) {
+	status.Batches++
+	status.Roots += int64(page.RootCount)
+	status.Bytes += int64(len(raw))
+	status.LastSHA256 = digest(raw)
+	status.Completed = page.NextCursor == ""
+	*cursor, *cutoff = page.NextCursor, page.Cutoff
+}
+
+func digest(raw []byte) string { sum := sha256.Sum256(raw); return hex.EncodeToString(sum[:]) }
+func batchName(n int) string   { return fmt.Sprintf("batch-%06d.json", n) }
+func decodeJobJSON(raw []byte, value any) error {
+	d := json.NewDecoder(bytes.NewReader(raw))
+	d.DisallowUnknownFields()
+	if err := d.Decode(value); err != nil {
+		return err
+	}
+	if err := d.Decode(new(any)); err != io.EOF {
+		return errors.New("trailing archive JSON")
+	}
+	return nil
+}
+
+func readJobFile(root *os.Root, name string, limit int64) ([]byte, error) {
+	info, err := root.Lstat(name)
+	if err != nil {
+		return nil, err
+	}
+	if !info.Mode().IsRegular() || info.Mode().Perm()&0077 != 0 || info.Size() > limit {
+		return nil, errors.New("archive file must be bounded, regular and private (0600)")
+	}
+	f, err := root.Open(name)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	raw, err := io.ReadAll(io.LimitReader(f, limit+1))
+	if err != nil {
+		return nil, err
+	}
+	if int64(len(raw)) > limit {
+		return nil, errors.New("archive file too large")
+	}
+	return raw, nil
+}
+
+func discardStaging(root *os.Root) error {
+	if _, err := root.Lstat(".write.tmp"); errors.Is(err, os.ErrNotExist) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+	if _, err := readJobFile(root, ".write.tmp", maxBatchFileBytes); err != nil {
+		return err
+	}
+	return root.Remove(".write.tmp")
+}
+
+// publishFile 先写私有临时文件并 fsync，再以硬链接原子发布（拒绝覆盖），最后同步目录。
+// 仅依赖本地 POSIX 文件系统语义；网络文件系统和多节点共享盘须独立验证。
+func publishFile(root *os.Root, name string, raw []byte) error {
+	f, err := root.OpenFile(".write.tmp", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
+	if err != nil {
+		return err
+	}
+	defer root.Remove(".write.tmp")
+	if _, err := f.Write(raw); err != nil {
+		f.Close()
+		return err
+	}
+	err = f.Sync()
+	closeErr := f.Close()
+	if err != nil {
+		return err
+	}
+	if closeErr != nil {
+		return closeErr
+	}
+	if err := root.Link(".write.tmp", name); err != nil {
+		return err
+	}
+	if err := root.Remove(".write.tmp"); err != nil {
+		return err
+	}
+	return syncJobDirectory(root)
+}
+
+func syncJobDirectory(root *os.Root) error {
+	dir, err := root.Open(".")
+	if err != nil {
+		return err
+	}
+	defer dir.Close()
+	return dir.Sync()
+}
--- a/internal/retention/job_lock_other.go
+++ b/internal/retention/job_lock_other.go
@@ -0,0 +1,12 @@
+//go:build !linux && !darwin
+
+package retention
+
+import (
+	"errors"
+	"os"
+)
+
+func lockJob(root *os.Root) (*os.File, error) {
+	return nil, errors.New("archive file jobs require Linux or macOS POSIX locking")
+}
--- a/internal/retention/job_lock_unix.go
+++ b/internal/retention/job_lock_unix.go
@@ -0,0 +1,27 @@
+//go:build linux || darwin
+
+package retention
+
+import (
+	"errors"
+	"os"
+	"syscall"
+)
+
+// lockJob 的非阻塞文件锁随进程退出自动释放，不使用永不过期的“锁目录”。
+func lockJob(root *os.Root) (*os.File, error) {
+	f, err := root.OpenFile(".lock", os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0600)
+	if err != nil {
+		return nil, err
+	}
+	info, err := f.Stat()
+	if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0077 != 0 {
+		f.Close()
+		return nil, errors.New("invalid archive lock file")
+	}
+	if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+		f.Close()
+		return nil, errors.New("archive job is locked or filesystem does not support locking")
+	}
+	return f, nil
+}
--- a/internal/retention/job_test.go
+++ b/internal/retention/job_test.go
@@ -0,0 +1,261 @@
+package retention
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	core "git.yuntuo.hk/k2safe/orbit/model"
+)
+
+func testJobSpec() JobSpec {
+	return JobSpec{NamespaceID: "11111111111111111111111111111111", Kind: "tasks", PolicyRevision: 1, BatchSize: 1}
+}
+func testJobOptions() JobOptions { return JobOptions{MaxBatches: 10, MaxBytes: 16 << 20} }
+func testJobSnapshot(t *testing.T, cursor string, payload int) ArchiveSnapshot {
+	t.Helper()
+	p := Defaults(testJobSpec().NamespaceID)
+	p.Enabled = true
+	p.Revision = 1
+	next := "next-page"
+	if cursor != "" {
+		next = ""
+	}
+	raw, err := json.Marshal(map[string]any{"policy": p, "kind": "tasks", "cutoff": time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), "root_count": 1, "source_retained": true, "next_cursor": next, "payload": strings.Repeat("x", payload)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return ArchiveSnapshot{1, digest(raw), string(raw)}
+}
+func testJobExporter(t *testing.T, calls *int) ExportBatch {
+	return func(ctx context.Context, job core.ID, in ArchiveInput) (ArchiveSnapshot, error) {
+		*calls++
+		if job.Validate() != nil || in.ExpectedRevision != 1 || in.Kind != "tasks" || in.Limit != 1 {
+			t.Fatal("export request changed identity")
+		}
+		return testJobSnapshot(t, in.Cursor, 0), nil
+	}
+}
+
+func TestArchiveJobResumeAndOfflineVerification(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "job")
+	calls := 0
+	options := testJobOptions()
+	options.MaxBatches = 1
+	first, err := RunArchive(context.Background(), dir, testJobSpec(), options, testJobExporter(t, &calls))
+	if err != nil || first.Completed || first.Roots != 1 || first.Batches != 1 {
+		t.Fatalf("first=%+v err=%v", first, err)
+	}
+	// 模拟强杀前只写了一部分临时文件；它不是有效检查点。
+	if err := os.WriteFile(filepath.Join(dir, ".write.tmp"), []byte(`{"unfinished":`), 0600); err != nil {
+		t.Fatal(err)
+	}
+	second, err := RunArchive(context.Background(), dir, testJobSpec(), options, testJobExporter(t, &calls))
+	if err != nil || !second.Completed || second.Roots != 2 || second.Batches != 2 || second.JobID != first.JobID || calls != 2 {
+		t.Fatalf("resume=%+v calls=%d err=%v", second, calls, err)
+	}
+	verified, err := VerifyArchive(context.Background(), dir)
+	if err != nil || verified != second {
+		t.Fatalf("verify=%+v err=%v", verified, err)
+	}
+	repeated, err := RunArchive(context.Background(), dir, testJobSpec(), options, testJobExporter(t, &calls))
+	if err != nil || repeated != second || calls != 2 {
+		t.Fatalf("completed job re-exported: %+v err=%v", repeated, err)
+	}
+	spec := testJobSpec()
+	spec.PolicyRevision++
+	if _, err := RunArchive(context.Background(), dir, spec, options, testJobExporter(t, &calls)); err == nil || calls != 2 {
+		t.Fatal("changed scope accepted")
+	}
+	for _, name := range []string{"manifest.json", "batch-000001.json", "batch-000002.json"} {
+		info, err := os.Stat(filepath.Join(dir, name))
+		if err != nil || info.Mode().Perm() != 0600 {
+			t.Fatalf("file permissions %s: %v", name, err)
+		}
+	}
+}
+
+func TestArchiveJobRejectsDamagedChain(t *testing.T) {
+	for _, kind := range []string{"body", "missing", "swapped", "symlink", "unexpected"} {
+		t.Run(kind, func(t *testing.T) {
+			dir := filepath.Join(t.TempDir(), "job")
+			calls := 0
+			if _, err := RunArchive(context.Background(), dir, testJobSpec(), testJobOptions(), testJobExporter(t, &calls)); err != nil {
+				t.Fatal(err)
+			}
+			first := filepath.Join(dir, "batch-000001.json")
+			second := filepath.Join(dir, "batch-000002.json")
+			var err error
+			switch kind {
+			case "body":
+				raw, _ := os.ReadFile(first)
+				var packet JobBatch
+				if err = json.Unmarshal(raw, &packet); err != nil {
+					t.Fatal(err)
+				}
+				packet.Snapshot.Content += " "
+				raw, _ = json.Marshal(packet)
+				err = os.WriteFile(first, raw, 0600)
+			case "missing":
+				err = os.Remove(first)
+			case "swapped":
+				raw, _ := os.ReadFile(second)
+				err = os.WriteFile(first, raw, 0600)
+			case "symlink":
+				if err = os.Remove(first); err == nil {
+					err = os.Symlink(second, first)
+				}
+			case "unexpected":
+				err = os.WriteFile(filepath.Join(dir, "unrelated.json"), []byte("{}"), 0600)
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			if _, err := VerifyArchive(context.Background(), dir); err == nil {
+				t.Fatal("damaged archive accepted")
+			}
+			if _, err := RunArchive(context.Background(), dir, testJobSpec(), testJobOptions(), testJobExporter(t, &calls)); err == nil || calls != 2 {
+				t.Fatalf("resume skipped corrupt history: calls=%d err=%v", calls, err)
+			}
+		})
+	}
+}
+
+func TestArchiveJobQuotaFailureAndCancellationPreserveProgress(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "quota")
+	options := testJobOptions()
+	options.MaxBytes = 1 << 20
+	calls := 0
+	export := func(ctx context.Context, id core.ID, in ArchiveInput) (ArchiveSnapshot, error) {
+		calls++
+		return testJobSnapshot(t, in.Cursor, 700<<10), nil
+	}
+	status, err := RunArchive(context.Background(), dir, testJobSpec(), options, export)
+	if err == nil || status.Completed || status.Batches != 1 {
+		t.Fatalf("quota=%+v err=%v", status, err)
+	}
+	first, _ := os.ReadFile(filepath.Join(dir, "batch-000001.json"))
+	options.MaxBytes = 4 << 20
+	final, err := RunArchive(context.Background(), dir, testJobSpec(), options, export)
+	if err != nil || !final.Completed || final.Batches != 2 || calls != 3 {
+		t.Fatalf("quota resume=%+v calls=%d err=%v", final, calls, err)
+	}
+	after, _ := os.ReadFile(filepath.Join(dir, "batch-000001.json"))
+	if string(first) != string(after) {
+		t.Fatal("prior file overwritten")
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	status, err = RunArchive(ctx, filepath.Join(t.TempDir(), "cancel"), testJobSpec(), testJobOptions(), func(ctx context.Context, id core.ID, in ArchiveInput) (ArchiveSnapshot, error) {
+		cancel()
+		return testJobSnapshot(t, "", 0), nil
+	})
+	if !errors.Is(err, context.Canceled) || status.Batches != 0 {
+		t.Fatalf("cancellation advanced progress: %+v %v", status, err)
+	}
+}
+
+func TestArchiveJobExclusiveLock(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "job")
+	root, lock, err := openJob(dir, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer root.Close()
+	defer lock.Close()
+	calls := 0
+	if _, err := RunArchive(context.Background(), dir, testJobSpec(), testJobOptions(), testJobExporter(t, &calls)); err == nil || calls != 0 {
+		t.Fatal("parallel writer accepted")
+	}
+}
+
+func TestArchiveJobProcessKillAndResume(t *testing.T) {
+	if dir := os.Getenv("ORBIT_ARCHIVE_KILL_TEST_DIR"); dir != "" {
+		_, err := RunArchive(context.Background(), dir, testJobSpec(), testJobOptions(), func(ctx context.Context, id core.ID, in ArchiveInput) (ArchiveSnapshot, error) {
+			if in.Cursor != "" {
+				os.Stdout.WriteString("first-batch-durable\n")
+				select {}
+			}
+			return testJobSnapshot(t, "", 0), nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		return
+	}
+	dir := filepath.Join(t.TempDir(), "job")
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestArchiveJobProcessKillAndResume$")
+	cmd.Env = append(os.Environ(), "ORBIT_ARCHIVE_KILL_TEST_DIR="+dir)
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cmd.Stderr = os.Stderr
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		if cmd.ProcessState == nil {
+			cmd.Process.Kill()
+			cmd.Wait()
+		}
+	}()
+	scanner := bufio.NewScanner(stdout)
+	if !scanner.Scan() || scanner.Text() != "first-batch-durable" {
+		t.Fatal("child did not publish first batch")
+	}
+	if err := cmd.Process.Kill(); err != nil {
+		t.Fatal(err)
+	}
+	if err := cmd.Wait(); err == nil {
+		t.Fatal("child was not killed")
+	}
+	// 等待旧进程真正退出后，新进程重新取得锁，只导出第二批。
+	calls := 0
+	status, err := RunArchive(context.Background(), dir, testJobSpec(), testJobOptions(), testJobExporter(t, &calls))
+	if err != nil || !status.Completed || status.Roots != 2 || status.Batches != 2 || calls != 1 {
+		t.Fatalf("killed-process recovery=%+v calls=%d err=%v", status, calls, err)
+	}
+}
+
+func TestArchiveJobWriteFailureDoesNotAdvance(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("permission failure requires an unprivileged process")
+	}
+	dir := filepath.Join(t.TempDir(), "job")
+	calls := 0
+	options := testJobOptions()
+	options.MaxBatches = 1
+	first, err := RunArchive(context.Background(), dir, testJobSpec(), options, testJobExporter(t, &calls))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chmod(dir, 0500); err != nil {
+		t.Fatal(err)
+	}
+	defer os.Chmod(dir, 0700)
+	after, err := RunArchive(context.Background(), dir, testJobSpec(), options, testJobExporter(t, &calls))
+	if err == nil || after != first {
+		t.Fatalf("failed write advanced checkpoint: %+v err=%v", after, err)
+	}
+	verified, err := VerifyArchive(context.Background(), dir)
+	if err != nil || verified != first {
+		t.Fatalf("saved archive changed: %+v err=%v", verified, err)
+	}
+	if err := os.Chmod(dir, 0700); err != nil {
+		t.Fatal(err)
+	}
+	final, err := RunArchive(context.Background(), dir, testJobSpec(), options, testJobExporter(t, &calls))
+	if err != nil || !final.Completed || final.Roots != 2 {
+		t.Fatalf("write recovery=%+v err=%v", final, err)
+	}
+}