k2safe/orbit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: schedule persistent archive jobs with console management

Browse Source
This commit is contained in:
k2safe
2026-09-23 11:29:32 +08:00
parent 6eb429e21f
commit 0140c78ea7
32 changed files with 2077 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
internal/retention/job.go
View File

@@ -16,6 +16,12 @@ import (
core "git.yuntuo.hk/k2safe/orbit/model"
)
// ErrArchiveLimit 表示达到显式容量边界，源数据与已发布文件均保留。
var ErrArchiveLimit = errors.New("archive byte limit reached; source and saved progress retained")
// ErrArchiveBusy 表示原作业被另一个进程持锁，不允许并发写入。
var ErrArchiveBusy = errors.New("archive job is locked")
const maxJobBatches = 10000
const maxBatchFileBytes = 64 << 20
@@ -47,8 +53,11 @@ type JobBatch struct {
// JobOptions 控制本次进程的工作量，重试时可显式提高上限，不改变作业身份。
type JobOptions struct {
MaxBatches int // MaxBatches 本次最多新增批次数，1–1000。
MaxBytes int64 // MaxBytes 作业清单和全部批次文件总字节上限，1 MiB–100 GiB；不自动删除旧文件腾空间。
JobID core.ID // JobID 周期作业预分配身份；空值保持CLI自动生成行为。
RequireExisting bool // RequireExisting 接管时必须存在原清单，禁止卷丢失后重建。
Checkpoint *JobStatus // Checkpoint 数据库已确认的链前缀，校验后才允许继续导出。
MaxBatches int // MaxBatches 本次最多新增批次数，1–1000。
MaxBytes int64 // MaxBytes 作业清单和全部批次文件总字节上限，1 MiB–100 GiB；不自动删除旧文件腾空间。
}
// JobStatus 是可输出到日志的摘要，不包含业务正文或游标。
@@ -84,7 +93,10 @@ func RunArchive(ctx context.Context, directory string, spec JobSpec, options Job
if options.MaxBatches < 1 || options.MaxBatches > 1000 || options.MaxBytes < 1<<20 || options.MaxBytes > 100<<30 || export == nil {
return JobStatus{}, errors.New("invalid archive execution limits")
}
root, lock, err := openJob(directory, true)
if options.JobID != "" && options.JobID.Validate() != nil {
return JobStatus{}, errors.New("invalid archive job id")
}
root, lock, err := openJob(directory, !options.RequireExisting)
if err != nil {
return JobStatus{}, err
}
@@ -94,14 +106,20 @@ func RunArchive(ctx context.Context, directory string, spec JobSpec, options Job
return JobStatus{}, err
}
if _, err := root.Lstat("manifest.json"); errors.Is(err, os.ErrNotExist) {
if options.RequireExisting {
return JobStatus{}, errors.New("saved archive manifest is missing")
}
// 不在未知目录或缺少清单的残留批次上悄悄创建新身份。
entries, err := jobEntries(root)
if err != nil || len(entries) != 1 || entries[0].Name() != ".lock" {
return JobStatus{}, errors.New("archive directory is not empty or manifest is missing")
}
id, err := core.NewID()
if err != nil {
return JobStatus{}, err
id := options.JobID
if id == "" {
id, err = core.NewID()
if err != nil {
return JobStatus{}, err
}
}
manifest := JobManifest{1, id, time.Now().UTC(), spec}
raw, err := json.Marshal(manifest)
@@ -118,9 +136,23 @@ func RunArchive(ctx context.Context, directory string, spec JobSpec, options Job
if err != nil {
return status, err
}
if manifest.Spec != spec {
if manifest.Spec != spec || (options.JobID != "" && manifest.JobID != options.JobID) {
return status, errors.New("archive scope differs from saved manifest; resume with original arguments")
}
if saved := options.Checkpoint; saved != nil && saved.LastSHA256 != "" {
if saved.JobID != status.JobID || saved.Batches > status.Batches || saved.Roots > status.Roots || saved.Bytes > status.Bytes {
return status, errors.New("archive progress is behind database checkpoint")
}
name := "manifest.json"
if saved.Batches > 0 {
name = batchName(saved.Batches)
}
raw, err := readJobFile(root, name, maxBatchFileBytes)
if err != nil || digest(raw) != saved.LastSHA256 {
return status, errors.New("archive prefix differs from database checkpoint")
}
}
if err := syncJobDirectory(root); err != nil {
return status, err
}
@@ -145,7 +177,7 @@ func RunArchive(ctx context.Context, directory string, spec JobSpec, options Job
return status, err
}
if len(raw) > maxBatchFileBytes || int64(len(raw)) > options.MaxBytes-status.Bytes {
return status, errors.New("archive byte limit reached; source and saved progress retained")
return status, ErrArchiveLimit
}
if err := ctx.Err(); err != nil {
return status, err
@@ -273,7 +305,7 @@ func readJob(ctx context.Context, root *os.Root, maxBytes int64) (JobManifest, J
return fail(err)
}
if int64(len(raw)) > maxBytes-status.Bytes {
return fail(errors.New("archive byte limit reached"))
return fail(ErrArchiveLimit)
}
var packet JobBatch
if decodeJobJSON(raw, &packet) != nil {