446 lines
16 KiB
Go
446 lines
16 KiB
Go
package retention
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"os"
|
||
"path/filepath"
|
||
"time"
|
||
|
||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||
)
|
||
|
||
// ErrArchiveLimit 表示达到显式容量边界，源数据与已发布文件均保留。
|
||
var ErrArchiveLimit = errors.New("archive byte limit reached; source and saved progress retained")
|
||
|
||
// ErrArchiveBusy 表示原作业被另一个进程持锁，不允许并发写入。
|
||
var ErrArchiveBusy = errors.New("archive job is locked")
|
||
|
||
const maxJobBatches = 10000
|
||
const maxBatchFileBytes = 64 << 20
|
||
|
||
// JobSpec 是首次落盘后不可变的作业范围，不包含凭据、DSN 或任意 SQL。
|
||
type JobSpec struct {
|
||
NamespaceID core.ID `json:"namespace_id"` // NamespaceID 唯一命名空间。
|
||
Kind string `json:"kind"` // Kind 固定的历史类别。
|
||
PolicyRevision uint64 `json:"policy_revision,string"` // PolicyRevision 已在控制台确认的策略版本，必须非零。
|
||
BatchSize int `json:"batch_size"` // BatchSize 每批根记录上限，1–100。
|
||
}
|
||
|
||
// JobManifest 为作业初始化清单；后续进度由连续且校验通过的批次文件重建。
|
||
type JobManifest struct {
|
||
FormatVersion int `json:"format_version"` // FormatVersion 当前为1。
|
||
JobID core.ID `json:"job_id"` // JobID 随机且持久化的作业身份，也是数据库导出审计的执行者标识。
|
||
CreatedAt time.Time `json:"created_at"` // CreatedAt 清单创建时间，UTC，不作为数据库截止时间。
|
||
Spec JobSpec `json:"spec"` // Spec 不可变范围，恢复时必须完全一致。
|
||
}
|
||
|
||
// JobBatch 是一个原子发布的归档批次，正文和摘要保持服务端原样。
|
||
type JobBatch struct {
|
||
FormatVersion int `json:"format_version"` // FormatVersion 当前为1。
|
||
JobID core.ID `json:"job_id"` // JobID 与清单一致。
|
||
Sequence int `json:"sequence"` // Sequence 从1开始的连续编号，缺批拒绝恢复。
|
||
PreviousSHA256 string `json:"previous_sha256"` // PreviousSHA256 首批绑定清单文件摘要，后续绑定前一批文件摘要。
|
||
InputCursor string `json:"input_cursor"` // InputCursor 此次导出使用的游标，首批为空。
|
||
Snapshot ArchiveSnapshot `json:"snapshot"` // Snapshot 包含原始正文与摘要。
|
||
}
|
||
|
||
// JobOptions 控制本次进程的工作量，重试时可显式提高上限，不改变作业身份。
|
||
type JobOptions struct {
|
||
JobID core.ID // JobID 周期作业预分配身份；空值保持CLI自动生成行为。
|
||
RequireExisting bool // RequireExisting 接管时必须存在原清单，禁止卷丢失后重建。
|
||
Checkpoint *JobStatus // Checkpoint 数据库已确认的链前缀，校验后才允许继续导出。
|
||
MaxBatches int // MaxBatches 本次最多新增批次数，1–1000。
|
||
MaxBytes int64 // MaxBytes 作业清单和全部批次文件总字节上限，1 MiB–100 GiB；不自动删除旧文件腾空间。
|
||
}
|
||
|
||
// JobStatus 是可输出到日志的摘要，不包含业务正文或游标。
|
||
type JobStatus struct {
|
||
JobID core.ID `json:"job_id"` // JobID 原作业标识，恢复不会生成新标识。
|
||
Completed bool `json:"completed"` // Completed 仅在末批成功持久化且游标为空时为真。
|
||
Batches int `json:"batches"` // Batches 已校验的连续文件数。
|
||
Roots int64 `json:"roots"` // Roots 已归档根记录总数，不包含关联历史行。
|
||
Bytes int64 `json:"bytes"` // Bytes 清单及批次文件实际总字节，不包含临时文件和锁。
|
||
LastSHA256 string `json:"last_sha256"` // LastSHA256 最新批次或初始清单的摘要，用于独立保存核对。
|
||
}
|
||
|
||
// ExportBatch 由运维调用方提供，数据库读取和授权仍由存储层完成。
|
||
type ExportBatch func(context.Context, core.ID, ArchiveInput) (ArchiveSnapshot, error)
|
||
|
||
func (s JobSpec) Validate() error {
|
||
if s.NamespaceID.Validate() != nil || s.PolicyRevision == 0 || s.BatchSize < 1 || s.BatchSize > 100 {
|
||
return errors.New("invalid archive namespace, revision or batch size")
|
||
}
|
||
switch s.Kind {
|
||
case "tasks", "workflows", "events", "audit", "deliveries", "inbox", "idempotency":
|
||
return nil
|
||
}
|
||
return errors.New("unsupported archive kind")
|
||
}
|
||
|
||
// RunArchive 自动处理有限批次，先验证全部已有文件再恢复。每个已发布文件就是检查点，
|
||
// 不存在“文件成功但内存游标尚未落盘”的跨资源提交窗口；崩溃留下的临时文件不计为成功。
|
||
func RunArchive(ctx context.Context, directory string, spec JobSpec, options JobOptions, export ExportBatch) (JobStatus, error) {
|
||
if err := spec.Validate(); err != nil {
|
||
return JobStatus{}, err
|
||
}
|
||
if options.MaxBatches < 1 || options.MaxBatches > 1000 || options.MaxBytes < 1<<20 || options.MaxBytes > 100<<30 || export == nil {
|
||
return JobStatus{}, errors.New("invalid archive execution limits")
|
||
}
|
||
if options.JobID != "" && options.JobID.Validate() != nil {
|
||
return JobStatus{}, errors.New("invalid archive job id")
|
||
}
|
||
root, lock, err := openJob(directory, !options.RequireExisting)
|
||
if err != nil {
|
||
return JobStatus{}, err
|
||
}
|
||
defer root.Close()
|
||
defer lock.Close()
|
||
if err := discardStaging(root); err != nil {
|
||
return JobStatus{}, err
|
||
}
|
||
if _, err := root.Lstat("manifest.json"); errors.Is(err, os.ErrNotExist) {
|
||
if options.RequireExisting {
|
||
return JobStatus{}, errors.New("saved archive manifest is missing")
|
||
}
|
||
// 不在未知目录或缺少清单的残留批次上悄悄创建新身份。
|
||
entries, err := jobEntries(root)
|
||
if err != nil || len(entries) != 1 || entries[0].Name() != ".lock" {
|
||
return JobStatus{}, errors.New("archive directory is not empty or manifest is missing")
|
||
}
|
||
id := options.JobID
|
||
if id == "" {
|
||
id, err = core.NewID()
|
||
if err != nil {
|
||
return JobStatus{}, err
|
||
}
|
||
}
|
||
manifest := JobManifest{1, id, time.Now().UTC(), spec}
|
||
raw, err := json.Marshal(manifest)
|
||
if err != nil {
|
||
return JobStatus{}, err
|
||
}
|
||
if err := publishFile(root, "manifest.json", raw); err != nil {
|
||
return JobStatus{}, err
|
||
}
|
||
} else if err != nil {
|
||
return JobStatus{}, err
|
||
}
|
||
manifest, status, cursor, cutoff, err := readJob(ctx, root, options.MaxBytes)
|
||
if err != nil {
|
||
return status, err
|
||
}
|
||
if manifest.Spec != spec || (options.JobID != "" && manifest.JobID != options.JobID) {
|
||
return status, errors.New("archive scope differs from saved manifest; resume with original arguments")
|
||
}
|
||
|
||
if saved := options.Checkpoint; saved != nil && saved.LastSHA256 != "" {
|
||
if saved.JobID != status.JobID || saved.Batches > status.Batches || saved.Roots > status.Roots || saved.Bytes > status.Bytes {
|
||
return status, errors.New("archive progress is behind database checkpoint")
|
||
}
|
||
name := "manifest.json"
|
||
if saved.Batches > 0 {
|
||
name = batchName(saved.Batches)
|
||
}
|
||
raw, err := readJobFile(root, name, maxBatchFileBytes)
|
||
if err != nil || digest(raw) != saved.LastSHA256 {
|
||
return status, errors.New("archive prefix differs from database checkpoint")
|
||
}
|
||
}
|
||
if err := syncJobDirectory(root); err != nil {
|
||
return status, err
|
||
}
|
||
for i := 0; i < options.MaxBatches && !status.Completed; i++ {
|
||
if err := ctx.Err(); err != nil {
|
||
return status, err
|
||
}
|
||
if status.Batches >= maxJobBatches {
|
||
return status, errors.New("archive batch count limit reached")
|
||
}
|
||
snapshot, err := export(ctx, manifest.JobID, ArchiveInput{Kind: spec.Kind, Limit: spec.BatchSize, Cursor: cursor, ExpectedRevision: spec.PolicyRevision})
|
||
if err != nil {
|
||
return status, err
|
||
}
|
||
packet := JobBatch{1, manifest.JobID, status.Batches + 1, status.LastSHA256, cursor, snapshot}
|
||
next, err := verifyBatch(manifest, packet, status, cursor, cutoff)
|
||
if err != nil {
|
||
return status, err
|
||
}
|
||
raw, err := json.Marshal(packet)
|
||
if err != nil {
|
||
return status, err
|
||
}
|
||
if len(raw) > maxBatchFileBytes || int64(len(raw)) > options.MaxBytes-status.Bytes {
|
||
return status, ErrArchiveLimit
|
||
}
|
||
if err := ctx.Err(); err != nil {
|
||
return status, err
|
||
}
|
||
if err := publishFile(root, batchName(packet.Sequence), raw); err != nil {
|
||
return status, err
|
||
}
|
||
// 重新读取原始字节核验，磁盘错误不能推进内存检查点或输出成功。
|
||
saved, err := readJobFile(root, batchName(packet.Sequence), maxBatchFileBytes)
|
||
if err != nil || !bytes.Equal(saved, raw) {
|
||
return status, errors.New("archive file readback failed; verify job before resuming")
|
||
}
|
||
advanceJob(&status, &cursor, &cutoff, next, raw)
|
||
}
|
||
return status, nil
|
||
}
|
||
|
||
// VerifyArchive 离线逐批校验整个作业，无需数据库或控制台会话；未到末批会返回 Completed=false。
|
||
func VerifyArchive(ctx context.Context, directory string) (JobStatus, error) {
|
||
root, lock, err := openJob(directory, false)
|
||
if err != nil {
|
||
return JobStatus{}, err
|
||
}
|
||
defer root.Close()
|
||
defer lock.Close()
|
||
_, status, _, _, err := readJob(ctx, root, 100<<30)
|
||
return status, err
|
||
}
|
||
|
||
func openJob(directory string, create bool) (*os.Root, *os.File, error) {
|
||
if !filepath.IsAbs(directory) {
|
||
return nil, nil, errors.New("archive directory must be an absolute path")
|
||
}
|
||
if create {
|
||
if err := os.Mkdir(directory, 0700); err == nil {
|
||
parent, err := os.Open(filepath.Dir(directory))
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
err = parent.Sync()
|
||
parent.Close()
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
} else if !errors.Is(err, os.ErrExist) {
|
||
return nil, nil, err
|
||
}
|
||
}
|
||
info, err := os.Lstat(directory)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
if !info.IsDir() || info.Mode().Perm()&0077 != 0 {
|
||
return nil, nil, errors.New("archive directory must be a private directory (0700), not a symlink")
|
||
}
|
||
root, err := os.OpenRoot(directory)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
lock, err := lockJob(root)
|
||
if err != nil {
|
||
root.Close()
|
||
return nil, nil, err
|
||
}
|
||
return root, lock, nil
|
||
}
|
||
|
||
func jobEntries(root *os.Root) ([]os.DirEntry, error) {
|
||
dir, err := root.Open(".")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer dir.Close()
|
||
entries, err := dir.ReadDir(maxJobBatches + 4)
|
||
if err != nil && !errors.Is(err, io.EOF) {
|
||
return nil, err
|
||
}
|
||
if len(entries) >= maxJobBatches+4 {
|
||
return nil, errors.New("archive directory contains too many files")
|
||
}
|
||
return entries, nil
|
||
}
|
||
|
||
func readJob(ctx context.Context, root *os.Root, maxBytes int64) (JobManifest, JobStatus, string, time.Time, error) {
|
||
var manifest JobManifest
|
||
var status JobStatus
|
||
var cursor string
|
||
var cutoff time.Time
|
||
fail := func(err error) (JobManifest, JobStatus, string, time.Time, error) {
|
||
return manifest, status, cursor, cutoff, err
|
||
}
|
||
raw, err := readJobFile(root, "manifest.json", 65536)
|
||
if err != nil {
|
||
return fail(err)
|
||
}
|
||
if decodeJobJSON(raw, &manifest) != nil || manifest.FormatVersion != 1 || manifest.JobID.Validate() != nil || manifest.CreatedAt.IsZero() || manifest.Spec.Validate() != nil {
|
||
return fail(errors.New("invalid archive manifest"))
|
||
}
|
||
status.JobID, status.Bytes, status.LastSHA256 = manifest.JobID, int64(len(raw)), digest(raw)
|
||
entries, err := jobEntries(root)
|
||
if err != nil {
|
||
return fail(err)
|
||
}
|
||
files := make(map[string]bool, len(entries))
|
||
for _, e := range entries {
|
||
files[e.Name()] = true
|
||
}
|
||
delete(files, ".lock")
|
||
delete(files, "manifest.json")
|
||
// 临时文件不属于已发布链，离线校验不会删除它；恢复运行时再清理。
|
||
if files[".write.tmp"] {
|
||
if _, err := readJobFile(root, ".write.tmp", maxBatchFileBytes); err != nil {
|
||
return fail(err)
|
||
}
|
||
delete(files, ".write.tmp")
|
||
}
|
||
seen := map[string]bool{}
|
||
for files[batchName(status.Batches+1)] {
|
||
if err := ctx.Err(); err != nil {
|
||
return fail(err)
|
||
}
|
||
name := batchName(status.Batches + 1)
|
||
raw, err := readJobFile(root, name, maxBatchFileBytes)
|
||
if err != nil {
|
||
return fail(err)
|
||
}
|
||
if int64(len(raw)) > maxBytes-status.Bytes {
|
||
return fail(ErrArchiveLimit)
|
||
}
|
||
var packet JobBatch
|
||
if decodeJobJSON(raw, &packet) != nil {
|
||
return fail(errors.New("invalid archive batch JSON"))
|
||
}
|
||
next, err := verifyBatch(manifest, packet, status, cursor, cutoff)
|
||
if err != nil {
|
||
return fail(err)
|
||
}
|
||
if next.NextCursor != "" && seen[next.NextCursor] {
|
||
return fail(errors.New("archive cursor cycle detected"))
|
||
}
|
||
seen[next.NextCursor] = true
|
||
advanceJob(&status, &cursor, &cutoff, next, raw)
|
||
delete(files, name)
|
||
}
|
||
if len(files) != 0 {
|
||
return fail(errors.New("archive has missing batches or unexpected files"))
|
||
}
|
||
return manifest, status, cursor, cutoff, nil
|
||
}
|
||
|
||
type archivePage struct {
|
||
Policy Policy `json:"policy"` // Policy 本批数据库策略。
|
||
Kind string `json:"kind"` // Kind 本批根类别。
|
||
Cutoff time.Time `json:"cutoff"` // Cutoff 所有批次冻结的UTC边界。
|
||
RootCount int `json:"root_count"` // RootCount 本批根数量。
|
||
SourceRetained bool `json:"source_retained"` // SourceRetained 必须为真。
|
||
NextCursor string `json:"next_cursor"` // NextCursor 空表示末批。
|
||
}
|
||
|
||
func verifyBatch(m JobManifest, b JobBatch, status JobStatus, cursor string, cutoff time.Time) (archivePage, error) {
|
||
var page archivePage
|
||
if status.Completed || b.FormatVersion != 1 || b.JobID != m.JobID || b.Sequence != status.Batches+1 || b.PreviousSHA256 != status.LastSHA256 || b.InputCursor != cursor || b.Snapshot.FormatVersion != 1 || len(b.Snapshot.Content) > 8<<20 || digest([]byte(b.Snapshot.Content)) != b.Snapshot.SHA256 {
|
||
return page, errors.New("archive batch identity or digest mismatch")
|
||
}
|
||
if json.Unmarshal([]byte(b.Snapshot.Content), &page) != nil || page.Policy.NamespaceID != m.Spec.NamespaceID || page.Policy.Revision != m.Spec.PolicyRevision || !page.Policy.Enabled || page.Kind != m.Spec.Kind || !page.SourceRetained || page.Cutoff.IsZero() || (!cutoff.IsZero() && !page.Cutoff.Equal(cutoff)) || page.RootCount < 0 || page.RootCount > m.Spec.BatchSize || len(page.NextCursor) > 2048 || (page.NextCursor != "" && (page.NextCursor == cursor || page.RootCount == 0)) {
|
||
return page, errors.New("archive batch scope or continuation mismatch")
|
||
}
|
||
return page, nil
|
||
}
|
||
|
||
func advanceJob(status *JobStatus, cursor *string, cutoff *time.Time, page archivePage, raw []byte) {
|
||
status.Batches++
|
||
status.Roots += int64(page.RootCount)
|
||
status.Bytes += int64(len(raw))
|
||
status.LastSHA256 = digest(raw)
|
||
status.Completed = page.NextCursor == ""
|
||
*cursor, *cutoff = page.NextCursor, page.Cutoff
|
||
}
|
||
|
||
func digest(raw []byte) string { sum := sha256.Sum256(raw); return hex.EncodeToString(sum[:]) }
|
||
func batchName(n int) string { return fmt.Sprintf("batch-%06d.json", n) }
|
||
func decodeJobJSON(raw []byte, value any) error {
|
||
d := json.NewDecoder(bytes.NewReader(raw))
|
||
d.DisallowUnknownFields()
|
||
if err := d.Decode(value); err != nil {
|
||
return err
|
||
}
|
||
if err := d.Decode(new(any)); err != io.EOF {
|
||
return errors.New("trailing archive JSON")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func readJobFile(root *os.Root, name string, limit int64) ([]byte, error) {
|
||
info, err := root.Lstat(name)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if !info.Mode().IsRegular() || info.Mode().Perm()&0077 != 0 || info.Size() > limit {
|
||
return nil, errors.New("archive file must be bounded, regular and private (0600)")
|
||
}
|
||
f, err := root.Open(name)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer f.Close()
|
||
raw, err := io.ReadAll(io.LimitReader(f, limit+1))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if int64(len(raw)) > limit {
|
||
return nil, errors.New("archive file too large")
|
||
}
|
||
return raw, nil
|
||
}
|
||
|
||
func discardStaging(root *os.Root) error {
|
||
if _, err := root.Lstat(".write.tmp"); errors.Is(err, os.ErrNotExist) {
|
||
return nil
|
||
} else if err != nil {
|
||
return err
|
||
}
|
||
if _, err := readJobFile(root, ".write.tmp", maxBatchFileBytes); err != nil {
|
||
return err
|
||
}
|
||
return root.Remove(".write.tmp")
|
||
}
|
||
|
||
// publishFile 先写私有临时文件并 fsync，再以硬链接原子发布（拒绝覆盖），最后同步目录。
|
||
// 仅依赖本地 POSIX 文件系统语义；网络文件系统和多节点共享盘须独立验证。
|
||
func publishFile(root *os.Root, name string, raw []byte) error {
|
||
f, err := root.OpenFile(".write.tmp", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer root.Remove(".write.tmp")
|
||
if _, err := f.Write(raw); err != nil {
|
||
f.Close()
|
||
return err
|
||
}
|
||
err = f.Sync()
|
||
closeErr := f.Close()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if closeErr != nil {
|
||
return closeErr
|
||
}
|
||
if err := root.Link(".write.tmp", name); err != nil {
|
||
return err
|
||
}
|
||
if err := root.Remove(".write.tmp"); err != nil {
|
||
return err
|
||
}
|
||
return syncJobDirectory(root)
|
||
}
|
||
|
||
func syncJobDirectory(root *os.Root) error {
|
||
dir, err := root.Open(".")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer dir.Close()
|
||
return dir.Sync()
|
||
}
|