feat: schedule persistent archive jobs with console management
This commit is contained in:
209
internal/retention/runtime.go
Normal file
209
internal/retention/runtime.go
Normal file
@@ -0,0 +1,209 @@
|
||||
package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
)
|
||||
|
||||
// RuntimeStore 只供受信任的归档执行器使用，控制台不能直接领取或确认租约。
|
||||
type RuntimeStore interface {
|
||||
ScheduleArchive(context.Context, core.ID) error
|
||||
ClaimArchive(context.Context, core.ID) (*ManagedJob, error)
|
||||
FinishArchive(context.Context, ManagedJob, JobStatus, string) error
|
||||
ExportRetentionForJob(context.Context, core.ID, core.ID, ArchiveInput) (ArchiveSnapshot, error)
|
||||
}
|
||||
|
||||
// ArchiveRuntime 固定一个部署卷，每次最多运行一批；所有共享该卷的节点串行占用文件锁。
|
||||
type ArchiveRuntime struct {
|
||||
Directory string // Directory 管理员配置的绝对路径，0700权限，不可使用容器临时层。
|
||||
StorageID core.ID // StorageID 卷内标记必须相同；不是共享存储的替代品。
|
||||
MaxBytes int64 // MaxBytes 整个卷内归档文件总字节上限，包含未完成作业。
|
||||
Store RuntimeStore // Store 持久化计划、领取和检查点。
|
||||
}
|
||||
|
||||
// Init 仅在空私有目录建立卷标记，已有文件缺少标记时拒绝猜测身份。
|
||||
func (r *ArchiveRuntime) Init() error {
|
||||
if r.StorageID.Validate() != nil || !filepath.IsAbs(r.Directory) || r.MaxBytes < 1<<20 || r.Store == nil {
|
||||
return errors.New("invalid archive runtime configuration")
|
||||
}
|
||||
|
||||
root, lock, err := openJob(r.Directory, true)
|
||||
for attempt := 0; errors.Is(err, ErrArchiveBusy) && attempt < 20; attempt++ {
|
||||
// 已有卷身份文件不可变；其他节点正在执行时，启动节点可以只读核对，
|
||||
// 不必等待整个文件批次释放卷锁。首次发布尚未结束时短暂等待。
|
||||
reader, readErr := os.OpenRoot(r.Directory)
|
||||
if readErr == nil {
|
||||
marker, e := readJobFile(reader, ".storage-id", 64)
|
||||
reader.Close()
|
||||
if e == nil {
|
||||
if string(marker) != string(r.StorageID) {
|
||||
return errors.New("archive volume identity mismatch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
root, lock, err = openJob(r.Directory, true)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer root.Close()
|
||||
defer lock.Close()
|
||||
if err := discardStaging(root); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := readJobFile(root, ".storage-id", 64)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
entries, e := jobEntries(root)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if len(entries) != 1 || entries[0].Name() != ".lock" {
|
||||
return errors.New("archive volume identity is missing")
|
||||
}
|
||||
return publishFile(root, ".storage-id", []byte(r.StorageID))
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if string(data) != string(r.StorageID) {
|
||||
return errors.New("archive volume identity mismatch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Tick 不在数据库事务中进行文件I/O；发布后确认失败时，由同一作业验证文件再收敛。
|
||||
func (r *ArchiveRuntime) Tick(ctx context.Context) error {
|
||||
root, lock, err := openJob(r.Directory, false)
|
||||
if errors.Is(err, ErrArchiveBusy) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer root.Close()
|
||||
defer lock.Close()
|
||||
marker, err := readJobFile(root, ".storage-id", 64)
|
||||
if err != nil || string(marker) != string(r.StorageID) {
|
||||
return errors.New("archive volume identity mismatch")
|
||||
}
|
||||
if err = r.Store.ScheduleArchive(ctx, r.StorageID); err != nil {
|
||||
return err
|
||||
}
|
||||
job, err := r.Store.ClaimArchive(ctx, r.StorageID)
|
||||
if err != nil || job == nil {
|
||||
return err
|
||||
}
|
||||
name := string(job.NamespaceID) + "-" + string(job.ID)
|
||||
files := 0
|
||||
total, current, err := archiveVolumeUsage(ctx, root, ".", name, &files)
|
||||
|
||||
code := ""
|
||||
var status JobStatus
|
||||
if err != nil {
|
||||
code = "FILE_CHECK_FAILED"
|
||||
if errors.Is(err, ErrArchiveLimit) {
|
||||
code = "STORAGE_LIMIT"
|
||||
}
|
||||
} else {
|
||||
budget := min(job.MaxBytes, r.MaxBytes-total+current)
|
||||
if budget < 1<<20 {
|
||||
code = "STORAGE_LIMIT"
|
||||
} else {
|
||||
checkpoint := JobStatus{JobID: job.ID, Batches: job.Batches, Roots: job.Roots, Bytes: job.Bytes, LastSHA256: job.LastSHA256}
|
||||
options := JobOptions{JobID: job.ID, RequireExisting: job.Attempts > 1, Checkpoint: &checkpoint, MaxBatches: 1, MaxBytes: budget}
|
||||
status, err = RunArchive(ctx, filepath.Join(r.Directory, name), JobSpec{job.NamespaceID, job.Kind, job.PolicyRevision, job.BatchSize}, options, func(ctx context.Context, id core.ID, in ArchiveInput) (ArchiveSnapshot, error) {
|
||||
out, err := r.Store.ExportRetentionForJob(ctx, job.NamespaceID, id, in)
|
||||
if err != nil {
|
||||
code = "EXPORT_FAILED"
|
||||
switch fault.Code(err) {
|
||||
case "STATE_CONFLICT":
|
||||
code = "POLICY_CHANGED"
|
||||
case "PERMISSION_DENIED":
|
||||
code = "PERMISSION_DENIED"
|
||||
}
|
||||
}
|
||||
return out, err
|
||||
})
|
||||
if err != nil && code == "" {
|
||||
code = "FILE_CHECK_FAILED"
|
||||
if errors.Is(err, ErrArchiveLimit) {
|
||||
code = "STORAGE_LIMIT"
|
||||
}
|
||||
if errors.Is(err, ErrArchiveBusy) {
|
||||
code = "STORAGE_BUSY"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// 关闭/超时不写成文件损坏；保留租约，下一节点在到期后从同一文件链接管。
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
finish, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
return r.Store.FinishArchive(finish, *job, status, code)
|
||||
}
|
||||
|
||||
// archiveVolumeUsage 分块读取目录项，避免 WalkDir 在大目录中先一次载入全部文件名。
|
||||
// 平台卷只有根目录和单层作业目录，意外嵌套或链接一律拒绝。
|
||||
func archiveVolumeUsage(ctx context.Context, root *os.Root, directory, currentJob string, count *int) (total, current int64, err error) {
|
||||
dir, err := root.Open(directory)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
defer dir.Close()
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
entries, readErr := dir.ReadDir(256)
|
||||
if readErr != nil && !errors.Is(readErr, io.EOF) {
|
||||
return 0, 0, readErr
|
||||
}
|
||||
for _, entry := range entries {
|
||||
*count++
|
||||
if *count > 100000 {
|
||||
return 0, 0, ErrArchiveLimit
|
||||
}
|
||||
if entry.Type()&fs.ModeSymlink != 0 {
|
||||
return 0, 0, errors.New("archive volume contains a symlink")
|
||||
}
|
||||
if entry.IsDir() {
|
||||
if directory != "." {
|
||||
return 0, 0, errors.New("archive volume contains nested directories")
|
||||
}
|
||||
n, j, err := archiveVolumeUsage(ctx, root, entry.Name(), currentJob, count)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
total += n
|
||||
current += j
|
||||
continue
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return 0, 0, errors.New("archive volume contains a special file")
|
||||
}
|
||||
total += info.Size()
|
||||
if directory == currentJob {
|
||||
current += info.Size()
|
||||
}
|
||||
}
|
||||
if errors.Is(readErr, io.EOF) {
|
||||
return total, current, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user