210 lines
6.5 KiB
Go
210 lines
6.5 KiB
Go
package retention
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"io"
|
||
"io/fs"
|
||
"os"
|
||
"path/filepath"
|
||
"time"
|
||
|
||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||
)
|
||
|
||
// RuntimeStore 只供受信任的归档执行器使用，控制台不能直接领取或确认租约。
|
||
type RuntimeStore interface {
|
||
ScheduleArchive(context.Context, core.ID) error
|
||
ClaimArchive(context.Context, core.ID) (*ManagedJob, error)
|
||
FinishArchive(context.Context, ManagedJob, JobStatus, string) error
|
||
ExportRetentionForJob(context.Context, core.ID, core.ID, ArchiveInput) (ArchiveSnapshot, error)
|
||
}
|
||
|
||
// ArchiveRuntime 固定一个部署卷，每次最多运行一批；所有共享该卷的节点串行占用文件锁。
|
||
type ArchiveRuntime struct {
|
||
Directory string // Directory 管理员配置的绝对路径，0700权限，不可使用容器临时层。
|
||
StorageID core.ID // StorageID 卷内标记必须相同；不是共享存储的替代品。
|
||
MaxBytes int64 // MaxBytes 整个卷内归档文件总字节上限，包含未完成作业。
|
||
Store RuntimeStore // Store 持久化计划、领取和检查点。
|
||
}
|
||
|
||
// Init 仅在空私有目录建立卷标记，已有文件缺少标记时拒绝猜测身份。
|
||
func (r *ArchiveRuntime) Init() error {
|
||
if r.StorageID.Validate() != nil || !filepath.IsAbs(r.Directory) || r.MaxBytes < 1<<20 || r.Store == nil {
|
||
return errors.New("invalid archive runtime configuration")
|
||
}
|
||
|
||
root, lock, err := openJob(r.Directory, true)
|
||
for attempt := 0; errors.Is(err, ErrArchiveBusy) && attempt < 20; attempt++ {
|
||
// 已有卷身份文件不可变；其他节点正在执行时，启动节点可以只读核对，
|
||
// 不必等待整个文件批次释放卷锁。首次发布尚未结束时短暂等待。
|
||
reader, readErr := os.OpenRoot(r.Directory)
|
||
if readErr == nil {
|
||
marker, e := readJobFile(reader, ".storage-id", 64)
|
||
reader.Close()
|
||
if e == nil {
|
||
if string(marker) != string(r.StorageID) {
|
||
return errors.New("archive volume identity mismatch")
|
||
}
|
||
return nil
|
||
}
|
||
}
|
||
time.Sleep(100 * time.Millisecond)
|
||
root, lock, err = openJob(r.Directory, true)
|
||
}
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer root.Close()
|
||
defer lock.Close()
|
||
if err := discardStaging(root); err != nil {
|
||
return err
|
||
}
|
||
data, err := readJobFile(root, ".storage-id", 64)
|
||
if errors.Is(err, fs.ErrNotExist) {
|
||
entries, e := jobEntries(root)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
if len(entries) != 1 || entries[0].Name() != ".lock" {
|
||
return errors.New("archive volume identity is missing")
|
||
}
|
||
return publishFile(root, ".storage-id", []byte(r.StorageID))
|
||
}
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if string(data) != string(r.StorageID) {
|
||
return errors.New("archive volume identity mismatch")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Tick 不在数据库事务中进行文件I/O；发布后确认失败时，由同一作业验证文件再收敛。
|
||
func (r *ArchiveRuntime) Tick(ctx context.Context) error {
|
||
root, lock, err := openJob(r.Directory, false)
|
||
if errors.Is(err, ErrArchiveBusy) {
|
||
return nil
|
||
}
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer root.Close()
|
||
defer lock.Close()
|
||
marker, err := readJobFile(root, ".storage-id", 64)
|
||
if err != nil || string(marker) != string(r.StorageID) {
|
||
return errors.New("archive volume identity mismatch")
|
||
}
|
||
if err = r.Store.ScheduleArchive(ctx, r.StorageID); err != nil {
|
||
return err
|
||
}
|
||
job, err := r.Store.ClaimArchive(ctx, r.StorageID)
|
||
if err != nil || job == nil {
|
||
return err
|
||
}
|
||
name := string(job.NamespaceID) + "-" + string(job.ID)
|
||
files := 0
|
||
total, current, err := archiveVolumeUsage(ctx, root, ".", name, &files)
|
||
|
||
code := ""
|
||
var status JobStatus
|
||
if err != nil {
|
||
code = "FILE_CHECK_FAILED"
|
||
if errors.Is(err, ErrArchiveLimit) {
|
||
code = "STORAGE_LIMIT"
|
||
}
|
||
} else {
|
||
budget := min(job.MaxBytes, r.MaxBytes-total+current)
|
||
if budget < 1<<20 {
|
||
code = "STORAGE_LIMIT"
|
||
} else {
|
||
checkpoint := JobStatus{JobID: job.ID, Batches: job.Batches, Roots: job.Roots, Bytes: job.Bytes, LastSHA256: job.LastSHA256}
|
||
options := JobOptions{JobID: job.ID, RequireExisting: job.Attempts > 1, Checkpoint: &checkpoint, MaxBatches: 1, MaxBytes: budget}
|
||
status, err = RunArchive(ctx, filepath.Join(r.Directory, name), JobSpec{job.NamespaceID, job.Kind, job.PolicyRevision, job.BatchSize}, options, func(ctx context.Context, id core.ID, in ArchiveInput) (ArchiveSnapshot, error) {
|
||
out, err := r.Store.ExportRetentionForJob(ctx, job.NamespaceID, id, in)
|
||
if err != nil {
|
||
code = "EXPORT_FAILED"
|
||
switch fault.Code(err) {
|
||
case "STATE_CONFLICT":
|
||
code = "POLICY_CHANGED"
|
||
case "PERMISSION_DENIED":
|
||
code = "PERMISSION_DENIED"
|
||
}
|
||
}
|
||
return out, err
|
||
})
|
||
if err != nil && code == "" {
|
||
code = "FILE_CHECK_FAILED"
|
||
if errors.Is(err, ErrArchiveLimit) {
|
||
code = "STORAGE_LIMIT"
|
||
}
|
||
if errors.Is(err, ErrArchiveBusy) {
|
||
code = "STORAGE_BUSY"
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// 关闭/超时不写成文件损坏；保留租约，下一节点在到期后从同一文件链接管。
|
||
if ctx.Err() != nil {
|
||
return ctx.Err()
|
||
}
|
||
finish, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||
defer cancel()
|
||
return r.Store.FinishArchive(finish, *job, status, code)
|
||
}
|
||
|
||
// archiveVolumeUsage 分块读取目录项，避免 WalkDir 在大目录中先一次载入全部文件名。
|
||
// 平台卷只有根目录和单层作业目录，意外嵌套或链接一律拒绝。
|
||
func archiveVolumeUsage(ctx context.Context, root *os.Root, directory, currentJob string, count *int) (total, current int64, err error) {
|
||
dir, err := root.Open(directory)
|
||
if err != nil {
|
||
return 0, 0, err
|
||
}
|
||
defer dir.Close()
|
||
for {
|
||
if err := ctx.Err(); err != nil {
|
||
return 0, 0, err
|
||
}
|
||
entries, readErr := dir.ReadDir(256)
|
||
if readErr != nil && !errors.Is(readErr, io.EOF) {
|
||
return 0, 0, readErr
|
||
}
|
||
for _, entry := range entries {
|
||
*count++
|
||
if *count > 100000 {
|
||
return 0, 0, ErrArchiveLimit
|
||
}
|
||
if entry.Type()&fs.ModeSymlink != 0 {
|
||
return 0, 0, errors.New("archive volume contains a symlink")
|
||
}
|
||
if entry.IsDir() {
|
||
if directory != "." {
|
||
return 0, 0, errors.New("archive volume contains nested directories")
|
||
}
|
||
n, j, err := archiveVolumeUsage(ctx, root, entry.Name(), currentJob, count)
|
||
if err != nil {
|
||
return 0, 0, err
|
||
}
|
||
total += n
|
||
current += j
|
||
continue
|
||
}
|
||
info, err := entry.Info()
|
||
if err != nil {
|
||
return 0, 0, err
|
||
}
|
||
if !info.Mode().IsRegular() {
|
||
return 0, 0, errors.New("archive volume contains a special file")
|
||
}
|
||
total += info.Size()
|
||
if directory == currentJob {
|
||
current += info.Size()
|
||
}
|
||
}
|
||
if errors.Is(readErr, io.EOF) {
|
||
return total, current, nil
|
||
}
|
||
}
|
||
}
|