feat: schedule persistent archive jobs with console management
This commit is contained in:
@@ -3,8 +3,10 @@ package config
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
@@ -12,6 +14,10 @@ import (
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
ArchiveRoot string // ArchiveRoot 私有持久归档卷绝对路径,空表示关闭后台执行器。
|
||||
ArchiveStorageID core.ID // ArchiveStorageID 32位小写十六进制卷身份,各节点须挂载同一物理卷。
|
||||
ArchiveMaxBytes int64 // ArchiveMaxBytes 全卷文件预算,默认10GiB。
|
||||
ArchiveTickInterval time.Duration // ArchiveTickInterval 后台扫描间隔,默认10秒。
|
||||
RetentionDefaults retention.Policy // RetentionDefaults 未保存数据库策略时使用的部署默认值。
|
||||
HTTPAddr string
|
||||
MySQLDSN string
|
||||
@@ -32,6 +38,7 @@ type Config struct {
|
||||
func Load() (Config, error) { return FromEnv(os.Getenv) }
|
||||
func FromEnv(get func(string) string) (Config, error) {
|
||||
c := Config{
|
||||
ArchiveRoot: get("ORBIT_ARCHIVE_ROOT"), ArchiveStorageID: core.ID(get("ORBIT_ARCHIVE_STORAGE_ID")), ArchiveMaxBytes: 10 << 30, ArchiveTickInterval: 10 * time.Second,
|
||||
HTTPAddr: get("ORBIT_HTTP_ADDR"),
|
||||
MySQLDSN: get("ORBIT_MYSQL_DSN"),
|
||||
MaxOpen: 20,
|
||||
@@ -123,6 +130,26 @@ func FromEnv(get func(string) string) (Config, error) {
|
||||
return Config{}, errors.New("invalid ORBIT_METRICS_ADDR")
|
||||
}
|
||||
}
|
||||
|
||||
if c.ArchiveRoot != "" || c.ArchiveStorageID != "" {
|
||||
if !filepath.IsAbs(c.ArchiveRoot) || c.ArchiveStorageID.Validate() != nil {
|
||||
return Config{}, errors.New("ORBIT_ARCHIVE_ROOT and ORBIT_ARCHIVE_STORAGE_ID must be configured together")
|
||||
}
|
||||
}
|
||||
if raw := get("ORBIT_ARCHIVE_MAX_BYTES"); raw != "" {
|
||||
n, err := strconv.ParseInt(raw, 10, 64)
|
||||
if err != nil || n < 1<<20 || n > 1<<40 {
|
||||
return Config{}, errors.New("ORBIT_ARCHIVE_MAX_BYTES must be 1MiB to 1TiB")
|
||||
}
|
||||
c.ArchiveMaxBytes = n
|
||||
}
|
||||
if raw := get("ORBIT_ARCHIVE_TICK_INTERVAL"); raw != "" {
|
||||
d, err := time.ParseDuration(raw)
|
||||
if err != nil || d < time.Second || d > time.Minute {
|
||||
return Config{}, errors.New("ORBIT_ARCHIVE_TICK_INTERVAL must be 1s to 1m")
|
||||
}
|
||||
c.ArchiveTickInterval = d
|
||||
}
|
||||
var err error
|
||||
c.RetentionDefaults, err = retention.FromEnv(get)
|
||||
if err != nil {
|
||||
|
||||
@@ -65,3 +65,28 @@ func TestMetricsAddressIsExplicitAndValidated(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveRuntimeRequiresBoundedPersistentVolumeConfig(t *testing.T) {
|
||||
base := map[string]string{"ORBIT_MYSQL_DSN": "test"}
|
||||
get := func(k string) string { return base[k] }
|
||||
cfg, err := FromEnv(get)
|
||||
if err != nil || cfg.ArchiveRoot != "" || cfg.ArchiveMaxBytes != 10<<30 {
|
||||
t.Fatal(cfg, err)
|
||||
}
|
||||
base["ORBIT_ARCHIVE_ROOT"] = "/private/archive"
|
||||
if _, err = FromEnv(get); err == nil {
|
||||
t.Fatal("missing volume identity accepted")
|
||||
}
|
||||
base["ORBIT_ARCHIVE_STORAGE_ID"] = "11111111111111111111111111111111"
|
||||
if _, err = FromEnv(get); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for k, v := range map[string]string{"ORBIT_ARCHIVE_MAX_BYTES": "0", "ORBIT_ARCHIVE_TICK_INTERVAL": "100ms", "ORBIT_ARCHIVE_ROOT": "relative", "ORBIT_ARCHIVE_STORAGE_ID": "not-an-id"} {
|
||||
old := base[k]
|
||||
base[k] = v
|
||||
if _, err = FromEnv(get); err == nil {
|
||||
t.Fatal("bad archive config", k)
|
||||
}
|
||||
base[k] = old
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,12 @@ import (
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
)
|
||||
|
||||
// ErrArchiveLimit 表示达到显式容量边界,源数据与已发布文件均保留。
|
||||
var ErrArchiveLimit = errors.New("archive byte limit reached; source and saved progress retained")
|
||||
|
||||
// ErrArchiveBusy 表示原作业被另一个进程持锁,不允许并发写入。
|
||||
var ErrArchiveBusy = errors.New("archive job is locked")
|
||||
|
||||
const maxJobBatches = 10000
|
||||
const maxBatchFileBytes = 64 << 20
|
||||
|
||||
@@ -47,8 +53,11 @@ type JobBatch struct {
|
||||
|
||||
// JobOptions 控制本次进程的工作量,重试时可显式提高上限,不改变作业身份。
|
||||
type JobOptions struct {
|
||||
MaxBatches int // MaxBatches 本次最多新增批次数,1–1000。
|
||||
MaxBytes int64 // MaxBytes 作业清单和全部批次文件总字节上限,1 MiB–100 GiB;不自动删除旧文件腾空间。
|
||||
JobID core.ID // JobID 周期作业预分配身份;空值保持CLI自动生成行为。
|
||||
RequireExisting bool // RequireExisting 接管时必须存在原清单,禁止卷丢失后重建。
|
||||
Checkpoint *JobStatus // Checkpoint 数据库已确认的链前缀,校验后才允许继续导出。
|
||||
MaxBatches int // MaxBatches 本次最多新增批次数,1–1000。
|
||||
MaxBytes int64 // MaxBytes 作业清单和全部批次文件总字节上限,1 MiB–100 GiB;不自动删除旧文件腾空间。
|
||||
}
|
||||
|
||||
// JobStatus 是可输出到日志的摘要,不包含业务正文或游标。
|
||||
@@ -84,7 +93,10 @@ func RunArchive(ctx context.Context, directory string, spec JobSpec, options Job
|
||||
if options.MaxBatches < 1 || options.MaxBatches > 1000 || options.MaxBytes < 1<<20 || options.MaxBytes > 100<<30 || export == nil {
|
||||
return JobStatus{}, errors.New("invalid archive execution limits")
|
||||
}
|
||||
root, lock, err := openJob(directory, true)
|
||||
if options.JobID != "" && options.JobID.Validate() != nil {
|
||||
return JobStatus{}, errors.New("invalid archive job id")
|
||||
}
|
||||
root, lock, err := openJob(directory, !options.RequireExisting)
|
||||
if err != nil {
|
||||
return JobStatus{}, err
|
||||
}
|
||||
@@ -94,14 +106,20 @@ func RunArchive(ctx context.Context, directory string, spec JobSpec, options Job
|
||||
return JobStatus{}, err
|
||||
}
|
||||
if _, err := root.Lstat("manifest.json"); errors.Is(err, os.ErrNotExist) {
|
||||
if options.RequireExisting {
|
||||
return JobStatus{}, errors.New("saved archive manifest is missing")
|
||||
}
|
||||
// 不在未知目录或缺少清单的残留批次上悄悄创建新身份。
|
||||
entries, err := jobEntries(root)
|
||||
if err != nil || len(entries) != 1 || entries[0].Name() != ".lock" {
|
||||
return JobStatus{}, errors.New("archive directory is not empty or manifest is missing")
|
||||
}
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
return JobStatus{}, err
|
||||
id := options.JobID
|
||||
if id == "" {
|
||||
id, err = core.NewID()
|
||||
if err != nil {
|
||||
return JobStatus{}, err
|
||||
}
|
||||
}
|
||||
manifest := JobManifest{1, id, time.Now().UTC(), spec}
|
||||
raw, err := json.Marshal(manifest)
|
||||
@@ -118,9 +136,23 @@ func RunArchive(ctx context.Context, directory string, spec JobSpec, options Job
|
||||
if err != nil {
|
||||
return status, err
|
||||
}
|
||||
if manifest.Spec != spec {
|
||||
if manifest.Spec != spec || (options.JobID != "" && manifest.JobID != options.JobID) {
|
||||
return status, errors.New("archive scope differs from saved manifest; resume with original arguments")
|
||||
}
|
||||
|
||||
if saved := options.Checkpoint; saved != nil && saved.LastSHA256 != "" {
|
||||
if saved.JobID != status.JobID || saved.Batches > status.Batches || saved.Roots > status.Roots || saved.Bytes > status.Bytes {
|
||||
return status, errors.New("archive progress is behind database checkpoint")
|
||||
}
|
||||
name := "manifest.json"
|
||||
if saved.Batches > 0 {
|
||||
name = batchName(saved.Batches)
|
||||
}
|
||||
raw, err := readJobFile(root, name, maxBatchFileBytes)
|
||||
if err != nil || digest(raw) != saved.LastSHA256 {
|
||||
return status, errors.New("archive prefix differs from database checkpoint")
|
||||
}
|
||||
}
|
||||
if err := syncJobDirectory(root); err != nil {
|
||||
return status, err
|
||||
}
|
||||
@@ -145,7 +177,7 @@ func RunArchive(ctx context.Context, directory string, spec JobSpec, options Job
|
||||
return status, err
|
||||
}
|
||||
if len(raw) > maxBatchFileBytes || int64(len(raw)) > options.MaxBytes-status.Bytes {
|
||||
return status, errors.New("archive byte limit reached; source and saved progress retained")
|
||||
return status, ErrArchiveLimit
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return status, err
|
||||
@@ -273,7 +305,7 @@ func readJob(ctx context.Context, root *os.Root, maxBytes int64) (JobManifest, J
|
||||
return fail(err)
|
||||
}
|
||||
if int64(len(raw)) > maxBytes-status.Bytes {
|
||||
return fail(errors.New("archive byte limit reached"))
|
||||
return fail(ErrArchiveLimit)
|
||||
}
|
||||
var packet JobBatch
|
||||
if decodeJobJSON(raw, &packet) != nil {
|
||||
|
||||
@@ -21,7 +21,10 @@ func lockJob(root *os.Root) (*os.File, error) {
|
||||
}
|
||||
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
|
||||
f.Close()
|
||||
return nil, errors.New("archive job is locked or filesystem does not support locking")
|
||||
if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) {
|
||||
return nil, ErrArchiveBusy
|
||||
}
|
||||
return nil, errors.New("archive filesystem does not support locking")
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
@@ -259,3 +259,33 @@ func TestArchiveJobWriteFailureDoesNotAdvance(t *testing.T) {
|
||||
t.Fatalf("write recovery=%+v err=%v", final, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedArchiveChecksDatabasePrefixAndMissingVolume(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := filepath.Join(t.TempDir(), "job")
|
||||
id, _ := core.NewID()
|
||||
calls := 0
|
||||
options := testJobOptions()
|
||||
options.JobID = id
|
||||
options.MaxBatches = 1
|
||||
first, err := RunArchive(ctx, dir, testJobSpec(), options, testJobExporter(t, &calls))
|
||||
if err != nil || first.JobID != id {
|
||||
t.Fatal(first, err)
|
||||
}
|
||||
options.Checkpoint = &first
|
||||
options.RequireExisting = true
|
||||
forged := first
|
||||
forged.LastSHA256 = strings.Repeat("0", 64)
|
||||
options.Checkpoint = &forged
|
||||
if _, err = RunArchive(ctx, dir, testJobSpec(), options, testJobExporter(t, &calls)); err == nil || calls != 1 {
|
||||
t.Fatal("untrusted prefix accepted")
|
||||
}
|
||||
options.Checkpoint = &first
|
||||
if _, err = RunArchive(ctx, filepath.Join(t.TempDir(), "missing"), testJobSpec(), options, testJobExporter(t, &calls)); err == nil || calls != 1 {
|
||||
t.Fatal("missing volume silently recreated")
|
||||
}
|
||||
second, err := RunArchive(ctx, dir, testJobSpec(), options, testJobExporter(t, &calls))
|
||||
if err != nil || !second.Completed || second.JobID != id {
|
||||
t.Fatal(second, err)
|
||||
}
|
||||
}
|
||||
|
||||
209
internal/retention/runtime.go
Normal file
209
internal/retention/runtime.go
Normal file
@@ -0,0 +1,209 @@
|
||||
package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
)
|
||||
|
||||
// RuntimeStore 只供受信任的归档执行器使用,控制台不能直接领取或确认租约。
|
||||
type RuntimeStore interface {
|
||||
ScheduleArchive(context.Context, core.ID) error
|
||||
ClaimArchive(context.Context, core.ID) (*ManagedJob, error)
|
||||
FinishArchive(context.Context, ManagedJob, JobStatus, string) error
|
||||
ExportRetentionForJob(context.Context, core.ID, core.ID, ArchiveInput) (ArchiveSnapshot, error)
|
||||
}
|
||||
|
||||
// ArchiveRuntime 固定一个部署卷,每次最多运行一批;所有共享该卷的节点串行占用文件锁。
|
||||
type ArchiveRuntime struct {
|
||||
Directory string // Directory 管理员配置的绝对路径,0700权限,不可使用容器临时层。
|
||||
StorageID core.ID // StorageID 卷内标记必须相同;不是共享存储的替代品。
|
||||
MaxBytes int64 // MaxBytes 整个卷内归档文件总字节上限,包含未完成作业。
|
||||
Store RuntimeStore // Store 持久化计划、领取和检查点。
|
||||
}
|
||||
|
||||
// Init 仅在空私有目录建立卷标记,已有文件缺少标记时拒绝猜测身份。
|
||||
func (r *ArchiveRuntime) Init() error {
|
||||
if r.StorageID.Validate() != nil || !filepath.IsAbs(r.Directory) || r.MaxBytes < 1<<20 || r.Store == nil {
|
||||
return errors.New("invalid archive runtime configuration")
|
||||
}
|
||||
|
||||
root, lock, err := openJob(r.Directory, true)
|
||||
for attempt := 0; errors.Is(err, ErrArchiveBusy) && attempt < 20; attempt++ {
|
||||
// 已有卷身份文件不可变;其他节点正在执行时,启动节点可以只读核对,
|
||||
// 不必等待整个文件批次释放卷锁。首次发布尚未结束时短暂等待。
|
||||
reader, readErr := os.OpenRoot(r.Directory)
|
||||
if readErr == nil {
|
||||
marker, e := readJobFile(reader, ".storage-id", 64)
|
||||
reader.Close()
|
||||
if e == nil {
|
||||
if string(marker) != string(r.StorageID) {
|
||||
return errors.New("archive volume identity mismatch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
root, lock, err = openJob(r.Directory, true)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer root.Close()
|
||||
defer lock.Close()
|
||||
if err := discardStaging(root); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := readJobFile(root, ".storage-id", 64)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
entries, e := jobEntries(root)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if len(entries) != 1 || entries[0].Name() != ".lock" {
|
||||
return errors.New("archive volume identity is missing")
|
||||
}
|
||||
return publishFile(root, ".storage-id", []byte(r.StorageID))
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if string(data) != string(r.StorageID) {
|
||||
return errors.New("archive volume identity mismatch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Tick 不在数据库事务中进行文件I/O;发布后确认失败时,由同一作业验证文件再收敛。
|
||||
func (r *ArchiveRuntime) Tick(ctx context.Context) error {
|
||||
root, lock, err := openJob(r.Directory, false)
|
||||
if errors.Is(err, ErrArchiveBusy) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer root.Close()
|
||||
defer lock.Close()
|
||||
marker, err := readJobFile(root, ".storage-id", 64)
|
||||
if err != nil || string(marker) != string(r.StorageID) {
|
||||
return errors.New("archive volume identity mismatch")
|
||||
}
|
||||
if err = r.Store.ScheduleArchive(ctx, r.StorageID); err != nil {
|
||||
return err
|
||||
}
|
||||
job, err := r.Store.ClaimArchive(ctx, r.StorageID)
|
||||
if err != nil || job == nil {
|
||||
return err
|
||||
}
|
||||
name := string(job.NamespaceID) + "-" + string(job.ID)
|
||||
files := 0
|
||||
total, current, err := archiveVolumeUsage(ctx, root, ".", name, &files)
|
||||
|
||||
code := ""
|
||||
var status JobStatus
|
||||
if err != nil {
|
||||
code = "FILE_CHECK_FAILED"
|
||||
if errors.Is(err, ErrArchiveLimit) {
|
||||
code = "STORAGE_LIMIT"
|
||||
}
|
||||
} else {
|
||||
budget := min(job.MaxBytes, r.MaxBytes-total+current)
|
||||
if budget < 1<<20 {
|
||||
code = "STORAGE_LIMIT"
|
||||
} else {
|
||||
checkpoint := JobStatus{JobID: job.ID, Batches: job.Batches, Roots: job.Roots, Bytes: job.Bytes, LastSHA256: job.LastSHA256}
|
||||
options := JobOptions{JobID: job.ID, RequireExisting: job.Attempts > 1, Checkpoint: &checkpoint, MaxBatches: 1, MaxBytes: budget}
|
||||
status, err = RunArchive(ctx, filepath.Join(r.Directory, name), JobSpec{job.NamespaceID, job.Kind, job.PolicyRevision, job.BatchSize}, options, func(ctx context.Context, id core.ID, in ArchiveInput) (ArchiveSnapshot, error) {
|
||||
out, err := r.Store.ExportRetentionForJob(ctx, job.NamespaceID, id, in)
|
||||
if err != nil {
|
||||
code = "EXPORT_FAILED"
|
||||
switch fault.Code(err) {
|
||||
case "STATE_CONFLICT":
|
||||
code = "POLICY_CHANGED"
|
||||
case "PERMISSION_DENIED":
|
||||
code = "PERMISSION_DENIED"
|
||||
}
|
||||
}
|
||||
return out, err
|
||||
})
|
||||
if err != nil && code == "" {
|
||||
code = "FILE_CHECK_FAILED"
|
||||
if errors.Is(err, ErrArchiveLimit) {
|
||||
code = "STORAGE_LIMIT"
|
||||
}
|
||||
if errors.Is(err, ErrArchiveBusy) {
|
||||
code = "STORAGE_BUSY"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// 关闭/超时不写成文件损坏;保留租约,下一节点在到期后从同一文件链接管。
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
finish, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
return r.Store.FinishArchive(finish, *job, status, code)
|
||||
}
|
||||
|
||||
// archiveVolumeUsage 分块读取目录项,避免 WalkDir 在大目录中先一次载入全部文件名。
|
||||
// 平台卷只有根目录和单层作业目录,意外嵌套或链接一律拒绝。
|
||||
func archiveVolumeUsage(ctx context.Context, root *os.Root, directory, currentJob string, count *int) (total, current int64, err error) {
|
||||
dir, err := root.Open(directory)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
defer dir.Close()
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
entries, readErr := dir.ReadDir(256)
|
||||
if readErr != nil && !errors.Is(readErr, io.EOF) {
|
||||
return 0, 0, readErr
|
||||
}
|
||||
for _, entry := range entries {
|
||||
*count++
|
||||
if *count > 100000 {
|
||||
return 0, 0, ErrArchiveLimit
|
||||
}
|
||||
if entry.Type()&fs.ModeSymlink != 0 {
|
||||
return 0, 0, errors.New("archive volume contains a symlink")
|
||||
}
|
||||
if entry.IsDir() {
|
||||
if directory != "." {
|
||||
return 0, 0, errors.New("archive volume contains nested directories")
|
||||
}
|
||||
n, j, err := archiveVolumeUsage(ctx, root, entry.Name(), currentJob, count)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
total += n
|
||||
current += j
|
||||
continue
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return 0, 0, errors.New("archive volume contains a special file")
|
||||
}
|
||||
total += info.Size()
|
||||
if directory == currentJob {
|
||||
current += info.Size()
|
||||
}
|
||||
}
|
||||
if errors.Is(readErr, io.EOF) {
|
||||
return total, current, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
96
internal/retention/runtime_test.go
Normal file
96
internal/retention/runtime_test.go
Normal file
@@ -0,0 +1,96 @@
|
||||
package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
)
|
||||
|
||||
type idleArchiveStore struct {
|
||||
RuntimeStore
|
||||
scheduled int
|
||||
}
|
||||
|
||||
func (s *idleArchiveStore) ScheduleArchive(context.Context, core.ID) error { s.scheduled++; return nil }
|
||||
func (s *idleArchiveStore) ClaimArchive(context.Context, core.ID) (*ManagedJob, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestArchiveVolumeIdentityLockAndInitializationRecovery(t *testing.T) {
|
||||
store := &idleArchiveStore{}
|
||||
id, _ := core.NewID()
|
||||
runtime := &ArchiveRuntime{Directory: filepath.Join(t.TempDir(), "volume"), StorageID: id, MaxBytes: 8 << 20, Store: store}
|
||||
if err := runtime.Init(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root, lock, err := openJob(runtime.Directory, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 其他节点正在执行时,新节点仍可读取不可变卷身份并正常启动。
|
||||
if err = runtime.Init(); err != nil {
|
||||
t.Fatal("another node could not initialize", err)
|
||||
}
|
||||
other := *runtime
|
||||
other.StorageID, _ = core.NewID()
|
||||
if err = other.Init(); err == nil {
|
||||
t.Fatal("wrong volume accepted while busy")
|
||||
}
|
||||
if err = runtime.Tick(context.Background()); err != nil || store.scheduled != 0 {
|
||||
t.Fatal("busy volume scheduled work", err)
|
||||
}
|
||||
lock.Close()
|
||||
root.Close()
|
||||
if err = runtime.Tick(context.Background()); err != nil || store.scheduled != 1 {
|
||||
t.Fatal("unlocked volume did not schedule", err)
|
||||
}
|
||||
if err = other.Init(); err == nil {
|
||||
t.Fatal("wrong volume accepted")
|
||||
}
|
||||
// 首次身份文件原子发布前崩溃,只允许丢弃本工具的安全临时文件。
|
||||
fresh := *runtime
|
||||
fresh.Directory = filepath.Join(t.TempDir(), "interrupted")
|
||||
if err = os.Mkdir(fresh.Directory, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.WriteFile(filepath.Join(fresh.Directory, ".write.tmp"), []byte(id), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = fresh.Init(); err != nil {
|
||||
t.Fatal("volume init did not recover", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveVolumeScanCountsAllJobsAndRejectsLinks(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
job := "job"
|
||||
if err := os.Mkdir(filepath.Join(dir, job), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "other"), []byte("volume"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, job, "batch"), []byte("job"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root, err := os.OpenRoot(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer root.Close()
|
||||
count := 0
|
||||
total, current, err := archiveVolumeUsage(context.Background(), root, ".", job, &count)
|
||||
if err != nil || total != 9 || current != 3 || count != 3 {
|
||||
t.Fatal(total, current, count, err)
|
||||
}
|
||||
if err = os.Symlink("other", filepath.Join(dir, "unexpected")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
count = 0
|
||||
if _, _, err = archiveVolumeUsage(context.Background(), root, ".", job, &count); err == nil {
|
||||
t.Fatal("volume symlink accepted")
|
||||
}
|
||||
}
|
||||
101
internal/retention/scheduler.go
Normal file
101
internal/retention/scheduler.go
Normal file
@@ -0,0 +1,101 @@
|
||||
package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
)
|
||||
|
||||
// ArchiveKinds 是周期归档支持的固定类别,不接受任意表名或 SQL。
|
||||
var ArchiveKinds = [...]string{"tasks", "workflows", "events", "audit", "deliveries", "inbox", "idempotency"}
|
||||
|
||||
// ArchivePlan 是命名空间内单一类别的周期计划;失败作业阻止本类别再生成新作业。
|
||||
type ArchivePlan struct {
|
||||
NamespaceID core.ID `json:"namespace_id"` // NamespaceID 由登录会话确定。
|
||||
Kind string `json:"kind"` // Kind 同一命名空间内唯一的归档类别。
|
||||
Revision uint64 `json:"revision,string"` // Revision 配置版本;尚未保存时为0,后台触发不改变配置版本。
|
||||
Enabled bool `json:"enabled"` // Enabled 仅控制新作业触发,关闭不会中断已有作业。
|
||||
PolicyRevision uint64 `json:"policy_revision,string"` // PolicyRevision 管理员确认的保留策略版本,变更策略后须重新保存计划。
|
||||
StorageID core.ID `json:"storage_id"` // StorageID 固定归档卷身份,由部署配置提供,不接受浏览器任意路径。
|
||||
IntervalSeconds uint32 `json:"interval_seconds"` // IntervalSeconds 周期秒数,3600至2592000;错过多次只合并触发一次。
|
||||
BatchSize int `json:"batch_size"` // BatchSize 单批根记录数,1至100。
|
||||
MaxBytes int64 `json:"max_bytes"` // MaxBytes 每个作业的文件字节上限,1 MiB至1 GiB。
|
||||
NextRunAt *time.Time `json:"next_run_at"` // NextRunAt 下次触发UTC时间,未保存时为空;首次启用立即到期。
|
||||
ActiveJobID *core.ID `json:"active_job_id"` // ActiveJobID 未完成或待处理失败作业;成功或人工关闭后清空。
|
||||
UpdatedAt *time.Time `json:"updated_at"` // UpdatedAt 最后配置修改UTC时间,未保存为空。
|
||||
}
|
||||
|
||||
// PlanInput 完整替换计划配置;保存与管理审计、幂等回执同事务。
|
||||
type PlanInput struct {
|
||||
ExpectedRevision uint64 `json:"expected_revision,string" required:"true"` // ExpectedRevision 已读取的计划版本。
|
||||
PolicyRevision uint64 `json:"policy_revision,string" required:"true"` // PolicyRevision 已确认的保留策略版本。
|
||||
Enabled bool `json:"enabled" required:"true"` // Enabled 是否开始周期触发。
|
||||
IntervalSeconds uint32 `json:"interval_seconds" required:"true"` // IntervalSeconds 周期秒数。
|
||||
BatchSize int `json:"batch_size" required:"true"` // BatchSize 每批根记录数。
|
||||
MaxBytes int64 `json:"max_bytes" required:"true"` // MaxBytes 单作业总文件预算,字节。
|
||||
Reason string `json:"reason" required:"true"` // Reason 单行操作原因,最多1024个UTF-8字节。
|
||||
}
|
||||
|
||||
// ManagedJob 保存周期作业权威状态;文件内容和游标不通过管理列表返回。
|
||||
type ManagedJob struct {
|
||||
ID core.ID `json:"id"` // ID 与目录名及文件清单作业ID一致。
|
||||
NamespaceID core.ID `json:"namespace_id"` // NamespaceID 所属命名空间。
|
||||
Kind string `json:"kind"` // Kind 固定归档类别。
|
||||
PlanRevision uint64 `json:"plan_revision,string"` // PlanRevision 创建时固定的计划配置版本。
|
||||
PolicyRevision uint64 `json:"policy_revision,string"` // PolicyRevision 作业期间不可替换的保留策略版本。
|
||||
StorageID core.ID `json:"storage_id"` // StorageID 固定归档卷身份,接管节点必须挂载同一物理卷。
|
||||
BatchSize int `json:"batch_size"` // BatchSize 创建时固定的单批根记录上限。
|
||||
MaxBytes int64 `json:"max_bytes"` // MaxBytes 累计字节预算,仅失败重试时允许提高。
|
||||
State string `json:"state"` // State 为PENDING/RUNNING/SUCCEEDED/FAILED/ABANDONED。
|
||||
Revision uint64 `json:"revision,string"` // Revision 每次领取、确认、人工操作递增,用于防止迟到覆盖。
|
||||
Attempts uint32 `json:"attempts"` // Attempts 累计批次领取次数,包含崩溃接管,不等于业务任务重试次数。
|
||||
Batches int `json:"batches"` // Batches 已验证并入库的连续批次数。
|
||||
Roots int64 `json:"roots"` // Roots 已验证根记录总数。
|
||||
Bytes int64 `json:"bytes"` // Bytes 已验证清单和批次总字节数。
|
||||
LastSHA256 string `json:"last_sha256"` // LastSHA256 数据库确认的文件链尾摘要;恢复必须验证此前缀。
|
||||
ErrorCode string `json:"error_code"` // ErrorCode 固定错误分类,不保存原始路径、业务正文或数据库错误。
|
||||
LeaseID core.ID `json:"-"` // LeaseID 当前领取随机代次,不向浏览器返回。
|
||||
LeaseUntil *time.Time `json:"lease_until"` // LeaseUntil UTC租约截止,非执行中为空;过期可接管。
|
||||
CreatedAt time.Time `json:"created_at"` // CreatedAt 作业受理UTC时间。
|
||||
UpdatedAt time.Time `json:"updated_at"` // UpdatedAt 最近状态变更UTC时间。
|
||||
}
|
||||
|
||||
// JobActionInput 是失败作业的人工恢复或关闭输入,必须留有原因。
|
||||
type JobActionInput struct {
|
||||
ExpectedRevision uint64 `json:"expected_revision,string" required:"true"` // ExpectedRevision 已读取的作业版本。
|
||||
Action string `json:"action" required:"true"` // Action 仅retry或abandon;后者保留已有文件与审计。
|
||||
MaxBytes int64 `json:"max_bytes" required:"true"` // MaxBytes 重试可提高预算,关闭时保持原值。
|
||||
Reason string `json:"reason" required:"true"` // Reason 人工操作原因。
|
||||
}
|
||||
|
||||
// JobPage 采用已返回作业ID作为翻页位置;位置必须属于当前命名空间。
|
||||
type JobPage struct {
|
||||
Items []ManagedJob `json:"items"` // Items 本页最多20个作业。
|
||||
NextBefore string `json:"next_before"` // NextBefore 下一页位置;空表示当前已到末页。
|
||||
}
|
||||
|
||||
// SchedulerStore 将控制台管理与后台执行的持久事务统一到存储层。
|
||||
type SchedulerStore interface {
|
||||
ListArchivePlans(context.Context, auth.Token) ([]ArchivePlan, error)
|
||||
SaveArchivePlan(context.Context, auth.Token, string, string, core.ID, PlanInput) (ArchivePlan, error)
|
||||
ListArchiveJobs(context.Context, auth.Token, string) (JobPage, error)
|
||||
ActArchiveJob(context.Context, auth.Token, core.ID, string, JobActionInput) (ManagedJob, error)
|
||||
}
|
||||
|
||||
func ValidReason(s string) bool {
|
||||
return strings.TrimSpace(s) != "" && utf8.ValidString(s) && len(s) <= 1024 && !strings.ContainsAny(s, "\r\n")
|
||||
}
|
||||
func (in PlanInput) Validate(namespace core.ID, kind string) error {
|
||||
if err := (JobSpec{namespace, kind, in.PolicyRevision, in.BatchSize}).Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if in.IntervalSeconds < 3600 || in.IntervalSeconds > 2592000 || in.MaxBytes < 1<<20 || in.MaxBytes > 1<<30 || !ValidReason(in.Reason) {
|
||||
return errors.New("invalid archive interval, byte budget or reason")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -94,7 +94,7 @@ func TestIntegrationApplicationResourcesUpgrade(t *testing.T) {
|
||||
f.exec(`ALTER TABLE pulse_worker_sessions DROP COLUMN concurrency_keys`)
|
||||
// 新增的控制台和保留策略迁移也要还原到历史 v9,避免升级测试把
|
||||
// 已经存在的 v29/v30 表误当成迁移重复执行。
|
||||
f.exec(`DROP TABLE orbit_retention_policies,orbit_console_sessions,orbit_console_users`)
|
||||
f.exec(`DROP TABLE orbit_archive_jobs,orbit_archive_plans,orbit_retention_policies,orbit_console_sessions,orbit_console_users`)
|
||||
f.exec(`DELETE FROM orbit_schema_migrations WHERE version>=10`)
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
441
internal/storage/mysql/archive_scheduler.go
Normal file
441
internal/storage/mysql/archive_scheduler.go
Normal file
@@ -0,0 +1,441 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/retention"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
)
|
||||
|
||||
const archivePlanColumns = `namespace_id,kind,revision,enabled,policy_revision,storage_id,interval_seconds,batch_size,max_bytes,next_run_at,active_job_id,updated_at`
|
||||
const archiveJobColumns = `namespace_id,id,kind,plan_revision,policy_revision,storage_id,batch_size,max_bytes,state,revision,attempts,batches,roots,bytes,last_sha256,error_code,lease_id,lease_until,created_at,updated_at`
|
||||
|
||||
type archiveScanner interface{ Scan(...any) error }
|
||||
|
||||
func scanArchivePlan(row archiveScanner) (p retention.ArchivePlan, err error) {
|
||||
err = row.Scan(&p.NamespaceID, &p.Kind, &p.Revision, &p.Enabled, &p.PolicyRevision, &p.StorageID, &p.IntervalSeconds, &p.BatchSize, &p.MaxBytes, &p.NextRunAt, &p.ActiveJobID, &p.UpdatedAt)
|
||||
return
|
||||
}
|
||||
func scanArchiveJob(row archiveScanner) (j retention.ManagedJob, err error) {
|
||||
var lease *core.ID
|
||||
err = row.Scan(&j.NamespaceID, &j.ID, &j.Kind, &j.PlanRevision, &j.PolicyRevision, &j.StorageID, &j.BatchSize, &j.MaxBytes, &j.State, &j.Revision, &j.Attempts, &j.Batches, &j.Roots, &j.Bytes, &j.LastSHA256, &j.ErrorCode, &lease, &j.LeaseUntil, &j.CreatedAt, &j.UpdatedAt)
|
||||
if lease != nil {
|
||||
j.LeaseID = *lease
|
||||
}
|
||||
return
|
||||
}
|
||||
func readArchivePlan(ctx context.Context, tx *sql.Tx, ns core.ID, kind string) (retention.ArchivePlan, error) {
|
||||
p, err := scanArchivePlan(tx.QueryRowContext(ctx, `SELECT `+archivePlanColumns+` FROM orbit_archive_plans WHERE namespace_id=? AND kind=? FOR UPDATE`, ns, kind))
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return retention.ArchivePlan{NamespaceID: ns, Kind: kind, IntervalSeconds: 86400, BatchSize: 20, MaxBytes: 256 << 20}, nil
|
||||
}
|
||||
return p, err
|
||||
}
|
||||
func readArchiveJob(ctx context.Context, tx *sql.Tx, ns, id core.ID) (retention.ManagedJob, error) {
|
||||
j, err := scanArchiveJob(tx.QueryRowContext(ctx, `SELECT `+archiveJobColumns+` FROM orbit_archive_jobs WHERE namespace_id=? AND id=? FOR UPDATE`, ns, id))
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
err = fault.NotFound
|
||||
}
|
||||
return j, err
|
||||
}
|
||||
func archiveAudit(ctx context.Context, tx *sql.Tx, ns, resource, op core.ID, actor, action, kind, reason string, before, after any) error {
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := json.Marshal(before)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a, err := json.Marshal(after)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, "INSERT INTO orbit_audit_events(namespace_id,id,operation_id,actor_id,action,resource_kind,resource_id,reason,`before`,`after`) VALUES (?,?,?,?,?,?,?,?,?,?)", ns, id, op, actor, action, kind, resource, reason, string(b), string(a))
|
||||
return err
|
||||
}
|
||||
func archiveKey(key string) error {
|
||||
if strings.TrimSpace(key) == "" || len(key) > 128 {
|
||||
return fault.Invalid("idempotency key must be 1-128 bytes")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *RetentionPolicyStore) ListArchivePlans(ctx context.Context, t auth.Token) (out []retention.ArchivePlan, err error) {
|
||||
err = WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
||||
if _, err := retentionActor(ctx, tx, t); err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, `SELECT `+archivePlanColumns+` FROM orbit_archive_plans WHERE namespace_id=? ORDER BY kind`, t.NamespaceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
found := map[string]retention.ArchivePlan{}
|
||||
for rows.Next() {
|
||||
p, err := scanArchivePlan(rows)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found[p.Kind] = p
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
out = []retention.ArchivePlan{}
|
||||
for _, kind := range retention.ArchiveKinds {
|
||||
p, ok := found[kind]
|
||||
if !ok {
|
||||
p = retention.ArchivePlan{NamespaceID: t.NamespaceID, Kind: kind, IntervalSeconds: 86400, BatchSize: 20, MaxBytes: 256 << 20}
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return
|
||||
}
|
||||
func (s *RetentionPolicyStore) SaveArchivePlan(ctx context.Context, t auth.Token, key, kind string, storage core.ID, in retention.PlanInput) (out retention.ArchivePlan, err error) {
|
||||
if err = in.Validate(t.NamespaceID, kind); err != nil {
|
||||
return out, fault.Invalid(err.Error())
|
||||
}
|
||||
if err = archiveKey(key); err != nil {
|
||||
return
|
||||
}
|
||||
// 卷身份属于幂等请求摘要;跨不同卷的节点不能把同一保存解释成不同目标。
|
||||
input := struct {
|
||||
Kind string
|
||||
Storage core.ID
|
||||
Input retention.PlanInput
|
||||
}{kind, storage, in}
|
||||
err = WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
||||
user, err := retentionActor(ctx, tx, t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "retention.plan.update", kind, key, "archive_plan", t.NamespaceID, input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cached != nil {
|
||||
return json.Unmarshal(cached, &out)
|
||||
}
|
||||
policy, err := s.readPolicy(ctx, tx, t.NamespaceID, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if policy.Revision != in.PolicyRevision || (in.Enabled && !policy.Enabled) {
|
||||
return fault.Conflict
|
||||
}
|
||||
before, err := readArchivePlan(ctx, tx, t.NamespaceID, kind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if before.Revision != in.ExpectedRevision {
|
||||
return fault.Conflict
|
||||
}
|
||||
target := storage
|
||||
if target == "" && !in.Enabled {
|
||||
target = before.StorageID
|
||||
}
|
||||
if target.Validate() != nil {
|
||||
return fault.Invalid("archive storage is not configured")
|
||||
}
|
||||
if before.ActiveJobID != nil && before.StorageID != target {
|
||||
return fault.Conflict
|
||||
}
|
||||
if before.Revision == 0 {
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO orbit_archive_plans(namespace_id,kind,enabled,policy_revision,storage_id,interval_seconds,batch_size,max_bytes,next_run_at) VALUES(?,?,?,?,?,?,?,?,UTC_TIMESTAMP(6))`, t.NamespaceID, kind, in.Enabled, in.PolicyRevision, target, in.IntervalSeconds, in.BatchSize, in.MaxBytes)
|
||||
} else {
|
||||
_, err = tx.ExecContext(ctx, `UPDATE orbit_archive_plans SET revision=revision+1,next_run_at=IF(enabled=FALSE AND ?=TRUE,UTC_TIMESTAMP(6),next_run_at),enabled=?,policy_revision=?,storage_id=?,interval_seconds=?,batch_size=?,max_bytes=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND kind=?`, in.Enabled, in.Enabled, in.PolicyRevision, target, in.IntervalSeconds, in.BatchSize, in.MaxBytes, t.NamespaceID, kind)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out, err = readArchivePlan(ctx, tx, t.NamespaceID, kind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = archiveAudit(ctx, tx, t.NamespaceID, t.NamespaceID, identity, "console:"+string(user), "retention.plan.update", "archive_plan", in.Reason, before, out); err != nil {
|
||||
return err
|
||||
}
|
||||
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
|
||||
})
|
||||
return
|
||||
}
|
||||
func (s *RetentionPolicyStore) ListArchiveJobs(ctx context.Context, t auth.Token, before string) (out retention.JobPage, err error) {
|
||||
if before != "" && core.ID(before).Validate() != nil {
|
||||
return out, fault.Invalid("invalid archive page position")
|
||||
}
|
||||
err = WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
||||
if _, err := retentionActor(ctx, tx, t); err != nil {
|
||||
return err
|
||||
}
|
||||
q := `SELECT ` + archiveJobColumns + ` FROM orbit_archive_jobs WHERE namespace_id=?`
|
||||
args := []any{t.NamespaceID}
|
||||
if before != "" {
|
||||
var at time.Time
|
||||
if err := tx.QueryRowContext(ctx, `SELECT created_at FROM orbit_archive_jobs WHERE namespace_id=? AND id=?`, t.NamespaceID, core.ID(before)).Scan(&at); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return fault.NotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
q += ` AND (created_at<? OR (created_at=? AND id<?))`
|
||||
args = append(args, at, at, core.ID(before))
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, q+` ORDER BY created_at DESC,id DESC LIMIT 21`, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
out = retention.JobPage{Items: []retention.ManagedJob{}}
|
||||
for rows.Next() {
|
||||
j, err := scanArchiveJob(rows)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out.Items = append(out.Items, j)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(out.Items) > 20 {
|
||||
out.Items = out.Items[:20]
|
||||
out.NextBefore = string(out.Items[19].ID)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return
|
||||
}
|
||||
func (s *RetentionPolicyStore) ActArchiveJob(ctx context.Context, t auth.Token, id core.ID, key string, in retention.JobActionInput) (out retention.ManagedJob, err error) {
|
||||
if id.Validate() != nil || (in.Action != "retry" && in.Action != "abandon") || !retention.ValidReason(in.Reason) || in.MaxBytes < 1<<20 || in.MaxBytes > 1<<30 {
|
||||
return out, fault.Invalid("invalid archive action, budget or reason")
|
||||
}
|
||||
if err = archiveKey(key); err != nil {
|
||||
return
|
||||
}
|
||||
err = WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
||||
user, err := retentionActor(ctx, tx, t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "retention.job.action", string(id), key, "archive_job", id, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cached != nil {
|
||||
return json.Unmarshal(cached, &out)
|
||||
}
|
||||
var kind string
|
||||
if err := tx.QueryRowContext(ctx, `SELECT kind FROM orbit_archive_jobs WHERE namespace_id=? AND id=?`, t.NamespaceID, id).Scan(&kind); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return fault.NotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
p, err := readArchivePlan(ctx, tx, t.NamespaceID, kind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
j, err := readArchiveJob(ctx, tx, t.NamespaceID, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if p.ActiveJobID == nil || *p.ActiveJobID != id || j.State != "FAILED" || j.Revision != in.ExpectedRevision || in.MaxBytes < j.MaxBytes {
|
||||
return fault.Conflict
|
||||
}
|
||||
next := "PENDING"
|
||||
if in.Action == "abandon" {
|
||||
next = "ABANDONED"
|
||||
if in.MaxBytes != j.MaxBytes {
|
||||
return fault.Conflict
|
||||
}
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE orbit_archive_jobs SET state=?,revision=revision+1,max_bytes=?,error_code='',updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, next, in.MaxBytes, t.NamespaceID, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if next == "ABANDONED" {
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE orbit_archive_plans SET active_job_id=NULL WHERE namespace_id=? AND kind=? AND active_job_id=?`, t.NamespaceID, kind, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
out, err = readArchiveJob(ctx, tx, t.NamespaceID, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = archiveAudit(ctx, tx, t.NamespaceID, id, identity, "console:"+string(user), "retention.job."+in.Action, "archive_job", in.Reason, j, out); err != nil {
|
||||
return err
|
||||
}
|
||||
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// ScheduleArchive 在数据库时钟下合并错过的时间槽;策略、计划和新作业同事务核验。
|
||||
func (s *RetentionPolicyStore) ScheduleArchive(ctx context.Context, storage core.ID) error {
|
||||
if storage.Validate() != nil {
|
||||
return fault.Invalid("invalid storage id")
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT p.namespace_id,p.kind FROM orbit_archive_plans p JOIN orbit_retention_policies r ON r.namespace_id=p.namespace_id AND r.revision=p.policy_revision AND r.enabled=TRUE JOIN orbit_namespaces n ON n.id=p.namespace_id AND n.enabled=TRUE JOIN orbit_applications a ON a.id=n.application_id AND a.enabled=TRUE WHERE p.storage_id=? AND p.enabled=TRUE AND p.active_job_id IS NULL AND p.next_run_at<=UTC_TIMESTAMP(6) ORDER BY p.next_run_at LIMIT 16`, storage)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type candidate struct {
|
||||
ns core.ID
|
||||
kind string
|
||||
}
|
||||
var candidates []candidate
|
||||
for rows.Next() {
|
||||
var c candidate
|
||||
if err = rows.Scan(&c.ns, &c.kind); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
candidates = append(candidates, c)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, c := range candidates {
|
||||
err = WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
||||
var enabled bool
|
||||
if err := tx.QueryRowContext(ctx, `SELECT a.enabled AND n.enabled FROM orbit_namespaces n JOIN orbit_applications a ON a.id=n.application_id WHERE n.id=? FOR SHARE`, c.ns).Scan(&enabled); err != nil {
|
||||
return err
|
||||
}
|
||||
if !enabled {
|
||||
return nil
|
||||
}
|
||||
policy, err := s.readPolicy(ctx, tx, c.ns, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p, err := readArchivePlan(ctx, tx, c.ns, c.kind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !p.Enabled || p.ActiveJobID != nil || p.StorageID != storage || p.NextRunAt == nil || p.NextRunAt.After(now) || !policy.Enabled || policy.Revision != p.PolicyRevision {
|
||||
return nil
|
||||
}
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO orbit_archive_jobs(namespace_id,id,kind,plan_revision,policy_revision,storage_id,batch_size,max_bytes) VALUES(?,?,?,?,?,?,?,?)`, c.ns, id, c.kind, p.Revision, p.PolicyRevision, storage, p.BatchSize, p.MaxBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE orbit_archive_plans SET active_job_id=?,next_run_at=? WHERE namespace_id=? AND kind=?`, id, now.Add(time.Duration(p.IntervalSeconds)*time.Second), c.ns, c.kind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
j, err := readArchiveJob(ctx, tx, c.ns, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return archiveAudit(ctx, tx, c.ns, id, id, "archive:scheduler", "retention.job.created", "archive_job", "周期归档到期", nil, j)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClaimArchive 领取一批工作;五分钟租约超时后重领,以随机代次隔离迟到确认。
|
||||
func (s *RetentionPolicyStore) ClaimArchive(ctx context.Context, storage core.ID) (job *retention.ManagedJob, err error) {
|
||||
err = WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
||||
job = nil
|
||||
j, err := scanArchiveJob(tx.QueryRowContext(ctx, `SELECT `+archiveJobColumns+` FROM orbit_archive_jobs WHERE storage_id=? AND (state='PENDING' OR (state='RUNNING' AND lease_until<=UTC_TIMESTAMP(6))) ORDER BY updated_at,id LIMIT 1 FOR UPDATE SKIP LOCKED`, storage))
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lease, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE orbit_archive_jobs SET state='RUNNING',revision=revision+1,attempts=attempts+1,lease_id=?,lease_until=TIMESTAMPADD(SECOND,300,UTC_TIMESTAMP(6)),updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, lease, j.NamespaceID, j.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
claimed, err := readArchiveJob(ctx, tx, j.NamespaceID, j.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
job = &claimed
|
||||
return nil
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// FinishArchive 仅确认当前未过期领取;失败不释放计划占位,避免不断生成重复失败作业。
|
||||
func (s *RetentionPolicyStore) FinishArchive(ctx context.Context, claim retention.ManagedJob, status retention.JobStatus, code string) error {
|
||||
switch code {
|
||||
case "", "FILE_CHECK_FAILED", "STORAGE_LIMIT", "STORAGE_BUSY", "POLICY_CHANGED", "PERMISSION_DENIED", "EXPORT_FAILED", "INTERRUPTED":
|
||||
default:
|
||||
return fault.Invalid("invalid archive result code")
|
||||
}
|
||||
return WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
||||
p, err := readArchivePlan(ctx, tx, claim.NamespaceID, claim.Kind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
j, err := readArchiveJob(ctx, tx, claim.NamespaceID, claim.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if p.ActiveJobID == nil || *p.ActiveJobID != j.ID || j.State != "RUNNING" || j.LeaseID != claim.LeaseID || j.Revision != claim.Revision || j.LeaseUntil == nil || !j.LeaseUntil.After(now) {
|
||||
return fault.Conflict
|
||||
}
|
||||
next := "PENDING"
|
||||
if code != "" {
|
||||
next = "FAILED"
|
||||
} else if status.Completed {
|
||||
next = "SUCCEEDED"
|
||||
}
|
||||
if code == "" && (status.JobID != j.ID || status.Batches < j.Batches || status.Roots < j.Roots || status.Bytes < j.Bytes || len(status.LastSHA256) != 64) {
|
||||
return fault.Conflict
|
||||
}
|
||||
// 校验失败的部分文件不能覆盖最后一次数据库确认的可信检查点。
|
||||
if code != "" {
|
||||
status = retention.JobStatus{JobID: j.ID, Batches: j.Batches, Roots: j.Roots, Bytes: j.Bytes, LastSHA256: j.LastSHA256}
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE orbit_archive_jobs SET state=?,revision=revision+1,batches=?,roots=?,bytes=?,last_sha256=?,error_code=?,lease_id=NULL,lease_until=NULL,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, next, status.Batches, status.Roots, status.Bytes, status.LastSHA256, code, j.NamespaceID, j.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if next == "SUCCEEDED" {
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE orbit_archive_plans SET active_job_id=NULL WHERE namespace_id=? AND kind=? AND active_job_id=?`, j.NamespaceID, j.Kind, j.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if next == "PENDING" {
|
||||
return nil
|
||||
}
|
||||
out, err := readArchiveJob(ctx, tx, j.NamespaceID, j.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return archiveAudit(ctx, tx, j.NamespaceID, j.ID, j.LeaseID, "archive:"+string(j.ID), "retention.job."+strings.ToLower(next), "archive_job", "归档执行结果", j, out)
|
||||
})
|
||||
}
|
||||
380
internal/storage/mysql/archive_scheduler_integration_test.go
Normal file
380
internal/storage/mysql/archive_scheduler_integration_test.go
Normal file
@@ -0,0 +1,380 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/retention"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
)
|
||||
|
||||
func archiveFixture(t *testing.T, payloadBytes ...int) (context.Context, *sql.DB, *RetentionPolicyStore, auth.Token, core.ID) {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
db, err := Open(ctx, dsn, 12, 4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
if err = Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
app := testID(t)
|
||||
username := "archive-" + string(app)
|
||||
var boot BootstrapResult
|
||||
if err = Bootstrap(ctx, db, BootstrapInput{Application: string(app), Namespace: "archive", Environment: "test", Queue: "archive", ConsoleUsername: username, ConsolePassword: "test-only-strong-password-123"}, func(b BootstrapResult) error { boot = b; return nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
login, err := NewConsoleAuthStore(db).Login(ctx, username, "test-only-strong-password-123", "archive-test", "127.0.0.1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
token, err := auth.Parse(login.Token)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := NewRetentionPolicyStore(db)
|
||||
_, err = store.UpdateRetentionPolicy(ctx, token, "initial", retention.UpdateInput{Enabled: true, IdempotencyDays: 90, TaskHistoryDays: 1, WorkflowHistoryDays: 1, EventLogDays: 1, DeliveryHistoryDays: 1, InboxHistoryDays: 1, AuditLogDays: 1, Reason: "isolated archive test"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 三条过期审计作为真实归档数据,分两批读取,避免凭空伪造任务生命周期。
|
||||
size := 0
|
||||
if len(payloadBytes) > 0 {
|
||||
size = payloadBytes[0]
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
_, err = db.ExecContext(ctx, "INSERT INTO orbit_audit_events(namespace_id,id,operation_id,actor_id,action,resource_kind,resource_id,reason,`before`,created_at) VALUES(?,?,?,'test','fixture','archive_job',?,'source retained',JSON_OBJECT('fixture',REPEAT('x',?)),TIMESTAMPADD(DAY,-3,UTC_TIMESTAMP(6)))", boot.NamespaceID, testID(t), testID(t), testID(t), size)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return ctx, db, store, token, testID(t)
|
||||
}
|
||||
func archivePlanInput() retention.PlanInput {
|
||||
return retention.PlanInput{PolicyRevision: 1, Enabled: true, IntervalSeconds: 3600, BatchSize: 2, MaxBytes: 4 << 20, Reason: "enable isolated schedule"}
|
||||
}
|
||||
func mustArchiveJobs(t *testing.T, ctx context.Context, s *RetentionPolicyStore, token auth.Token) []retention.ManagedJob {
|
||||
t.Helper()
|
||||
p, err := s.ListArchiveJobs(ctx, token, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p.Items
|
||||
}
|
||||
|
||||
func TestIntegrationArchiveSchedulerRaceResumeAndFencing(t *testing.T) {
|
||||
ctx, db, s, token, volume := archiveFixture(t)
|
||||
in := archivePlanInput()
|
||||
p, err := s.SaveArchivePlan(ctx, token, "plan", "audit", volume, in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replay, err := s.SaveArchivePlan(ctx, token, "plan", "audit", volume, in)
|
||||
if err != nil || replay.Revision != p.Revision {
|
||||
t.Fatal("plan replay", err)
|
||||
}
|
||||
if _, err = s.SaveArchivePlan(ctx, token, "stale", "audit", volume, in); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("stale plan accepted", err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 8)
|
||||
for i := 0; i < 8; i++ {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); errs <- NewRetentionPolicyStore(db).ScheduleArchive(ctx, volume) }()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
jobs := mustArchiveJobs(t, ctx, s, token)
|
||||
if len(jobs) != 1 || jobs[0].PlanRevision != p.Revision {
|
||||
t.Fatalf("duplicate schedules: %+v", jobs)
|
||||
}
|
||||
claims := make(chan *retention.ManagedJob, 8)
|
||||
errs = make(chan error, 8)
|
||||
for i := 0; i < 8; i++ {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); j, e := s.ClaimArchive(ctx, volume); claims <- j; errs <- e }()
|
||||
}
|
||||
wg.Wait()
|
||||
close(claims)
|
||||
close(errs)
|
||||
for e := range errs {
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
var old *retention.ManagedJob
|
||||
for j := range claims {
|
||||
if j != nil {
|
||||
if old != nil {
|
||||
t.Fatal("duplicate claim")
|
||||
}
|
||||
old = j
|
||||
}
|
||||
}
|
||||
if old == nil {
|
||||
t.Fatal("no claim")
|
||||
}
|
||||
root := filepath.Join(t.TempDir(), "volume")
|
||||
runtime := &retention.ArchiveRuntime{Directory: root, StorageID: volume, MaxBytes: 16 << 20, Store: s}
|
||||
if err = runtime.Init(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := filepath.Join(root, string(token.NamespaceID)+"-"+string(old.ID))
|
||||
spec := retention.JobSpec{NamespaceID: token.NamespaceID, Kind: "audit", PolicyRevision: 1, BatchSize: 2}
|
||||
first, err := retention.RunArchive(ctx, dir, spec, retention.JobOptions{JobID: old.ID, MaxBatches: 1, MaxBytes: 4 << 20}, func(ctx context.Context, id core.ID, in retention.ArchiveInput) (retention.ArchiveSnapshot, error) {
|
||||
return s.ExportRetentionForJob(ctx, token.NamespaceID, id, in)
|
||||
})
|
||||
if err != nil || first.Batches != 1 || first.Roots != 2 || first.Completed {
|
||||
t.Fatal("first batch", first, err)
|
||||
}
|
||||
// 模拟文件已落盘但数据库确认尚未提交;租约到期后另一节点恢复同一作业。
|
||||
if _, err = db.ExecContext(ctx, `UPDATE orbit_archive_jobs SET lease_until=TIMESTAMPADD(SECOND,-1,UTC_TIMESTAMP(6)) WHERE namespace_id=? AND id=?`, token.NamespaceID, old.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newer, err := s.ClaimArchive(ctx, volume)
|
||||
if err != nil || newer == nil || newer.LeaseID == old.LeaseID {
|
||||
t.Fatal("reclaim", err)
|
||||
}
|
||||
if err = s.FinishArchive(ctx, *old, first, ""); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("stale claimant advanced", err)
|
||||
}
|
||||
if err = s.FinishArchive(ctx, *newer, first, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jobs = mustArchiveJobs(t, ctx, s, token)
|
||||
if len(jobs) != 1 || jobs[0].ID != old.ID || jobs[0].State != "SUCCEEDED" || jobs[0].Roots != 3 || jobs[0].Batches != 2 {
|
||||
t.Fatalf("resume result: %+v", jobs)
|
||||
}
|
||||
verified, err := retention.VerifyArchive(ctx, dir)
|
||||
if err != nil || verified.LastSHA256 != jobs[0].LastSHA256 || !verified.Completed {
|
||||
t.Fatal("offline mismatch", err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(mustArchiveJobs(t, ctx, s, token)) != 1 {
|
||||
t.Fatal("replayed due slot")
|
||||
}
|
||||
var sources, fks int
|
||||
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM orbit_audit_events WHERE namespace_id=? AND action='fixture'`, token.NamespaceID).Scan(&sources); err != nil || sources != 3 {
|
||||
t.Fatal("source deleted", err)
|
||||
}
|
||||
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&fks); err != nil || fks != 0 {
|
||||
t.Fatal("foreign keys introduced", err)
|
||||
}
|
||||
t.Log("8 schedulers -> 1 job; 8 claimants -> 1 lease; durable file prefix survives missing DB confirmation; old lease rejected; 3 source rows retained")
|
||||
}
|
||||
|
||||
func TestIntegrationArchiveFailureRetryPolicyAndScope(t *testing.T) {
|
||||
ctx, db, s, token, volume := archiveFixture(t)
|
||||
in := archivePlanInput()
|
||||
if _, err := s.SaveArchivePlan(ctx, token, "plan", "audit", volume, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime := &retention.ArchiveRuntime{Directory: filepath.Join(t.TempDir(), "volume"), StorageID: volume, MaxBytes: 16 << 20, Store: s}
|
||||
if err := runtime.Init(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := mustArchiveJobs(t, ctx, s, token)[0]
|
||||
if j.State != "PENDING" || j.Batches != 1 {
|
||||
t.Fatalf("first chunk %+v", j)
|
||||
}
|
||||
dir := filepath.Join(runtime.Directory, string(token.NamespaceID)+"-"+string(j.ID))
|
||||
file := filepath.Join(dir, "batch-000001.json")
|
||||
saved, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.WriteFile(file, []byte("broken"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed := mustArchiveJobs(t, ctx, s, token)[0]
|
||||
if failed.State != "FAILED" || failed.ErrorCode != "FILE_CHECK_FAILED" || failed.LastSHA256 != j.LastSHA256 || failed.Batches != 1 {
|
||||
t.Fatalf("failed checkpoint %+v", failed)
|
||||
}
|
||||
// 失败占位阻止到期计划另建作业。
|
||||
if _, err = db.ExecContext(ctx, `UPDATE orbit_archive_plans SET next_run_at=TIMESTAMPADD(DAY,-2,UTC_TIMESTAMP(6)) WHERE namespace_id=?`, token.NamespaceID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(mustArchiveJobs(t, ctx, s, token)) != 1 {
|
||||
t.Fatal("failure spawned another job")
|
||||
}
|
||||
action := retention.JobActionInput{ExpectedRevision: failed.Revision, Action: "retry", MaxBytes: failed.MaxBytes, Reason: "restored exact original archive file"}
|
||||
if err = os.WriteFile(file, saved, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retried, err := s.ActArchiveJob(ctx, token, j.ID, "retry-original", action)
|
||||
if err != nil || retried.ID != j.ID || retried.State != "PENDING" {
|
||||
t.Fatal("retry", err)
|
||||
}
|
||||
if _, err = s.ActArchiveJob(ctx, token, j.ID, "retry-original", action); err != nil {
|
||||
t.Fatal("retry replay", err)
|
||||
}
|
||||
if _, err = s.ActArchiveJob(ctx, token, j.ID, "retry-stale", action); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("stale retry", err)
|
||||
}
|
||||
// 改策略不允许旧作业静默采用新保留窗口。
|
||||
_, err = s.UpdateRetentionPolicy(ctx, token, "changed-policy", retention.UpdateInput{ExpectedRevision: 1, Enabled: true, IdempotencyDays: 90, TaskHistoryDays: 2, WorkflowHistoryDays: 2, EventLogDays: 2, DeliveryHistoryDays: 2, InboxHistoryDays: 2, AuditLogDays: 2, Reason: "new reviewed retention policy"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed = mustArchiveJobs(t, ctx, s, token)[0]
|
||||
if failed.State != "FAILED" || failed.ErrorCode != "POLICY_CHANGED" {
|
||||
t.Fatalf("changed policy %+v", failed)
|
||||
}
|
||||
action = retention.JobActionInput{ExpectedRevision: failed.Revision, Action: "abandon", MaxBytes: failed.MaxBytes, Reason: "close old version; preserve files"}
|
||||
closed, err := s.ActArchiveJob(ctx, token, j.ID, "close", action)
|
||||
if err != nil || closed.State != "ABANDONED" {
|
||||
t.Fatal("close", err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(mustArchiveJobs(t, ctx, s, token)) != 1 {
|
||||
t.Fatal("stale plan triggered")
|
||||
}
|
||||
// 无会话和异命名空间不能读取或处理作业。
|
||||
other := token
|
||||
other.NamespaceID = testID(t)
|
||||
if _, err = s.ListArchiveJobs(ctx, other, ""); err == nil {
|
||||
t.Fatal("cross namespace read")
|
||||
}
|
||||
if _, err = s.ActArchiveJob(ctx, other, j.ID, "cross", action); err == nil {
|
||||
t.Fatal("cross namespace action")
|
||||
}
|
||||
// 更新计划策略后,过期的多次周期只生成一个新作业。
|
||||
in.ExpectedRevision = 1
|
||||
in.PolicyRevision = 2
|
||||
if _, err = s.SaveArchivePlan(ctx, token, "plan-new-policy", "audit", volume, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jobs := mustArchiveJobs(t, ctx, s, token)
|
||||
if len(jobs) != 2 || jobs[0].PolicyRevision != 2 {
|
||||
t.Fatalf("new policy job %+v", jobs)
|
||||
}
|
||||
// 暂停计划不影响已有作业结清,但不再生成后续作业。
|
||||
in.ExpectedRevision = 2
|
||||
in.Enabled = false
|
||||
if _, err = s.SaveArchivePlan(ctx, token, "pause-plan", "audit", volume, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jobs = mustArchiveJobs(t, ctx, s, token)
|
||||
if len(jobs) != 2 || jobs[0].State != "SUCCEEDED" {
|
||||
t.Fatalf("pause blocked in-flight %+v", jobs)
|
||||
}
|
||||
var retries int
|
||||
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM orbit_audit_events WHERE namespace_id=? AND action='retention.job.retry'`, token.NamespaceID).Scan(&retries); err != nil || retries != 1 {
|
||||
t.Fatal("retry audit duplicated", err)
|
||||
}
|
||||
t.Log("corrupt file fails closed; restored file retries same identity exactly once; policy change blocks old job; audited close releases slot; disabled plan lets current job settle")
|
||||
}
|
||||
|
||||
func TestIntegrationArchiveBudgetsResumeAndPagination(t *testing.T) {
|
||||
ctx, db, s, token, volume := archiveFixture(t, 600000)
|
||||
in := archivePlanInput()
|
||||
in.MaxBytes = 1 << 20
|
||||
if _, err := s.SaveArchivePlan(ctx, token, "plan", "audit", volume, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime := &retention.ArchiveRuntime{Directory: filepath.Join(t.TempDir(), "volume"), StorageID: volume, MaxBytes: 8 << 20, Store: s}
|
||||
if err := runtime.Init(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed := mustArchiveJobs(t, ctx, s, token)[0]
|
||||
if failed.State != "FAILED" || failed.ErrorCode != "STORAGE_LIMIT" {
|
||||
t.Fatalf("budget was ignored %+v", failed)
|
||||
}
|
||||
_, err := s.ActArchiveJob(ctx, token, failed.ID, "raise-budget", retention.JobActionInput{ExpectedRevision: failed.Revision, Action: "retry", MaxBytes: 4 << 20, Reason: "reviewed file budget"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
complete := mustArchiveJobs(t, ctx, s, token)[0]
|
||||
if complete.ID != failed.ID || complete.State != "SUCCEEDED" || complete.Roots != 3 || complete.Bytes <= 1<<20 || complete.Bytes > 4<<20 {
|
||||
t.Fatalf("budget retry %+v", complete)
|
||||
}
|
||||
// 全卷预算独立于单作业预算:即使单作业允许4MiB,也不能突破剩余卷容量。
|
||||
if _, err = db.ExecContext(ctx, `UPDATE orbit_archive_plans SET next_run_at=TIMESTAMPADD(DAY,-1,UTC_TIMESTAMP(6)) WHERE namespace_id=?`, token.NamespaceID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime.MaxBytes = complete.Bytes + 100
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jobs := mustArchiveJobs(t, ctx, s, token)
|
||||
if len(jobs) != 2 || jobs[0].State != "FAILED" || jobs[0].ErrorCode != "STORAGE_LIMIT" || jobs[0].Bytes != 0 {
|
||||
t.Fatalf("volume budget %+v", jobs)
|
||||
}
|
||||
// 分页使用真实时间+ID顺序,数量限制由SQL完成;无效/异命名空间位置拒绝。
|
||||
for i := 0; i < 24; i++ {
|
||||
_, err = db.ExecContext(ctx, `INSERT INTO orbit_archive_jobs(namespace_id,id,kind,plan_revision,policy_revision,storage_id,batch_size,max_bytes,state,created_at) VALUES(?,?,'audit',1,1,?,2,1048576,'ABANDONED',TIMESTAMPADD(DAY,-2,UTC_TIMESTAMP(6)))`, token.NamespaceID, testID(t), volume)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
first, err := s.ListArchiveJobs(ctx, token, "")
|
||||
if err != nil || len(first.Items) != 20 || first.NextBefore == "" {
|
||||
t.Fatal("first page", err)
|
||||
}
|
||||
second, err := s.ListArchiveJobs(ctx, token, first.NextBefore)
|
||||
if err != nil || len(second.Items) != 6 || second.NextBefore != "" {
|
||||
t.Fatal("second page", err)
|
||||
}
|
||||
seen := map[core.ID]bool{}
|
||||
for _, j := range append(first.Items, second.Items...) {
|
||||
if seen[j.ID] {
|
||||
t.Fatal("duplicate page item")
|
||||
}
|
||||
seen[j.ID] = true
|
||||
}
|
||||
if _, err = s.ListArchiveJobs(ctx, token, string(testID(t))); fault.Code(err) != "NOT_FOUND" {
|
||||
t.Fatal("unknown page position", err)
|
||||
}
|
||||
t.Log("single-job byte budget stops and resumes same ID; volume budget blocks extra output; 26 jobs paginate 20+6 without duplicates")
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
-- Orbit fresh-install baseline, generated from migration version 30.
|
||||
-- Orbit fresh-install baseline, generated from migration version 31.
|
||||
-- This file intentionally contains no database-level relation constraints.
|
||||
CREATE TABLE `orbit_application_resources` (
|
||||
`application_id` binary(16) NOT NULL COMMENT '所属应用标识及主键,跨命名空间共享额度',
|
||||
@@ -878,3 +878,52 @@ CREATE TABLE orbit_retention_policies (
|
||||
CONSTRAINT ck_retention_policy_inbox CHECK (inbox_history_days BETWEEN 1 AND 3650),
|
||||
CONSTRAINT ck_retention_policy_audit CHECK (audit_log_days BETWEEN 1 AND 3650)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='命名空间历史数据保留策略,只配置窗口和开关,不自动删除';
|
||||
|
||||
-- 周期计划、固定范围作业和领取租约;不使用数据库外键,不删除历史源记录。
|
||||
CREATE TABLE orbit_archive_plans (
|
||||
namespace_id BINARY(16) NOT NULL COMMENT '所属命名空间,由事务核验',
|
||||
kind VARCHAR(16) NOT NULL COMMENT '固定历史类别,同一命名空间每类一个计划',
|
||||
revision BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '配置版本,后台触发不修改此版本',
|
||||
enabled BOOLEAN NOT NULL DEFAULT FALSE COMMENT '是否触发新作业,不中断已有作业',
|
||||
policy_revision BIGINT UNSIGNED NOT NULL COMMENT '管理员确认的保留策略版本',
|
||||
storage_id BINARY(16) NOT NULL COMMENT '固定归档卷身份,接管节点必须挂载同一物理卷',
|
||||
interval_seconds INT UNSIGNED NOT NULL COMMENT '周期秒数,3600至2592000,错过多次合并一次',
|
||||
batch_size INT UNSIGNED NOT NULL COMMENT '每批根记录上限,1至100',
|
||||
max_bytes BIGINT UNSIGNED NOT NULL COMMENT '新作业累计文件字节预算,1MiB至1GiB',
|
||||
next_run_at DATETIME(6) NOT NULL COMMENT '下次触发UTC时间,首次启用立即到期',
|
||||
active_job_id BINARY(16) NULL COMMENT '未完成或失败待处理作业,成功或关闭后为空',
|
||||
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '首次保存UTC时间',
|
||||
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '配置最后修改UTC时间',
|
||||
PRIMARY KEY(namespace_id,kind),
|
||||
KEY ix_archive_plan_due(storage_id,enabled,next_run_at),
|
||||
CONSTRAINT ck_archive_plan_kind CHECK(kind IN ('tasks','workflows','events','audit','deliveries','inbox','idempotency')),
|
||||
CONSTRAINT ck_archive_plan_config CHECK(revision>0 AND policy_revision>0 AND interval_seconds BETWEEN 3600 AND 2592000 AND batch_size BETWEEN 1 AND 100 AND max_bytes BETWEEN 1048576 AND 1073741824)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='按命名空间和类别配置的周期归档计划,无外键';
|
||||
CREATE TABLE orbit_archive_jobs (
|
||||
namespace_id BINARY(16) NOT NULL COMMENT '所属命名空间,由事务核验',
|
||||
id BINARY(16) NOT NULL COMMENT '作业及目录及文件清单的固定标识',
|
||||
kind VARCHAR(16) NOT NULL COMMENT '创建时固定的历史类别',
|
||||
plan_revision BIGINT UNSIGNED NOT NULL COMMENT '创建时固定的计划版本',
|
||||
policy_revision BIGINT UNSIGNED NOT NULL COMMENT '创建时固定的保留策略版本',
|
||||
storage_id BINARY(16) NOT NULL COMMENT '固定归档卷身份,不允许跨独立磁盘接管',
|
||||
batch_size INT UNSIGNED NOT NULL COMMENT '创建时固定的单批根记录上限',
|
||||
max_bytes BIGINT UNSIGNED NOT NULL COMMENT '累计文件字节预算,人工重试仅可提高',
|
||||
state VARCHAR(16) NOT NULL DEFAULT 'PENDING' COMMENT 'PENDING待领取RUNNING执行中SUCCEEDED完成FAILED待处理ABANDONED已关闭',
|
||||
revision BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '状态并发版本,领取确认和人工操作递增',
|
||||
attempts INT UNSIGNED NOT NULL DEFAULT 0 COMMENT '累计领取次数,包含分批继续和崩溃接管',
|
||||
batches INT UNSIGNED NOT NULL DEFAULT 0 COMMENT '已验证并入库的连续文件批次数',
|
||||
roots BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '已确认根记录总数',
|
||||
bytes BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '已确认清单及批次总字节数',
|
||||
last_sha256 CHAR(64) NOT NULL DEFAULT '' COMMENT '已确认文件链尾摘要,恢复必须验证此前缀',
|
||||
error_code VARCHAR(32) NOT NULL DEFAULT '' COMMENT '固定失败分类,不记录原始异常或业务内容',
|
||||
lease_id BINARY(16) NULL COMMENT '领取随机代次,非执行中为空',
|
||||
lease_until DATETIME(6) NULL COMMENT 'UTC租约截止,过期可接管,非执行中为空',
|
||||
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '作业创建UTC时间',
|
||||
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '状态最后变更UTC时间',
|
||||
PRIMARY KEY(namespace_id,id),
|
||||
KEY ix_archive_job_claim(storage_id,state,lease_until,updated_at),
|
||||
KEY ix_archive_job_list(namespace_id,created_at,id),
|
||||
CONSTRAINT ck_archive_job_state CHECK(state IN ('PENDING','RUNNING','SUCCEEDED','FAILED','ABANDONED')),
|
||||
CONSTRAINT ck_archive_job_config CHECK(revision>0 AND plan_revision>0 AND policy_revision>0 AND batch_size BETWEEN 1 AND 100 AND max_bytes BETWEEN 1048576 AND 1073741824),
|
||||
CONSTRAINT ck_archive_job_lease CHECK((state='RUNNING' AND lease_id IS NOT NULL AND lease_until IS NOT NULL) OR (state<>'RUNNING' AND lease_id IS NULL AND lease_until IS NULL))
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='持久归档作业及租约,只读源数据,无外键';
|
||||
|
||||
@@ -26,7 +26,7 @@ var migrationFiles embed.FS
|
||||
//go:embed baseline.sql
|
||||
var freshInstallBaseline string
|
||||
|
||||
const freshInstallBaselineVersion uint64 = 30
|
||||
const freshInstallBaselineVersion uint64 = 31
|
||||
|
||||
type Migration struct {
|
||||
Version uint64
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
-- 周期计划、固定范围作业和领取租约;不使用数据库外键,不删除历史源记录。
|
||||
CREATE TABLE orbit_archive_plans (
|
||||
namespace_id BINARY(16) NOT NULL COMMENT '所属命名空间,由事务核验',
|
||||
kind VARCHAR(16) NOT NULL COMMENT '固定历史类别,同一命名空间每类一个计划',
|
||||
revision BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '配置版本,后台触发不修改此版本',
|
||||
enabled BOOLEAN NOT NULL DEFAULT FALSE COMMENT '是否触发新作业,不中断已有作业',
|
||||
policy_revision BIGINT UNSIGNED NOT NULL COMMENT '管理员确认的保留策略版本',
|
||||
storage_id BINARY(16) NOT NULL COMMENT '固定归档卷身份,接管节点必须挂载同一物理卷',
|
||||
interval_seconds INT UNSIGNED NOT NULL COMMENT '周期秒数,3600至2592000,错过多次合并一次',
|
||||
batch_size INT UNSIGNED NOT NULL COMMENT '每批根记录上限,1至100',
|
||||
max_bytes BIGINT UNSIGNED NOT NULL COMMENT '新作业累计文件字节预算,1MiB至1GiB',
|
||||
next_run_at DATETIME(6) NOT NULL COMMENT '下次触发UTC时间,首次启用立即到期',
|
||||
active_job_id BINARY(16) NULL COMMENT '未完成或失败待处理作业,成功或关闭后为空',
|
||||
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '首次保存UTC时间',
|
||||
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '配置最后修改UTC时间',
|
||||
PRIMARY KEY(namespace_id,kind),
|
||||
KEY ix_archive_plan_due(storage_id,enabled,next_run_at),
|
||||
CONSTRAINT ck_archive_plan_kind CHECK(kind IN ('tasks','workflows','events','audit','deliveries','inbox','idempotency')),
|
||||
CONSTRAINT ck_archive_plan_config CHECK(revision>0 AND policy_revision>0 AND interval_seconds BETWEEN 3600 AND 2592000 AND batch_size BETWEEN 1 AND 100 AND max_bytes BETWEEN 1048576 AND 1073741824)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='按命名空间和类别配置的周期归档计划,无外键';
|
||||
CREATE TABLE orbit_archive_jobs (
|
||||
namespace_id BINARY(16) NOT NULL COMMENT '所属命名空间,由事务核验',
|
||||
id BINARY(16) NOT NULL COMMENT '作业及目录及文件清单的固定标识',
|
||||
kind VARCHAR(16) NOT NULL COMMENT '创建时固定的历史类别',
|
||||
plan_revision BIGINT UNSIGNED NOT NULL COMMENT '创建时固定的计划版本',
|
||||
policy_revision BIGINT UNSIGNED NOT NULL COMMENT '创建时固定的保留策略版本',
|
||||
storage_id BINARY(16) NOT NULL COMMENT '固定归档卷身份,不允许跨独立磁盘接管',
|
||||
batch_size INT UNSIGNED NOT NULL COMMENT '创建时固定的单批根记录上限',
|
||||
max_bytes BIGINT UNSIGNED NOT NULL COMMENT '累计文件字节预算,人工重试仅可提高',
|
||||
state VARCHAR(16) NOT NULL DEFAULT 'PENDING' COMMENT 'PENDING待领取RUNNING执行中SUCCEEDED完成FAILED待处理ABANDONED已关闭',
|
||||
revision BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '状态并发版本,领取确认和人工操作递增',
|
||||
attempts INT UNSIGNED NOT NULL DEFAULT 0 COMMENT '累计领取次数,包含分批继续和崩溃接管',
|
||||
batches INT UNSIGNED NOT NULL DEFAULT 0 COMMENT '已验证并入库的连续文件批次数',
|
||||
roots BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '已确认根记录总数',
|
||||
bytes BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '已确认清单及批次总字节数',
|
||||
last_sha256 CHAR(64) NOT NULL DEFAULT '' COMMENT '已确认文件链尾摘要,恢复必须验证此前缀',
|
||||
error_code VARCHAR(32) NOT NULL DEFAULT '' COMMENT '固定失败分类,不记录原始异常或业务内容',
|
||||
lease_id BINARY(16) NULL COMMENT '领取随机代次,非执行中为空',
|
||||
lease_until DATETIME(6) NULL COMMENT 'UTC租约截止,过期可接管,非执行中为空',
|
||||
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '作业创建UTC时间',
|
||||
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '状态最后变更UTC时间',
|
||||
PRIMARY KEY(namespace_id,id),
|
||||
KEY ix_archive_job_claim(storage_id,state,lease_until,updated_at),
|
||||
KEY ix_archive_job_list(namespace_id,created_at,id),
|
||||
CONSTRAINT ck_archive_job_state CHECK(state IN ('PENDING','RUNNING','SUCCEEDED','FAILED','ABANDONED')),
|
||||
CONSTRAINT ck_archive_job_config CHECK(revision>0 AND plan_revision>0 AND policy_revision>0 AND batch_size BETWEEN 1 AND 100 AND max_bytes BETWEEN 1048576 AND 1073741824),
|
||||
CONSTRAINT ck_archive_job_lease CHECK((state='RUNNING' AND lease_id IS NOT NULL AND lease_until IS NOT NULL) OR (state<>'RUNNING' AND lease_id IS NULL AND lease_until IS NULL))
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='持久归档作业及租约,只读源数据,无外键';
|
||||
@@ -147,7 +147,7 @@ func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
|
||||
indexes := func() []string {
|
||||
t.Helper()
|
||||
// 排除后续新增/替换索引;原有项在此逐项比较,v24/v25新增项由对应Query/Timeline升级测试核对。
|
||||
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events','orbit_console_users','orbit_console_sessions','orbit_retention_policies','weave_version_scopes') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result','ix_signal_expiry','ix_signal_activation','uq_wait_winner_signal','ix_task_workflow_owner','ix_signal_history','ix_event_workflow_history','ix_workflow_created','ix_workflow_state_created','ix_workflow_business_state_created','ix_workflow_definition_created','ix_workflow_definition_state_created','ix_workflow_definition_business_created','ix_workflow_definition_business_state_created','ix_task_observe_state_available','ix_delivery_observe_state_available','ix_workflow_observe_state_updated','ix_event_observe_type_occurred','ix_task_observe_terminal_completed','ix_workflow_observe_terminal_completed') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
|
||||
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events','orbit_console_users','orbit_console_sessions','orbit_retention_policies','orbit_archive_jobs','orbit_archive_plans','weave_version_scopes') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result','ix_signal_expiry','ix_signal_activation','uq_wait_winner_signal','ix_task_workflow_owner','ix_signal_history','ix_event_workflow_history','ix_workflow_created','ix_workflow_state_created','ix_workflow_business_state_created','ix_workflow_definition_created','ix_workflow_definition_state_created','ix_workflow_definition_business_created','ix_workflow_definition_business_state_created','ix_task_observe_state_available','ix_delivery_observe_state_available','ix_workflow_observe_state_updated','ix_event_observe_type_occurred','ix_task_observe_terminal_completed','ix_workflow_observe_terminal_completed') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
108
internal/transport/httpserver/archive_jobs.go
Normal file
108
internal/transport/httpserver/archive_jobs.go
Normal file
@@ -0,0 +1,108 @@
|
||||
package httpserver
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/retention"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
)
|
||||
|
||||
// RegisterConsoleArchiveJobs 注册周期归档管理;只接受管理员会话,目标目录由部署配置固定。
|
||||
func RegisterConsoleArchiveJobs(server *http.Server, consoleAuth RetentionConsoleAuth, store retention.SchedulerStore, storage core.ID) {
|
||||
if server == nil || consoleAuth == nil || store == nil {
|
||||
return
|
||||
}
|
||||
mux, ok := server.Handler.(*http.ServeMux)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
parent := mux
|
||||
mux = http.NewServeMux()
|
||||
for _, method := range []string{"GET", "PUT", "POST"} {
|
||||
parent.Handle(method+" /console/archive/", http.NewCrossOriginProtection().Handler(mux))
|
||||
}
|
||||
check := func(w http.ResponseWriter, r *http.Request) (auth.Token, bool) {
|
||||
t, ok := consoleCookieToken(r)
|
||||
if !ok || consoleAuth.ValidateSession(r.Context(), t) != nil {
|
||||
writeError(w, "", fault.Unauthorized)
|
||||
return auth.Token{}, false
|
||||
}
|
||||
return t, true
|
||||
}
|
||||
key := func(w http.ResponseWriter, r *http.Request) (string, bool) {
|
||||
keys := r.Header.Values("Idempotency-Key")
|
||||
if len(keys) != 1 {
|
||||
writeError(w, "", fault.Invalid("exactly one Idempotency-Key is required"))
|
||||
return "", false
|
||||
}
|
||||
return keys[0], true
|
||||
}
|
||||
mux.HandleFunc("GET /console/archive/plans", func(w http.ResponseWriter, r *http.Request) {
|
||||
t, ok := check(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
items, err := store.ListArchivePlans(r.Context(), t)
|
||||
if err != nil {
|
||||
writeError(w, "", err)
|
||||
return
|
||||
}
|
||||
writeConsoleJSON(w, 200, map[string]any{"data": map[string]any{"items": items, "executor_configured": storage != "", "storage_id": string(storage)}})
|
||||
})
|
||||
mux.HandleFunc("PUT /console/archive/plans/{kind}", func(w http.ResponseWriter, r *http.Request) {
|
||||
t, ok := check(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
k, ok := key(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var in retention.PlanInput
|
||||
if err := decodeBody(r, &in); err != nil {
|
||||
writeError(w, "", err)
|
||||
return
|
||||
}
|
||||
out, err := store.SaveArchivePlan(r.Context(), t, k, r.PathValue("kind"), storage, in)
|
||||
if err != nil {
|
||||
writeError(w, "", err)
|
||||
return
|
||||
}
|
||||
writeConsoleJSON(w, 200, map[string]any{"data": out})
|
||||
})
|
||||
mux.HandleFunc("GET /console/archive/jobs", func(w http.ResponseWriter, r *http.Request) {
|
||||
t, ok := check(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
out, err := store.ListArchiveJobs(r.Context(), t, r.URL.Query().Get("before"))
|
||||
if err != nil {
|
||||
writeError(w, "", err)
|
||||
return
|
||||
}
|
||||
writeConsoleJSON(w, 200, map[string]any{"data": out})
|
||||
})
|
||||
mux.HandleFunc("POST /console/archive/jobs/{id}/actions", func(w http.ResponseWriter, r *http.Request) {
|
||||
t, ok := check(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
k, ok := key(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var in retention.JobActionInput
|
||||
if err := decodeBody(r, &in); err != nil {
|
||||
writeError(w, "", err)
|
||||
return
|
||||
}
|
||||
out, err := store.ActArchiveJob(r.Context(), t, core.ID(r.PathValue("id")), k, in)
|
||||
if err != nil {
|
||||
writeError(w, "", err)
|
||||
return
|
||||
}
|
||||
writeConsoleJSON(w, 200, map[string]any{"data": out})
|
||||
})
|
||||
}
|
||||
69
internal/transport/httpserver/archive_jobs_test.go
Normal file
69
internal/transport/httpserver/archive_jobs_test.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package httpserver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/retention"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
)
|
||||
|
||||
type fakeArchiveManager struct {
|
||||
retention.SchedulerStore
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeArchiveManager) ListArchivePlans(context.Context, auth.Token) ([]retention.ArchivePlan, error) {
|
||||
f.calls++
|
||||
return []retention.ArchivePlan{}, nil
|
||||
}
|
||||
func TestArchiveRoutesRequireCookieAndRejectCrossOrigin(t *testing.T) {
|
||||
ns, _ := core.NewID()
|
||||
cred, _ := core.NewID()
|
||||
raw, _, _ := auth.Issue(ns, cred)
|
||||
volume, _ := core.NewID()
|
||||
authStore := &fakeConsoleAuth{valid: true}
|
||||
store := &fakeArchiveManager{}
|
||||
server := NewWithConsoleAuth("127.0.0.1:0", nil, authStore)
|
||||
RegisterConsoleArchiveJobs(server, authStore, store, volume)
|
||||
WrapConsoleAuth(server, authStore)
|
||||
tests := []struct {
|
||||
name, method, path, origin string
|
||||
cookie, bearer bool
|
||||
status int
|
||||
}{
|
||||
{"no session", "GET", "/console/archive/plans", "", false, false, 401},
|
||||
{"bearer alone", "GET", "/console/archive/plans", "", false, true, 401},
|
||||
{"cookie read", "GET", "/console/archive/plans", "", true, false, 200},
|
||||
{"missing key", "PUT", "/console/archive/plans/tasks", "", true, false, 400},
|
||||
{"cross origin", "PUT", "/console/archive/plans/tasks", "https://foreign.invalid", true, false, 403},
|
||||
{"cross origin retry", "POST", "/console/archive/jobs/11111111111111111111111111111111/actions", "https://foreign.invalid", true, false, 403},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := httptest.NewRequest(tt.method, tt.path, strings.NewReader(`{}`))
|
||||
r.Header.Set("Content-Type", "application/json")
|
||||
if tt.cookie {
|
||||
r.AddCookie(&http.Cookie{Name: consoleSessionCookie, Value: raw})
|
||||
}
|
||||
if tt.bearer {
|
||||
r.Header.Set("Authorization", "Bearer "+raw)
|
||||
}
|
||||
if tt.origin != "" {
|
||||
r.Header.Set("Origin", tt.origin)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
server.Handler.ServeHTTP(w, r)
|
||||
if w.Code != tt.status {
|
||||
t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
if store.calls != 1 {
|
||||
t.Fatal("unexpected store access", store.calls)
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
const state = { token: '', authenticated: false, queue: '', base: location.origin, consoleConfig: null, retention: null, timer: null, connected: false, loading: false, selectedTask: null, actionMode: '', selectedSchedule: null, selectedWorkflow: null, workflowActionMode: '', workflowRecoveryMode: '', workflowRecoveryContext: null, selectedSubscription: null, subscriptionActionMode: '', selectedDelivery: null };
|
||||
const $ = (id) => document.getElementById(id);
|
||||
const text = (id, value) => { const el = $(id); if (el) el.textContent = value; };
|
||||
const pageNames = { overview: '总览', tasks: '任务执行', schedules: '定时计划', workflows: '流程实例', subscriptions: '结果订阅', retention: '保留策略' };
|
||||
const pageNames = { overview: '总览', tasks: '任务执行', schedules: '定时计划', workflows: '流程实例', subscriptions: '结果订阅', retention: '保留策略', archives: '归档作业' };
|
||||
|
||||
function showToast(message) {
|
||||
const toast = $('toast'); toast.textContent = message; toast.classList.add('show');
|
||||
@@ -29,6 +29,7 @@
|
||||
if (updateHash && location.hash !== `#/${next}`) location.hash = `/${next}`;
|
||||
window.scrollTo({ top: 0, behavior: 'smooth' });
|
||||
if (next === 'retention' && state.authenticated) loadRetention();
|
||||
if (next === 'archives' && state.authenticated) loadArchives();
|
||||
}
|
||||
function redirectToLogin() {
|
||||
const returnTo = `${location.pathname}${location.search}${location.hash}`;
|
||||
@@ -46,7 +47,7 @@
|
||||
const raw = await response.text(); let body = {};
|
||||
try { body = raw ? JSON.parse(raw) : {}; } catch { body = { message: raw }; }
|
||||
if (response.status === 401) { state.authenticated = false; redirectToLogin(); throw new Error('登录已过期,请重新登录'); }
|
||||
if (!response.ok) throw new Error(body?.error?.message || body?.message || `HTTP ${response.status}`);
|
||||
if (!response.ok) { const error = new Error(body?.error?.message || body?.message || `HTTP ${response.status}`); error.status = response.status; throw error; }
|
||||
return body;
|
||||
}
|
||||
async function loadConsoleConfig() {
|
||||
@@ -186,6 +187,96 @@
|
||||
} catch (error) { text('retention-error', error.message || '归档快照导出失败'); }
|
||||
finally { retentionUI.busy = false; syncRetentionControls(); }
|
||||
}
|
||||
const archiveUI = { busy: false, loading: false, dirty: false, plans: [], jobs: [], configured: false, policy: null, editing: null, editPolicy: '0', before: '', next: '', pending: null, selected: null };
|
||||
const archiveLabels = { tasks: '任务历史', workflows: '流程历史', events: '事件日志', audit: '管理审计', deliveries: '交付历史', inbox: 'Inbox 历史', idempotency: '幂等记录' };
|
||||
const archiveStates = { PENDING: '等待执行', RUNNING: '执行中', SUCCEEDED: '已完成', FAILED: '待处理', ABANDONED: '已关闭' };
|
||||
const archiveErrors = { FILE_CHECK_FAILED: '文件检查未通过,请核查持久卷、原目录与文件完整性。', STORAGE_LIMIT: '归档容量达到上限,请核查单作业与归档卷容量。', STORAGE_BUSY: '文件正被其他执行器使用,确认旧进程退出后可重试。', POLICY_CHANGED: '保留策略已变更或关闭。原作业不能替换策略版本,可关闭后重新配置周期。', PERMISSION_DENIED: '所属应用或命名空间已停用。', EXPORT_FAILED: '数据库导出未完成,确认数据库恢复后可重试。', INTERRUPTED: '执行中断,核查后可从原作业续跑。' };
|
||||
function syncArchiveControls() {
|
||||
document.querySelectorAll('#archive-plan-form input, #archive-plan-form select, #archive-plan-form button, #archive-action-form input, #archive-action-form button, [data-archive-action]').forEach(el => { el.disabled = archiveUI.busy; });
|
||||
$('archive-plan-save').disabled = archiveUI.busy || !archiveUI.editing || archiveUI.editPolicy === '0';
|
||||
$('archive-action-budget').disabled = archiveUI.busy || archiveUI.selected?.action === 'abandon';
|
||||
$('archive-next').disabled = archiveUI.busy || archiveUI.loading || !archiveUI.next;
|
||||
$('archive-first').disabled = archiveUI.busy || archiveUI.loading || !archiveUI.before;
|
||||
}
|
||||
function renderArchiveEditor() {
|
||||
const p = archiveUI.plans.find(p => p.kind === $('archive-kind').value); if (!p) return;
|
||||
archiveUI.editing = p; archiveUI.editPolicy = String(archiveUI.policy?.revision || '0'); archiveUI.dirty = false;
|
||||
$('archive-enabled').checked = p.enabled; $('archive-interval').value = p.interval_seconds / 3600;
|
||||
$('archive-batch').value = p.batch_size; $('archive-budget').value = p.max_bytes / 1048576;
|
||||
text('archive-plan-revision', `计划版本 ${p.revision} · 保留策略 ${archiveUI.editPolicy}`);
|
||||
text('archive-next-run', p.active_job_id ? `当前作业 ${shortID(p.active_job_id)} · 完成或关闭后继续周期` : `下次执行 ${p.enabled ? formatTime(p.next_run_at) : '尚未开启'}`);
|
||||
const stale = p.enabled && String(p.policy_revision) !== archiveUI.editPolicy;
|
||||
text('archive-policy-note', !archiveUI.policy?.enabled ? '请先在“保留策略”中允许只读归档。' : stale ? '保留策略已变更,重新保存本计划后才会触发新作业。' : '每个作业固定创建时的策略版本。');
|
||||
text('archive-plan-hint', archiveUI.configured ? '关闭周期只停止新作业,已有作业继续结清。失败作业需先处理,后续周期才会继续。' : '当前节点尚未配置归档持久卷,可查看历史或关闭已保存的周期。');
|
||||
syncArchiveControls();
|
||||
}
|
||||
function archiveSize(bytes) { return bytes < 1048576 ? `${(bytes / 1024).toFixed(1)} KiB` : `${(bytes / 1048576).toFixed(2)} MiB`; }
|
||||
function renderArchiveJobs() {
|
||||
const jobs = archiveUI.jobs;
|
||||
text('archive-job-count', `${jobs.length} 项`);
|
||||
text('archive-enabled-count', String(archiveUI.plans.filter(p => p.enabled).length));
|
||||
text('archive-failed-count', String(jobs.filter(j => j.state === 'FAILED').length));
|
||||
text('archive-root-count', jobs.reduce((sum, j) => sum + j.roots, 0).toLocaleString('zh-CN'));
|
||||
text('archive-executor', archiveUI.configured ? '已配置持久卷' : '未配置持久卷');
|
||||
$('archive-executor').classList.toggle('ready', archiveUI.configured);
|
||||
$('archive-plan-chips').innerHTML = archiveUI.plans.map(p => `<span class="${p.enabled ? 'enabled' : ''}">${archiveLabels[p.kind]} · ${p.enabled ? '开启' : '关闭'}</span>`).join('');
|
||||
$('archive-job-list').innerHTML = jobs.length ? jobs.map(j => `<article class="archive-job"><div class="archive-job-head"><div><b>${escapeHTML(archiveLabels[j.kind] || j.kind)}</b><span class="state-pill ${stateClass(j.state)}">${escapeHTML(archiveStates[j.state] || j.state)}</span></div><time>${escapeHTML(formatTime(j.created_at))}</time></div><code class="archive-job-id">${escapeHTML(j.id)}</code><div class="archive-progress"><span><b>${j.batches}</b> 批已确认</span><span><b>${j.roots.toLocaleString('zh-CN')}</b> 条根记录</span><span><b>${archiveSize(j.bytes)}</b> / ${j.max_bytes / 1048576} MiB</span><span>策略 v${escapeHTML(j.policy_revision)}</span></div>${j.last_sha256 ? `<details><summary>查看文件校验摘要</summary><code class="archive-digest">${escapeHTML(j.last_sha256)}</code></details>` : ''}${j.error_code ? `<p class="archive-job-error">${escapeHTML(archiveErrors[j.error_code] || j.error_code)}</p>` : ''}${j.state === 'RUNNING' ? `<p class="archive-job-note">领取次数 ${j.attempts} · 租约至 ${escapeHTML(formatTime(j.lease_until))},中断后可接管。</p>` : ''}${j.state === 'FAILED' ? `<div class="archive-job-actions"><button type="button" class="secondary-button compact" data-archive-action="retry" data-id="${escapeHTML(j.id)}">从原作业重试 ↻</button><button type="button" class="text-button" data-archive-action="abandon" data-id="${escapeHTML(j.id)}">关闭此作业</button></div>` : ''}</article>`).join('') : '<div class="empty-row archive-empty"><span class="empty-icon">▤</span><b>还没有归档作业</b><small>保存并开启周期后,系统会按保留窗口生成作业。</small></div>';
|
||||
syncArchiveControls();
|
||||
}
|
||||
async function loadArchives() {
|
||||
if (archiveUI.loading || archiveUI.busy) return; archiveUI.loading = true;
|
||||
try {
|
||||
const [plans, policy, jobs] = await Promise.all([request('/console/archive/plans'), request('/console/retention'), request(`/console/archive/jobs${archiveUI.before ? `?before=${encodeURIComponent(archiveUI.before)}` : ''}`)]);
|
||||
const config = extractData(plans), page = extractData(jobs);
|
||||
archiveUI.plans = config.items || []; archiveUI.configured = config.executor_configured;
|
||||
archiveUI.policy = extractData(policy); archiveUI.jobs = page.items || []; archiveUI.next = page.next_before || '';
|
||||
if (!archiveUI.dirty) renderArchiveEditor(); renderArchiveJobs();
|
||||
} catch (error) { text('archive-error', error.message || '归档作业读取失败'); }
|
||||
finally { archiveUI.loading = false; syncArchiveControls(); }
|
||||
}
|
||||
async function archiveMutation(path, method, input) {
|
||||
const body = JSON.stringify(input), signature = `${method}:${path}:${body}`;
|
||||
if (archiveUI.pending && archiveUI.pending.signature !== signature) throw new Error('上次操作尚未确认,请保持原内容重试;确认结果后再修改。');
|
||||
if (!archiveUI.pending) archiveUI.pending = { signature, key: crypto.randomUUID() };
|
||||
try {
|
||||
const response = await request(path, { method, headers: { 'Content-Type': 'application/json', 'Idempotency-Key': archiveUI.pending.key }, body });
|
||||
archiveUI.pending = null; return response;
|
||||
} catch (error) {
|
||||
// 明确的客户端拒绝允许核对后更正;网络故障和服务端错误保留原幂等键。
|
||||
if (error.status >= 400 && error.status < 500) archiveUI.pending = null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
async function saveArchivePlan(event) {
|
||||
event.preventDefault(); if (archiveUI.busy || !archiveUI.editing) return;
|
||||
archiveUI.busy = true; syncArchiveControls(); text('archive-error', '');
|
||||
try {
|
||||
await archiveMutation(`/console/archive/plans/${encodeURIComponent(archiveUI.editing.kind)}`, 'PUT', { expected_revision: String(archiveUI.editing.revision), policy_revision: archiveUI.editPolicy, enabled: $('archive-enabled').checked, interval_seconds: Number($('archive-interval').value) * 3600, batch_size: Number($('archive-batch').value), max_bytes: Number($('archive-budget').value) * 1048576, reason: $('archive-reason').value });
|
||||
archiveUI.dirty = false; $('archive-reason').value = ''; showToast('周期配置已保存,操作已留存审计');
|
||||
} catch (error) { text('archive-error', `${error.message}。网络故障请保持原内容重试;版本冲突请刷新核对。`); }
|
||||
finally { archiveUI.busy = false; syncArchiveControls(); }
|
||||
if (!archiveUI.dirty) await loadArchives();
|
||||
}
|
||||
function openArchiveAction(id, action) {
|
||||
if (archiveUI.busy) return; const job = archiveUI.jobs.find(j => j.id === id); if (!job) return;
|
||||
archiveUI.selected = { job, action }; $('archive-action-form').hidden = false;
|
||||
text('archive-action-title', action === 'retry' ? '从原作业继续' : '关闭失败作业');
|
||||
text('archive-action-note', action === 'retry' ? `${shortID(id)} · 校验原文件后续跑,保留原策略与作业身份。容量不足时可提高上限。` : `${shortID(id)} · 保留已有文件和审计,不再重试本作业。已开启的计划可继续生成后续作业。`);
|
||||
$('archive-action-budget').value = job.max_bytes / 1048576; $('archive-action-budget').min = job.max_bytes / 1048576;
|
||||
$('archive-action-reason').value = ''; syncArchiveControls(); $('archive-action-reason').focus();
|
||||
}
|
||||
async function submitArchiveAction(event) {
|
||||
event.preventDefault(); if (archiveUI.busy || !archiveUI.selected) return;
|
||||
archiveUI.busy = true; syncArchiveControls(); text('archive-error', '');
|
||||
try {
|
||||
const { job, action } = archiveUI.selected;
|
||||
await archiveMutation(`/console/archive/jobs/${encodeURIComponent(job.id)}/actions`, 'POST', { expected_revision: String(job.revision), action, max_bytes: Number($('archive-action-budget').value) * 1048576, reason: $('archive-action-reason').value });
|
||||
$('archive-action-form').hidden = true; archiveUI.selected = null; showToast('作业处理已保存,操作已留存审计');
|
||||
} catch (error) { text('archive-error', `${error.message}。网络故障请保持原内容重试。`); }
|
||||
finally { archiveUI.busy = false; syncArchiveControls(); }
|
||||
await loadArchives();
|
||||
}
|
||||
|
||||
function renderConnectionSummary() {
|
||||
text('summary-api', state.base || location.origin);
|
||||
text('summary-api-hint', state.connected ? '接口响应正常,按 10 秒周期刷新' : 'API 地址自动跟随当前页面');
|
||||
@@ -597,6 +688,15 @@
|
||||
finally { state.loading = false; $('refresh-button').classList.remove('spinning'); }
|
||||
}
|
||||
async function logout() { await fetch('/console/auth/logout', { method: 'POST', credentials: 'same-origin' }); redirectToLogin(); }
|
||||
$('archive-plan-form').addEventListener('submit', saveArchivePlan);
|
||||
$('archive-plan-form').addEventListener('input', () => { archiveUI.dirty = true; });
|
||||
$('archive-kind').addEventListener('change', () => { if (archiveUI.pending) { $('archive-kind').value = archiveUI.editing.kind; showToast('请先保持原内容重试,确认上次操作'); return; } renderArchiveEditor(); $('archive-reason').value = ''; });
|
||||
$('archive-refresh').addEventListener('click', () => { if (!archiveUI.busy && !archiveUI.pending) { archiveUI.dirty = false; text('archive-error', ''); loadArchives(); } });
|
||||
$('archive-next').addEventListener('click', () => { if (archiveUI.loading || archiveUI.busy) return; archiveUI.before = archiveUI.next; loadArchives(); });
|
||||
$('archive-first').addEventListener('click', () => { if (archiveUI.loading || archiveUI.busy) return; archiveUI.before = ''; loadArchives(); });
|
||||
$('archive-job-list').addEventListener('click', event => { const button = event.target.closest('[data-archive-action]'); if (button) openArchiveAction(button.dataset.id, button.dataset.archiveAction); });
|
||||
$('archive-action-form').addEventListener('submit', submitArchiveAction);
|
||||
$('archive-action-cancel').addEventListener('click', () => { if (archiveUI.pending) { showToast('请先保持原内容重试,确认上次操作'); return; } $('archive-action-form').hidden = true; archiveUI.selected = null; });
|
||||
$('retention-kind').addEventListener('change', resetRetentionExport);
|
||||
$('retention-restart-button').addEventListener('click', resetRetentionExport);
|
||||
$('retention-form').addEventListener('input', event => { if (event.target.matches('input, textarea')) { retentionUI.dirty = true; syncRetentionControls(); } });
|
||||
@@ -607,6 +707,6 @@
|
||||
document.querySelectorAll('[data-view]').forEach(button => button.addEventListener('click', event => { event.preventDefault(); setView(button.dataset.view); }));
|
||||
window.addEventListener('hashchange', () => setView(currentView(), false));
|
||||
setView(currentView(), false);
|
||||
ensureSession().then(ok => { if (ok) return loadConsoleConfig().then(() => Promise.all([loadRetention(), refresh()])); }).catch(() => redirectToLogin());
|
||||
state.timer = setInterval(() => { if (state.authenticated) refresh(); }, 10000);
|
||||
ensureSession().then(ok => { if (ok) return loadConsoleConfig().then(() => Promise.all([loadRetention(), refresh(), currentView() === 'archives' ? loadArchives() : Promise.resolve()])); }).catch(() => redirectToLogin());
|
||||
state.timer = setInterval(() => { if (state.authenticated) { refresh(); if (currentView() === 'archives') loadArchives(); } }, 10000);
|
||||
})();
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
<button class="nav-item" data-view="workflows"><span class="nav-icon">⌘</span><span>流程实例</span><em>04</em></button>
|
||||
<button class="nav-item" data-view="subscriptions"><span class="nav-icon">⌁</span><span>结果订阅</span><em>05</em></button>
|
||||
<button class="nav-item" data-view="retention"><span class="nav-icon">⌛</span><span>保留策略</span><em>06</em></button>
|
||||
<button class="nav-item" data-view="archives"><span class="nav-icon">▤</span><span>归档作业</span><em>07</em></button>
|
||||
</nav>
|
||||
<div class="sidebar-bottom">
|
||||
<div class="rail-card">
|
||||
@@ -130,6 +131,41 @@
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<div class="page-view" id="view-archives" data-page="archives">
|
||||
<div class="archive-heading"><div><span class="panel-kicker">DATA / ARCHIVE JOBS</span><h1>让历史数据,有序归档。</h1><p>配置周期,自动分批保存历史;每次执行都有进度与校验记录。</p></div><button class="secondary-button compact" id="archive-refresh">刷新作业 ↻</button></div>
|
||||
<div class="archive-stats"><article><span>周期计划</span><strong id="archive-enabled-count">—</strong><small>已启用 / 7 个类别</small></article><article><span>本页待处理</span><strong id="archive-failed-count">—</strong><small>失败作业保留进度,等待处理</small></article><article><span>本页归档记录</span><strong id="archive-root-count">—</strong><small>数据库已确认的根记录数</small></article></div>
|
||||
<div id="archive-error" class="action-error" role="alert"></div>
|
||||
<section class="archive-grid">
|
||||
<article class="panel archive-config">
|
||||
<div class="panel-heading"><div><span class="panel-kicker">SCHEDULE</span><h2>周期配置</h2></div><span class="connection-state" id="archive-executor">正在读取</span></div>
|
||||
<form id="archive-plan-form">
|
||||
<label class="field-label" for="archive-kind">归档类别</label><select id="archive-kind"><option value="tasks">任务历史</option><option value="workflows">流程历史</option><option value="events">事件日志</option><option value="audit">管理审计</option><option value="deliveries">交付历史</option><option value="inbox">Inbox 历史</option><option value="idempotency">幂等记录</option></select>
|
||||
<div class="retention-switch"><div><b>开启周期归档</b><small>首次启用立即运行,之后按周期执行。</small></div><label class="switch"><input id="archive-enabled" type="checkbox" aria-label="开启周期归档"><span></span></label></div>
|
||||
<div class="archive-input-grid"><label for="archive-interval">执行间隔(小时)<input id="archive-interval" type="number" min="1" max="720" required></label><label for="archive-batch">每批根记录数<input id="archive-batch" type="number" min="1" max="100" required></label></div>
|
||||
<label class="field-label" for="archive-budget">单作业容量上限(MiB)</label><input id="archive-budget" type="number" min="1" max="1024" required>
|
||||
<div class="archive-plan-meta"><span id="archive-plan-revision">配置尚未保存</span><span id="archive-next-run">下次执行 —</span><span id="archive-policy-note"></span></div>
|
||||
<label class="field-label" for="archive-reason">变更原因</label><input id="archive-reason" type="text" maxlength="1024" placeholder="说明本次周期配置的调整原因" required>
|
||||
<button id="archive-plan-save" class="primary-button compact" type="submit">保存周期配置 <span>→</span></button>
|
||||
<p class="panel-description" id="archive-plan-hint">归档使用已保存的保留策略。关闭周期只停止新作业,已有作业继续结清。</p>
|
||||
</form>
|
||||
<div class="archive-plan-chips" id="archive-plan-chips"></div>
|
||||
<button class="text-button" data-view="retention">调整历史保留时间 ↗</button>
|
||||
</article>
|
||||
<article class="panel archive-history">
|
||||
<div class="panel-heading"><div><span class="panel-kicker">EXECUTION HISTORY</span><h2>归档作业</h2></div><span class="table-count" id="archive-job-count">0 项</span></div>
|
||||
<p class="panel-description">源数据库记录始终保留。文件写入服务端持久卷,页面展示已确认的批次和 SHA-256 摘要。</p>
|
||||
<div class="archive-job-list" id="archive-job-list"><div class="empty-row">正在读取归档作业…</div></div>
|
||||
<div class="archive-pager"><button class="secondary-button compact" id="archive-first">返回最新</button><button class="secondary-button compact" id="archive-next" disabled>更早作业 ↓</button></div>
|
||||
<form id="archive-action-form" hidden>
|
||||
<h3 id="archive-action-title">处理失败作业</h3><p class="panel-description" id="archive-action-note"></p>
|
||||
<label class="field-label" for="archive-action-budget">单作业容量上限(MiB)</label><input id="archive-action-budget" type="number" min="1" max="1024" required>
|
||||
<label class="field-label" for="archive-action-reason">处理原因</label><input id="archive-action-reason" type="text" maxlength="1024" placeholder="说明核查情况和处理原因" required>
|
||||
<div class="action-form-buttons"><button id="archive-action-submit" type="submit" class="primary-button compact">确认处理</button><button id="archive-action-cancel" type="button" class="secondary-button compact">取消</button></div>
|
||||
</form>
|
||||
</article>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<div class="page-view" id="view-retention" data-page="retention">
|
||||
<section class="content-grid single-column">
|
||||
<article class="panel wide-panel retention-panel">
|
||||
|
||||
@@ -10,3 +10,5 @@
|
||||
.retention-tools{display:flex;align-items:flex-end;justify-content:space-between;gap:16px;margin-top:22px;padding-top:18px;border-top:1px solid rgba(156,183,218,.1)}.retention-select{display:flex;align-items:center;gap:10px;color:#9eb4cb;font-size:10px}.retention-select select{min-width:150px;border:1px solid rgba(156,183,218,.15);border-radius:9px;background:#0a1a2e;color:#dceafb;font:inherit;padding:9px 11px;outline:0}.retention-select select:focus{border-color:rgba(98,230,212,.58)}.retention-tool-buttons{display:flex;gap:9px}.secondary-button.compact{min-height:37px;padding:0 14px}.retention-preview{margin-top:13px;padding:13px 14px;border:1px solid rgba(156,183,218,.1);border-radius:11px;background:rgba(4,13,24,.28);color:#7189a3;font-size:10px;line-height:1.5}.retention-preview-head{display:flex;justify-content:space-between;gap:10px;padding-bottom:8px;color:#c9d8e8}.retention-preview-head small{color:#7189a3}.retention-preview-row{display:grid;grid-template-columns:1fr auto 1.6fr;gap:12px;padding:7px 0;border-top:1px solid rgba(156,183,218,.08)}.retention-preview-row b{color:#9beade;font-weight:600}.retention-preview-row small{color:#7189a3;text-align:right}@media(max-width:780px){.retention-tools{align-items:stretch;flex-direction:column}.retention-tool-buttons{justify-content:flex-start}.retention-preview-row{grid-template-columns:1fr auto}.retention-preview-row small{grid-column:span 2;text-align:left}}
|
||||
|
||||
.retention-tool-buttons{flex-wrap:wrap}.secondary-button{display:inline-flex;align-items:center;justify-content:center;gap:8px;border:1px solid rgba(156,183,218,.18);border-radius:9px;background:rgba(23,46,65,.6);color:#b9cddd;font:inherit;font-size:10px;cursor:pointer}.secondary-button:hover{border-color:#62e6d4;color:#9beade}.retention-form button:disabled{opacity:.45;cursor:not-allowed}.retention-export-result{display:flex;flex-wrap:wrap;gap:10px;color:#9eb4cb;font-size:10px;line-height:1.6}.retention-export-result a{color:#62e6d4}.retention-export-result code{width:100%;font-size:9px;overflow-wrap:anywhere;color:#7189a3}.retention-export-result [hidden]{display:none}.retention-tools{flex-wrap:wrap}
|
||||
|
||||
.archive-heading{display:flex;justify-content:space-between;align-items:center;gap:20px;margin:22px 0}.archive-heading h1{font-size:28px;letter-spacing:-.7px;margin:12px 0;color:#e4effb}.archive-heading p{color:#849bb3;font-size:12px;line-height:1.7}.archive-stats{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:15px;margin-bottom:20px}.archive-stats article{padding:22px 24px;border:1px solid #20354a;border-radius:18px;background:linear-gradient(125deg,#112439,#0a1728)}.archive-stats article:first-child{background:linear-gradient(125deg,#13373e,#0b202d);border-color:#24535a}.archive-stats span,.archive-stats small{display:block;color:#8ba2b8;font-size:11px}.archive-stats strong{display:block;font-size:30px;margin:10px 0;color:#e5f9f5}.archive-grid{display:grid;grid-template-columns:minmax(290px,360px) minmax(0,1fr);gap:20px;align-items:start}.archive-config,.archive-history{padding:24px;min-width:0}.archive-config .panel-heading,.archive-history .panel-heading{padding:0;margin-bottom:20px}.archive-config input:not([type=checkbox]),.archive-config select,#archive-action-form input{width:100%;border:1px solid #294157;border-radius:9px;padding:11px 12px;background:#09192a;color:#dbe8f5;font:inherit;font-size:12px;outline:0}.archive-config input:focus,.archive-config select:focus,#archive-action-form input:focus{border-color:#61dfd0;box-shadow:0 0 0 3px #61dfd010}.archive-config .field-label,#archive-action-form .field-label{display:block;margin:18px 0 9px;font-size:11px}.archive-config .retention-switch{margin-top:19px;padding:13px 12px}.archive-input-grid{display:grid;grid-template-columns:1fr 1fr;gap:12px;margin-top:20px;color:#94adc4;font-size:11px}.archive-input-grid input{margin-top:9px}.archive-plan-meta{display:grid;gap:9px;margin:17px 0;color:#7893ae;font-size:10px;line-height:1.6}.archive-config .primary-button{width:100%;margin:17px 0 6px}.archive-config .panel-description{line-height:1.8;font-size:10px}.archive-plan-chips{display:flex;flex-wrap:wrap;gap:7px;margin:16px 0}.archive-plan-chips span{padding:6px 8px;border:1px solid #24374a;border-radius:7px;font-size:9px;color:#7590aa}.archive-plan-chips .enabled{color:#85e5d8;border-color:#2d6564;background:#184b4b44}.archive-history>.panel-description{margin-bottom:18px;line-height:1.8}.archive-job{border:1px solid #243a50;border-radius:13px;padding:18px;margin:12px 0;background:#09172766}.archive-job-head{display:flex;align-items:center;justify-content:space-between;gap:12px}.archive-job-head>div{display:flex;align-items:center;gap:12px}.archive-job-head b{font-size:13px;color:#d2e4f1}.archive-job-head time{font-size:10px;color:#7c98b1}.archive-job-id{display:block;margin:10px 0;color:#65849f;font-size:10px;overflow-wrap:anywhere}.archive-progress{display:flex;flex-wrap:wrap;gap:14px;margin:14px 0;color:#8eaac1;font-size:10px}.archive-progress b{color:#ccdee9;font-size:12px;font-weight:550}.archive-job details{font-size:10px;color:#7297b5;line-height:1.7}.archive-job summary{cursor:pointer}.archive-digest{display:block;overflow-wrap:anywhere;margin-top:8px;color:#9cbad0}.archive-job-error{padding:10px 12px;background:#59412338;border-radius:8px;color:#d9b484;font-size:11px;line-height:1.8}.archive-job-note{font-size:10px;color:#7698b2}.archive-job-actions,.archive-pager{display:flex;gap:14px;align-items:center;margin-top:16px}.archive-pager{justify-content:space-between}.archive-empty{padding:60px 20px}.archive-empty b,.archive-empty small{display:block;margin-top:15px}.archive-grid button:disabled{opacity:.4;cursor:not-allowed}.archive-history [hidden]{display:none}#archive-action-form{border:1px solid #36736f;border-radius:13px;padding:20px;margin-top:20px;background:#102c3544}#archive-action-form h3{font-size:16px;color:#cdeae6}.state-pill.abandoned{color:#9bb0c2;background:#52637933}@media(max-width:1150px){.archive-grid{grid-template-columns:1fr}.archive-stats{gap:10px}.archive-heading h1{font-size:24px}}@media(max-width:600px){.archive-stats{grid-template-columns:1fr}.archive-heading{align-items:flex-start;flex-direction:column}.archive-config,.archive-history{padding:18px}.archive-job-head{align-items:flex-start;flex-direction:column}}
|
||||
|
||||
Reference in New Issue
Block a user