feat: schedule persistent archive jobs with console management
This commit is contained in:
380
internal/storage/mysql/archive_scheduler_integration_test.go
Normal file
380
internal/storage/mysql/archive_scheduler_integration_test.go
Normal file
@@ -0,0 +1,380 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/retention"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
)
|
||||
|
||||
func archiveFixture(t *testing.T, payloadBytes ...int) (context.Context, *sql.DB, *RetentionPolicyStore, auth.Token, core.ID) {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
db, err := Open(ctx, dsn, 12, 4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
if err = Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
app := testID(t)
|
||||
username := "archive-" + string(app)
|
||||
var boot BootstrapResult
|
||||
if err = Bootstrap(ctx, db, BootstrapInput{Application: string(app), Namespace: "archive", Environment: "test", Queue: "archive", ConsoleUsername: username, ConsolePassword: "test-only-strong-password-123"}, func(b BootstrapResult) error { boot = b; return nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
login, err := NewConsoleAuthStore(db).Login(ctx, username, "test-only-strong-password-123", "archive-test", "127.0.0.1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
token, err := auth.Parse(login.Token)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := NewRetentionPolicyStore(db)
|
||||
_, err = store.UpdateRetentionPolicy(ctx, token, "initial", retention.UpdateInput{Enabled: true, IdempotencyDays: 90, TaskHistoryDays: 1, WorkflowHistoryDays: 1, EventLogDays: 1, DeliveryHistoryDays: 1, InboxHistoryDays: 1, AuditLogDays: 1, Reason: "isolated archive test"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 三条过期审计作为真实归档数据，分两批读取，避免凭空伪造任务生命周期。
|
||||
size := 0
|
||||
if len(payloadBytes) > 0 {
|
||||
size = payloadBytes[0]
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
_, err = db.ExecContext(ctx, "INSERT INTO orbit_audit_events(namespace_id,id,operation_id,actor_id,action,resource_kind,resource_id,reason,`before`,created_at) VALUES(?,?,?,'test','fixture','archive_job',?,'source retained',JSON_OBJECT('fixture',REPEAT('x',?)),TIMESTAMPADD(DAY,-3,UTC_TIMESTAMP(6)))", boot.NamespaceID, testID(t), testID(t), testID(t), size)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return ctx, db, store, token, testID(t)
|
||||
}
|
||||
func archivePlanInput() retention.PlanInput {
|
||||
return retention.PlanInput{PolicyRevision: 1, Enabled: true, IntervalSeconds: 3600, BatchSize: 2, MaxBytes: 4 << 20, Reason: "enable isolated schedule"}
|
||||
}
|
||||
func mustArchiveJobs(t *testing.T, ctx context.Context, s *RetentionPolicyStore, token auth.Token) []retention.ManagedJob {
|
||||
t.Helper()
|
||||
p, err := s.ListArchiveJobs(ctx, token, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p.Items
|
||||
}
|
||||
|
||||
func TestIntegrationArchiveSchedulerRaceResumeAndFencing(t *testing.T) {
|
||||
ctx, db, s, token, volume := archiveFixture(t)
|
||||
in := archivePlanInput()
|
||||
p, err := s.SaveArchivePlan(ctx, token, "plan", "audit", volume, in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replay, err := s.SaveArchivePlan(ctx, token, "plan", "audit", volume, in)
|
||||
if err != nil || replay.Revision != p.Revision {
|
||||
t.Fatal("plan replay", err)
|
||||
}
|
||||
if _, err = s.SaveArchivePlan(ctx, token, "stale", "audit", volume, in); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("stale plan accepted", err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 8)
|
||||
for i := 0; i < 8; i++ {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); errs <- NewRetentionPolicyStore(db).ScheduleArchive(ctx, volume) }()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
jobs := mustArchiveJobs(t, ctx, s, token)
|
||||
if len(jobs) != 1 || jobs[0].PlanRevision != p.Revision {
|
||||
t.Fatalf("duplicate schedules: %+v", jobs)
|
||||
}
|
||||
claims := make(chan *retention.ManagedJob, 8)
|
||||
errs = make(chan error, 8)
|
||||
for i := 0; i < 8; i++ {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); j, e := s.ClaimArchive(ctx, volume); claims <- j; errs <- e }()
|
||||
}
|
||||
wg.Wait()
|
||||
close(claims)
|
||||
close(errs)
|
||||
for e := range errs {
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
var old *retention.ManagedJob
|
||||
for j := range claims {
|
||||
if j != nil {
|
||||
if old != nil {
|
||||
t.Fatal("duplicate claim")
|
||||
}
|
||||
old = j
|
||||
}
|
||||
}
|
||||
if old == nil {
|
||||
t.Fatal("no claim")
|
||||
}
|
||||
root := filepath.Join(t.TempDir(), "volume")
|
||||
runtime := &retention.ArchiveRuntime{Directory: root, StorageID: volume, MaxBytes: 16 << 20, Store: s}
|
||||
if err = runtime.Init(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := filepath.Join(root, string(token.NamespaceID)+"-"+string(old.ID))
|
||||
spec := retention.JobSpec{NamespaceID: token.NamespaceID, Kind: "audit", PolicyRevision: 1, BatchSize: 2}
|
||||
first, err := retention.RunArchive(ctx, dir, spec, retention.JobOptions{JobID: old.ID, MaxBatches: 1, MaxBytes: 4 << 20}, func(ctx context.Context, id core.ID, in retention.ArchiveInput) (retention.ArchiveSnapshot, error) {
|
||||
return s.ExportRetentionForJob(ctx, token.NamespaceID, id, in)
|
||||
})
|
||||
if err != nil || first.Batches != 1 || first.Roots != 2 || first.Completed {
|
||||
t.Fatal("first batch", first, err)
|
||||
}
|
||||
// 模拟文件已落盘但数据库确认尚未提交；租约到期后另一节点恢复同一作业。
|
||||
if _, err = db.ExecContext(ctx, `UPDATE orbit_archive_jobs SET lease_until=TIMESTAMPADD(SECOND,-1,UTC_TIMESTAMP(6)) WHERE namespace_id=? AND id=?`, token.NamespaceID, old.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newer, err := s.ClaimArchive(ctx, volume)
|
||||
if err != nil || newer == nil || newer.LeaseID == old.LeaseID {
|
||||
t.Fatal("reclaim", err)
|
||||
}
|
||||
if err = s.FinishArchive(ctx, *old, first, ""); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("stale claimant advanced", err)
|
||||
}
|
||||
if err = s.FinishArchive(ctx, *newer, first, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jobs = mustArchiveJobs(t, ctx, s, token)
|
||||
if len(jobs) != 1 || jobs[0].ID != old.ID || jobs[0].State != "SUCCEEDED" || jobs[0].Roots != 3 || jobs[0].Batches != 2 {
|
||||
t.Fatalf("resume result: %+v", jobs)
|
||||
}
|
||||
verified, err := retention.VerifyArchive(ctx, dir)
|
||||
if err != nil || verified.LastSHA256 != jobs[0].LastSHA256 || !verified.Completed {
|
||||
t.Fatal("offline mismatch", err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(mustArchiveJobs(t, ctx, s, token)) != 1 {
|
||||
t.Fatal("replayed due slot")
|
||||
}
|
||||
var sources, fks int
|
||||
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM orbit_audit_events WHERE namespace_id=? AND action='fixture'`, token.NamespaceID).Scan(&sources); err != nil || sources != 3 {
|
||||
t.Fatal("source deleted", err)
|
||||
}
|
||||
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&fks); err != nil || fks != 0 {
|
||||
t.Fatal("foreign keys introduced", err)
|
||||
}
|
||||
t.Log("8 schedulers -> 1 job; 8 claimants -> 1 lease; durable file prefix survives missing DB confirmation; old lease rejected; 3 source rows retained")
|
||||
}
|
||||
|
||||
func TestIntegrationArchiveFailureRetryPolicyAndScope(t *testing.T) {
|
||||
ctx, db, s, token, volume := archiveFixture(t)
|
||||
in := archivePlanInput()
|
||||
if _, err := s.SaveArchivePlan(ctx, token, "plan", "audit", volume, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime := &retention.ArchiveRuntime{Directory: filepath.Join(t.TempDir(), "volume"), StorageID: volume, MaxBytes: 16 << 20, Store: s}
|
||||
if err := runtime.Init(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := mustArchiveJobs(t, ctx, s, token)[0]
|
||||
if j.State != "PENDING" || j.Batches != 1 {
|
||||
t.Fatalf("first chunk %+v", j)
|
||||
}
|
||||
dir := filepath.Join(runtime.Directory, string(token.NamespaceID)+"-"+string(j.ID))
|
||||
file := filepath.Join(dir, "batch-000001.json")
|
||||
saved, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.WriteFile(file, []byte("broken"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed := mustArchiveJobs(t, ctx, s, token)[0]
|
||||
if failed.State != "FAILED" || failed.ErrorCode != "FILE_CHECK_FAILED" || failed.LastSHA256 != j.LastSHA256 || failed.Batches != 1 {
|
||||
t.Fatalf("failed checkpoint %+v", failed)
|
||||
}
|
||||
// 失败占位阻止到期计划另建作业。
|
||||
if _, err = db.ExecContext(ctx, `UPDATE orbit_archive_plans SET next_run_at=TIMESTAMPADD(DAY,-2,UTC_TIMESTAMP(6)) WHERE namespace_id=?`, token.NamespaceID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(mustArchiveJobs(t, ctx, s, token)) != 1 {
|
||||
t.Fatal("failure spawned another job")
|
||||
}
|
||||
action := retention.JobActionInput{ExpectedRevision: failed.Revision, Action: "retry", MaxBytes: failed.MaxBytes, Reason: "restored exact original archive file"}
|
||||
if err = os.WriteFile(file, saved, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retried, err := s.ActArchiveJob(ctx, token, j.ID, "retry-original", action)
|
||||
if err != nil || retried.ID != j.ID || retried.State != "PENDING" {
|
||||
t.Fatal("retry", err)
|
||||
}
|
||||
if _, err = s.ActArchiveJob(ctx, token, j.ID, "retry-original", action); err != nil {
|
||||
t.Fatal("retry replay", err)
|
||||
}
|
||||
if _, err = s.ActArchiveJob(ctx, token, j.ID, "retry-stale", action); fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal("stale retry", err)
|
||||
}
|
||||
// 改策略不允许旧作业静默采用新保留窗口。
|
||||
_, err = s.UpdateRetentionPolicy(ctx, token, "changed-policy", retention.UpdateInput{ExpectedRevision: 1, Enabled: true, IdempotencyDays: 90, TaskHistoryDays: 2, WorkflowHistoryDays: 2, EventLogDays: 2, DeliveryHistoryDays: 2, InboxHistoryDays: 2, AuditLogDays: 2, Reason: "new reviewed retention policy"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed = mustArchiveJobs(t, ctx, s, token)[0]
|
||||
if failed.State != "FAILED" || failed.ErrorCode != "POLICY_CHANGED" {
|
||||
t.Fatalf("changed policy %+v", failed)
|
||||
}
|
||||
action = retention.JobActionInput{ExpectedRevision: failed.Revision, Action: "abandon", MaxBytes: failed.MaxBytes, Reason: "close old version; preserve files"}
|
||||
closed, err := s.ActArchiveJob(ctx, token, j.ID, "close", action)
|
||||
if err != nil || closed.State != "ABANDONED" {
|
||||
t.Fatal("close", err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(mustArchiveJobs(t, ctx, s, token)) != 1 {
|
||||
t.Fatal("stale plan triggered")
|
||||
}
|
||||
// 无会话和异命名空间不能读取或处理作业。
|
||||
other := token
|
||||
other.NamespaceID = testID(t)
|
||||
if _, err = s.ListArchiveJobs(ctx, other, ""); err == nil {
|
||||
t.Fatal("cross namespace read")
|
||||
}
|
||||
if _, err = s.ActArchiveJob(ctx, other, j.ID, "cross", action); err == nil {
|
||||
t.Fatal("cross namespace action")
|
||||
}
|
||||
// 更新计划策略后，过期的多次周期只生成一个新作业。
|
||||
in.ExpectedRevision = 1
|
||||
in.PolicyRevision = 2
|
||||
if _, err = s.SaveArchivePlan(ctx, token, "plan-new-policy", "audit", volume, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jobs := mustArchiveJobs(t, ctx, s, token)
|
||||
if len(jobs) != 2 || jobs[0].PolicyRevision != 2 {
|
||||
t.Fatalf("new policy job %+v", jobs)
|
||||
}
|
||||
// 暂停计划不影响已有作业结清，但不再生成后续作业。
|
||||
in.ExpectedRevision = 2
|
||||
in.Enabled = false
|
||||
if _, err = s.SaveArchivePlan(ctx, token, "pause-plan", "audit", volume, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jobs = mustArchiveJobs(t, ctx, s, token)
|
||||
if len(jobs) != 2 || jobs[0].State != "SUCCEEDED" {
|
||||
t.Fatalf("pause blocked in-flight %+v", jobs)
|
||||
}
|
||||
var retries int
|
||||
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM orbit_audit_events WHERE namespace_id=? AND action='retention.job.retry'`, token.NamespaceID).Scan(&retries); err != nil || retries != 1 {
|
||||
t.Fatal("retry audit duplicated", err)
|
||||
}
|
||||
t.Log("corrupt file fails closed; restored file retries same identity exactly once; policy change blocks old job; audited close releases slot; disabled plan lets current job settle")
|
||||
}
|
||||
|
||||
func TestIntegrationArchiveBudgetsResumeAndPagination(t *testing.T) {
|
||||
ctx, db, s, token, volume := archiveFixture(t, 600000)
|
||||
in := archivePlanInput()
|
||||
in.MaxBytes = 1 << 20
|
||||
if _, err := s.SaveArchivePlan(ctx, token, "plan", "audit", volume, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime := &retention.ArchiveRuntime{Directory: filepath.Join(t.TempDir(), "volume"), StorageID: volume, MaxBytes: 8 << 20, Store: s}
|
||||
if err := runtime.Init(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed := mustArchiveJobs(t, ctx, s, token)[0]
|
||||
if failed.State != "FAILED" || failed.ErrorCode != "STORAGE_LIMIT" {
|
||||
t.Fatalf("budget was ignored %+v", failed)
|
||||
}
|
||||
_, err := s.ActArchiveJob(ctx, token, failed.ID, "raise-budget", retention.JobActionInput{ExpectedRevision: failed.Revision, Action: "retry", MaxBytes: 4 << 20, Reason: "reviewed file budget"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
complete := mustArchiveJobs(t, ctx, s, token)[0]
|
||||
if complete.ID != failed.ID || complete.State != "SUCCEEDED" || complete.Roots != 3 || complete.Bytes <= 1<<20 || complete.Bytes > 4<<20 {
|
||||
t.Fatalf("budget retry %+v", complete)
|
||||
}
|
||||
// 全卷预算独立于单作业预算：即使单作业允许4MiB，也不能突破剩余卷容量。
|
||||
if _, err = db.ExecContext(ctx, `UPDATE orbit_archive_plans SET next_run_at=TIMESTAMPADD(DAY,-1,UTC_TIMESTAMP(6)) WHERE namespace_id=?`, token.NamespaceID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime.MaxBytes = complete.Bytes + 100
|
||||
if err = runtime.Tick(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jobs := mustArchiveJobs(t, ctx, s, token)
|
||||
if len(jobs) != 2 || jobs[0].State != "FAILED" || jobs[0].ErrorCode != "STORAGE_LIMIT" || jobs[0].Bytes != 0 {
|
||||
t.Fatalf("volume budget %+v", jobs)
|
||||
}
|
||||
// 分页使用真实时间+ID顺序，数量限制由SQL完成；无效/异命名空间位置拒绝。
|
||||
for i := 0; i < 24; i++ {
|
||||
_, err = db.ExecContext(ctx, `INSERT INTO orbit_archive_jobs(namespace_id,id,kind,plan_revision,policy_revision,storage_id,batch_size,max_bytes,state,created_at) VALUES(?,?,'audit',1,1,?,2,1048576,'ABANDONED',TIMESTAMPADD(DAY,-2,UTC_TIMESTAMP(6)))`, token.NamespaceID, testID(t), volume)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
first, err := s.ListArchiveJobs(ctx, token, "")
|
||||
if err != nil || len(first.Items) != 20 || first.NextBefore == "" {
|
||||
t.Fatal("first page", err)
|
||||
}
|
||||
second, err := s.ListArchiveJobs(ctx, token, first.NextBefore)
|
||||
if err != nil || len(second.Items) != 6 || second.NextBefore != "" {
|
||||
t.Fatal("second page", err)
|
||||
}
|
||||
seen := map[core.ID]bool{}
|
||||
for _, j := range append(first.Items, second.Items...) {
|
||||
if seen[j.ID] {
|
||||
t.Fatal("duplicate page item")
|
||||
}
|
||||
seen[j.ID] = true
|
||||
}
|
||||
if _, err = s.ListArchiveJobs(ctx, token, string(testID(t))); fault.Code(err) != "NOT_FOUND" {
|
||||
t.Fatal("unknown page position", err)
|
||||
}
|
||||
t.Log("single-job byte budget stops and resumes same ID; volume budget blocks extra output; 26 jobs paginate 20+6 without duplicates")
|
||||
}
|
||||
Reference in New Issue
Block a user