k2safe/orbit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add resumable file archive jobs and offline verification

Browse Source
This commit is contained in:
k2safe
2026-09-23 10:50:49 +08:00
parent 2d8467e50c
commit 6eb429e21f
19 changed files with 1160 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
internal/storage/mysql/retention_archive.go
View File

@@ -120,6 +120,32 @@ type archiveData struct {
}
func (s *RetentionPolicyStore) ExportRetention(ctx context.Context, t auth.Token, in retention.ArchiveInput) (retention.ArchiveSnapshot, error) {
return s.exportRetention(ctx, t.NamespaceID, in, func(ctx context.Context, tx *sql.Tx) (string, error) {
actor, err := retentionActor(ctx, tx, t)
return "console:" + string(actor), err
})
}
// ExportRetentionForJob 仅供持有数据库管理权限的归档 CLI 使用，不注册为 HTTP 路由。
// 每批核验应用/命名空间启用状态和数据库策略版本；作业标识进入审计，不能冒充控制台用户。
func (s *RetentionPolicyStore) ExportRetentionForJob(ctx context.Context, namespace, job core.ID, in retention.ArchiveInput) (retention.ArchiveSnapshot, error) {
if namespace.Validate() != nil || job.Validate() != nil {
return retention.ArchiveSnapshot{}, fault.Invalid("invalid archive job identity")
}
return s.exportRetention(ctx, namespace, in, func(ctx context.Context, tx *sql.Tx) (string, error) {
var enabled bool
err := tx.QueryRowContext(ctx, `SELECT a.enabled AND n.enabled FROM orbit_namespaces n JOIN orbit_applications a ON a.id=n.application_id WHERE n.id=? FOR SHARE`, namespace).Scan(&enabled)
if err != nil {
return "", err
}
if !enabled {
return "", fault.Forbidden
}
return "archive:" + string(job), nil
})
}
func (s *RetentionPolicyStore) exportRetention(ctx context.Context, namespace core.ID, in retention.ArchiveInput, authorize func(context.Context, *sql.Tx) (string, error)) (retention.ArchiveSnapshot, error) {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if in.Limit == 0 {
@@ -134,11 +160,11 @@ func (s *RetentionPolicyStore) ExportRetention(ctx context.Context, t auth.Token
return retention.ArchiveSnapshot{}, err
}
defer tx.Rollback()
actor, err := retentionActor(ctx, tx, t)
actor, err := authorize(ctx, tx)
if err != nil {
return retention.ArchiveSnapshot{}, err
}
p, err := s.readPolicy(ctx, tx, t.NamespaceID, true)
p, err := s.readPolicy(ctx, tx, namespace, true)
if err != nil {
return retention.ArchiveSnapshot{}, err
}
@@ -166,13 +192,13 @@ func (s *RetentionPolicyStore) ExportRetention(ctx context.Context, t auth.Token
}
b, e := base64.RawURLEncoding.DecodeString(in.Cursor)
var c archiveCursor
if e != nil || json.Unmarshal(b, &c) != nil || c.Namespace != t.NamespaceID || c.Kind != in.Kind || c.Revision != p.Revision || c.ID.Validate() != nil || c.Cutoff.After(cutoff) || c.Time.IsZero() || !c.Time.Before(c.Cutoff) {
if e != nil || json.Unmarshal(b, &c) != nil || c.Namespace != namespace || c.Kind != in.Kind || c.Revision != p.Revision || c.ID.Validate() != nil || c.Cutoff.After(cutoff) || c.Time.IsZero() || !c.Time.Before(c.Cutoff) {
return retention.ArchiveSnapshot{}, fault.Invalid("invalid archive cursor")
}
cursor = &c
cutoff = c.Cutoff
}
roots, err := archiveCandidates(ctx, tx, t.NamespaceID, source, cutoff, in.Limit+1, cursor)
roots, err := archiveCandidates(ctx, tx, namespace, source, cutoff, in.Limit+1, cursor)
if err != nil {
return retention.ArchiveSnapshot{}, err
}
@@ -180,7 +206,7 @@ func (s *RetentionPolicyStore) ExportRetention(ctx context.Context, t auth.Token
if len(roots) > in.Limit {
roots = roots[:in.Limit]
last := roots[len(roots)-1]
raw, _ := json.Marshal(archiveCursor{t.NamespaceID, in.Kind, p.Revision, cutoff, last.at, last.id})
raw, _ := json.Marshal(archiveCursor{namespace, in.Kind, p.Revision, cutoff, last.at, last.id})
data.NextCursor = base64.RawURLEncoding.EncodeToString(raw)
}
data.RootCount = len(roots)
@@ -206,10 +232,10 @@ func (s *RetentionPolicyStore) ExportRetention(ctx context.Context, t auth.Token
}
}
// 流程固定版本与全部历史关联任务也一起导出，不能只保存实例头。
if err := archiveRows(ctx, tx, &data, &budget, "weave_definition_versions", "id IN (SELECT definition_version_id FROM weave_instances WHERE namespace_id=? AND id=?)", t.NamespaceID, root.id); err != nil {
if err := archiveRows(ctx, tx, &data, &budget, "weave_definition_versions", "id IN (SELECT definition_version_id FROM weave_instances WHERE namespace_id=? AND id=?)", namespace, root.id); err != nil {
return retention.ArchiveSnapshot{}, err
}
taskRows, err := tx.QueryContext(ctx, `SELECT id FROM pulse_tasks WHERE namespace_id=? AND gate_id=(SELECT gate_id FROM weave_instances WHERE namespace_id=? AND id=?) LIMIT 10001`, t.NamespaceID, t.NamespaceID, root.id)
taskRows, err := tx.QueryContext(ctx, `SELECT id FROM pulse_tasks WHERE namespace_id=? AND gate_id=(SELECT gate_id FROM weave_instances WHERE namespace_id=? AND id=?) LIMIT 10001`, namespace, namespace, root.id)
if err != nil {
return retention.ArchiveSnapshot{}, err
}
@@ -265,7 +291,7 @@ func (s *RetentionPolicyStore) ExportRetention(ctx context.Context, t auth.Token
return retention.ArchiveSnapshot{}, err
}
info, _ := json.Marshal(map[string]any{"kind": in.Kind, "policy_revision": p.Revision, "root_count": len(roots), "rows": budget.rows, "sha256": digest, "source_retained": true})
_, err = tx.ExecContext(ctx, "INSERT INTO orbit_audit_events(namespace_id,id,operation_id,actor_id,action,resource_kind,resource_id,reason,`after`) VALUES (?,?,?,?,'retention.archive.export','retention_policy',?,'export retained history snapshot',?)", t.NamespaceID, audit, op, "console:"+string(actor), t.NamespaceID, string(info))
_, err = tx.ExecContext(ctx, "INSERT INTO orbit_audit_events(namespace_id,id,operation_id,actor_id,action,resource_kind,resource_id,reason,`after`) VALUES (?,?,?,?,'retention.archive.export','retention_policy',?,'export retained history snapshot',?)", namespace, audit, op, actor, namespace, string(info))
if err != nil {
return retention.ArchiveSnapshot{}, err
}

36
internal/storage/mysql/retention_policy_integration_test.go
View File

@@ -8,12 +8,14 @@ import (
"encoding/hex"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.yuntuo.hk/k2safe/orbit/internal/fault"
"git.yuntuo.hk/k2safe/orbit/internal/retention"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/platform/auth"
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
"git.yuntuo.hk/k2safe/orbit/pulse/service"
@@ -300,6 +302,40 @@ func TestIntegrationRetentionDiskGrowthAndPreview(t *testing.T) {
if _, err := store.ExportRetention(ctx, apiToken, retention.ArchiveInput{Kind: "tasks", ExpectedRevision: 1}); err == nil {
t.Fatal("business bearer token accepted as console admin")
}
jobSpec := retention.JobSpec{NamespaceID: boot.NamespaceID, Kind: "tasks", PolicyRevision: 1, BatchSize: 37}
jobDir := filepath.Join(t.TempDir(), "retention-job")
jobOptions := retention.JobOptions{MaxBatches: 1, MaxBytes: 64 << 20}
exportJob := func(ctx context.Context, job core.ID, in retention.ArchiveInput) (retention.ArchiveSnapshot, error) {
return store.ExportRetentionForJob(ctx, boot.NamespaceID, job, in)
}
firstJob, err := retention.RunArchive(ctx, jobDir, jobSpec, jobOptions, exportJob)
if err != nil || firstJob.Completed || firstJob.Roots != 37 {
t.Fatalf("archive job first batch=%+v err=%v", firstJob, err)
}
jobOptions.MaxBatches = 10
finishedJob, err := retention.RunArchive(ctx, jobDir, jobSpec, jobOptions, exportJob)
if err != nil || !finishedJob.Completed || finishedJob.Roots != 254 || finishedJob.Batches != 7 || finishedJob.JobID != firstJob.JobID {
t.Fatalf("archive job resumed=%+v err=%v", finishedJob, err)
}
verifiedJob, err := retention.VerifyArchive(ctx, jobDir)
if err != nil || verifiedJob != finishedJob {
t.Fatalf("offline job verification=%+v err=%v", verifiedJob, err)
}
var jobAudits int
if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM orbit_audit_events WHERE namespace_id=? AND actor_id=? AND action='retention.archive.export'", boot.NamespaceID, "archive:"+string(finishedJob.JobID)).Scan(&jobAudits); err != nil || jobAudits != 7 {
t.Fatalf("job audit count=%d err=%v", jobAudits, err)
}
if _, err := db.ExecContext(ctx, "UPDATE orbit_namespaces SET enabled=FALSE WHERE id=?", boot.NamespaceID); err != nil {
t.Fatal(err)
}
if _, err := store.ExportRetentionForJob(ctx, boot.NamespaceID, finishedJob.JobID, retention.ArchiveInput{Kind: "tasks", ExpectedRevision: 1}); faultCode(err) != "PERMISSION_DENIED" {
t.Fatalf("disabled namespace exported: %v", err)
}
if _, err := db.ExecContext(ctx, "UPDATE orbit_namespaces SET enabled=TRUE WHERE id=?", boot.NamespaceID); err != nil {
t.Fatal(err)
}
t.Logf("durable archive job: roots=%d batches=%d audit_rows=%d offline_verified=true", finishedJob.Roots, finishedJob.Batches, jobAudits)
policyChange := retention.UpdateInput{ExpectedRevision: 1, Enabled: true, IdempotencyDays: 90, TaskHistoryDays: 2, WorkflowHistoryDays: 30, EventLogDays: 30, DeliveryHistoryDays: 30, InboxHistoryDays: 30, AuditLogDays: 365, Reason: "verify frozen cursor boundary"}
if _, err := store.UpdateRetentionPolicy(ctx, consoleToken, "growth-retention-revise", policyChange); err != nil {
t.Fatal(err)