package mysql import ( "context" "database/sql" "encoding/json" "errors" "strings" "time" "git.yuntuo.hk/k2safe/orbit/internal/fault" "git.yuntuo.hk/k2safe/orbit/internal/retention" core "git.yuntuo.hk/k2safe/orbit/model" "git.yuntuo.hk/k2safe/orbit/platform/auth" ) const archivePlanColumns = `namespace_id,kind,revision,enabled,policy_revision,storage_id,interval_seconds,batch_size,max_bytes,next_run_at,active_job_id,updated_at` const archiveJobColumns = `namespace_id,id,kind,plan_revision,policy_revision,storage_id,batch_size,max_bytes,state,revision,attempts,batches,roots,bytes,last_sha256,error_code,lease_id,lease_until,created_at,updated_at` type archiveScanner interface{ Scan(...any) error } func scanArchivePlan(row archiveScanner) (p retention.ArchivePlan, err error) { err = row.Scan(&p.NamespaceID, &p.Kind, &p.Revision, &p.Enabled, &p.PolicyRevision, &p.StorageID, &p.IntervalSeconds, &p.BatchSize, &p.MaxBytes, &p.NextRunAt, &p.ActiveJobID, &p.UpdatedAt) return } func scanArchiveJob(row archiveScanner) (j retention.ManagedJob, err error) { var lease *core.ID err = row.Scan(&j.NamespaceID, &j.ID, &j.Kind, &j.PlanRevision, &j.PolicyRevision, &j.StorageID, &j.BatchSize, &j.MaxBytes, &j.State, &j.Revision, &j.Attempts, &j.Batches, &j.Roots, &j.Bytes, &j.LastSHA256, &j.ErrorCode, &lease, &j.LeaseUntil, &j.CreatedAt, &j.UpdatedAt) if lease != nil { j.LeaseID = *lease } return } func readArchivePlan(ctx context.Context, tx *sql.Tx, ns core.ID, kind string) (retention.ArchivePlan, error) { p, err := scanArchivePlan(tx.QueryRowContext(ctx, `SELECT `+archivePlanColumns+` FROM orbit_archive_plans WHERE namespace_id=? AND kind=? FOR UPDATE`, ns, kind)) if errors.Is(err, sql.ErrNoRows) { return retention.ArchivePlan{NamespaceID: ns, Kind: kind, IntervalSeconds: 86400, BatchSize: 20, MaxBytes: 256 << 20}, nil } return p, err } func readArchiveJob(ctx context.Context, tx *sql.Tx, ns, id core.ID) (retention.ManagedJob, error) { j, err := scanArchiveJob(tx.QueryRowContext(ctx, `SELECT `+archiveJobColumns+` FROM orbit_archive_jobs WHERE namespace_id=? AND id=? FOR UPDATE`, ns, id)) if errors.Is(err, sql.ErrNoRows) { err = fault.NotFound } return j, err } func archiveAudit(ctx context.Context, tx *sql.Tx, ns, resource, op core.ID, actor, action, kind, reason string, before, after any) error { id, err := core.NewID() if err != nil { return err } b, err := json.Marshal(before) if err != nil { return err } a, err := json.Marshal(after) if err != nil { return err } _, err = tx.ExecContext(ctx, "INSERT INTO orbit_audit_events(namespace_id,id,operation_id,actor_id,action,resource_kind,resource_id,reason,`before`,`after`) VALUES (?,?,?,?,?,?,?,?,?,?)", ns, id, op, actor, action, kind, resource, reason, string(b), string(a)) return err } func archiveKey(key string) error { if strings.TrimSpace(key) == "" || len(key) > 128 { return fault.Invalid("idempotency key must be 1-128 bytes") } return nil } func (s *RetentionPolicyStore) ListArchivePlans(ctx context.Context, t auth.Token) (out []retention.ArchivePlan, err error) { err = WithTx(ctx, s.db, func(tx *sql.Tx) error { if _, err := retentionActor(ctx, tx, t); err != nil { return err } rows, err := tx.QueryContext(ctx, `SELECT `+archivePlanColumns+` FROM orbit_archive_plans WHERE namespace_id=? ORDER BY kind`, t.NamespaceID) if err != nil { return err } defer rows.Close() found := map[string]retention.ArchivePlan{} for rows.Next() { p, err := scanArchivePlan(rows) if err != nil { return err } found[p.Kind] = p } if err := rows.Err(); err != nil { return err } out = []retention.ArchivePlan{} for _, kind := range retention.ArchiveKinds { p, ok := found[kind] if !ok { p = retention.ArchivePlan{NamespaceID: t.NamespaceID, Kind: kind, IntervalSeconds: 86400, BatchSize: 20, MaxBytes: 256 << 20} } out = append(out, p) } return nil }) return } func (s *RetentionPolicyStore) SaveArchivePlan(ctx context.Context, t auth.Token, key, kind string, storage core.ID, in retention.PlanInput) (out retention.ArchivePlan, err error) { if err = in.Validate(t.NamespaceID, kind); err != nil { return out, fault.Invalid(err.Error()) } if err = archiveKey(key); err != nil { return } // 卷身份属于幂等请求摘要;跨不同卷的节点不能把同一保存解释成不同目标。 input := struct { Kind string Storage core.ID Input retention.PlanInput }{kind, storage, in} err = WithTx(ctx, s.db, func(tx *sql.Tx) error { user, err := retentionActor(ctx, tx, t) if err != nil { return err } identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "retention.plan.update", kind, key, "archive_plan", t.NamespaceID, input) if err != nil { return err } if cached != nil { return json.Unmarshal(cached, &out) } policy, err := s.readPolicy(ctx, tx, t.NamespaceID, true) if err != nil { return err } if policy.Revision != in.PolicyRevision || (in.Enabled && !policy.Enabled) { return fault.Conflict } before, err := readArchivePlan(ctx, tx, t.NamespaceID, kind) if err != nil { return err } if before.Revision != in.ExpectedRevision { return fault.Conflict } target := storage if target == "" && !in.Enabled { target = before.StorageID } if target.Validate() != nil { return fault.Invalid("archive storage is not configured") } if before.ActiveJobID != nil && before.StorageID != target { return fault.Conflict } if before.Revision == 0 { _, err = tx.ExecContext(ctx, `INSERT INTO orbit_archive_plans(namespace_id,kind,enabled,policy_revision,storage_id,interval_seconds,batch_size,max_bytes,next_run_at) VALUES(?,?,?,?,?,?,?,?,UTC_TIMESTAMP(6))`, t.NamespaceID, kind, in.Enabled, in.PolicyRevision, target, in.IntervalSeconds, in.BatchSize, in.MaxBytes) } else { _, err = tx.ExecContext(ctx, `UPDATE orbit_archive_plans SET revision=revision+1,next_run_at=IF(enabled=FALSE AND ?=TRUE,UTC_TIMESTAMP(6),next_run_at),enabled=?,policy_revision=?,storage_id=?,interval_seconds=?,batch_size=?,max_bytes=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND kind=?`, in.Enabled, in.Enabled, in.PolicyRevision, target, in.IntervalSeconds, in.BatchSize, in.MaxBytes, t.NamespaceID, kind) } if err != nil { return err } out, err = readArchivePlan(ctx, tx, t.NamespaceID, kind) if err != nil { return err } if err = archiveAudit(ctx, tx, t.NamespaceID, t.NamespaceID, identity, "console:"+string(user), "retention.plan.update", "archive_plan", in.Reason, before, out); err != nil { return err } return saveIdentity(ctx, tx, t.NamespaceID, identity, out) }) return } func (s *RetentionPolicyStore) ListArchiveJobs(ctx context.Context, t auth.Token, before string) (out retention.JobPage, err error) { if before != "" && core.ID(before).Validate() != nil { return out, fault.Invalid("invalid archive page position") } err = WithTx(ctx, s.db, func(tx *sql.Tx) error { if _, err := retentionActor(ctx, tx, t); err != nil { return err } q := `SELECT ` + archiveJobColumns + ` FROM orbit_archive_jobs WHERE namespace_id=?` args := []any{t.NamespaceID} if before != "" { var at time.Time if err := tx.QueryRowContext(ctx, `SELECT created_at FROM orbit_archive_jobs WHERE namespace_id=? AND id=?`, t.NamespaceID, core.ID(before)).Scan(&at); err != nil { if errors.Is(err, sql.ErrNoRows) { return fault.NotFound } return err } q += ` AND (created_at 20 { out.Items = out.Items[:20] out.NextBefore = string(out.Items[19].ID) } return nil }) return } func (s *RetentionPolicyStore) ActArchiveJob(ctx context.Context, t auth.Token, id core.ID, key string, in retention.JobActionInput) (out retention.ManagedJob, err error) { if id.Validate() != nil || (in.Action != "retry" && in.Action != "abandon") || !retention.ValidReason(in.Reason) || in.MaxBytes < 1<<20 || in.MaxBytes > 1<<30 { return out, fault.Invalid("invalid archive action, budget or reason") } if err = archiveKey(key); err != nil { return } err = WithTx(ctx, s.db, func(tx *sql.Tx) error { user, err := retentionActor(ctx, tx, t) if err != nil { return err } identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "retention.job.action", string(id), key, "archive_job", id, in) if err != nil { return err } if cached != nil { return json.Unmarshal(cached, &out) } var kind string if err := tx.QueryRowContext(ctx, `SELECT kind FROM orbit_archive_jobs WHERE namespace_id=? AND id=?`, t.NamespaceID, id).Scan(&kind); err != nil { if errors.Is(err, sql.ErrNoRows) { return fault.NotFound } return err } p, err := readArchivePlan(ctx, tx, t.NamespaceID, kind) if err != nil { return err } j, err := readArchiveJob(ctx, tx, t.NamespaceID, id) if err != nil { return err } if p.ActiveJobID == nil || *p.ActiveJobID != id || j.State != "FAILED" || j.Revision != in.ExpectedRevision || in.MaxBytes < j.MaxBytes { return fault.Conflict } next := "PENDING" if in.Action == "abandon" { next = "ABANDONED" if in.MaxBytes != j.MaxBytes { return fault.Conflict } } _, err = tx.ExecContext(ctx, `UPDATE orbit_archive_jobs SET state=?,revision=revision+1,max_bytes=?,error_code='',updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, next, in.MaxBytes, t.NamespaceID, id) if err != nil { return err } if next == "ABANDONED" { if _, err = tx.ExecContext(ctx, `UPDATE orbit_archive_plans SET active_job_id=NULL WHERE namespace_id=? AND kind=? AND active_job_id=?`, t.NamespaceID, kind, id); err != nil { return err } } out, err = readArchiveJob(ctx, tx, t.NamespaceID, id) if err != nil { return err } if err = archiveAudit(ctx, tx, t.NamespaceID, id, identity, "console:"+string(user), "retention.job."+in.Action, "archive_job", in.Reason, j, out); err != nil { return err } return saveIdentity(ctx, tx, t.NamespaceID, identity, out) }) return } // ScheduleArchive 在数据库时钟下合并错过的时间槽;策略、计划和新作业同事务核验。 func (s *RetentionPolicyStore) ScheduleArchive(ctx context.Context, storage core.ID) error { if storage.Validate() != nil { return fault.Invalid("invalid storage id") } rows, err := s.db.QueryContext(ctx, `SELECT p.namespace_id,p.kind FROM orbit_archive_plans p JOIN orbit_retention_policies r ON r.namespace_id=p.namespace_id AND r.revision=p.policy_revision AND r.enabled=TRUE JOIN orbit_namespaces n ON n.id=p.namespace_id AND n.enabled=TRUE JOIN orbit_applications a ON a.id=n.application_id AND a.enabled=TRUE WHERE p.storage_id=? AND p.enabled=TRUE AND p.active_job_id IS NULL AND p.next_run_at<=UTC_TIMESTAMP(6) ORDER BY p.next_run_at LIMIT 16`, storage) if err != nil { return err } type candidate struct { ns core.ID kind string } var candidates []candidate for rows.Next() { var c candidate if err = rows.Scan(&c.ns, &c.kind); err != nil { rows.Close() return err } candidates = append(candidates, c) } err = rows.Err() rows.Close() if err != nil { return err } for _, c := range candidates { err = WithTx(ctx, s.db, func(tx *sql.Tx) error { var enabled bool if err := tx.QueryRowContext(ctx, `SELECT a.enabled AND n.enabled FROM orbit_namespaces n JOIN orbit_applications a ON a.id=n.application_id WHERE n.id=? FOR SHARE`, c.ns).Scan(&enabled); err != nil { return err } if !enabled { return nil } policy, err := s.readPolicy(ctx, tx, c.ns, true) if err != nil { return err } p, err := readArchivePlan(ctx, tx, c.ns, c.kind) if err != nil { return err } now, err := dbNow(ctx, tx) if err != nil { return err } if !p.Enabled || p.ActiveJobID != nil || p.StorageID != storage || p.NextRunAt == nil || p.NextRunAt.After(now) || !policy.Enabled || policy.Revision != p.PolicyRevision { return nil } id, err := core.NewID() if err != nil { return err } _, err = tx.ExecContext(ctx, `INSERT INTO orbit_archive_jobs(namespace_id,id,kind,plan_revision,policy_revision,storage_id,batch_size,max_bytes) VALUES(?,?,?,?,?,?,?,?)`, c.ns, id, c.kind, p.Revision, p.PolicyRevision, storage, p.BatchSize, p.MaxBytes) if err != nil { return err } _, err = tx.ExecContext(ctx, `UPDATE orbit_archive_plans SET active_job_id=?,next_run_at=? WHERE namespace_id=? AND kind=?`, id, now.Add(time.Duration(p.IntervalSeconds)*time.Second), c.ns, c.kind) if err != nil { return err } j, err := readArchiveJob(ctx, tx, c.ns, id) if err != nil { return err } return archiveAudit(ctx, tx, c.ns, id, id, "archive:scheduler", "retention.job.created", "archive_job", "周期归档到期", nil, j) }) if err != nil { return err } } return nil } // ClaimArchive 领取一批工作;五分钟租约超时后重领,以随机代次隔离迟到确认。 func (s *RetentionPolicyStore) ClaimArchive(ctx context.Context, storage core.ID) (job *retention.ManagedJob, err error) { err = WithTx(ctx, s.db, func(tx *sql.Tx) error { job = nil j, err := scanArchiveJob(tx.QueryRowContext(ctx, `SELECT `+archiveJobColumns+` FROM orbit_archive_jobs WHERE storage_id=? AND (state='PENDING' OR (state='RUNNING' AND lease_until<=UTC_TIMESTAMP(6))) ORDER BY updated_at,id LIMIT 1 FOR UPDATE SKIP LOCKED`, storage)) if errors.Is(err, sql.ErrNoRows) { return nil } if err != nil { return err } lease, err := core.NewID() if err != nil { return err } _, err = tx.ExecContext(ctx, `UPDATE orbit_archive_jobs SET state='RUNNING',revision=revision+1,attempts=attempts+1,lease_id=?,lease_until=TIMESTAMPADD(SECOND,300,UTC_TIMESTAMP(6)),updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, lease, j.NamespaceID, j.ID) if err != nil { return err } claimed, err := readArchiveJob(ctx, tx, j.NamespaceID, j.ID) if err != nil { return err } job = &claimed return nil }) return } // FinishArchive 仅确认当前未过期领取;失败不释放计划占位,避免不断生成重复失败作业。 func (s *RetentionPolicyStore) FinishArchive(ctx context.Context, claim retention.ManagedJob, status retention.JobStatus, code string) error { switch code { case "", "FILE_CHECK_FAILED", "STORAGE_LIMIT", "STORAGE_BUSY", "POLICY_CHANGED", "PERMISSION_DENIED", "EXPORT_FAILED", "INTERRUPTED": default: return fault.Invalid("invalid archive result code") } return WithTx(ctx, s.db, func(tx *sql.Tx) error { p, err := readArchivePlan(ctx, tx, claim.NamespaceID, claim.Kind) if err != nil { return err } j, err := readArchiveJob(ctx, tx, claim.NamespaceID, claim.ID) if err != nil { return err } now, err := dbNow(ctx, tx) if err != nil { return err } if p.ActiveJobID == nil || *p.ActiveJobID != j.ID || j.State != "RUNNING" || j.LeaseID != claim.LeaseID || j.Revision != claim.Revision || j.LeaseUntil == nil || !j.LeaseUntil.After(now) { return fault.Conflict } next := "PENDING" if code != "" { next = "FAILED" } else if status.Completed { next = "SUCCEEDED" } if code == "" && (status.JobID != j.ID || status.Batches < j.Batches || status.Roots < j.Roots || status.Bytes < j.Bytes || len(status.LastSHA256) != 64) { return fault.Conflict } // 校验失败的部分文件不能覆盖最后一次数据库确认的可信检查点。 if code != "" { status = retention.JobStatus{JobID: j.ID, Batches: j.Batches, Roots: j.Roots, Bytes: j.Bytes, LastSHA256: j.LastSHA256} } _, err = tx.ExecContext(ctx, `UPDATE orbit_archive_jobs SET state=?,revision=revision+1,batches=?,roots=?,bytes=?,last_sha256=?,error_code=?,lease_id=NULL,lease_until=NULL,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, next, status.Batches, status.Roots, status.Bytes, status.LastSHA256, code, j.NamespaceID, j.ID) if err != nil { return err } if next == "SUCCEEDED" { if _, err = tx.ExecContext(ctx, `UPDATE orbit_archive_plans SET active_job_id=NULL WHERE namespace_id=? AND kind=? AND active_job_id=?`, j.NamespaceID, j.Kind, j.ID); err != nil { return err } } if next == "PENDING" { return nil } out, err := readArchiveJob(ctx, tx, j.NamespaceID, j.ID) if err != nil { return err } return archiveAudit(ctx, tx, j.NamespaceID, j.ID, j.LeaseID, "archive:"+string(j.ID), "retention.job."+strings.ToLower(next), "archive_job", "归档执行结果", j, out) }) }