442 lines
17 KiB
Go
442 lines
17 KiB
Go
package mysql
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
|
"git.yuntuo.hk/k2safe/orbit/internal/retention"
|
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
|
)
|
|
|
|
const archivePlanColumns = `namespace_id,kind,revision,enabled,policy_revision,storage_id,interval_seconds,batch_size,max_bytes,next_run_at,active_job_id,updated_at`
|
|
const archiveJobColumns = `namespace_id,id,kind,plan_revision,policy_revision,storage_id,batch_size,max_bytes,state,revision,attempts,batches,roots,bytes,last_sha256,error_code,lease_id,lease_until,created_at,updated_at`
|
|
|
|
type archiveScanner interface{ Scan(...any) error }
|
|
|
|
func scanArchivePlan(row archiveScanner) (p retention.ArchivePlan, err error) {
|
|
err = row.Scan(&p.NamespaceID, &p.Kind, &p.Revision, &p.Enabled, &p.PolicyRevision, &p.StorageID, &p.IntervalSeconds, &p.BatchSize, &p.MaxBytes, &p.NextRunAt, &p.ActiveJobID, &p.UpdatedAt)
|
|
return
|
|
}
|
|
func scanArchiveJob(row archiveScanner) (j retention.ManagedJob, err error) {
|
|
var lease *core.ID
|
|
err = row.Scan(&j.NamespaceID, &j.ID, &j.Kind, &j.PlanRevision, &j.PolicyRevision, &j.StorageID, &j.BatchSize, &j.MaxBytes, &j.State, &j.Revision, &j.Attempts, &j.Batches, &j.Roots, &j.Bytes, &j.LastSHA256, &j.ErrorCode, &lease, &j.LeaseUntil, &j.CreatedAt, &j.UpdatedAt)
|
|
if lease != nil {
|
|
j.LeaseID = *lease
|
|
}
|
|
return
|
|
}
|
|
func readArchivePlan(ctx context.Context, tx *sql.Tx, ns core.ID, kind string) (retention.ArchivePlan, error) {
|
|
p, err := scanArchivePlan(tx.QueryRowContext(ctx, `SELECT `+archivePlanColumns+` FROM orbit_archive_plans WHERE namespace_id=? AND kind=? FOR UPDATE`, ns, kind))
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return retention.ArchivePlan{NamespaceID: ns, Kind: kind, IntervalSeconds: 86400, BatchSize: 20, MaxBytes: 256 << 20}, nil
|
|
}
|
|
return p, err
|
|
}
|
|
func readArchiveJob(ctx context.Context, tx *sql.Tx, ns, id core.ID) (retention.ManagedJob, error) {
|
|
j, err := scanArchiveJob(tx.QueryRowContext(ctx, `SELECT `+archiveJobColumns+` FROM orbit_archive_jobs WHERE namespace_id=? AND id=? FOR UPDATE`, ns, id))
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
err = fault.NotFound
|
|
}
|
|
return j, err
|
|
}
|
|
func archiveAudit(ctx context.Context, tx *sql.Tx, ns, resource, op core.ID, actor, action, kind, reason string, before, after any) error {
|
|
id, err := core.NewID()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
b, err := json.Marshal(before)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
a, err := json.Marshal(after)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = tx.ExecContext(ctx, "INSERT INTO orbit_audit_events(namespace_id,id,operation_id,actor_id,action,resource_kind,resource_id,reason,`before`,`after`) VALUES (?,?,?,?,?,?,?,?,?,?)", ns, id, op, actor, action, kind, resource, reason, string(b), string(a))
|
|
return err
|
|
}
|
|
func archiveKey(key string) error {
|
|
if strings.TrimSpace(key) == "" || len(key) > 128 {
|
|
return fault.Invalid("idempotency key must be 1-128 bytes")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *RetentionPolicyStore) ListArchivePlans(ctx context.Context, t auth.Token) (out []retention.ArchivePlan, err error) {
|
|
err = WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
|
if _, err := retentionActor(ctx, tx, t); err != nil {
|
|
return err
|
|
}
|
|
rows, err := tx.QueryContext(ctx, `SELECT `+archivePlanColumns+` FROM orbit_archive_plans WHERE namespace_id=? ORDER BY kind`, t.NamespaceID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer rows.Close()
|
|
found := map[string]retention.ArchivePlan{}
|
|
for rows.Next() {
|
|
p, err := scanArchivePlan(rows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
found[p.Kind] = p
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return err
|
|
}
|
|
out = []retention.ArchivePlan{}
|
|
for _, kind := range retention.ArchiveKinds {
|
|
p, ok := found[kind]
|
|
if !ok {
|
|
p = retention.ArchivePlan{NamespaceID: t.NamespaceID, Kind: kind, IntervalSeconds: 86400, BatchSize: 20, MaxBytes: 256 << 20}
|
|
}
|
|
out = append(out, p)
|
|
}
|
|
return nil
|
|
})
|
|
return
|
|
}
|
|
func (s *RetentionPolicyStore) SaveArchivePlan(ctx context.Context, t auth.Token, key, kind string, storage core.ID, in retention.PlanInput) (out retention.ArchivePlan, err error) {
|
|
if err = in.Validate(t.NamespaceID, kind); err != nil {
|
|
return out, fault.Invalid(err.Error())
|
|
}
|
|
if err = archiveKey(key); err != nil {
|
|
return
|
|
}
|
|
// 卷身份属于幂等请求摘要；跨不同卷的节点不能把同一保存解释成不同目标。
|
|
input := struct {
|
|
Kind string
|
|
Storage core.ID
|
|
Input retention.PlanInput
|
|
}{kind, storage, in}
|
|
err = WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
|
user, err := retentionActor(ctx, tx, t)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "retention.plan.update", kind, key, "archive_plan", t.NamespaceID, input)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if cached != nil {
|
|
return json.Unmarshal(cached, &out)
|
|
}
|
|
policy, err := s.readPolicy(ctx, tx, t.NamespaceID, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if policy.Revision != in.PolicyRevision || (in.Enabled && !policy.Enabled) {
|
|
return fault.Conflict
|
|
}
|
|
before, err := readArchivePlan(ctx, tx, t.NamespaceID, kind)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if before.Revision != in.ExpectedRevision {
|
|
return fault.Conflict
|
|
}
|
|
target := storage
|
|
if target == "" && !in.Enabled {
|
|
target = before.StorageID
|
|
}
|
|
if target.Validate() != nil {
|
|
return fault.Invalid("archive storage is not configured")
|
|
}
|
|
if before.ActiveJobID != nil && before.StorageID != target {
|
|
return fault.Conflict
|
|
}
|
|
if before.Revision == 0 {
|
|
_, err = tx.ExecContext(ctx, `INSERT INTO orbit_archive_plans(namespace_id,kind,enabled,policy_revision,storage_id,interval_seconds,batch_size,max_bytes,next_run_at) VALUES(?,?,?,?,?,?,?,?,UTC_TIMESTAMP(6))`, t.NamespaceID, kind, in.Enabled, in.PolicyRevision, target, in.IntervalSeconds, in.BatchSize, in.MaxBytes)
|
|
} else {
|
|
_, err = tx.ExecContext(ctx, `UPDATE orbit_archive_plans SET revision=revision+1,next_run_at=IF(enabled=FALSE AND ?=TRUE,UTC_TIMESTAMP(6),next_run_at),enabled=?,policy_revision=?,storage_id=?,interval_seconds=?,batch_size=?,max_bytes=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND kind=?`, in.Enabled, in.Enabled, in.PolicyRevision, target, in.IntervalSeconds, in.BatchSize, in.MaxBytes, t.NamespaceID, kind)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
out, err = readArchivePlan(ctx, tx, t.NamespaceID, kind)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err = archiveAudit(ctx, tx, t.NamespaceID, t.NamespaceID, identity, "console:"+string(user), "retention.plan.update", "archive_plan", in.Reason, before, out); err != nil {
|
|
return err
|
|
}
|
|
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
|
|
})
|
|
return
|
|
}
|
|
func (s *RetentionPolicyStore) ListArchiveJobs(ctx context.Context, t auth.Token, before string) (out retention.JobPage, err error) {
|
|
if before != "" && core.ID(before).Validate() != nil {
|
|
return out, fault.Invalid("invalid archive page position")
|
|
}
|
|
err = WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
|
if _, err := retentionActor(ctx, tx, t); err != nil {
|
|
return err
|
|
}
|
|
q := `SELECT ` + archiveJobColumns + ` FROM orbit_archive_jobs WHERE namespace_id=?`
|
|
args := []any{t.NamespaceID}
|
|
if before != "" {
|
|
var at time.Time
|
|
if err := tx.QueryRowContext(ctx, `SELECT created_at FROM orbit_archive_jobs WHERE namespace_id=? AND id=?`, t.NamespaceID, core.ID(before)).Scan(&at); err != nil {
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return fault.NotFound
|
|
}
|
|
return err
|
|
}
|
|
q += ` AND (created_at<? OR (created_at=? AND id<?))`
|
|
args = append(args, at, at, core.ID(before))
|
|
}
|
|
rows, err := tx.QueryContext(ctx, q+` ORDER BY created_at DESC,id DESC LIMIT 21`, args...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer rows.Close()
|
|
out = retention.JobPage{Items: []retention.ManagedJob{}}
|
|
for rows.Next() {
|
|
j, err := scanArchiveJob(rows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
out.Items = append(out.Items, j)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return err
|
|
}
|
|
if len(out.Items) > 20 {
|
|
out.Items = out.Items[:20]
|
|
out.NextBefore = string(out.Items[19].ID)
|
|
}
|
|
return nil
|
|
})
|
|
return
|
|
}
|
|
func (s *RetentionPolicyStore) ActArchiveJob(ctx context.Context, t auth.Token, id core.ID, key string, in retention.JobActionInput) (out retention.ManagedJob, err error) {
|
|
if id.Validate() != nil || (in.Action != "retry" && in.Action != "abandon") || !retention.ValidReason(in.Reason) || in.MaxBytes < 1<<20 || in.MaxBytes > 1<<30 {
|
|
return out, fault.Invalid("invalid archive action, budget or reason")
|
|
}
|
|
if err = archiveKey(key); err != nil {
|
|
return
|
|
}
|
|
err = WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
|
user, err := retentionActor(ctx, tx, t)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "retention.job.action", string(id), key, "archive_job", id, in)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if cached != nil {
|
|
return json.Unmarshal(cached, &out)
|
|
}
|
|
var kind string
|
|
if err := tx.QueryRowContext(ctx, `SELECT kind FROM orbit_archive_jobs WHERE namespace_id=? AND id=?`, t.NamespaceID, id).Scan(&kind); err != nil {
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return fault.NotFound
|
|
}
|
|
return err
|
|
}
|
|
p, err := readArchivePlan(ctx, tx, t.NamespaceID, kind)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
j, err := readArchiveJob(ctx, tx, t.NamespaceID, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if p.ActiveJobID == nil || *p.ActiveJobID != id || j.State != "FAILED" || j.Revision != in.ExpectedRevision || in.MaxBytes < j.MaxBytes {
|
|
return fault.Conflict
|
|
}
|
|
next := "PENDING"
|
|
if in.Action == "abandon" {
|
|
next = "ABANDONED"
|
|
if in.MaxBytes != j.MaxBytes {
|
|
return fault.Conflict
|
|
}
|
|
}
|
|
_, err = tx.ExecContext(ctx, `UPDATE orbit_archive_jobs SET state=?,revision=revision+1,max_bytes=?,error_code='',updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, next, in.MaxBytes, t.NamespaceID, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if next == "ABANDONED" {
|
|
if _, err = tx.ExecContext(ctx, `UPDATE orbit_archive_plans SET active_job_id=NULL WHERE namespace_id=? AND kind=? AND active_job_id=?`, t.NamespaceID, kind, id); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
out, err = readArchiveJob(ctx, tx, t.NamespaceID, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err = archiveAudit(ctx, tx, t.NamespaceID, id, identity, "console:"+string(user), "retention.job."+in.Action, "archive_job", in.Reason, j, out); err != nil {
|
|
return err
|
|
}
|
|
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
|
|
})
|
|
return
|
|
}
|
|
|
|
// ScheduleArchive 在数据库时钟下合并错过的时间槽；策略、计划和新作业同事务核验。
|
|
func (s *RetentionPolicyStore) ScheduleArchive(ctx context.Context, storage core.ID) error {
|
|
if storage.Validate() != nil {
|
|
return fault.Invalid("invalid storage id")
|
|
}
|
|
rows, err := s.db.QueryContext(ctx, `SELECT p.namespace_id,p.kind FROM orbit_archive_plans p JOIN orbit_retention_policies r ON r.namespace_id=p.namespace_id AND r.revision=p.policy_revision AND r.enabled=TRUE JOIN orbit_namespaces n ON n.id=p.namespace_id AND n.enabled=TRUE JOIN orbit_applications a ON a.id=n.application_id AND a.enabled=TRUE WHERE p.storage_id=? AND p.enabled=TRUE AND p.active_job_id IS NULL AND p.next_run_at<=UTC_TIMESTAMP(6) ORDER BY p.next_run_at LIMIT 16`, storage)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
type candidate struct {
|
|
ns core.ID
|
|
kind string
|
|
}
|
|
var candidates []candidate
|
|
for rows.Next() {
|
|
var c candidate
|
|
if err = rows.Scan(&c.ns, &c.kind); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
candidates = append(candidates, c)
|
|
}
|
|
err = rows.Err()
|
|
rows.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, c := range candidates {
|
|
err = WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
|
var enabled bool
|
|
if err := tx.QueryRowContext(ctx, `SELECT a.enabled AND n.enabled FROM orbit_namespaces n JOIN orbit_applications a ON a.id=n.application_id WHERE n.id=? FOR SHARE`, c.ns).Scan(&enabled); err != nil {
|
|
return err
|
|
}
|
|
if !enabled {
|
|
return nil
|
|
}
|
|
policy, err := s.readPolicy(ctx, tx, c.ns, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
p, err := readArchivePlan(ctx, tx, c.ns, c.kind)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
now, err := dbNow(ctx, tx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !p.Enabled || p.ActiveJobID != nil || p.StorageID != storage || p.NextRunAt == nil || p.NextRunAt.After(now) || !policy.Enabled || policy.Revision != p.PolicyRevision {
|
|
return nil
|
|
}
|
|
id, err := core.NewID()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = tx.ExecContext(ctx, `INSERT INTO orbit_archive_jobs(namespace_id,id,kind,plan_revision,policy_revision,storage_id,batch_size,max_bytes) VALUES(?,?,?,?,?,?,?,?)`, c.ns, id, c.kind, p.Revision, p.PolicyRevision, storage, p.BatchSize, p.MaxBytes)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = tx.ExecContext(ctx, `UPDATE orbit_archive_plans SET active_job_id=?,next_run_at=? WHERE namespace_id=? AND kind=?`, id, now.Add(time.Duration(p.IntervalSeconds)*time.Second), c.ns, c.kind)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
j, err := readArchiveJob(ctx, tx, c.ns, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return archiveAudit(ctx, tx, c.ns, id, id, "archive:scheduler", "retention.job.created", "archive_job", "周期归档到期", nil, j)
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ClaimArchive 领取一批工作；五分钟租约超时后重领，以随机代次隔离迟到确认。
|
|
func (s *RetentionPolicyStore) ClaimArchive(ctx context.Context, storage core.ID) (job *retention.ManagedJob, err error) {
|
|
err = WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
|
job = nil
|
|
j, err := scanArchiveJob(tx.QueryRowContext(ctx, `SELECT `+archiveJobColumns+` FROM orbit_archive_jobs WHERE storage_id=? AND (state='PENDING' OR (state='RUNNING' AND lease_until<=UTC_TIMESTAMP(6))) ORDER BY updated_at,id LIMIT 1 FOR UPDATE SKIP LOCKED`, storage))
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
lease, err := core.NewID()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = tx.ExecContext(ctx, `UPDATE orbit_archive_jobs SET state='RUNNING',revision=revision+1,attempts=attempts+1,lease_id=?,lease_until=TIMESTAMPADD(SECOND,300,UTC_TIMESTAMP(6)),updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, lease, j.NamespaceID, j.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
claimed, err := readArchiveJob(ctx, tx, j.NamespaceID, j.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
job = &claimed
|
|
return nil
|
|
})
|
|
return
|
|
}
|
|
|
|
// FinishArchive 仅确认当前未过期领取；失败不释放计划占位，避免不断生成重复失败作业。
|
|
func (s *RetentionPolicyStore) FinishArchive(ctx context.Context, claim retention.ManagedJob, status retention.JobStatus, code string) error {
|
|
switch code {
|
|
case "", "FILE_CHECK_FAILED", "STORAGE_LIMIT", "STORAGE_BUSY", "POLICY_CHANGED", "PERMISSION_DENIED", "EXPORT_FAILED", "INTERRUPTED":
|
|
default:
|
|
return fault.Invalid("invalid archive result code")
|
|
}
|
|
return WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
|
p, err := readArchivePlan(ctx, tx, claim.NamespaceID, claim.Kind)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
j, err := readArchiveJob(ctx, tx, claim.NamespaceID, claim.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
now, err := dbNow(ctx, tx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if p.ActiveJobID == nil || *p.ActiveJobID != j.ID || j.State != "RUNNING" || j.LeaseID != claim.LeaseID || j.Revision != claim.Revision || j.LeaseUntil == nil || !j.LeaseUntil.After(now) {
|
|
return fault.Conflict
|
|
}
|
|
next := "PENDING"
|
|
if code != "" {
|
|
next = "FAILED"
|
|
} else if status.Completed {
|
|
next = "SUCCEEDED"
|
|
}
|
|
if code == "" && (status.JobID != j.ID || status.Batches < j.Batches || status.Roots < j.Roots || status.Bytes < j.Bytes || len(status.LastSHA256) != 64) {
|
|
return fault.Conflict
|
|
}
|
|
// 校验失败的部分文件不能覆盖最后一次数据库确认的可信检查点。
|
|
if code != "" {
|
|
status = retention.JobStatus{JobID: j.ID, Batches: j.Batches, Roots: j.Roots, Bytes: j.Bytes, LastSHA256: j.LastSHA256}
|
|
}
|
|
_, err = tx.ExecContext(ctx, `UPDATE orbit_archive_jobs SET state=?,revision=revision+1,batches=?,roots=?,bytes=?,last_sha256=?,error_code=?,lease_id=NULL,lease_until=NULL,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, next, status.Batches, status.Roots, status.Bytes, status.LastSHA256, code, j.NamespaceID, j.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if next == "SUCCEEDED" {
|
|
if _, err = tx.ExecContext(ctx, `UPDATE orbit_archive_plans SET active_job_id=NULL WHERE namespace_id=? AND kind=? AND active_job_id=?`, j.NamespaceID, j.Kind, j.ID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if next == "PENDING" {
|
|
return nil
|
|
}
|
|
out, err := readArchiveJob(ctx, tx, j.NamespaceID, j.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return archiveAudit(ctx, tx, j.NamespaceID, j.ID, j.LeaseID, "archive:"+string(j.ID), "retention.job."+strings.ToLower(next), "archive_job", "归档执行结果", j, out)
|
|
})
|
|
}
|