package mysql import ( "context" "database/sql" "encoding/json" "errors" "fmt" "time" "git.yuntuo.hk/k2safe/orbit/internal/fault" core "git.yuntuo.hk/k2safe/orbit/model" "git.yuntuo.hk/k2safe/orbit/pulse/schedule" ) // ScheduleStats 是本次已确认提交的批次计数,事务失败不计作成功。 type ScheduleStats struct { Examined int // Examined 已检查的候选计划数。 Fired int // Fired 本轮创建的任务数,合并范围计一个。 Skipped int // Skipped 本轮保存的跳过范围数,不是被覆盖触发点总数。 } // Tick 按持久化 check_at 扫描,一次最多 100 个计划,每个计划独立短事务。 func (s *ScheduleStore) Tick(ctx context.Context, limit int) (ScheduleStats, error) { var out ScheduleStats if limit < 1 || limit > 100 { return out, fault.Invalid("schedule batch limit must be 1 to 100") } type candidate struct{ ns, id, queue core.ID } candidates := []candidate{} rows, err := s.db.QueryContext(ctx, `SELECT s.namespace_id,s.id,s.queue_id FROM pulse_schedules s JOIN orbit_namespaces n ON n.id=s.namespace_id JOIN orbit_applications a ON a.id=n.application_id WHERE s.paused=FALSE AND s.check_at<=UTC_TIMESTAMP(6) AND n.enabled=TRUE AND a.enabled=TRUE ORDER BY s.check_at,s.namespace_id,s.id LIMIT ?`, limit) if err != nil { return out, err } for rows.Next() { var c candidate if err := rows.Scan(&c.ns, &c.id, &c.queue); err != nil { rows.Close() return out, err } candidates = append(candidates, c) } err = rows.Err() rows.Close() if err != nil { return out, err } var failures []error for _, c := range candidates { out.Examined++ fired, skipped, err := s.tickOne(ctx, c.ns, c.id, c.queue) if err != nil { failures = append(failures, err) if ctx.Err() != nil { break } continue } out.Fired += fired out.Skipped += skipped } return out, errors.Join(failures...) } func (s *ScheduleStore) tickOne(ctx context.Context, ns, id, queue core.ID) (int, int, error) { fired, skipped := 0, 0 err := WithTx(ctx, s.db, func(tx *sql.Tx) error { var allowed core.ID err := tx.QueryRowContext(ctx, `SELECT n.id FROM orbit_namespaces n JOIN orbit_applications a ON a.id=n.application_id WHERE n.id=? AND n.enabled=TRUE AND a.enabled=TRUE FOR SHARE`, ns).Scan(&allowed) if errors.Is(err, sql.ErrNoRows) { return nil } if err != nil { return err } // 应用额度 → 队列 → 计划;忙碌资源留给下一批。 if ok, err := lockApplicationResources(ctx, tx, ns, true); err != nil || !ok { return err } err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, ns, queue).Scan(&allowed) if errors.Is(err, sql.ErrNoRows) { return nil } if err != nil { return err } v, err := readSchedule(ctx, tx, ns, id, true) if err != nil { return err } now, err := dbNow(ctx, tx) if err != nil { return err } if v.Paused || v.CheckAt == nil || v.CheckAt.After(now) { return nil } if v.QueueID != queue { return errors.New("schedule queue changed") } cursor, next := v.ProcessingVersion, v.NextFireAt needsInitialization := false // 单批最多跨过 16 个空历史版本,避免长事务;下次从已保存版本继续。 for n := 0; n < 16; n++ { version, err := readScheduleVersion(ctx, tx, ns, id, cursor) if err != nil { return err } c, err := schedule.Compile(version.Definition.Rule) if err != nil { return err } var before *time.Time if cursor < v.CurrentVersion { newer, err := readScheduleVersion(ctx, tx, ns, id, cursor+1) if err != nil { return err } before = &newer.EffectiveAt } needsInitialization = false if next == nil { next, err = c.AtOrAfter(version.EffectiveAt) if err != nil { return err } } if next != nil && next.Before(version.EffectiveAt) { return errors.New("schedule cursor predates its version") } if next == nil || (before != nil && !next.Before(*before)) { next = nil if cursor < v.CurrentVersion { cursor++ needsInitialization = true continue } break } if next.After(now) { break } var active bool if version.Definition.Policy.Overlap != "PARALLEL" { if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pulse_tasks WHERE namespace_id=? AND owner_kind='schedule' AND owner_id=? AND state IN ('WAITING','LEASED','RUNNING','NEEDS_REVIEW'))`, ns, id).Scan(&active); err != nil { return err } } policy := version.Definition.Policy // 大参数也要保持事务有界;每批复制的业务参数最多约 1 MiB。 if size := len(version.Definition.Task.Payload); size > 0 { if max := uint32((1 << 20) / size); max < policy.CatchupLimit { policy.CatchupLimit = max } } batch, err := schedule.Plan(c, policy, *next, now, before, active) if err != nil { return err } backpressured := false for _, decision := range batch.Decisions { if err := s.saveOccurrence(ctx, tx, v, version, decision, now); err != nil { if errors.Is(err, fault.QueueFull) || errors.Is(err, fault.ApplicationFull) { // 已提交本批较早的决定仍保留;满额的原触发点留待下次按冻结策略处理。 t := decision.ScheduledAt next, backpressured = &t, true break } return err } if decision.Kind == "SKIPPED" { skipped++ } else { fired++ } } if !backpressured { next = batch.Next } if next == nil && cursor < v.CurrentVersion { cursor++ needsInitialization = true } break // 每计划每批只创建一个版本的有限任务,跨版本也不绕过速率上限。 } var check *time.Time if next != nil || cursor < v.CurrentVersion || needsInitialization { t := now.Add(time.Second) check = &t if next != nil && next.After(t) { check = next } } result, err := tx.ExecContext(ctx, `UPDATE pulse_schedules SET processing_version=?,next_fire_at=?,check_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, cursor, next, check, now, ns, id, v.Revision) if err := exactlyOne(result, err); err != nil { return err } return emitEvent(ctx, tx, ns, id, v.Revision+1, "schedule", "schedule.advanced", map[string]any{"node_id": s.node, "processing_version": fmt.Sprint(cursor), "next_fire_at": next, "fired": fired, "skipped": skipped}) }) if err != nil { return 0, 0, err } return fired, skipped, nil } func (s *ScheduleStore) saveOccurrence(ctx context.Context, tx *sql.Tx, v schedule.Summary, version schedule.VersionView, d schedule.Decision, now time.Time) error { id, err := core.NewID() if err != nil { return err } var task *core.ID if d.Kind != "SKIPPED" { value, err := core.NewID() if err != nil { return err } task = &value in := version.Definition.Task if in.QueueID != v.QueueID { return errors.New("schedule template queue mismatch") } if err := reserveQueueTask(ctx, tx, v.NamespaceID, v.QueueID); err != nil { return err } policy, err := json.Marshal(in.Retry) if err != nil { return err } _, err = tx.ExecContext(ctx, `INSERT INTO pulse_tasks(namespace_id,id,queue_id,handler,handler_version,payload_version,payload,policy_snapshot,recovery_safety,business_key,concurrency_key,concurrency_key_digest,priority,owner_kind,owner_id,available_at,deadline_at,attempt_timeout_ms,max_attempts,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,'schedule',?,?,?,?,?,?,?)`, v.NamespaceID, value, v.QueueID, in.Handler, in.HandlerVersion, in.PayloadVersion, string(in.Payload), string(policy), in.RecoverySafety, businessKeyValue(in.BusinessKey), concurrencyKeyValue(in.ConcurrencyKey), concurrencyDigest(in.ConcurrencyKey), in.Priority, v.ID, now, now.Add(time.Duration(in.TaskTimeoutMS)*time.Millisecond), in.Retry.AttemptTimeoutMS, in.Retry.MaxAttempts, now, now) if err != nil { return err } if err := emitEvent(ctx, tx, v.NamespaceID, value, 1, "task", "task.accepted", map[string]any{"schedule_id": v.ID, "occurrence_id": id, "schedule_version": fmt.Sprint(version.Version), "scheduled_at": d.ScheduledAt, "node_id": s.node}); err != nil { return err } } _, err = tx.ExecContext(ctx, `INSERT INTO pulse_schedule_occurrences(namespace_id,id,schedule_id,schedule_version,scheduled_at,decision,task_id,covered_until,reason,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)`, v.NamespaceID, id, v.ID, version.Version, d.ScheduledAt, d.Kind, task, d.CoveredUntil, d.Reason, now, now) if err != nil { return err } return emitEvent(ctx, tx, v.NamespaceID, id, 1, "schedule_occurrence", "schedule.decided", map[string]any{"schedule_id": v.ID, "version": fmt.Sprint(version.Version), "scheduled_at": d.ScheduledAt, "decision": d.Kind, "covered_until": d.CoveredUntil, "reason": d.Reason, "task_id": task, "node_id": s.node}) }