241 lines
8.8 KiB
Go
241 lines
8.8 KiB
Go
package mysql
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
|
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
|
|
)
|
|
|
|
// ScheduleStats 是本次已确认提交的批次计数，事务失败不计作成功。
|
|
type ScheduleStats struct {
|
|
Examined int // Examined 已检查的候选计划数。
|
|
Fired int // Fired 本轮创建的任务数，合并范围计一个。
|
|
Skipped int // Skipped 本轮保存的跳过范围数，不是被覆盖触发点总数。
|
|
}
|
|
|
|
// Tick 按持久化 check_at 扫描，一次最多 100 个计划，每个计划独立短事务。
|
|
func (s *ScheduleStore) Tick(ctx context.Context, limit int) (ScheduleStats, error) {
|
|
var out ScheduleStats
|
|
if limit < 1 || limit > 100 {
|
|
return out, fault.Invalid("schedule batch limit must be 1 to 100")
|
|
}
|
|
type candidate struct{ ns, id, queue core.ID }
|
|
candidates := []candidate{}
|
|
rows, err := s.db.QueryContext(ctx, `SELECT s.namespace_id,s.id,s.queue_id FROM pulse_schedules s JOIN orbit_namespaces n ON n.id=s.namespace_id JOIN orbit_applications a ON a.id=n.application_id WHERE s.paused=FALSE AND s.check_at<=UTC_TIMESTAMP(6) AND n.enabled=TRUE AND a.enabled=TRUE ORDER BY s.check_at,s.namespace_id,s.id LIMIT ?`, limit)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
for rows.Next() {
|
|
var c candidate
|
|
if err := rows.Scan(&c.ns, &c.id, &c.queue); err != nil {
|
|
rows.Close()
|
|
return out, err
|
|
}
|
|
candidates = append(candidates, c)
|
|
}
|
|
err = rows.Err()
|
|
rows.Close()
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
var failures []error
|
|
for _, c := range candidates {
|
|
out.Examined++
|
|
fired, skipped, err := s.tickOne(ctx, c.ns, c.id, c.queue)
|
|
if err != nil {
|
|
failures = append(failures, err)
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
continue
|
|
}
|
|
out.Fired += fired
|
|
out.Skipped += skipped
|
|
}
|
|
return out, errors.Join(failures...)
|
|
}
|
|
func (s *ScheduleStore) tickOne(ctx context.Context, ns, id, queue core.ID) (int, int, error) {
|
|
fired, skipped := 0, 0
|
|
err := WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
|
var allowed core.ID
|
|
err := tx.QueryRowContext(ctx, `SELECT n.id FROM orbit_namespaces n JOIN orbit_applications a ON a.id=n.application_id WHERE n.id=? AND n.enabled=TRUE AND a.enabled=TRUE FOR SHARE`, ns).Scan(&allowed)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// 应用额度 → 队列 → 计划；忙碌资源留给下一批。
|
|
if ok, err := lockApplicationResources(ctx, tx, ns, true); err != nil || !ok {
|
|
return err
|
|
}
|
|
err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, ns, queue).Scan(&allowed)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
v, err := readSchedule(ctx, tx, ns, id, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
now, err := dbNow(ctx, tx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if v.Paused || v.CheckAt == nil || v.CheckAt.After(now) {
|
|
return nil
|
|
}
|
|
if v.QueueID != queue {
|
|
return errors.New("schedule queue changed")
|
|
}
|
|
cursor, next := v.ProcessingVersion, v.NextFireAt
|
|
needsInitialization := false
|
|
// 单批最多跨过 16 个空历史版本，避免长事务；下次从已保存版本继续。
|
|
for n := 0; n < 16; n++ {
|
|
version, err := readScheduleVersion(ctx, tx, ns, id, cursor)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
c, err := schedule.Compile(version.Definition.Rule)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var before *time.Time
|
|
if cursor < v.CurrentVersion {
|
|
newer, err := readScheduleVersion(ctx, tx, ns, id, cursor+1)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
before = &newer.EffectiveAt
|
|
}
|
|
needsInitialization = false
|
|
if next == nil {
|
|
next, err = c.AtOrAfter(version.EffectiveAt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if next != nil && next.Before(version.EffectiveAt) {
|
|
return errors.New("schedule cursor predates its version")
|
|
}
|
|
if next == nil || (before != nil && !next.Before(*before)) {
|
|
next = nil
|
|
if cursor < v.CurrentVersion {
|
|
cursor++
|
|
needsInitialization = true
|
|
continue
|
|
}
|
|
break
|
|
}
|
|
if next.After(now) {
|
|
break
|
|
}
|
|
var active bool
|
|
if version.Definition.Policy.Overlap != "PARALLEL" {
|
|
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pulse_tasks WHERE namespace_id=? AND owner_kind='schedule' AND owner_id=? AND state IN ('WAITING','LEASED','RUNNING','NEEDS_REVIEW'))`, ns, id).Scan(&active); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
policy := version.Definition.Policy
|
|
// 大参数也要保持事务有界；每批复制的业务参数最多约 1 MiB。
|
|
if size := len(version.Definition.Task.Payload); size > 0 {
|
|
if max := uint32((1 << 20) / size); max < policy.CatchupLimit {
|
|
policy.CatchupLimit = max
|
|
}
|
|
}
|
|
batch, err := schedule.Plan(c, policy, *next, now, before, active)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
backpressured := false
|
|
for _, decision := range batch.Decisions {
|
|
if err := s.saveOccurrence(ctx, tx, v, version, decision, now); err != nil {
|
|
if errors.Is(err, fault.QueueFull) || errors.Is(err, fault.ApplicationFull) {
|
|
// 已提交本批较早的决定仍保留；满额的原触发点留待下次按冻结策略处理。
|
|
t := decision.ScheduledAt
|
|
next, backpressured = &t, true
|
|
break
|
|
}
|
|
return err
|
|
}
|
|
if decision.Kind == "SKIPPED" {
|
|
skipped++
|
|
} else {
|
|
fired++
|
|
}
|
|
}
|
|
if !backpressured {
|
|
next = batch.Next
|
|
}
|
|
if next == nil && cursor < v.CurrentVersion {
|
|
cursor++
|
|
needsInitialization = true
|
|
}
|
|
break // 每计划每批只创建一个版本的有限任务，跨版本也不绕过速率上限。
|
|
}
|
|
var check *time.Time
|
|
if next != nil || cursor < v.CurrentVersion || needsInitialization {
|
|
t := now.Add(time.Second)
|
|
check = &t
|
|
if next != nil && next.After(t) {
|
|
check = next
|
|
}
|
|
}
|
|
result, err := tx.ExecContext(ctx, `UPDATE pulse_schedules SET processing_version=?,next_fire_at=?,check_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, cursor, next, check, now, ns, id, v.Revision)
|
|
if err := exactlyOne(result, err); err != nil {
|
|
return err
|
|
}
|
|
return emitEvent(ctx, tx, ns, id, v.Revision+1, "schedule", "schedule.advanced", map[string]any{"node_id": s.node, "processing_version": fmt.Sprint(cursor), "next_fire_at": next, "fired": fired, "skipped": skipped})
|
|
})
|
|
if err != nil {
|
|
return 0, 0, err
|
|
}
|
|
return fired, skipped, nil
|
|
}
|
|
func (s *ScheduleStore) saveOccurrence(ctx context.Context, tx *sql.Tx, v schedule.Summary, version schedule.VersionView, d schedule.Decision, now time.Time) error {
|
|
id, err := core.NewID()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var task *core.ID
|
|
if d.Kind != "SKIPPED" {
|
|
value, err := core.NewID()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
task = &value
|
|
in := version.Definition.Task
|
|
if in.QueueID != v.QueueID {
|
|
return errors.New("schedule template queue mismatch")
|
|
}
|
|
if err := reserveQueueTask(ctx, tx, v.NamespaceID, v.QueueID); err != nil {
|
|
return err
|
|
}
|
|
policy, err := json.Marshal(in.Retry)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_tasks(namespace_id,id,queue_id,handler,handler_version,payload_version,payload,policy_snapshot,recovery_safety,business_key,concurrency_key,concurrency_key_digest,priority,owner_kind,owner_id,available_at,deadline_at,attempt_timeout_ms,max_attempts,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,'schedule',?,?,?,?,?,?,?)`, v.NamespaceID, value, v.QueueID, in.Handler, in.HandlerVersion, in.PayloadVersion, string(in.Payload), string(policy), in.RecoverySafety, businessKeyValue(in.BusinessKey), concurrencyKeyValue(in.ConcurrencyKey), concurrencyDigest(in.ConcurrencyKey), in.Priority, v.ID, now, now.Add(time.Duration(in.TaskTimeoutMS)*time.Millisecond), in.Retry.AttemptTimeoutMS, in.Retry.MaxAttempts, now, now)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := emitEvent(ctx, tx, v.NamespaceID, value, 1, "task", "task.accepted", map[string]any{"schedule_id": v.ID, "occurrence_id": id, "schedule_version": fmt.Sprint(version.Version), "scheduled_at": d.ScheduledAt, "node_id": s.node}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_schedule_occurrences(namespace_id,id,schedule_id,schedule_version,scheduled_at,decision,task_id,covered_until,reason,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)`, v.NamespaceID, id, v.ID, version.Version, d.ScheduledAt, d.Kind, task, d.CoveredUntil, d.Reason, now, now)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return emitEvent(ctx, tx, v.NamespaceID, id, 1, "schedule_occurrence", "schedule.decided", map[string]any{"schedule_id": v.ID, "version": fmt.Sprint(version.Version), "scheduled_at": d.ScheduledAt, "decision": d.Kind, "covered_until": d.CoveredUntil, "reason": d.Reason, "task_id": task, "node_id": s.node})
|
|
}
|