feat: add durable schedule management and multi-node crash recovery

This commit is contained in:
2026-09-17 16:28:58 +08:00
parent d65cf96ea7
commit 6805506450
43 changed files with 2952 additions and 29 deletions

View File

@@ -14,11 +14,12 @@ import (
// BootstrapInput 仅供持有数据库管理权限的初始化命令使用,不暴露为公共 HTTP。
type BootstrapInput struct {
Application string // Application 应用名称,每次初始化创建独立应用。
Namespace string // Namespace 命名空间名称。
Environment string // Environment 环境名称,必须显式指定。
Queue string // Queue 初始队列名称。
WorkerAccess bool // WorkerAccess 显式增加开发用 Worker 接入及消费权限。
Application string // Application 应用名称,每次初始化创建独立应用。
Namespace string // Namespace 命名空间名称。
Environment string // Environment 环境名称,必须显式指定。
Queue string // Queue 初始队列名称。
WorkerAccess bool // WorkerAccess 显式增加开发用 Worker 接入及消费权限。
ScheduleAccess bool // ScheduleAccess 显式增加开发用计划查询及管理权限。
}
// BootstrapResult 包含一次性交付的原始凭据,禁止记录到日志或提交到版本库。
@@ -60,6 +61,9 @@ func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func(
if in.WorkerAccess {
permissions.Actions = append(permissions.Actions, "workers:connect", "workers:consume")
}
if in.ScheduleAccess {
permissions.Actions = append(permissions.Actions, "schedules:read", "schedules:manage")
}
raw, err := json.Marshal(permissions)
if err != nil {
return err

View File

@@ -0,0 +1,18 @@
-- 旧版本只定义了结构,没有公开创建计划的入口。未绑定队列的历史行不启用扫描。
ALTER TABLE pulse_schedules
ADD COLUMN queue_id BINARY(16) NULL COMMENT '计划终身绑定队列;旧结构人工行为空,不允许执行',
ADD COLUMN processing_version BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '当前处理积压的不可变版本,不因发布新版本丢弃旧进度',
ADD COLUMN check_at DATETIME(6) NULL COMMENT '下一可扫描时间,UTC;补跑批次之间持久化至少一秒间隔,为空表示没有扫描工作',
ADD CONSTRAINT ck_schedule_cursor CHECK (processing_version>0 AND processing_version<=current_version),
ADD CONSTRAINT ck_schedule_enabled_scan CHECK (queue_id IS NOT NULL OR check_at IS NULL),
ADD CONSTRAINT fk_schedule_queue FOREIGN KEY(namespace_id,queue_id) REFERENCES pulse_queues(namespace_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT,
ADD KEY ix_schedule_scan(paused,check_at,namespace_id,id),
ADD KEY ix_schedule_list(namespace_id,queue_id,created_at,id);
ALTER TABLE pulse_schedule_versions
ADD COLUMN catchup_window_ms BIGINT UNSIGNED NOT NULL DEFAULT 86400000 COMMENT '未处理触发点最大保留时长,单位毫秒,过旧范围记录跳过',
ADD COLUMN grace_ms BIGINT UNSIGNED NOT NULL DEFAULT 5000 COMMENT '正常派发允许延迟的毫秒数,超过后按漏跑策略处理',
ADD CONSTRAINT ck_schedule_window CHECK(catchup_window_ms BETWEEN 1000 AND 2592000000 AND grace_ms BETWEEN 1000 AND 60000 AND grace_ms<=catchup_window_ms);
ALTER TABLE pulse_schedule_occurrences
MODIFY COLUMN covered_until DATETIME(6) NULL COMMENT '合并或跳过范围的排除边界,范围为原计划时间至该边界,不包含边界',
ADD CONSTRAINT ck_occurrence_range CHECK(covered_until IS NULL OR covered_until>scheduled_at);
ALTER TABLE pulse_tasks ADD KEY ix_task_owner_state(namespace_id,owner_kind,owner_id,state,id);

View File

@@ -0,0 +1,139 @@
package mysql
import (
"bytes"
"context"
"database/sql"
"encoding/base64"
"encoding/json"
"time"
"git.yuntuo.hk/k2safe/orbit/internal/fault"
"git.yuntuo.hk/k2safe/orbit/internal/jsonvalue"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/platform/auth"
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
)
type scheduleCursor struct {
Version int `json:"v"`
Kind string `json:"k"`
Namespace core.ID `json:"n"`
Parent core.ID `json:"p"`
At time.Time `json:"t"`
ID core.ID `json:"i"`
}
func parseScheduleCursor(raw string, ns, parent core.ID, kind string) (scheduleCursor, error) {
var c scheduleCursor
b, err := base64.RawURLEncoding.Strict().DecodeString(raw)
if err != nil || len(b) > 768 {
return c, fault.Invalid("invalid schedule cursor")
}
if _, err := jsonvalue.Canonical(b); err != nil {
return c, fault.Invalid("invalid schedule cursor")
}
var fields map[string]json.RawMessage
if json.Unmarshal(b, &fields) != nil || len(fields) != 6 {
return c, fault.Invalid("invalid schedule cursor")
}
for _, key := range []string{"v", "k", "n", "p", "t", "i"} {
if _, ok := fields[key]; !ok {
return c, fault.Invalid("invalid schedule cursor")
}
}
d := json.NewDecoder(bytes.NewReader(b))
d.DisallowUnknownFields()
if d.Decode(&c) != nil || c.Version != 1 || c.Namespace != ns || c.Parent != parent || c.Kind != kind || c.ID.Validate() != nil || c.At.Year() < 2000 || c.At.Year() > 9998 || c.At.Nanosecond()%1000 != 0 {
return c, fault.Invalid("cursor does not match schedule query")
}
return c, nil
}
func makeScheduleCursor(ns, parent core.ID, kind string, at time.Time, id core.ID) (string, error) {
b, err := json.Marshal(scheduleCursor{1, kind, ns, parent, at, id})
return base64.RawURLEncoding.EncodeToString(b), err
}
func (s *ScheduleStore) List(ctx context.Context, t auth.Token, queue core.ID, in schedule.PageInput) (schedule.Page, error) {
out := schedule.Page{Items: []schedule.Summary{}}
err := s.transaction(ctx, t, "schedules:read", func(tx *sql.Tx, p auth.Permissions) error {
if !p.AllowsQueue(queue) {
return fault.Forbidden
}
q := `SELECT ` + scheduleColumns + ` FROM pulse_schedules WHERE namespace_id=? AND queue_id=?`
args := []any{t.NamespaceID, queue}
if in.Cursor != "" {
c, err := parseScheduleCursor(in.Cursor, t.NamespaceID, queue, "schedules")
if err != nil {
return err
}
q += ` AND (created_at<? OR (created_at=? AND id<?))`
args = append(args, c.At, c.At, c.ID)
}
q += ` ORDER BY created_at DESC,id DESC LIMIT ?`
args = append(args, in.Limit+1)
rows, err := tx.QueryContext(ctx, q, args...)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
v, err := scanSchedule(rows)
if err != nil {
return err
}
out.Items = append(out.Items, v)
}
if err := rows.Err(); err != nil {
return err
}
if len(out.Items) > in.Limit {
out.Items = out.Items[:in.Limit]
last := out.Items[len(out.Items)-1]
out.NextCursor, err = makeScheduleCursor(t.NamespaceID, queue, "schedules", last.CreatedAt, last.ID)
}
return err
})
return out, err
}
func (s *ScheduleStore) Occurrences(ctx context.Context, t auth.Token, id core.ID, in schedule.PageInput) (schedule.OccurrencePage, error) {
out := schedule.OccurrencePage{Items: []schedule.OccurrenceView{}}
err := s.transaction(ctx, t, "schedules:read", func(tx *sql.Tx, p auth.Permissions) error {
if _, err := authorizeSchedule(ctx, tx, t.NamespaceID, id, p); err != nil {
return err
}
q := `SELECT id,schedule_id,schedule_version,scheduled_at,decision,task_id,covered_until,COALESCE(reason,''),created_at FROM pulse_schedule_occurrences WHERE namespace_id=? AND schedule_id=?`
args := []any{t.NamespaceID, id}
if in.Cursor != "" {
c, err := parseScheduleCursor(in.Cursor, t.NamespaceID, id, "occurrences")
if err != nil {
return err
}
q += ` AND (scheduled_at<? OR (scheduled_at=? AND id<?))`
args = append(args, c.At, c.At, c.ID)
}
q += ` ORDER BY scheduled_at DESC,id DESC LIMIT ?`
args = append(args, in.Limit+1)
rows, err := tx.QueryContext(ctx, q, args...)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var v schedule.OccurrenceView
if err := rows.Scan(&v.ID, &v.ScheduleID, &v.ScheduleVersion, &v.ScheduledAt, &v.Decision, &v.TaskID, &v.CoveredUntil, &v.Reason, &v.CreatedAt); err != nil {
return err
}
out.Items = append(out.Items, v)
}
if err := rows.Err(); err != nil {
return err
}
if len(out.Items) > in.Limit {
out.Items = out.Items[:in.Limit]
last := out.Items[len(out.Items)-1]
out.NextCursor, err = makeScheduleCursor(t.NamespaceID, id, "occurrences", last.ScheduledAt, last.ID)
}
return err
})
return out, err
}

View File

@@ -0,0 +1,400 @@
package mysql
import (
"context"
"encoding/json"
"os"
"sync"
"testing"
"time"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/platform/auth"
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
"git.yuntuo.hk/k2safe/orbit/pulse/service"
)
type scheduleFixture struct {
*executionFixture
store *ScheduleStore
api *schedule.Service
}
func newScheduleFixture(t *testing.T, f *executionFixture) *scheduleFixture {
t.Helper()
raw, _ := json.Marshal(auth.Permissions{Actions: []string{"tasks:submit", "tasks:read", "tasks:cancel", "queues:read", "schedules:read", "schedules:manage", "workers:connect", "workers:consume"}, Queues: []string{"*"}})
f.exec(`UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?`, string(raw), f.resources.NamespaceID, f.resources.CredentialID)
store, err := NewScheduleStore(f.db, executionID(t))
if err != nil {
t.Fatal(err)
}
return &scheduleFixture{f, store, schedule.New(store)}
}
func (f *scheduleFixture) definition() schedule.Definition {
return schedule.Definition{Rule: schedule.Rule{Kind: "FIXED_RATE", Timezone: "UTC", StartAt: time.Now().UTC().Truncate(time.Microsecond).Add(-time.Hour), IntervalMS: 1000}, Policy: schedule.Policy{Misfire: "CATCH_UP", Overlap: "PARALLEL", CatchupLimit: 3, CatchupWindowMS: 30000, GraceMS: 1000}, Task: service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"number":9007199254740993}`)}}
}
func (f *scheduleFixture) create(in schedule.Definition) schedule.View {
f.t.Helper()
out, err := f.api.Create(f.ctx, f.token, string(executionID(f.t)), schedule.CreateInput{Name: "plan-" + string(executionID(f.t)), Definition: in})
if err != nil {
f.t.Fatal(err)
}
return out.Schedule
}
func (f *scheduleFixture) current(id core.ID) schedule.View {
f.t.Helper()
out, err := f.api.Get(f.ctx, f.token, id)
if err != nil {
f.t.Fatal(err)
}
return out
}
func (f *scheduleFixture) occurrences(id core.ID) []schedule.OccurrenceView {
f.t.Helper()
out, err := f.api.Occurrences(f.ctx, f.token, id, schedule.PageInput{Limit: 100})
if err != nil {
f.t.Fatal(err)
}
return out.Items
}
// 等待真实时间形成积压,不将游标改到计划实际生效之前。
func (f *scheduleFixture) due(v schedule.View, age time.Duration) {
f.t.Helper()
if delay := time.Until(v.NextFireAt.Add(age)); delay > 0 {
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-timer.C:
case <-f.ctx.Done():
f.t.Fatal(f.ctx.Err())
}
}
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
}
func (f *scheduleFixture) tick() ScheduleStats {
f.t.Helper()
out, err := f.store.Tick(f.ctx, 100)
if err != nil {
f.t.Fatal(err)
}
return out
}
func TestIntegrationSchedules(t *testing.T) {
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("requires disposable MySQL")
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
db, err := Open(ctx, dsn, 32, 8)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
fixture := func(t *testing.T) *scheduleFixture { return newScheduleFixture(t, newExecutionFixture(t, ctx, db)) }
// 连续发布经过“下一版本尚待初始化”和空版本时,不能丢失积压或重新开始已结束版本。
for _, emptyMiddle := range []bool{false, true} {
name := "publish_preserves_uninitialized_version"
if emptyMiddle {
name = "empty_version_then_finished_latest_stays_finished"
}
t.Run(name, func(t *testing.T) {
f := fixture(t)
finite := func(delay time.Duration) schedule.Definition {
in := f.definition()
in.Rule.StartAt = time.Now().UTC().Truncate(time.Microsecond).Add(delay)
end := in.Rule.StartAt.Add(time.Millisecond)
in.Rule.EndAt = &end
return in
}
publish := func(id core.ID, in schedule.Definition) schedule.View {
t.Helper()
current := f.current(id)
out, err := f.api.Publish(ctx, f.token, id, string(executionID(t)), schedule.PublishInput{ExpectedRevision: current.Revision, Reason: "version boundary regression", Definition: in})
if err != nil {
t.Fatal(err)
}
return out.Schedule
}
waitUntil := func(at time.Time) {
t.Helper()
timer := time.NewTimer(time.Until(at.Add(10 * time.Millisecond)))
defer timer.Stop()
select {
case <-timer.C:
case <-ctx.Done():
t.Fatal(ctx.Err())
}
}
v := f.create(finite(300 * time.Millisecond))
waitUntil(*v.NextFireAt)
delay := 300 * time.Millisecond
if emptyMiddle {
delay = 30 * time.Second
}
middle := finite(delay)
publish(v.ID, middle)
var latest schedule.Definition
if emptyMiddle {
latest = finite(300 * time.Millisecond)
publish(v.ID, latest)
}
f.tick() // 第一版本结束,第二版本尚未计算第一个触发点。
pending := f.current(v.ID)
if pending.ProcessingVersion != 2 || pending.NextFireAt != nil || pending.CheckAt == nil {
t.Fatal("expected pending initialization of version 2")
}
if !emptyMiddle {
waitUntil(middle.Rule.StartAt)
latest = finite(300 * time.Millisecond)
published := publish(v.ID, latest)
if published.ProcessingVersion != 2 {
t.Fatal("publishing discarded an uninitialized version")
}
}
waitUntil(latest.Rule.StartAt)
for i := 0; i < 3 && f.current(v.ID).CheckAt != nil; i++ {
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
f.tick()
}
finished := f.current(v.ID)
if finished.ProcessingVersion != 3 || finished.CheckAt != nil || finished.NextFireAt != nil {
t.Fatal("finished latest version retained scan work")
}
got := f.occurrences(v.ID)
want := map[uint64]time.Time{1: *v.NextFireAt, 3: latest.Rule.StartAt}
if !emptyMiddle {
want[2] = middle.Rule.StartAt
}
if len(got) != len(want) {
t.Fatalf("got %d occurrences, want %d", len(got), len(want))
}
for _, o := range got {
at, ok := want[o.ScheduleVersion]
if !ok || !o.ScheduledAt.Equal(at) || o.Decision != "FIRED" || o.TaskID == nil {
t.Fatal("version occurrence lost or duplicated")
}
delete(want, o.ScheduleVersion)
}
f.tick()
if f.current(v.ID).Revision != finished.Revision {
t.Fatal("finished schedule advanced again")
}
})
}
t.Run("multi_node_atomic_trigger_and_rate", func(t *testing.T) {
f := fixture(t)
v := f.create(f.definition())
f.due(v, 5*time.Second)
other, err := NewScheduleStore(db, executionID(t))
if err != nil {
t.Fatal(err)
}
var wg sync.WaitGroup
failures := make(chan error, 2)
for _, s := range []*ScheduleStore{f.store, other} {
wg.Add(1)
go func(s *ScheduleStore) { defer wg.Done(); _, err := s.Tick(ctx, 100); failures <- err }(s)
}
wg.Wait()
close(failures)
for err := range failures {
if err != nil {
t.Fatal(err)
}
}
got := f.occurrences(v.ID)
if len(got) != 3 || f.count("pulse_tasks", "owner_kind='schedule' AND owner_id=?", v.ID) != 3 {
t.Fatal("duplicate or partial trigger")
}
current := f.current(v.ID)
if current.CheckAt == nil || !current.CheckAt.After(current.UpdatedAt) {
t.Fatal("catchup rate not durable")
}
f.tick()
if len(f.occurrences(v.ID)) != 3 {
t.Fatal("repeat scan bypassed cooldown")
}
for _, o := range got {
if o.TaskID == nil || o.ScheduleVersion != 1 {
t.Fatal("missing ownership")
}
task := f.get(*o.TaskID)
if string(task.Payload) != `{"number":9007199254740993}` || task.AttemptCount != 0 {
t.Fatal("template snapshot damaged")
}
}
f.exec(`UPDATE orbit_namespaces SET enabled=FALSE WHERE id=?`, f.resources.NamespaceID)
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
f.tick()
if f.count("pulse_tasks", "owner_kind='schedule' AND owner_id=?", v.ID) != 3 {
t.Fatal("disabled namespace generated tasks")
}
f.exec(`UPDATE orbit_namespaces SET enabled=TRUE WHERE id=?`, f.resources.NamespaceID)
})
t.Run("idempotency_pause_scope_and_version", func(t *testing.T) {
f := fixture(t)
in := schedule.CreateInput{Name: "stable", Definition: f.definition(), Paused: true}
key := string(executionID(t))
out, err := f.api.Create(ctx, f.token, key, in)
if err != nil {
t.Fatal(err)
}
again, err := f.api.Create(ctx, f.token, key, in)
if err != nil || !again.Deduplicated || again.Schedule.ID != out.Schedule.ID {
t.Fatal("creation replay")
}
in.Paused = false
_, err = f.api.Create(ctx, f.token, key, in)
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
v := out.Schedule
f.due(v, 4*time.Second)
f.tick()
if len(f.occurrences(v.ID)) != 0 {
t.Fatal("paused schedule fired")
}
next := f.definition()
next.Task.Payload = json.RawMessage(`{"version":2}`)
published, err := f.api.Publish(ctx, f.token, v.ID, string(executionID(t)), schedule.PublishInput{ExpectedRevision: v.Revision, Reason: "new template", Definition: next})
if err != nil {
t.Fatal(err)
}
if published.Schedule.ProcessingVersion != 1 || published.Schedule.CurrentVersion != 2 {
t.Fatal("publish erased old backlog")
}
old, err := f.api.Version(ctx, f.token, v.ID, 1)
if err != nil || string(old.Definition.Task.Payload) != `{"number":9007199254740993}` {
t.Fatal("old version changed")
}
resumeKey := string(executionID(t))
resume := schedule.ControlInput{ExpectedRevision: published.Schedule.Revision, Reason: "resume work"}
resumed, err := f.api.Control(ctx, f.token, v.ID, resumeKey, resume, false)
if err != nil {
t.Fatal(err)
}
repeated, err := f.api.Control(ctx, f.token, v.ID, resumeKey, resume, false)
if err != nil || !repeated.Deduplicated || repeated.Schedule.Revision != resumed.Schedule.Revision {
t.Fatal("control replay")
}
f.tick()
for _, o := range f.occurrences(v.ID) {
if o.ScheduleVersion != 1 || !o.ScheduledAt.Before(published.Schedule.Current.EffectiveAt) {
t.Fatal("historical version boundary lost")
}
}
for i := 0; i < 3 && f.current(v.ID).ProcessingVersion == 1; i++ {
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
f.tick()
}
if f.current(v.ID).ProcessingVersion != 2 {
t.Fatal("old backlog never advanced to published version")
}
for _, o := range f.occurrences(v.ID) {
if o.ScheduleVersion == 1 && !o.ScheduledAt.Before(published.Schedule.Current.EffectiveAt) {
t.Fatal("old version crossed effective boundary")
}
}
// 另一命名空间知道 ID 也不可读或管理。
stranger := fixture(t)
_, err = stranger.api.Get(ctx, stranger.token, v.ID)
expectWorkerCode(t, err, "NOT_FOUND")
raw, _ := json.Marshal(auth.Permissions{Actions: []string{"schedules:read", "schedules:manage"}, Queues: []string{string(executionID(t))}})
f.exec(`UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?`, string(raw), f.resources.NamespaceID, f.resources.CredentialID)
_, err = f.api.Get(ctx, f.token, v.ID)
expectWorkerCode(t, err, "NOT_FOUND")
})
t.Run("persisted_skip_and_coalesce_ranges", func(t *testing.T) {
f := fixture(t)
in := f.definition()
in.Policy.CatchupWindowMS = 2000
in.Policy.Misfire = "COALESCE"
coalesce := f.create(in)
in.Policy.Misfire = "SKIP"
skip := f.create(in)
f.due(coalesce, 4*time.Second)
f.tick()
for _, v := range []schedule.View{coalesce, skip} {
items := f.occurrences(v.ID)
skipped, merged := false, false
for _, o := range items {
if o.Decision == "SKIPPED" {
skipped = true
if o.TaskID != nil || o.CoveredUntil == nil || !o.CoveredUntil.After(o.ScheduledAt) {
t.Fatal("invalid skipped range")
}
}
if o.Decision == "COALESCED" {
merged = true
if o.TaskID == nil || o.CoveredUntil == nil {
t.Fatal("coalesced range lost task")
}
}
}
if !skipped || (v.ID == coalesce.ID && !merged) {
t.Fatal("misfire decisions not persisted")
}
}
})
t.Run("overlap_serial_unknown_and_waiting_cancel", func(t *testing.T) {
f := fixture(t)
in := f.definition()
in.Policy.Overlap = "SERIAL"
v := f.create(in)
f.due(v, 5*time.Second)
f.tick()
items := f.occurrences(v.ID)
if len(items) != 1 || items[0].TaskID == nil {
t.Fatal("serial did not start one")
}
first := f.current(v.ID)
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
f.tick()
if len(f.occurrences(v.ID)) != 1 || !f.current(v.ID).NextFireAt.Equal(*first.NextFireAt) {
t.Fatal("serial skipped waiting task")
}
task := f.get(*items[0].TaskID)
if _, err := f.tasks.CancelTask(ctx, f.token, task.ID, string(executionID(t)), service.CancelInput{ExpectedRevision: task.Revision, Reason: "cancel pending scheduled task"}); err != nil {
t.Fatal(err)
}
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
f.tick()
if len(f.occurrences(v.ID)) != 2 {
t.Fatal("serial did not continue after terminal task")
}
retryFinish(t, f.executionFixture, retryStart(t, f.executionFixture), "UNKNOWN", false)
waiting := f.current(v.ID)
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
f.tick()
if len(f.occurrences(v.ID)) != 2 || !f.current(v.ID).NextFireAt.Equal(*waiting.NextFireAt) {
t.Fatal("unknown outcome did not block serial schedule")
}
})
t.Run("transaction_rollback_and_read_pagination", func(t *testing.T) {
f := fixture(t)
v := f.create(f.definition())
f.due(v, 6*time.Second)
prior := f.current(v.ID)
f.exec(`CREATE TRIGGER orbit_test_schedule_event BEFORE INSERT ON pulse_schedule_occurrences FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='controlled trigger failure'`)
_, err := f.store.Tick(ctx, 100)
f.exec(`DROP TRIGGER orbit_test_schedule_event`)
if err == nil || len(f.occurrences(v.ID)) != 0 || f.count("pulse_tasks", "owner_id=?", v.ID) != 0 || !f.current(v.ID).NextFireAt.Equal(*prior.NextFireAt) {
t.Fatal("partial schedule transaction")
}
f.tick()
page, err := f.api.Occurrences(ctx, f.token, v.ID, schedule.PageInput{Limit: 1})
if err != nil || len(page.Items) != 1 || page.NextCursor == "" {
t.Fatal("missing page")
}
second, err := f.api.Occurrences(ctx, f.token, v.ID, schedule.PageInput{Limit: 1, Cursor: page.NextCursor})
if err != nil || len(second.Items) != 1 || second.Items[0].ID == page.Items[0].ID {
t.Fatal("duplicate page")
}
another := f.create(f.definition())
_, err = f.api.Occurrences(ctx, f.token, another.ID, schedule.PageInput{Cursor: page.NextCursor})
expectWorkerCode(t, err, "INVALID_ARGUMENT")
})
}

View File

@@ -0,0 +1,294 @@
package mysql
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"time"
"git.yuntuo.hk/k2safe/orbit/internal/fault"
"git.yuntuo.hk/k2safe/orbit/internal/jsonwire"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/platform/auth"
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
driver "github.com/go-sql-driver/mysql"
)
// ScheduleStore 管理计划控制与扫描;共享任务存储的事务认证,不在事务外推进游标。
type ScheduleStore struct {
*TaskStore
node core.ID
}
func NewScheduleStore(db *sql.DB, node core.ID) (*ScheduleStore, error) {
if node.Validate() != nil {
return nil, errors.New("scheduler node id required")
}
return &ScheduleStore{TaskStore: NewTaskStore(db), node: node}, nil
}
var _ schedule.Store = (*ScheduleStore)(nil)
const scheduleColumns = `id,namespace_id,queue_id,name,revision,current_version,processing_version,paused,next_fire_at,check_at,created_at,updated_at`
func scanSchedule(row scanner) (schedule.Summary, error) {
var v schedule.Summary
err := row.Scan(&v.ID, &v.NamespaceID, &v.QueueID, &v.Name, &v.Revision, &v.CurrentVersion, &v.ProcessingVersion, &v.Paused, &v.NextFireAt, &v.CheckAt, &v.CreatedAt, &v.UpdatedAt)
return v, err
}
func readSchedule(ctx context.Context, tx *sql.Tx, ns, id core.ID, lock bool) (schedule.Summary, error) {
q := `SELECT ` + scheduleColumns + ` FROM pulse_schedules WHERE namespace_id=? AND id=? AND queue_id IS NOT NULL`
if lock {
q += ` FOR UPDATE`
}
v, err := scanSchedule(tx.QueryRowContext(ctx, q, ns, id))
if errors.Is(err, sql.ErrNoRows) {
err = fault.NotFound
}
return v, err
}
func readScheduleVersion(ctx context.Context, tx *sql.Tx, ns, id core.ID, version uint64) (schedule.VersionView, error) {
var v schedule.VersionView
var rule, task []byte
err := tx.QueryRowContext(ctx, `SELECT version,effective_at,specification,task_template,misfire_policy,overlap_policy,catchup_limit,catchup_window_ms,grace_ms FROM pulse_schedule_versions WHERE namespace_id=? AND schedule_id=? AND version=?`, ns, id, version).Scan(&v.Version, &v.EffectiveAt, &rule, &task, &v.Definition.Policy.Misfire, &v.Definition.Policy.Overlap, &v.Definition.Policy.CatchupLimit, &v.Definition.Policy.CatchupWindowMS, &v.Definition.Policy.GraceMS)
if errors.Is(err, sql.ErrNoRows) {
return v, fault.NotFound
}
if err != nil {
return v, err
}
if err = json.Unmarshal(rule, &v.Definition.Rule); err != nil {
return v, err
}
err = json.Unmarshal(task, &v.Definition.Task)
return v, err
}
func insertScheduleVersion(ctx context.Context, tx *sql.Tx, ns, id core.ID, v schedule.VersionView) error {
identity, err := core.NewID()
if err != nil {
return err
}
rule, err := json.Marshal(v.Definition.Rule)
if err != nil {
return err
}
task, err := jsonwire.Marshal(v.Definition.Task)
if err != nil {
return err
}
p := v.Definition.Policy
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_schedule_versions(namespace_id,id,schedule_id,version,effective_at,timezone,specification,task_template,misfire_policy,overlap_policy,catchup_limit,catchup_window_ms,grace_ms) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)`, ns, identity, id, v.Version, v.EffectiveAt, v.Definition.Rule.Timezone, string(rule), string(task), p.Misfire, p.Overlap, p.CatchupLimit, p.CatchupWindowMS, p.GraceMS)
return err
}
func scheduleView(ctx context.Context, tx *sql.Tx, ns, id core.ID) (schedule.View, error) {
summary, err := readSchedule(ctx, tx, ns, id, false)
if err != nil {
return schedule.View{}, err
}
current, err := readScheduleVersion(ctx, tx, ns, id, summary.CurrentVersion)
return schedule.View{Summary: summary, Current: current}, err
}
func authorizeSchedule(ctx context.Context, tx *sql.Tx, ns, id core.ID, p auth.Permissions) (schedule.Summary, error) {
v, err := readSchedule(ctx, tx, ns, id, false)
if err != nil {
return v, err
}
if !p.AllowsQueue(v.QueueID) {
return v, fault.NotFound
}
return v, nil
}
func (s *ScheduleStore) Create(ctx context.Context, t auth.Token, key string, in schedule.CreateInput) (schedule.Result, error) {
var out schedule.Result
err := s.transaction(ctx, t, "schedules:manage", func(tx *sql.Tx, p auth.Permissions) error {
if !p.Allows("tasks:submit") || !p.AllowsQueue(in.Definition.Task.QueueID) {
return fault.Forbidden
}
id, err := core.NewID()
if err != nil {
return err
}
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "schedule.create", "", key, "schedule", id, in)
if err != nil {
return err
}
if cached != nil {
if err := json.Unmarshal(cached, &out); err != nil {
return err
}
out.Deduplicated = true
return nil
}
var queue core.ID
err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR SHARE`, t.NamespaceID, in.Definition.Task.QueueID).Scan(&queue)
if errors.Is(err, sql.ErrNoRows) {
return fault.NotFound
}
if err != nil {
return err
}
now, err := dbNow(ctx, tx)
if err != nil {
return err
}
c, err := schedule.Compile(in.Definition.Rule)
if err != nil {
return err
}
next, err := c.AtOrAfter(now)
if err != nil {
return err
}
if next == nil {
return fault.Invalid("schedule has no future occurrence")
}
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_schedules(namespace_id,id,queue_id,name,current_version,processing_version,paused,next_fire_at,check_at,created_at,updated_at) VALUES (?,?,?,?,1,1,?,?,?,?,?)`, t.NamespaceID, id, queue, in.Name, in.Paused, next, next, now, now)
var me *driver.MySQLError
if errors.As(err, &me) && me.Number == 1062 {
return fault.Conflict
}
if err != nil {
return err
}
if err := insertScheduleVersion(ctx, tx, t.NamespaceID, id, schedule.VersionView{Version: 1, EffectiveAt: now, Definition: in.Definition}); err != nil {
return err
}
out.Schedule, err = scheduleView(ctx, tx, t.NamespaceID, id)
if err != nil {
return err
}
if err := emitEvent(ctx, tx, t.NamespaceID, id, 1, "schedule", "schedule.created", map[string]any{"queue_id": queue, "version": "1", "paused": in.Paused}); err != nil {
return err
}
if err := appendAudit(ctx, tx, t, identity, id, "schedule", "schedule.create", "create schedule", nil, out.Schedule.Summary); err != nil {
return err
}
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
})
return out, err
}
func (s *ScheduleStore) Get(ctx context.Context, t auth.Token, id core.ID) (schedule.View, error) {
var out schedule.View
err := s.transaction(ctx, t, "schedules:read", func(tx *sql.Tx, p auth.Permissions) error {
if _, err := authorizeSchedule(ctx, tx, t.NamespaceID, id, p); err != nil {
return err
}
var err error
out, err = scheduleView(ctx, tx, t.NamespaceID, id)
return err
})
return out, err
}
func (s *ScheduleStore) Publish(ctx context.Context, t auth.Token, id core.ID, key string, in schedule.PublishInput) (schedule.Result, error) {
return s.mutate(ctx, t, id, key, "schedule.publish", in, in.ExpectedRevision, in.Reason, func(tx *sql.Tx, v schedule.Summary, now time.Time) error {
if in.Definition.Task.QueueID != v.QueueID {
return fault.Invalid("schedule queue cannot change")
}
last, err := readScheduleVersion(ctx, tx, t.NamespaceID, id, v.CurrentVersion)
if err != nil {
return err
}
if !now.After(last.EffectiveAt) {
return fault.Unavailable
}
if err := insertScheduleVersion(ctx, tx, t.NamespaceID, id, schedule.VersionView{Version: v.CurrentVersion + 1, EffectiveAt: now, Definition: in.Definition}); err != nil {
return err
}
cursor, next := v.ProcessingVersion, v.NextFireAt
// nil 游标也可能表示刚进入最新版本、尚未初始化;只有没有扫描工作才表示结束。
if (next == nil && cursor == v.CurrentVersion && v.CheckAt == nil) || (next != nil && !next.Before(now)) {
c, err := schedule.Compile(in.Definition.Rule)
if err != nil {
return err
}
next, err = c.AtOrAfter(now)
if err != nil {
return err
}
cursor = v.CurrentVersion + 1
}
check := &now
if next != nil && next.After(now) {
check = next
} else if next == nil && cursor == v.CurrentVersion+1 {
check = nil
}
_, err = tx.ExecContext(ctx, `UPDATE pulse_schedules SET current_version=current_version+1,processing_version=?,next_fire_at=?,check_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, cursor, next, check, now, t.NamespaceID, id)
return err
})
}
func (s *ScheduleStore) Control(ctx context.Context, t auth.Token, id core.ID, key string, in schedule.ControlInput, paused bool) (schedule.Result, error) {
operation := "schedule.resume"
if paused {
operation = "schedule.pause"
}
return s.mutate(ctx, t, id, key, operation, in, in.ExpectedRevision, in.Reason, func(tx *sql.Tx, v schedule.Summary, now time.Time) error {
if v.Paused == paused {
return fault.Conflict
}
_, err := tx.ExecContext(ctx, `UPDATE pulse_schedules SET paused=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, paused, now, t.NamespaceID, id)
return err
})
}
func (s *ScheduleStore) mutate(ctx context.Context, t auth.Token, id core.ID, key, operation string, input any, revision uint64, reason string, change func(*sql.Tx, schedule.Summary, time.Time) error) (schedule.Result, error) {
var out schedule.Result
err := s.transaction(ctx, t, "schedules:manage", func(tx *sql.Tx, p auth.Permissions) error {
if operation == "schedule.publish" && !p.Allows("tasks:submit") {
return fault.Forbidden
}
if _, err := authorizeSchedule(ctx, tx, t.NamespaceID, id, p); err != nil {
return err
}
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, operation, string(id), key, "schedule", id, input)
if err != nil {
return err
}
if cached != nil {
if err := json.Unmarshal(cached, &out); err != nil {
return err
}
out.Deduplicated = true
return nil
}
v, err := readSchedule(ctx, tx, t.NamespaceID, id, true)
if err != nil {
return err
}
if v.Revision != revision {
return fault.Conflict
}
now, err := dbNow(ctx, tx)
if err != nil {
return err
}
if err := change(tx, v, now); err != nil {
return err
}
out.Schedule, err = scheduleView(ctx, tx, t.NamespaceID, id)
if err != nil {
return err
}
if err := appendAudit(ctx, tx, t, identity, id, "schedule", operation, reason, v, out.Schedule.Summary); err != nil {
return err
}
if err := emitEvent(ctx, tx, t.NamespaceID, id, out.Schedule.Revision, "schedule", operation, map[string]any{"version": fmt.Sprint(out.Schedule.CurrentVersion), "paused": out.Schedule.Paused}); err != nil {
return err
}
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
})
return out, err
}
func (s *ScheduleStore) Version(ctx context.Context, t auth.Token, id core.ID, version uint64) (schedule.VersionView, error) {
var out schedule.VersionView
err := s.transaction(ctx, t, "schedules:read", func(tx *sql.Tx, p auth.Permissions) error {
if _, err := authorizeSchedule(ctx, tx, t.NamespaceID, id, p); err != nil {
return err
}
var err error
out, err = readScheduleVersion(ctx, tx, t.NamespaceID, id, version)
return err
})
return out, err
}

View File

@@ -0,0 +1,225 @@
package mysql
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"time"
"git.yuntuo.hk/k2safe/orbit/internal/fault"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
)
// ScheduleStats 是本次已确认提交的批次计数,事务失败不计作成功。
type ScheduleStats struct {
Examined int // Examined 已检查的候选计划数。
Fired int // Fired 本轮创建的任务数,合并范围计一个。
Skipped int // Skipped 本轮保存的跳过范围数,不是被覆盖触发点总数。
}
// Tick 按持久化 check_at 扫描,一次最多 100 个计划,每个计划独立短事务。
func (s *ScheduleStore) Tick(ctx context.Context, limit int) (ScheduleStats, error) {
var out ScheduleStats
if limit < 1 || limit > 100 {
return out, fault.Invalid("schedule batch limit must be 1 to 100")
}
type candidate struct{ ns, id, queue core.ID }
candidates := []candidate{}
rows, err := s.db.QueryContext(ctx, `SELECT s.namespace_id,s.id,s.queue_id FROM pulse_schedules s JOIN orbit_namespaces n ON n.id=s.namespace_id JOIN orbit_applications a ON a.id=n.application_id WHERE s.paused=FALSE AND s.check_at<=UTC_TIMESTAMP(6) AND n.enabled=TRUE AND a.enabled=TRUE ORDER BY s.check_at,s.namespace_id,s.id LIMIT ?`, limit)
if err != nil {
return out, err
}
for rows.Next() {
var c candidate
if err := rows.Scan(&c.ns, &c.id, &c.queue); err != nil {
rows.Close()
return out, err
}
candidates = append(candidates, c)
}
err = rows.Err()
rows.Close()
if err != nil {
return out, err
}
var failures []error
for _, c := range candidates {
out.Examined++
fired, skipped, err := s.tickOne(ctx, c.ns, c.id, c.queue)
if err != nil {
failures = append(failures, err)
if ctx.Err() != nil {
break
}
continue
}
out.Fired += fired
out.Skipped += skipped
}
return out, errors.Join(failures...)
}
func (s *ScheduleStore) tickOne(ctx context.Context, ns, id, queue core.ID) (int, int, error) {
fired, skipped := 0, 0
err := WithTx(ctx, s.db, func(tx *sql.Tx) error {
var allowed core.ID
err := tx.QueryRowContext(ctx, `SELECT n.id FROM orbit_namespaces n JOIN orbit_applications a ON a.id=n.application_id WHERE n.id=? AND n.enabled=TRUE AND a.enabled=TRUE FOR SHARE`, ns).Scan(&allowed)
if errors.Is(err, sql.ErrNoRows) {
return nil
}
if err != nil {
return err
}
// 和任务执行一致先锁队列,再锁计划;其他扫描节点不等待同一计划所在队列。
err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, ns, queue).Scan(&allowed)
if errors.Is(err, sql.ErrNoRows) {
return nil
}
if err != nil {
return err
}
v, err := readSchedule(ctx, tx, ns, id, true)
if err != nil {
return err
}
now, err := dbNow(ctx, tx)
if err != nil {
return err
}
if v.Paused || v.CheckAt == nil || v.CheckAt.After(now) {
return nil
}
if v.QueueID != queue {
return errors.New("schedule queue changed")
}
cursor, next := v.ProcessingVersion, v.NextFireAt
needsInitialization := false
// 单批最多跨过 16 个空历史版本,避免长事务;下次从已保存版本继续。
for n := 0; n < 16; n++ {
version, err := readScheduleVersion(ctx, tx, ns, id, cursor)
if err != nil {
return err
}
c, err := schedule.Compile(version.Definition.Rule)
if err != nil {
return err
}
var before *time.Time
if cursor < v.CurrentVersion {
newer, err := readScheduleVersion(ctx, tx, ns, id, cursor+1)
if err != nil {
return err
}
before = &newer.EffectiveAt
}
needsInitialization = false
if next == nil {
next, err = c.AtOrAfter(version.EffectiveAt)
if err != nil {
return err
}
}
if next != nil && next.Before(version.EffectiveAt) {
return errors.New("schedule cursor predates its version")
}
if next == nil || (before != nil && !next.Before(*before)) {
next = nil
if cursor < v.CurrentVersion {
cursor++
needsInitialization = true
continue
}
break
}
if next.After(now) {
break
}
var active bool
if version.Definition.Policy.Overlap != "PARALLEL" {
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pulse_tasks WHERE namespace_id=? AND owner_kind='schedule' AND owner_id=? AND state IN ('WAITING','LEASED','RUNNING','NEEDS_REVIEW'))`, ns, id).Scan(&active); err != nil {
return err
}
}
policy := version.Definition.Policy
// 大参数也要保持事务有界;每批复制的业务参数最多约 1 MiB。
if size := len(version.Definition.Task.Payload); size > 0 {
if max := uint32((1 << 20) / size); max < policy.CatchupLimit {
policy.CatchupLimit = max
}
}
batch, err := schedule.Plan(c, policy, *next, now, before, active)
if err != nil {
return err
}
for _, decision := range batch.Decisions {
if err := s.saveOccurrence(ctx, tx, v, version, decision, now); err != nil {
return err
}
if decision.Kind == "SKIPPED" {
skipped++
} else {
fired++
}
}
next = batch.Next
if next == nil && cursor < v.CurrentVersion {
cursor++
needsInitialization = true
}
break // 每计划每批只创建一个版本的有限任务,跨版本也不绕过速率上限。
}
var check *time.Time
if next != nil || cursor < v.CurrentVersion || needsInitialization {
t := now.Add(time.Second)
check = &t
if next != nil && next.After(t) {
check = next
}
}
result, err := tx.ExecContext(ctx, `UPDATE pulse_schedules SET processing_version=?,next_fire_at=?,check_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, cursor, next, check, now, ns, id, v.Revision)
if err := exactlyOne(result, err); err != nil {
return err
}
return emitEvent(ctx, tx, ns, id, v.Revision+1, "schedule", "schedule.advanced", map[string]any{"node_id": s.node, "processing_version": fmt.Sprint(cursor), "next_fire_at": next, "fired": fired, "skipped": skipped})
})
if err != nil {
return 0, 0, err
}
return fired, skipped, nil
}
func (s *ScheduleStore) saveOccurrence(ctx context.Context, tx *sql.Tx, v schedule.Summary, version schedule.VersionView, d schedule.Decision, now time.Time) error {
id, err := core.NewID()
if err != nil {
return err
}
var task *core.ID
if d.Kind != "SKIPPED" {
value, err := core.NewID()
if err != nil {
return err
}
task = &value
in := version.Definition.Task
if in.QueueID != v.QueueID {
return errors.New("schedule template queue mismatch")
}
policy, err := json.Marshal(in.Retry)
if err != nil {
return err
}
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_tasks(namespace_id,id,queue_id,handler,handler_version,payload_version,payload,policy_snapshot,recovery_safety,business_key,owner_kind,owner_id,available_at,deadline_at,attempt_timeout_ms,max_attempts,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,NULLIF(?,''),'schedule',?,?,?,?,?,?,?)`, v.NamespaceID, value, v.QueueID, in.Handler, in.HandlerVersion, in.PayloadVersion, string(in.Payload), string(policy), in.RecoverySafety, in.BusinessKey, v.ID, now, now.Add(time.Duration(in.TaskTimeoutMS)*time.Millisecond), in.Retry.AttemptTimeoutMS, in.Retry.MaxAttempts, now, now)
if err != nil {
return err
}
if err := emitEvent(ctx, tx, v.NamespaceID, value, 1, "task", "task.accepted", map[string]any{"schedule_id": v.ID, "occurrence_id": id, "schedule_version": fmt.Sprint(version.Version), "scheduled_at": d.ScheduledAt, "node_id": s.node}); err != nil {
return err
}
}
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_schedule_occurrences(namespace_id,id,schedule_id,schedule_version,scheduled_at,decision,task_id,covered_until,reason,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)`, v.NamespaceID, id, v.ID, version.Version, d.ScheduledAt, d.Kind, task, d.CoveredUntil, d.Reason, now, now)
if err != nil {
return err
}
return emitEvent(ctx, tx, v.NamespaceID, id, 1, "schedule_occurrence", "schedule.decided", map[string]any{"schedule_id": v.ID, "version": fmt.Sprint(version.Version), "scheduled_at": d.ScheduledAt, "decision": d.Kind, "covered_until": d.CoveredUntil, "reason": d.Reason, "task_id": task, "node_id": s.node})
}

View File

@@ -354,7 +354,7 @@ func (s *TaskStore) CancelTask(ctx context.Context, token auth.Token, id core.ID
if err := tx.QueryRowContext(ctx, "SELECT owner_kind FROM pulse_tasks WHERE namespace_id=? AND id=?", token.NamespaceID, id).Scan(&owner); err != nil {
return err
}
if owner.Valid {
if owner.Valid && owner.String != "schedule" {
return fault.Conflict
}
now, err := dbNow(ctx, tx)