feat: add durable schedule management and multi-node crash recovery
This commit is contained in:
@@ -14,11 +14,12 @@ import (
|
||||
|
||||
// BootstrapInput 仅供持有数据库管理权限的初始化命令使用,不暴露为公共 HTTP。
|
||||
type BootstrapInput struct {
|
||||
Application string // Application 应用名称,每次初始化创建独立应用。
|
||||
Namespace string // Namespace 命名空间名称。
|
||||
Environment string // Environment 环境名称,必须显式指定。
|
||||
Queue string // Queue 初始队列名称。
|
||||
WorkerAccess bool // WorkerAccess 显式增加开发用 Worker 接入及消费权限。
|
||||
Application string // Application 应用名称,每次初始化创建独立应用。
|
||||
Namespace string // Namespace 命名空间名称。
|
||||
Environment string // Environment 环境名称,必须显式指定。
|
||||
Queue string // Queue 初始队列名称。
|
||||
WorkerAccess bool // WorkerAccess 显式增加开发用 Worker 接入及消费权限。
|
||||
ScheduleAccess bool // ScheduleAccess 显式增加开发用计划查询及管理权限。
|
||||
}
|
||||
|
||||
// BootstrapResult 包含一次性交付的原始凭据,禁止记录到日志或提交到版本库。
|
||||
@@ -60,6 +61,9 @@ func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func(
|
||||
if in.WorkerAccess {
|
||||
permissions.Actions = append(permissions.Actions, "workers:connect", "workers:consume")
|
||||
}
|
||||
if in.ScheduleAccess {
|
||||
permissions.Actions = append(permissions.Actions, "schedules:read", "schedules:manage")
|
||||
}
|
||||
raw, err := json.Marshal(permissions)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
-- 旧版本只定义了结构,没有公开创建计划的入口。未绑定队列的历史行不启用扫描。
|
||||
ALTER TABLE pulse_schedules
|
||||
ADD COLUMN queue_id BINARY(16) NULL COMMENT '计划终身绑定队列;旧结构人工行为空,不允许执行',
|
||||
ADD COLUMN processing_version BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '当前处理积压的不可变版本,不因发布新版本丢弃旧进度',
|
||||
ADD COLUMN check_at DATETIME(6) NULL COMMENT '下一可扫描时间,UTC;补跑批次之间持久化至少一秒间隔,为空表示没有扫描工作',
|
||||
ADD CONSTRAINT ck_schedule_cursor CHECK (processing_version>0 AND processing_version<=current_version),
|
||||
ADD CONSTRAINT ck_schedule_enabled_scan CHECK (queue_id IS NOT NULL OR check_at IS NULL),
|
||||
ADD CONSTRAINT fk_schedule_queue FOREIGN KEY(namespace_id,queue_id) REFERENCES pulse_queues(namespace_id,id) ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
ADD KEY ix_schedule_scan(paused,check_at,namespace_id,id),
|
||||
ADD KEY ix_schedule_list(namespace_id,queue_id,created_at,id);
|
||||
ALTER TABLE pulse_schedule_versions
|
||||
ADD COLUMN catchup_window_ms BIGINT UNSIGNED NOT NULL DEFAULT 86400000 COMMENT '未处理触发点最大保留时长,单位毫秒,过旧范围记录跳过',
|
||||
ADD COLUMN grace_ms BIGINT UNSIGNED NOT NULL DEFAULT 5000 COMMENT '正常派发允许延迟的毫秒数,超过后按漏跑策略处理',
|
||||
ADD CONSTRAINT ck_schedule_window CHECK(catchup_window_ms BETWEEN 1000 AND 2592000000 AND grace_ms BETWEEN 1000 AND 60000 AND grace_ms<=catchup_window_ms);
|
||||
ALTER TABLE pulse_schedule_occurrences
|
||||
MODIFY COLUMN covered_until DATETIME(6) NULL COMMENT '合并或跳过范围的排除边界,范围为原计划时间至该边界,不包含边界',
|
||||
ADD CONSTRAINT ck_occurrence_range CHECK(covered_until IS NULL OR covered_until>scheduled_at);
|
||||
ALTER TABLE pulse_tasks ADD KEY ix_task_owner_state(namespace_id,owner_kind,owner_id,state,id);
|
||||
139
internal/storage/mysql/schedule_history.go
Normal file
139
internal/storage/mysql/schedule_history.go
Normal file
@@ -0,0 +1,139 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/jsonvalue"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
|
||||
)
|
||||
|
||||
type scheduleCursor struct {
|
||||
Version int `json:"v"`
|
||||
Kind string `json:"k"`
|
||||
Namespace core.ID `json:"n"`
|
||||
Parent core.ID `json:"p"`
|
||||
At time.Time `json:"t"`
|
||||
ID core.ID `json:"i"`
|
||||
}
|
||||
|
||||
func parseScheduleCursor(raw string, ns, parent core.ID, kind string) (scheduleCursor, error) {
|
||||
var c scheduleCursor
|
||||
b, err := base64.RawURLEncoding.Strict().DecodeString(raw)
|
||||
if err != nil || len(b) > 768 {
|
||||
return c, fault.Invalid("invalid schedule cursor")
|
||||
}
|
||||
if _, err := jsonvalue.Canonical(b); err != nil {
|
||||
return c, fault.Invalid("invalid schedule cursor")
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(b, &fields) != nil || len(fields) != 6 {
|
||||
return c, fault.Invalid("invalid schedule cursor")
|
||||
}
|
||||
for _, key := range []string{"v", "k", "n", "p", "t", "i"} {
|
||||
if _, ok := fields[key]; !ok {
|
||||
return c, fault.Invalid("invalid schedule cursor")
|
||||
}
|
||||
}
|
||||
d := json.NewDecoder(bytes.NewReader(b))
|
||||
d.DisallowUnknownFields()
|
||||
if d.Decode(&c) != nil || c.Version != 1 || c.Namespace != ns || c.Parent != parent || c.Kind != kind || c.ID.Validate() != nil || c.At.Year() < 2000 || c.At.Year() > 9998 || c.At.Nanosecond()%1000 != 0 {
|
||||
return c, fault.Invalid("cursor does not match schedule query")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
func makeScheduleCursor(ns, parent core.ID, kind string, at time.Time, id core.ID) (string, error) {
|
||||
b, err := json.Marshal(scheduleCursor{1, kind, ns, parent, at, id})
|
||||
return base64.RawURLEncoding.EncodeToString(b), err
|
||||
}
|
||||
func (s *ScheduleStore) List(ctx context.Context, t auth.Token, queue core.ID, in schedule.PageInput) (schedule.Page, error) {
|
||||
out := schedule.Page{Items: []schedule.Summary{}}
|
||||
err := s.transaction(ctx, t, "schedules:read", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
if !p.AllowsQueue(queue) {
|
||||
return fault.Forbidden
|
||||
}
|
||||
q := `SELECT ` + scheduleColumns + ` FROM pulse_schedules WHERE namespace_id=? AND queue_id=?`
|
||||
args := []any{t.NamespaceID, queue}
|
||||
if in.Cursor != "" {
|
||||
c, err := parseScheduleCursor(in.Cursor, t.NamespaceID, queue, "schedules")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q += ` AND (created_at<? OR (created_at=? AND id<?))`
|
||||
args = append(args, c.At, c.At, c.ID)
|
||||
}
|
||||
q += ` ORDER BY created_at DESC,id DESC LIMIT ?`
|
||||
args = append(args, in.Limit+1)
|
||||
rows, err := tx.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
v, err := scanSchedule(rows)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out.Items = append(out.Items, v)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(out.Items) > in.Limit {
|
||||
out.Items = out.Items[:in.Limit]
|
||||
last := out.Items[len(out.Items)-1]
|
||||
out.NextCursor, err = makeScheduleCursor(t.NamespaceID, queue, "schedules", last.CreatedAt, last.ID)
|
||||
}
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
func (s *ScheduleStore) Occurrences(ctx context.Context, t auth.Token, id core.ID, in schedule.PageInput) (schedule.OccurrencePage, error) {
|
||||
out := schedule.OccurrencePage{Items: []schedule.OccurrenceView{}}
|
||||
err := s.transaction(ctx, t, "schedules:read", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
if _, err := authorizeSchedule(ctx, tx, t.NamespaceID, id, p); err != nil {
|
||||
return err
|
||||
}
|
||||
q := `SELECT id,schedule_id,schedule_version,scheduled_at,decision,task_id,covered_until,COALESCE(reason,''),created_at FROM pulse_schedule_occurrences WHERE namespace_id=? AND schedule_id=?`
|
||||
args := []any{t.NamespaceID, id}
|
||||
if in.Cursor != "" {
|
||||
c, err := parseScheduleCursor(in.Cursor, t.NamespaceID, id, "occurrences")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q += ` AND (scheduled_at<? OR (scheduled_at=? AND id<?))`
|
||||
args = append(args, c.At, c.At, c.ID)
|
||||
}
|
||||
q += ` ORDER BY scheduled_at DESC,id DESC LIMIT ?`
|
||||
args = append(args, in.Limit+1)
|
||||
rows, err := tx.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var v schedule.OccurrenceView
|
||||
if err := rows.Scan(&v.ID, &v.ScheduleID, &v.ScheduleVersion, &v.ScheduledAt, &v.Decision, &v.TaskID, &v.CoveredUntil, &v.Reason, &v.CreatedAt); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Items = append(out.Items, v)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(out.Items) > in.Limit {
|
||||
out.Items = out.Items[:in.Limit]
|
||||
last := out.Items[len(out.Items)-1]
|
||||
out.NextCursor, err = makeScheduleCursor(t.NamespaceID, id, "occurrences", last.ScheduledAt, last.ID)
|
||||
}
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
400
internal/storage/mysql/schedule_integration_test.go
Normal file
400
internal/storage/mysql/schedule_integration_test.go
Normal file
@@ -0,0 +1,400 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
)
|
||||
|
||||
type scheduleFixture struct {
|
||||
*executionFixture
|
||||
store *ScheduleStore
|
||||
api *schedule.Service
|
||||
}
|
||||
|
||||
func newScheduleFixture(t *testing.T, f *executionFixture) *scheduleFixture {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(auth.Permissions{Actions: []string{"tasks:submit", "tasks:read", "tasks:cancel", "queues:read", "schedules:read", "schedules:manage", "workers:connect", "workers:consume"}, Queues: []string{"*"}})
|
||||
f.exec(`UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?`, string(raw), f.resources.NamespaceID, f.resources.CredentialID)
|
||||
store, err := NewScheduleStore(f.db, executionID(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &scheduleFixture{f, store, schedule.New(store)}
|
||||
}
|
||||
func (f *scheduleFixture) definition() schedule.Definition {
|
||||
return schedule.Definition{Rule: schedule.Rule{Kind: "FIXED_RATE", Timezone: "UTC", StartAt: time.Now().UTC().Truncate(time.Microsecond).Add(-time.Hour), IntervalMS: 1000}, Policy: schedule.Policy{Misfire: "CATCH_UP", Overlap: "PARALLEL", CatchupLimit: 3, CatchupWindowMS: 30000, GraceMS: 1000}, Task: service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"number":9007199254740993}`)}}
|
||||
}
|
||||
func (f *scheduleFixture) create(in schedule.Definition) schedule.View {
|
||||
f.t.Helper()
|
||||
out, err := f.api.Create(f.ctx, f.token, string(executionID(f.t)), schedule.CreateInput{Name: "plan-" + string(executionID(f.t)), Definition: in})
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return out.Schedule
|
||||
}
|
||||
func (f *scheduleFixture) current(id core.ID) schedule.View {
|
||||
f.t.Helper()
|
||||
out, err := f.api.Get(f.ctx, f.token, id)
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func (f *scheduleFixture) occurrences(id core.ID) []schedule.OccurrenceView {
|
||||
f.t.Helper()
|
||||
out, err := f.api.Occurrences(f.ctx, f.token, id, schedule.PageInput{Limit: 100})
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return out.Items
|
||||
}
|
||||
|
||||
// 等待真实时间形成积压,不将游标改到计划实际生效之前。
|
||||
func (f *scheduleFixture) due(v schedule.View, age time.Duration) {
|
||||
f.t.Helper()
|
||||
if delay := time.Until(v.NextFireAt.Add(age)); delay > 0 {
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
case <-f.ctx.Done():
|
||||
f.t.Fatal(f.ctx.Err())
|
||||
}
|
||||
}
|
||||
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||
}
|
||||
func (f *scheduleFixture) tick() ScheduleStats {
|
||||
f.t.Helper()
|
||||
out, err := f.store.Tick(f.ctx, 100)
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func TestIntegrationSchedules(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
db, err := Open(ctx, dsn, 32, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fixture := func(t *testing.T) *scheduleFixture { return newScheduleFixture(t, newExecutionFixture(t, ctx, db)) }
|
||||
// 连续发布经过“下一版本尚待初始化”和空版本时,不能丢失积压或重新开始已结束版本。
|
||||
for _, emptyMiddle := range []bool{false, true} {
|
||||
name := "publish_preserves_uninitialized_version"
|
||||
if emptyMiddle {
|
||||
name = "empty_version_then_finished_latest_stays_finished"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
finite := func(delay time.Duration) schedule.Definition {
|
||||
in := f.definition()
|
||||
in.Rule.StartAt = time.Now().UTC().Truncate(time.Microsecond).Add(delay)
|
||||
end := in.Rule.StartAt.Add(time.Millisecond)
|
||||
in.Rule.EndAt = &end
|
||||
return in
|
||||
}
|
||||
publish := func(id core.ID, in schedule.Definition) schedule.View {
|
||||
t.Helper()
|
||||
current := f.current(id)
|
||||
out, err := f.api.Publish(ctx, f.token, id, string(executionID(t)), schedule.PublishInput{ExpectedRevision: current.Revision, Reason: "version boundary regression", Definition: in})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.Schedule
|
||||
}
|
||||
waitUntil := func(at time.Time) {
|
||||
t.Helper()
|
||||
timer := time.NewTimer(time.Until(at.Add(10 * time.Millisecond)))
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
}
|
||||
v := f.create(finite(300 * time.Millisecond))
|
||||
waitUntil(*v.NextFireAt)
|
||||
delay := 300 * time.Millisecond
|
||||
if emptyMiddle {
|
||||
delay = 30 * time.Second
|
||||
}
|
||||
middle := finite(delay)
|
||||
publish(v.ID, middle)
|
||||
var latest schedule.Definition
|
||||
if emptyMiddle {
|
||||
latest = finite(300 * time.Millisecond)
|
||||
publish(v.ID, latest)
|
||||
}
|
||||
f.tick() // 第一版本结束,第二版本尚未计算第一个触发点。
|
||||
pending := f.current(v.ID)
|
||||
if pending.ProcessingVersion != 2 || pending.NextFireAt != nil || pending.CheckAt == nil {
|
||||
t.Fatal("expected pending initialization of version 2")
|
||||
}
|
||||
if !emptyMiddle {
|
||||
waitUntil(middle.Rule.StartAt)
|
||||
latest = finite(300 * time.Millisecond)
|
||||
published := publish(v.ID, latest)
|
||||
if published.ProcessingVersion != 2 {
|
||||
t.Fatal("publishing discarded an uninitialized version")
|
||||
}
|
||||
}
|
||||
waitUntil(latest.Rule.StartAt)
|
||||
for i := 0; i < 3 && f.current(v.ID).CheckAt != nil; i++ {
|
||||
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||
f.tick()
|
||||
}
|
||||
finished := f.current(v.ID)
|
||||
if finished.ProcessingVersion != 3 || finished.CheckAt != nil || finished.NextFireAt != nil {
|
||||
t.Fatal("finished latest version retained scan work")
|
||||
}
|
||||
got := f.occurrences(v.ID)
|
||||
want := map[uint64]time.Time{1: *v.NextFireAt, 3: latest.Rule.StartAt}
|
||||
if !emptyMiddle {
|
||||
want[2] = middle.Rule.StartAt
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("got %d occurrences, want %d", len(got), len(want))
|
||||
}
|
||||
for _, o := range got {
|
||||
at, ok := want[o.ScheduleVersion]
|
||||
if !ok || !o.ScheduledAt.Equal(at) || o.Decision != "FIRED" || o.TaskID == nil {
|
||||
t.Fatal("version occurrence lost or duplicated")
|
||||
}
|
||||
delete(want, o.ScheduleVersion)
|
||||
}
|
||||
f.tick()
|
||||
if f.current(v.ID).Revision != finished.Revision {
|
||||
t.Fatal("finished schedule advanced again")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("multi_node_atomic_trigger_and_rate", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
v := f.create(f.definition())
|
||||
f.due(v, 5*time.Second)
|
||||
other, err := NewScheduleStore(db, executionID(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
failures := make(chan error, 2)
|
||||
for _, s := range []*ScheduleStore{f.store, other} {
|
||||
wg.Add(1)
|
||||
go func(s *ScheduleStore) { defer wg.Done(); _, err := s.Tick(ctx, 100); failures <- err }(s)
|
||||
}
|
||||
wg.Wait()
|
||||
close(failures)
|
||||
for err := range failures {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got := f.occurrences(v.ID)
|
||||
if len(got) != 3 || f.count("pulse_tasks", "owner_kind='schedule' AND owner_id=?", v.ID) != 3 {
|
||||
t.Fatal("duplicate or partial trigger")
|
||||
}
|
||||
current := f.current(v.ID)
|
||||
if current.CheckAt == nil || !current.CheckAt.After(current.UpdatedAt) {
|
||||
t.Fatal("catchup rate not durable")
|
||||
}
|
||||
f.tick()
|
||||
if len(f.occurrences(v.ID)) != 3 {
|
||||
t.Fatal("repeat scan bypassed cooldown")
|
||||
}
|
||||
for _, o := range got {
|
||||
if o.TaskID == nil || o.ScheduleVersion != 1 {
|
||||
t.Fatal("missing ownership")
|
||||
}
|
||||
task := f.get(*o.TaskID)
|
||||
if string(task.Payload) != `{"number":9007199254740993}` || task.AttemptCount != 0 {
|
||||
t.Fatal("template snapshot damaged")
|
||||
}
|
||||
}
|
||||
f.exec(`UPDATE orbit_namespaces SET enabled=FALSE WHERE id=?`, f.resources.NamespaceID)
|
||||
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||
f.tick()
|
||||
if f.count("pulse_tasks", "owner_kind='schedule' AND owner_id=?", v.ID) != 3 {
|
||||
t.Fatal("disabled namespace generated tasks")
|
||||
}
|
||||
f.exec(`UPDATE orbit_namespaces SET enabled=TRUE WHERE id=?`, f.resources.NamespaceID)
|
||||
})
|
||||
t.Run("idempotency_pause_scope_and_version", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
in := schedule.CreateInput{Name: "stable", Definition: f.definition(), Paused: true}
|
||||
key := string(executionID(t))
|
||||
out, err := f.api.Create(ctx, f.token, key, in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := f.api.Create(ctx, f.token, key, in)
|
||||
if err != nil || !again.Deduplicated || again.Schedule.ID != out.Schedule.ID {
|
||||
t.Fatal("creation replay")
|
||||
}
|
||||
in.Paused = false
|
||||
_, err = f.api.Create(ctx, f.token, key, in)
|
||||
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
|
||||
v := out.Schedule
|
||||
f.due(v, 4*time.Second)
|
||||
f.tick()
|
||||
if len(f.occurrences(v.ID)) != 0 {
|
||||
t.Fatal("paused schedule fired")
|
||||
}
|
||||
next := f.definition()
|
||||
next.Task.Payload = json.RawMessage(`{"version":2}`)
|
||||
published, err := f.api.Publish(ctx, f.token, v.ID, string(executionID(t)), schedule.PublishInput{ExpectedRevision: v.Revision, Reason: "new template", Definition: next})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if published.Schedule.ProcessingVersion != 1 || published.Schedule.CurrentVersion != 2 {
|
||||
t.Fatal("publish erased old backlog")
|
||||
}
|
||||
old, err := f.api.Version(ctx, f.token, v.ID, 1)
|
||||
if err != nil || string(old.Definition.Task.Payload) != `{"number":9007199254740993}` {
|
||||
t.Fatal("old version changed")
|
||||
}
|
||||
resumeKey := string(executionID(t))
|
||||
resume := schedule.ControlInput{ExpectedRevision: published.Schedule.Revision, Reason: "resume work"}
|
||||
resumed, err := f.api.Control(ctx, f.token, v.ID, resumeKey, resume, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repeated, err := f.api.Control(ctx, f.token, v.ID, resumeKey, resume, false)
|
||||
if err != nil || !repeated.Deduplicated || repeated.Schedule.Revision != resumed.Schedule.Revision {
|
||||
t.Fatal("control replay")
|
||||
}
|
||||
f.tick()
|
||||
for _, o := range f.occurrences(v.ID) {
|
||||
if o.ScheduleVersion != 1 || !o.ScheduledAt.Before(published.Schedule.Current.EffectiveAt) {
|
||||
t.Fatal("historical version boundary lost")
|
||||
}
|
||||
}
|
||||
for i := 0; i < 3 && f.current(v.ID).ProcessingVersion == 1; i++ {
|
||||
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||
f.tick()
|
||||
}
|
||||
if f.current(v.ID).ProcessingVersion != 2 {
|
||||
t.Fatal("old backlog never advanced to published version")
|
||||
}
|
||||
for _, o := range f.occurrences(v.ID) {
|
||||
if o.ScheduleVersion == 1 && !o.ScheduledAt.Before(published.Schedule.Current.EffectiveAt) {
|
||||
t.Fatal("old version crossed effective boundary")
|
||||
}
|
||||
}
|
||||
// 另一命名空间知道 ID 也不可读或管理。
|
||||
stranger := fixture(t)
|
||||
_, err = stranger.api.Get(ctx, stranger.token, v.ID)
|
||||
expectWorkerCode(t, err, "NOT_FOUND")
|
||||
raw, _ := json.Marshal(auth.Permissions{Actions: []string{"schedules:read", "schedules:manage"}, Queues: []string{string(executionID(t))}})
|
||||
f.exec(`UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?`, string(raw), f.resources.NamespaceID, f.resources.CredentialID)
|
||||
_, err = f.api.Get(ctx, f.token, v.ID)
|
||||
expectWorkerCode(t, err, "NOT_FOUND")
|
||||
})
|
||||
t.Run("persisted_skip_and_coalesce_ranges", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
in := f.definition()
|
||||
in.Policy.CatchupWindowMS = 2000
|
||||
in.Policy.Misfire = "COALESCE"
|
||||
coalesce := f.create(in)
|
||||
in.Policy.Misfire = "SKIP"
|
||||
skip := f.create(in)
|
||||
f.due(coalesce, 4*time.Second)
|
||||
f.tick()
|
||||
for _, v := range []schedule.View{coalesce, skip} {
|
||||
items := f.occurrences(v.ID)
|
||||
skipped, merged := false, false
|
||||
for _, o := range items {
|
||||
if o.Decision == "SKIPPED" {
|
||||
skipped = true
|
||||
if o.TaskID != nil || o.CoveredUntil == nil || !o.CoveredUntil.After(o.ScheduledAt) {
|
||||
t.Fatal("invalid skipped range")
|
||||
}
|
||||
}
|
||||
if o.Decision == "COALESCED" {
|
||||
merged = true
|
||||
if o.TaskID == nil || o.CoveredUntil == nil {
|
||||
t.Fatal("coalesced range lost task")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !skipped || (v.ID == coalesce.ID && !merged) {
|
||||
t.Fatal("misfire decisions not persisted")
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("overlap_serial_unknown_and_waiting_cancel", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
in := f.definition()
|
||||
in.Policy.Overlap = "SERIAL"
|
||||
v := f.create(in)
|
||||
f.due(v, 5*time.Second)
|
||||
f.tick()
|
||||
items := f.occurrences(v.ID)
|
||||
if len(items) != 1 || items[0].TaskID == nil {
|
||||
t.Fatal("serial did not start one")
|
||||
}
|
||||
first := f.current(v.ID)
|
||||
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||
f.tick()
|
||||
if len(f.occurrences(v.ID)) != 1 || !f.current(v.ID).NextFireAt.Equal(*first.NextFireAt) {
|
||||
t.Fatal("serial skipped waiting task")
|
||||
}
|
||||
task := f.get(*items[0].TaskID)
|
||||
if _, err := f.tasks.CancelTask(ctx, f.token, task.ID, string(executionID(t)), service.CancelInput{ExpectedRevision: task.Revision, Reason: "cancel pending scheduled task"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||
f.tick()
|
||||
if len(f.occurrences(v.ID)) != 2 {
|
||||
t.Fatal("serial did not continue after terminal task")
|
||||
}
|
||||
retryFinish(t, f.executionFixture, retryStart(t, f.executionFixture), "UNKNOWN", false)
|
||||
waiting := f.current(v.ID)
|
||||
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||
f.tick()
|
||||
if len(f.occurrences(v.ID)) != 2 || !f.current(v.ID).NextFireAt.Equal(*waiting.NextFireAt) {
|
||||
t.Fatal("unknown outcome did not block serial schedule")
|
||||
}
|
||||
})
|
||||
t.Run("transaction_rollback_and_read_pagination", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
v := f.create(f.definition())
|
||||
f.due(v, 6*time.Second)
|
||||
prior := f.current(v.ID)
|
||||
f.exec(`CREATE TRIGGER orbit_test_schedule_event BEFORE INSERT ON pulse_schedule_occurrences FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='controlled trigger failure'`)
|
||||
_, err := f.store.Tick(ctx, 100)
|
||||
f.exec(`DROP TRIGGER orbit_test_schedule_event`)
|
||||
if err == nil || len(f.occurrences(v.ID)) != 0 || f.count("pulse_tasks", "owner_id=?", v.ID) != 0 || !f.current(v.ID).NextFireAt.Equal(*prior.NextFireAt) {
|
||||
t.Fatal("partial schedule transaction")
|
||||
}
|
||||
f.tick()
|
||||
page, err := f.api.Occurrences(ctx, f.token, v.ID, schedule.PageInput{Limit: 1})
|
||||
if err != nil || len(page.Items) != 1 || page.NextCursor == "" {
|
||||
t.Fatal("missing page")
|
||||
}
|
||||
second, err := f.api.Occurrences(ctx, f.token, v.ID, schedule.PageInput{Limit: 1, Cursor: page.NextCursor})
|
||||
if err != nil || len(second.Items) != 1 || second.Items[0].ID == page.Items[0].ID {
|
||||
t.Fatal("duplicate page")
|
||||
}
|
||||
another := f.create(f.definition())
|
||||
_, err = f.api.Occurrences(ctx, f.token, another.ID, schedule.PageInput{Cursor: page.NextCursor})
|
||||
expectWorkerCode(t, err, "INVALID_ARGUMENT")
|
||||
})
|
||||
}
|
||||
294
internal/storage/mysql/schedule_store.go
Normal file
294
internal/storage/mysql/schedule_store.go
Normal file
@@ -0,0 +1,294 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/jsonwire"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
|
||||
driver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// ScheduleStore 管理计划控制与扫描;共享任务存储的事务认证,不在事务外推进游标。
|
||||
type ScheduleStore struct {
|
||||
*TaskStore
|
||||
node core.ID
|
||||
}
|
||||
|
||||
func NewScheduleStore(db *sql.DB, node core.ID) (*ScheduleStore, error) {
|
||||
if node.Validate() != nil {
|
||||
return nil, errors.New("scheduler node id required")
|
||||
}
|
||||
return &ScheduleStore{TaskStore: NewTaskStore(db), node: node}, nil
|
||||
}
|
||||
|
||||
var _ schedule.Store = (*ScheduleStore)(nil)
|
||||
|
||||
const scheduleColumns = `id,namespace_id,queue_id,name,revision,current_version,processing_version,paused,next_fire_at,check_at,created_at,updated_at`
|
||||
|
||||
func scanSchedule(row scanner) (schedule.Summary, error) {
|
||||
var v schedule.Summary
|
||||
err := row.Scan(&v.ID, &v.NamespaceID, &v.QueueID, &v.Name, &v.Revision, &v.CurrentVersion, &v.ProcessingVersion, &v.Paused, &v.NextFireAt, &v.CheckAt, &v.CreatedAt, &v.UpdatedAt)
|
||||
return v, err
|
||||
}
|
||||
func readSchedule(ctx context.Context, tx *sql.Tx, ns, id core.ID, lock bool) (schedule.Summary, error) {
|
||||
q := `SELECT ` + scheduleColumns + ` FROM pulse_schedules WHERE namespace_id=? AND id=? AND queue_id IS NOT NULL`
|
||||
if lock {
|
||||
q += ` FOR UPDATE`
|
||||
}
|
||||
v, err := scanSchedule(tx.QueryRowContext(ctx, q, ns, id))
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
err = fault.NotFound
|
||||
}
|
||||
return v, err
|
||||
}
|
||||
func readScheduleVersion(ctx context.Context, tx *sql.Tx, ns, id core.ID, version uint64) (schedule.VersionView, error) {
|
||||
var v schedule.VersionView
|
||||
var rule, task []byte
|
||||
err := tx.QueryRowContext(ctx, `SELECT version,effective_at,specification,task_template,misfire_policy,overlap_policy,catchup_limit,catchup_window_ms,grace_ms FROM pulse_schedule_versions WHERE namespace_id=? AND schedule_id=? AND version=?`, ns, id, version).Scan(&v.Version, &v.EffectiveAt, &rule, &task, &v.Definition.Policy.Misfire, &v.Definition.Policy.Overlap, &v.Definition.Policy.CatchupLimit, &v.Definition.Policy.CatchupWindowMS, &v.Definition.Policy.GraceMS)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return v, fault.NotFound
|
||||
}
|
||||
if err != nil {
|
||||
return v, err
|
||||
}
|
||||
if err = json.Unmarshal(rule, &v.Definition.Rule); err != nil {
|
||||
return v, err
|
||||
}
|
||||
err = json.Unmarshal(task, &v.Definition.Task)
|
||||
return v, err
|
||||
}
|
||||
func insertScheduleVersion(ctx context.Context, tx *sql.Tx, ns, id core.ID, v schedule.VersionView) error {
|
||||
identity, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rule, err := json.Marshal(v.Definition.Rule)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task, err := jsonwire.Marshal(v.Definition.Task)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p := v.Definition.Policy
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_schedule_versions(namespace_id,id,schedule_id,version,effective_at,timezone,specification,task_template,misfire_policy,overlap_policy,catchup_limit,catchup_window_ms,grace_ms) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)`, ns, identity, id, v.Version, v.EffectiveAt, v.Definition.Rule.Timezone, string(rule), string(task), p.Misfire, p.Overlap, p.CatchupLimit, p.CatchupWindowMS, p.GraceMS)
|
||||
return err
|
||||
}
|
||||
func scheduleView(ctx context.Context, tx *sql.Tx, ns, id core.ID) (schedule.View, error) {
|
||||
summary, err := readSchedule(ctx, tx, ns, id, false)
|
||||
if err != nil {
|
||||
return schedule.View{}, err
|
||||
}
|
||||
current, err := readScheduleVersion(ctx, tx, ns, id, summary.CurrentVersion)
|
||||
return schedule.View{Summary: summary, Current: current}, err
|
||||
}
|
||||
func authorizeSchedule(ctx context.Context, tx *sql.Tx, ns, id core.ID, p auth.Permissions) (schedule.Summary, error) {
|
||||
v, err := readSchedule(ctx, tx, ns, id, false)
|
||||
if err != nil {
|
||||
return v, err
|
||||
}
|
||||
if !p.AllowsQueue(v.QueueID) {
|
||||
return v, fault.NotFound
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
func (s *ScheduleStore) Create(ctx context.Context, t auth.Token, key string, in schedule.CreateInput) (schedule.Result, error) {
|
||||
var out schedule.Result
|
||||
err := s.transaction(ctx, t, "schedules:manage", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
if !p.Allows("tasks:submit") || !p.AllowsQueue(in.Definition.Task.QueueID) {
|
||||
return fault.Forbidden
|
||||
}
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "schedule.create", "", key, "schedule", id, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cached != nil {
|
||||
if err := json.Unmarshal(cached, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Deduplicated = true
|
||||
return nil
|
||||
}
|
||||
var queue core.ID
|
||||
err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR SHARE`, t.NamespaceID, in.Definition.Task.QueueID).Scan(&queue)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return fault.NotFound
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c, err := schedule.Compile(in.Definition.Rule)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next, err := c.AtOrAfter(now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if next == nil {
|
||||
return fault.Invalid("schedule has no future occurrence")
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_schedules(namespace_id,id,queue_id,name,current_version,processing_version,paused,next_fire_at,check_at,created_at,updated_at) VALUES (?,?,?,?,1,1,?,?,?,?,?)`, t.NamespaceID, id, queue, in.Name, in.Paused, next, next, now, now)
|
||||
var me *driver.MySQLError
|
||||
if errors.As(err, &me) && me.Number == 1062 {
|
||||
return fault.Conflict
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := insertScheduleVersion(ctx, tx, t.NamespaceID, id, schedule.VersionView{Version: 1, EffectiveAt: now, Definition: in.Definition}); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Schedule, err = scheduleView(ctx, tx, t.NamespaceID, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, t.NamespaceID, id, 1, "schedule", "schedule.created", map[string]any{"queue_id": queue, "version": "1", "paused": in.Paused}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := appendAudit(ctx, tx, t, identity, id, "schedule", "schedule.create", "create schedule", nil, out.Schedule.Summary); err != nil {
|
||||
return err
|
||||
}
|
||||
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
func (s *ScheduleStore) Get(ctx context.Context, t auth.Token, id core.ID) (schedule.View, error) {
|
||||
var out schedule.View
|
||||
err := s.transaction(ctx, t, "schedules:read", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
if _, err := authorizeSchedule(ctx, tx, t.NamespaceID, id, p); err != nil {
|
||||
return err
|
||||
}
|
||||
var err error
|
||||
out, err = scheduleView(ctx, tx, t.NamespaceID, id)
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
func (s *ScheduleStore) Publish(ctx context.Context, t auth.Token, id core.ID, key string, in schedule.PublishInput) (schedule.Result, error) {
|
||||
return s.mutate(ctx, t, id, key, "schedule.publish", in, in.ExpectedRevision, in.Reason, func(tx *sql.Tx, v schedule.Summary, now time.Time) error {
|
||||
if in.Definition.Task.QueueID != v.QueueID {
|
||||
return fault.Invalid("schedule queue cannot change")
|
||||
}
|
||||
last, err := readScheduleVersion(ctx, tx, t.NamespaceID, id, v.CurrentVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !now.After(last.EffectiveAt) {
|
||||
return fault.Unavailable
|
||||
}
|
||||
if err := insertScheduleVersion(ctx, tx, t.NamespaceID, id, schedule.VersionView{Version: v.CurrentVersion + 1, EffectiveAt: now, Definition: in.Definition}); err != nil {
|
||||
return err
|
||||
}
|
||||
cursor, next := v.ProcessingVersion, v.NextFireAt
|
||||
// nil 游标也可能表示刚进入最新版本、尚未初始化;只有没有扫描工作才表示结束。
|
||||
if (next == nil && cursor == v.CurrentVersion && v.CheckAt == nil) || (next != nil && !next.Before(now)) {
|
||||
c, err := schedule.Compile(in.Definition.Rule)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next, err = c.AtOrAfter(now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cursor = v.CurrentVersion + 1
|
||||
}
|
||||
check := &now
|
||||
if next != nil && next.After(now) {
|
||||
check = next
|
||||
} else if next == nil && cursor == v.CurrentVersion+1 {
|
||||
check = nil
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE pulse_schedules SET current_version=current_version+1,processing_version=?,next_fire_at=?,check_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, cursor, next, check, now, t.NamespaceID, id)
|
||||
return err
|
||||
})
|
||||
}
|
||||
func (s *ScheduleStore) Control(ctx context.Context, t auth.Token, id core.ID, key string, in schedule.ControlInput, paused bool) (schedule.Result, error) {
|
||||
operation := "schedule.resume"
|
||||
if paused {
|
||||
operation = "schedule.pause"
|
||||
}
|
||||
return s.mutate(ctx, t, id, key, operation, in, in.ExpectedRevision, in.Reason, func(tx *sql.Tx, v schedule.Summary, now time.Time) error {
|
||||
if v.Paused == paused {
|
||||
return fault.Conflict
|
||||
}
|
||||
_, err := tx.ExecContext(ctx, `UPDATE pulse_schedules SET paused=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=?`, paused, now, t.NamespaceID, id)
|
||||
return err
|
||||
})
|
||||
}
|
||||
func (s *ScheduleStore) mutate(ctx context.Context, t auth.Token, id core.ID, key, operation string, input any, revision uint64, reason string, change func(*sql.Tx, schedule.Summary, time.Time) error) (schedule.Result, error) {
|
||||
var out schedule.Result
|
||||
err := s.transaction(ctx, t, "schedules:manage", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
if operation == "schedule.publish" && !p.Allows("tasks:submit") {
|
||||
return fault.Forbidden
|
||||
}
|
||||
if _, err := authorizeSchedule(ctx, tx, t.NamespaceID, id, p); err != nil {
|
||||
return err
|
||||
}
|
||||
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, operation, string(id), key, "schedule", id, input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cached != nil {
|
||||
if err := json.Unmarshal(cached, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Deduplicated = true
|
||||
return nil
|
||||
}
|
||||
v, err := readSchedule(ctx, tx, t.NamespaceID, id, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if v.Revision != revision {
|
||||
return fault.Conflict
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := change(tx, v, now); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Schedule, err = scheduleView(ctx, tx, t.NamespaceID, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := appendAudit(ctx, tx, t, identity, id, "schedule", operation, reason, v, out.Schedule.Summary); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, t.NamespaceID, id, out.Schedule.Revision, "schedule", operation, map[string]any{"version": fmt.Sprint(out.Schedule.CurrentVersion), "paused": out.Schedule.Paused}); err != nil {
|
||||
return err
|
||||
}
|
||||
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
func (s *ScheduleStore) Version(ctx context.Context, t auth.Token, id core.ID, version uint64) (schedule.VersionView, error) {
|
||||
var out schedule.VersionView
|
||||
err := s.transaction(ctx, t, "schedules:read", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
if _, err := authorizeSchedule(ctx, tx, t.NamespaceID, id, p); err != nil {
|
||||
return err
|
||||
}
|
||||
var err error
|
||||
out, err = readScheduleVersion(ctx, tx, t.NamespaceID, id, version)
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
225
internal/storage/mysql/schedule_tick.go
Normal file
225
internal/storage/mysql/schedule_tick.go
Normal file
@@ -0,0 +1,225 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
|
||||
)
|
||||
|
||||
// ScheduleStats 是本次已确认提交的批次计数,事务失败不计作成功。
|
||||
type ScheduleStats struct {
|
||||
Examined int // Examined 已检查的候选计划数。
|
||||
Fired int // Fired 本轮创建的任务数,合并范围计一个。
|
||||
Skipped int // Skipped 本轮保存的跳过范围数,不是被覆盖触发点总数。
|
||||
}
|
||||
|
||||
// Tick 按持久化 check_at 扫描,一次最多 100 个计划,每个计划独立短事务。
|
||||
func (s *ScheduleStore) Tick(ctx context.Context, limit int) (ScheduleStats, error) {
|
||||
var out ScheduleStats
|
||||
if limit < 1 || limit > 100 {
|
||||
return out, fault.Invalid("schedule batch limit must be 1 to 100")
|
||||
}
|
||||
type candidate struct{ ns, id, queue core.ID }
|
||||
candidates := []candidate{}
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT s.namespace_id,s.id,s.queue_id FROM pulse_schedules s JOIN orbit_namespaces n ON n.id=s.namespace_id JOIN orbit_applications a ON a.id=n.application_id WHERE s.paused=FALSE AND s.check_at<=UTC_TIMESTAMP(6) AND n.enabled=TRUE AND a.enabled=TRUE ORDER BY s.check_at,s.namespace_id,s.id LIMIT ?`, limit)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var c candidate
|
||||
if err := rows.Scan(&c.ns, &c.id, &c.queue); err != nil {
|
||||
rows.Close()
|
||||
return out, err
|
||||
}
|
||||
candidates = append(candidates, c)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
var failures []error
|
||||
for _, c := range candidates {
|
||||
out.Examined++
|
||||
fired, skipped, err := s.tickOne(ctx, c.ns, c.id, c.queue)
|
||||
if err != nil {
|
||||
failures = append(failures, err)
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
out.Fired += fired
|
||||
out.Skipped += skipped
|
||||
}
|
||||
return out, errors.Join(failures...)
|
||||
}
|
||||
func (s *ScheduleStore) tickOne(ctx context.Context, ns, id, queue core.ID) (int, int, error) {
|
||||
fired, skipped := 0, 0
|
||||
err := WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
||||
var allowed core.ID
|
||||
err := tx.QueryRowContext(ctx, `SELECT n.id FROM orbit_namespaces n JOIN orbit_applications a ON a.id=n.application_id WHERE n.id=? AND n.enabled=TRUE AND a.enabled=TRUE FOR SHARE`, ns).Scan(&allowed)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 和任务执行一致先锁队列,再锁计划;其他扫描节点不等待同一计划所在队列。
|
||||
err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, ns, queue).Scan(&allowed)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
v, err := readSchedule(ctx, tx, ns, id, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if v.Paused || v.CheckAt == nil || v.CheckAt.After(now) {
|
||||
return nil
|
||||
}
|
||||
if v.QueueID != queue {
|
||||
return errors.New("schedule queue changed")
|
||||
}
|
||||
cursor, next := v.ProcessingVersion, v.NextFireAt
|
||||
needsInitialization := false
|
||||
// 单批最多跨过 16 个空历史版本,避免长事务;下次从已保存版本继续。
|
||||
for n := 0; n < 16; n++ {
|
||||
version, err := readScheduleVersion(ctx, tx, ns, id, cursor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c, err := schedule.Compile(version.Definition.Rule)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var before *time.Time
|
||||
if cursor < v.CurrentVersion {
|
||||
newer, err := readScheduleVersion(ctx, tx, ns, id, cursor+1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
before = &newer.EffectiveAt
|
||||
}
|
||||
needsInitialization = false
|
||||
if next == nil {
|
||||
next, err = c.AtOrAfter(version.EffectiveAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if next != nil && next.Before(version.EffectiveAt) {
|
||||
return errors.New("schedule cursor predates its version")
|
||||
}
|
||||
if next == nil || (before != nil && !next.Before(*before)) {
|
||||
next = nil
|
||||
if cursor < v.CurrentVersion {
|
||||
cursor++
|
||||
needsInitialization = true
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if next.After(now) {
|
||||
break
|
||||
}
|
||||
var active bool
|
||||
if version.Definition.Policy.Overlap != "PARALLEL" {
|
||||
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pulse_tasks WHERE namespace_id=? AND owner_kind='schedule' AND owner_id=? AND state IN ('WAITING','LEASED','RUNNING','NEEDS_REVIEW'))`, ns, id).Scan(&active); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
policy := version.Definition.Policy
|
||||
// 大参数也要保持事务有界;每批复制的业务参数最多约 1 MiB。
|
||||
if size := len(version.Definition.Task.Payload); size > 0 {
|
||||
if max := uint32((1 << 20) / size); max < policy.CatchupLimit {
|
||||
policy.CatchupLimit = max
|
||||
}
|
||||
}
|
||||
batch, err := schedule.Plan(c, policy, *next, now, before, active)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, decision := range batch.Decisions {
|
||||
if err := s.saveOccurrence(ctx, tx, v, version, decision, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if decision.Kind == "SKIPPED" {
|
||||
skipped++
|
||||
} else {
|
||||
fired++
|
||||
}
|
||||
}
|
||||
next = batch.Next
|
||||
if next == nil && cursor < v.CurrentVersion {
|
||||
cursor++
|
||||
needsInitialization = true
|
||||
}
|
||||
break // 每计划每批只创建一个版本的有限任务,跨版本也不绕过速率上限。
|
||||
}
|
||||
var check *time.Time
|
||||
if next != nil || cursor < v.CurrentVersion || needsInitialization {
|
||||
t := now.Add(time.Second)
|
||||
check = &t
|
||||
if next != nil && next.After(t) {
|
||||
check = next
|
||||
}
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_schedules SET processing_version=?,next_fire_at=?,check_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, cursor, next, check, now, ns, id, v.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
return emitEvent(ctx, tx, ns, id, v.Revision+1, "schedule", "schedule.advanced", map[string]any{"node_id": s.node, "processing_version": fmt.Sprint(cursor), "next_fire_at": next, "fired": fired, "skipped": skipped})
|
||||
})
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return fired, skipped, nil
|
||||
}
|
||||
func (s *ScheduleStore) saveOccurrence(ctx context.Context, tx *sql.Tx, v schedule.Summary, version schedule.VersionView, d schedule.Decision, now time.Time) error {
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var task *core.ID
|
||||
if d.Kind != "SKIPPED" {
|
||||
value, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task = &value
|
||||
in := version.Definition.Task
|
||||
if in.QueueID != v.QueueID {
|
||||
return errors.New("schedule template queue mismatch")
|
||||
}
|
||||
policy, err := json.Marshal(in.Retry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_tasks(namespace_id,id,queue_id,handler,handler_version,payload_version,payload,policy_snapshot,recovery_safety,business_key,owner_kind,owner_id,available_at,deadline_at,attempt_timeout_ms,max_attempts,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,NULLIF(?,''),'schedule',?,?,?,?,?,?,?)`, v.NamespaceID, value, v.QueueID, in.Handler, in.HandlerVersion, in.PayloadVersion, string(in.Payload), string(policy), in.RecoverySafety, in.BusinessKey, v.ID, now, now.Add(time.Duration(in.TaskTimeoutMS)*time.Millisecond), in.Retry.AttemptTimeoutMS, in.Retry.MaxAttempts, now, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, v.NamespaceID, value, 1, "task", "task.accepted", map[string]any{"schedule_id": v.ID, "occurrence_id": id, "schedule_version": fmt.Sprint(version.Version), "scheduled_at": d.ScheduledAt, "node_id": s.node}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_schedule_occurrences(namespace_id,id,schedule_id,schedule_version,scheduled_at,decision,task_id,covered_until,reason,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)`, v.NamespaceID, id, v.ID, version.Version, d.ScheduledAt, d.Kind, task, d.CoveredUntil, d.Reason, now, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return emitEvent(ctx, tx, v.NamespaceID, id, 1, "schedule_occurrence", "schedule.decided", map[string]any{"schedule_id": v.ID, "version": fmt.Sprint(version.Version), "scheduled_at": d.ScheduledAt, "decision": d.Kind, "covered_until": d.CoveredUntil, "reason": d.Reason, "task_id": task, "node_id": s.node})
|
||||
}
|
||||
@@ -354,7 +354,7 @@ func (s *TaskStore) CancelTask(ctx context.Context, token auth.Token, id core.ID
|
||||
if err := tx.QueryRowContext(ctx, "SELECT owner_kind FROM pulse_tasks WHERE namespace_id=? AND id=?", token.NamespaceID, id).Scan(&owner); err != nil {
|
||||
return err
|
||||
}
|
||||
if owner.Valid {
|
||||
if owner.Valid && owner.String != "schedule" {
|
||||
return fault.Conflict
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
|
||||
Reference in New Issue
Block a user