413 lines
15 KiB
Go
413 lines
15 KiB
Go
package mysql
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
|
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
|
|
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
|
)
|
|
|
|
type scheduleFixture struct {
|
|
*executionFixture
|
|
store *ScheduleStore
|
|
api *schedule.Service
|
|
}
|
|
|
|
func newScheduleFixture(t *testing.T, f *executionFixture) *scheduleFixture {
|
|
t.Helper()
|
|
raw, _ := json.Marshal(auth.Permissions{Actions: []string{"tasks:submit", "tasks:read", "tasks:cancel", "queues:read", "schedules:read", "schedules:manage", "workers:connect", "workers:consume"}, Queues: []string{"*"}})
|
|
f.exec(`UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?`, string(raw), f.resources.NamespaceID, f.resources.CredentialID)
|
|
store, err := NewScheduleStore(f.db, executionID(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return &scheduleFixture{f, store, schedule.New(store)}
|
|
}
|
|
func (f *scheduleFixture) definition() schedule.Definition {
|
|
return schedule.Definition{Rule: schedule.Rule{Kind: "FIXED_RATE", Timezone: "UTC", StartAt: time.Now().UTC().Truncate(time.Microsecond).Add(-time.Hour), IntervalMS: 1000}, Policy: schedule.Policy{Misfire: "CATCH_UP", Overlap: "PARALLEL", CatchupLimit: 3, CatchupWindowMS: 30000, GraceMS: 1000}, Task: service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"number":9007199254740993}`)}}
|
|
}
|
|
func (f *scheduleFixture) create(in schedule.Definition) schedule.View {
|
|
f.t.Helper()
|
|
out, err := f.api.Create(f.ctx, f.token, string(executionID(f.t)), schedule.CreateInput{Name: "plan-" + string(executionID(f.t)), Definition: in})
|
|
if err != nil {
|
|
f.t.Fatal(err)
|
|
}
|
|
return out.Schedule
|
|
}
|
|
func (f *scheduleFixture) current(id core.ID) schedule.View {
|
|
f.t.Helper()
|
|
out, err := f.api.Get(f.ctx, f.token, id)
|
|
if err != nil {
|
|
f.t.Fatal(err)
|
|
}
|
|
return out
|
|
}
|
|
func (f *scheduleFixture) occurrences(id core.ID) []schedule.OccurrenceView {
|
|
f.t.Helper()
|
|
out, err := f.api.Occurrences(f.ctx, f.token, id, schedule.PageInput{Limit: 100})
|
|
if err != nil {
|
|
f.t.Fatal(err)
|
|
}
|
|
return out.Items
|
|
}
|
|
|
|
// 等待真实时间形成积压，不将游标改到计划实际生效之前。
|
|
func (f *scheduleFixture) due(v schedule.View, age time.Duration) {
|
|
f.t.Helper()
|
|
if delay := time.Until(v.NextFireAt.Add(age)); delay > 0 {
|
|
timer := time.NewTimer(delay)
|
|
defer timer.Stop()
|
|
select {
|
|
case <-timer.C:
|
|
case <-f.ctx.Done():
|
|
f.t.Fatal(f.ctx.Err())
|
|
}
|
|
}
|
|
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
|
}
|
|
func (f *scheduleFixture) tick() ScheduleStats {
|
|
f.t.Helper()
|
|
out, err := f.store.Tick(f.ctx, 100)
|
|
if err != nil {
|
|
f.t.Fatal(err)
|
|
}
|
|
return out
|
|
}
|
|
func TestIntegrationSchedules(t *testing.T) {
|
|
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
|
if dsn == "" {
|
|
t.Skip("requires disposable MySQL")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
|
defer cancel()
|
|
db, err := Open(ctx, dsn, 32, 8)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
if err := Migrate(ctx, db); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture := func(t *testing.T) *scheduleFixture { return newScheduleFixture(t, newExecutionFixture(t, ctx, db)) }
|
|
// 连续发布经过“下一版本尚待初始化”和空版本时，不能丢失积压或重新开始已结束版本。
|
|
for _, emptyMiddle := range []bool{false, true} {
|
|
name := "publish_preserves_uninitialized_version"
|
|
if emptyMiddle {
|
|
name = "empty_version_then_finished_latest_stays_finished"
|
|
}
|
|
t.Run(name, func(t *testing.T) {
|
|
f := fixture(t)
|
|
finite := func(delay time.Duration) schedule.Definition {
|
|
in := f.definition()
|
|
var now time.Time
|
|
if err := db.QueryRowContext(ctx, `SELECT UTC_TIMESTAMP(6)`).Scan(&now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
in.Rule.StartAt = now.Add(delay)
|
|
end := in.Rule.StartAt.Add(time.Millisecond)
|
|
in.Rule.EndAt = &end
|
|
return in
|
|
}
|
|
publish := func(id core.ID, in schedule.Definition) schedule.View {
|
|
t.Helper()
|
|
current := f.current(id)
|
|
out, err := f.api.Publish(ctx, f.token, id, string(executionID(t)), schedule.PublishInput{ExpectedRevision: current.Revision, Reason: "version boundary regression", Definition: in})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return out.Schedule
|
|
}
|
|
waitUntil := func(at time.Time) {
|
|
t.Helper()
|
|
// 调度使用数据库时间，不能依赖容器与宿主机时钟完全同步。
|
|
for {
|
|
var now time.Time
|
|
if err := db.QueryRowContext(ctx, `SELECT UTC_TIMESTAMP(6)`).Scan(&now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if now.After(at) {
|
|
return
|
|
}
|
|
select {
|
|
case <-time.After(10 * time.Millisecond):
|
|
case <-ctx.Done():
|
|
t.Fatal(ctx.Err())
|
|
}
|
|
}
|
|
}
|
|
v := f.create(finite(300 * time.Millisecond))
|
|
waitUntil(*v.NextFireAt)
|
|
delay := 300 * time.Millisecond
|
|
if emptyMiddle {
|
|
delay = 30 * time.Second
|
|
}
|
|
middle := finite(delay)
|
|
publish(v.ID, middle)
|
|
var latest schedule.Definition
|
|
if emptyMiddle {
|
|
latest = finite(300 * time.Millisecond)
|
|
publish(v.ID, latest)
|
|
}
|
|
f.tick() // 第一版本结束，第二版本尚未计算第一个触发点。
|
|
pending := f.current(v.ID)
|
|
if pending.ProcessingVersion != 2 || pending.NextFireAt != nil || pending.CheckAt == nil {
|
|
t.Fatalf("expected pending initialization of version 2: processing=%d next=%v check=%v current=%d occurrences=%d", pending.ProcessingVersion, pending.NextFireAt, pending.CheckAt, pending.CurrentVersion, len(f.occurrences(v.ID)))
|
|
}
|
|
if !emptyMiddle {
|
|
waitUntil(middle.Rule.StartAt)
|
|
latest = finite(300 * time.Millisecond)
|
|
published := publish(v.ID, latest)
|
|
if published.ProcessingVersion != 2 {
|
|
t.Fatal("publishing discarded an uninitialized version")
|
|
}
|
|
}
|
|
waitUntil(latest.Rule.StartAt)
|
|
for i := 0; i < 3 && f.current(v.ID).CheckAt != nil; i++ {
|
|
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
|
f.tick()
|
|
}
|
|
finished := f.current(v.ID)
|
|
if finished.ProcessingVersion != 3 || finished.CheckAt != nil || finished.NextFireAt != nil {
|
|
t.Fatal("finished latest version retained scan work")
|
|
}
|
|
got := f.occurrences(v.ID)
|
|
want := map[uint64]time.Time{1: *v.NextFireAt, 3: latest.Rule.StartAt}
|
|
if !emptyMiddle {
|
|
want[2] = middle.Rule.StartAt
|
|
}
|
|
if len(got) != len(want) {
|
|
t.Fatalf("got %d occurrences, want %d", len(got), len(want))
|
|
}
|
|
for _, o := range got {
|
|
at, ok := want[o.ScheduleVersion]
|
|
if !ok || !o.ScheduledAt.Equal(at) || o.Decision != "FIRED" || o.TaskID == nil {
|
|
t.Fatal("version occurrence lost or duplicated")
|
|
}
|
|
delete(want, o.ScheduleVersion)
|
|
}
|
|
f.tick()
|
|
if f.current(v.ID).Revision != finished.Revision {
|
|
t.Fatal("finished schedule advanced again")
|
|
}
|
|
})
|
|
}
|
|
t.Run("multi_node_atomic_trigger_and_rate", func(t *testing.T) {
|
|
f := fixture(t)
|
|
v := f.create(f.definition())
|
|
f.due(v, 5*time.Second)
|
|
other, err := NewScheduleStore(db, executionID(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var wg sync.WaitGroup
|
|
failures := make(chan error, 2)
|
|
for _, s := range []*ScheduleStore{f.store, other} {
|
|
wg.Add(1)
|
|
go func(s *ScheduleStore) { defer wg.Done(); _, err := s.Tick(ctx, 100); failures <- err }(s)
|
|
}
|
|
wg.Wait()
|
|
close(failures)
|
|
for err := range failures {
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
got := f.occurrences(v.ID)
|
|
if len(got) != 3 || f.count("pulse_tasks", "owner_kind='schedule' AND owner_id=?", v.ID) != 3 {
|
|
t.Fatal("duplicate or partial trigger")
|
|
}
|
|
current := f.current(v.ID)
|
|
if current.CheckAt == nil || !current.CheckAt.After(current.UpdatedAt) {
|
|
t.Fatal("catchup rate not durable")
|
|
}
|
|
f.tick()
|
|
if len(f.occurrences(v.ID)) != 3 {
|
|
t.Fatal("repeat scan bypassed cooldown")
|
|
}
|
|
for _, o := range got {
|
|
if o.TaskID == nil || o.ScheduleVersion != 1 {
|
|
t.Fatal("missing ownership")
|
|
}
|
|
task := f.get(*o.TaskID)
|
|
if string(task.Payload) != `{"number":9007199254740993}` || task.AttemptCount != 0 {
|
|
t.Fatal("template snapshot damaged")
|
|
}
|
|
}
|
|
f.exec(`UPDATE orbit_namespaces SET enabled=FALSE WHERE id=?`, f.resources.NamespaceID)
|
|
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
|
f.tick()
|
|
if f.count("pulse_tasks", "owner_kind='schedule' AND owner_id=?", v.ID) != 3 {
|
|
t.Fatal("disabled namespace generated tasks")
|
|
}
|
|
f.exec(`UPDATE orbit_namespaces SET enabled=TRUE WHERE id=?`, f.resources.NamespaceID)
|
|
})
|
|
t.Run("idempotency_pause_scope_and_version", func(t *testing.T) {
|
|
f := fixture(t)
|
|
in := schedule.CreateInput{Name: "stable", Definition: f.definition(), Paused: true}
|
|
key := string(executionID(t))
|
|
out, err := f.api.Create(ctx, f.token, key, in)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again, err := f.api.Create(ctx, f.token, key, in)
|
|
if err != nil || !again.Deduplicated || again.Schedule.ID != out.Schedule.ID {
|
|
t.Fatal("creation replay")
|
|
}
|
|
in.Paused = false
|
|
_, err = f.api.Create(ctx, f.token, key, in)
|
|
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
|
|
v := out.Schedule
|
|
f.due(v, 4*time.Second)
|
|
f.tick()
|
|
if len(f.occurrences(v.ID)) != 0 {
|
|
t.Fatal("paused schedule fired")
|
|
}
|
|
next := f.definition()
|
|
next.Task.Payload = json.RawMessage(`{"version":2}`)
|
|
published, err := f.api.Publish(ctx, f.token, v.ID, string(executionID(t)), schedule.PublishInput{ExpectedRevision: v.Revision, Reason: "new template", Definition: next})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if published.Schedule.ProcessingVersion != 1 || published.Schedule.CurrentVersion != 2 {
|
|
t.Fatal("publish erased old backlog")
|
|
}
|
|
old, err := f.api.Version(ctx, f.token, v.ID, 1)
|
|
if err != nil || string(old.Definition.Task.Payload) != `{"number":9007199254740993}` {
|
|
t.Fatal("old version changed")
|
|
}
|
|
resumeKey := string(executionID(t))
|
|
resume := schedule.ControlInput{ExpectedRevision: published.Schedule.Revision, Reason: "resume work"}
|
|
resumed, err := f.api.Control(ctx, f.token, v.ID, resumeKey, resume, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
repeated, err := f.api.Control(ctx, f.token, v.ID, resumeKey, resume, false)
|
|
if err != nil || !repeated.Deduplicated || repeated.Schedule.Revision != resumed.Schedule.Revision {
|
|
t.Fatal("control replay")
|
|
}
|
|
f.tick()
|
|
for _, o := range f.occurrences(v.ID) {
|
|
if o.ScheduleVersion != 1 || !o.ScheduledAt.Before(published.Schedule.Current.EffectiveAt) {
|
|
t.Fatal("historical version boundary lost")
|
|
}
|
|
}
|
|
for i := 0; i < 3 && f.current(v.ID).ProcessingVersion == 1; i++ {
|
|
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
|
f.tick()
|
|
}
|
|
if f.current(v.ID).ProcessingVersion != 2 {
|
|
t.Fatal("old backlog never advanced to published version")
|
|
}
|
|
for _, o := range f.occurrences(v.ID) {
|
|
if o.ScheduleVersion == 1 && !o.ScheduledAt.Before(published.Schedule.Current.EffectiveAt) {
|
|
t.Fatal("old version crossed effective boundary")
|
|
}
|
|
}
|
|
// 另一命名空间知道 ID 也不可读或管理。
|
|
stranger := fixture(t)
|
|
_, err = stranger.api.Get(ctx, stranger.token, v.ID)
|
|
expectWorkerCode(t, err, "NOT_FOUND")
|
|
raw, _ := json.Marshal(auth.Permissions{Actions: []string{"schedules:read", "schedules:manage"}, Queues: []string{string(executionID(t))}})
|
|
f.exec(`UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?`, string(raw), f.resources.NamespaceID, f.resources.CredentialID)
|
|
_, err = f.api.Get(ctx, f.token, v.ID)
|
|
expectWorkerCode(t, err, "NOT_FOUND")
|
|
})
|
|
t.Run("persisted_skip_and_coalesce_ranges", func(t *testing.T) {
|
|
f := fixture(t)
|
|
in := f.definition()
|
|
in.Policy.CatchupWindowMS = 2000
|
|
in.Policy.Misfire = "COALESCE"
|
|
coalesce := f.create(in)
|
|
in.Policy.Misfire = "SKIP"
|
|
skip := f.create(in)
|
|
f.due(coalesce, 4*time.Second)
|
|
f.tick()
|
|
for _, v := range []schedule.View{coalesce, skip} {
|
|
items := f.occurrences(v.ID)
|
|
skipped, merged := false, false
|
|
for _, o := range items {
|
|
if o.Decision == "SKIPPED" {
|
|
skipped = true
|
|
if o.TaskID != nil || o.CoveredUntil == nil || !o.CoveredUntil.After(o.ScheduledAt) {
|
|
t.Fatal("invalid skipped range")
|
|
}
|
|
}
|
|
if o.Decision == "COALESCED" {
|
|
merged = true
|
|
if o.TaskID == nil || o.CoveredUntil == nil {
|
|
t.Fatal("coalesced range lost task")
|
|
}
|
|
}
|
|
}
|
|
if !skipped || (v.ID == coalesce.ID && !merged) {
|
|
t.Fatal("misfire decisions not persisted")
|
|
}
|
|
}
|
|
})
|
|
t.Run("overlap_serial_unknown_and_waiting_cancel", func(t *testing.T) {
|
|
f := fixture(t)
|
|
in := f.definition()
|
|
in.Policy.Overlap = "SERIAL"
|
|
v := f.create(in)
|
|
f.due(v, 5*time.Second)
|
|
f.tick()
|
|
items := f.occurrences(v.ID)
|
|
if len(items) != 1 || items[0].TaskID == nil {
|
|
t.Fatal("serial did not start one")
|
|
}
|
|
first := f.current(v.ID)
|
|
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
|
f.tick()
|
|
if len(f.occurrences(v.ID)) != 1 || !f.current(v.ID).NextFireAt.Equal(*first.NextFireAt) {
|
|
t.Fatal("serial skipped waiting task")
|
|
}
|
|
task := f.get(*items[0].TaskID)
|
|
if _, err := f.tasks.CancelTask(ctx, f.token, task.ID, string(executionID(t)), service.CancelInput{ExpectedRevision: task.Revision, Reason: "cancel pending scheduled task"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
|
f.tick()
|
|
if len(f.occurrences(v.ID)) != 2 {
|
|
t.Fatal("serial did not continue after terminal task")
|
|
}
|
|
retryFinish(t, f.executionFixture, retryStart(t, f.executionFixture), "UNKNOWN", false)
|
|
waiting := f.current(v.ID)
|
|
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
|
f.tick()
|
|
if len(f.occurrences(v.ID)) != 2 || !f.current(v.ID).NextFireAt.Equal(*waiting.NextFireAt) {
|
|
t.Fatal("unknown outcome did not block serial schedule")
|
|
}
|
|
})
|
|
t.Run("transaction_rollback_and_read_pagination", func(t *testing.T) {
|
|
f := fixture(t)
|
|
v := f.create(f.definition())
|
|
f.due(v, 6*time.Second)
|
|
prior := f.current(v.ID)
|
|
f.exec(`CREATE TRIGGER orbit_test_schedule_event BEFORE INSERT ON pulse_schedule_occurrences FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='controlled trigger failure'`)
|
|
_, err := f.store.Tick(ctx, 100)
|
|
f.exec(`DROP TRIGGER orbit_test_schedule_event`)
|
|
if err == nil || len(f.occurrences(v.ID)) != 0 || f.count("pulse_tasks", "owner_id=?", v.ID) != 0 || !f.current(v.ID).NextFireAt.Equal(*prior.NextFireAt) {
|
|
t.Fatal("partial schedule transaction")
|
|
}
|
|
f.tick()
|
|
page, err := f.api.Occurrences(ctx, f.token, v.ID, schedule.PageInput{Limit: 1})
|
|
if err != nil || len(page.Items) != 1 || page.NextCursor == "" {
|
|
t.Fatal("missing page")
|
|
}
|
|
second, err := f.api.Occurrences(ctx, f.token, v.ID, schedule.PageInput{Limit: 1, Cursor: page.NextCursor})
|
|
if err != nil || len(second.Items) != 1 || second.Items[0].ID == page.Items[0].ID {
|
|
t.Fatal("duplicate page")
|
|
}
|
|
another := f.create(f.definition())
|
|
_, err = f.api.Occurrences(ctx, f.token, another.ID, schedule.PageInput{Cursor: page.NextCursor})
|
|
expectWorkerCode(t, err, "INVALID_ARGUMENT")
|
|
})
|
|
}
|