feat: verify bounded simultaneous schedule execution
This commit is contained in:
313
sdk/go/cmd/orbit-schedule-load/measure_test.go
Normal file
313
sdk/go/cmd/orbit-schedule-load/measure_test.go
Normal file
@@ -0,0 +1,313 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func testID(n int) string { return fmt.Sprintf("%032x", n) }
|
||||
func testConfig() config {
|
||||
return config{QueueID: testID(1), Requests: 2, Schedules: 1, Workers: 1, Capacity: 2, StartDelay: 5 * time.Second, Interval: time.Second, RunTimeout: time.Minute, Timeout: time.Second, PollInterval: 100 * time.Millisecond}
|
||||
}
|
||||
func TestDecodePayloadStrict(t *testing.T) {
|
||||
want := schedulePayload{RunID: "run", Schedule: 0}
|
||||
for _, raw := range []string{`{"run_id":"run","schedule":0}`, ` {"schedule":0,"run_id":"run"} `} {
|
||||
if got, ok := decodePayload(json.RawMessage(raw)); !ok || got != want {
|
||||
t.Fatalf("valid payload rejected %s", raw)
|
||||
}
|
||||
}
|
||||
for _, raw := range []string{`{}`, `null`, `[]`, `{"run_id":"run"}`, `{"schedule":0}`, `{"run_id":null,"schedule":0}`, `{"run_id":"run","schedule": null }`, `{"run_id":"run","schedule":0.0}`, `{"run_id":"run","schedule":0,"extra":1}`, `{"run_id":"run","Schedule":0}`, `{"run_id":"run","schedule":0,"schedule":0}`, `{"run_id":"run","schedule":0} {}`} {
|
||||
if _, ok := decodePayload(json.RawMessage(raw)); ok {
|
||||
t.Fatalf("invalid payload accepted %s", raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// memoryAPI 仅为观察/收尾提供持久快照替身；未使用方法通过嵌入接口留空。
|
||||
type memoryAPI struct {
|
||||
scheduleAPI
|
||||
task orbit.Task
|
||||
history orbit.AttemptPage
|
||||
source orbit.TaskSource
|
||||
current orbit.Schedule
|
||||
reads, pauses int
|
||||
keys []string
|
||||
pauseError error
|
||||
conflictOnce bool
|
||||
}
|
||||
|
||||
func (m *memoryAPI) GetTask(context.Context, string) (orbit.Task, error) {
|
||||
m.reads++
|
||||
return m.task, nil
|
||||
}
|
||||
func (m *memoryAPI) ListTaskAttempts(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error) {
|
||||
return m.history, nil
|
||||
}
|
||||
func (m *memoryAPI) GetTaskSource(context.Context, string) (orbit.TaskSource, error) {
|
||||
return m.source, nil
|
||||
}
|
||||
func (m *memoryAPI) GetSchedule(context.Context, string) (orbit.Schedule, error) {
|
||||
return m.current, nil
|
||||
}
|
||||
func (m *memoryAPI) PauseSchedule(_ context.Context, _ string, key string, in orbit.ControlScheduleRequest) (orbit.ScheduleResult, error) {
|
||||
m.pauses++
|
||||
m.keys = append(m.keys, key)
|
||||
if m.pauseError != nil {
|
||||
return orbit.ScheduleResult{}, m.pauseError
|
||||
}
|
||||
if m.conflictOnce && m.pauses == 1 {
|
||||
m.current.Revision++
|
||||
return orbit.ScheduleResult{}, &orbit.APIError{StatusCode: 409, Code: "STATE_CONFLICT"}
|
||||
}
|
||||
if in.ExpectedRevision != m.current.Revision {
|
||||
return orbit.ScheduleResult{}, errors.New("stale revision")
|
||||
}
|
||||
m.current.Paused = true
|
||||
return orbit.ScheduleResult{Schedule: m.current}, nil
|
||||
}
|
||||
func fixture() (*observations, *memoryAPI, orbit.ScheduleOccurrencePage) {
|
||||
c := testConfig()
|
||||
c.Requests = 1
|
||||
start := time.Date(2026, 9, 20, 0, 0, 0, 0, time.UTC)
|
||||
id, taskID := testID(2), testID(3)
|
||||
o := newObservations(c, "run")
|
||||
o.addPlan(id, 0, start, testID(4))
|
||||
item := orbit.ScheduleOccurrence{ID: testID(5), ScheduleID: id, ScheduleVersion: 1, ScheduledAt: start, Decision: "FIRED", TaskID: &taskID, Reason: "SCHEDULED", CreatedAt: start.Add(time.Second)}
|
||||
raw, _ := json.Marshal(schedulePayload{RunID: "run", Schedule: 0})
|
||||
authorized, end := start.Add(2*time.Second), start.Add(3*time.Second)
|
||||
task := orbit.Task{ID: taskID, NamespaceID: testID(4), QueueID: c.QueueID, State: orbit.TaskSucceeded, Handler: handlerName("run"), HandlerVersion: "v1", PayloadVersion: "v1", BusinessKey: businessKey("run", 0), Payload: raw, Result: raw, RecoverySafety: orbit.RecoveryReconcileOnly, Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: uint64(c.RunTimeout.Milliseconds())}, AttemptCount: 1, CreatedAt: item.CreatedAt, AvailableAt: item.CreatedAt, DeadlineAt: item.CreatedAt.Add(c.RunTimeout), CompletedAt: &end}
|
||||
attempt := orbit.AttemptSummary{ID: testID(6), TaskID: taskID, Epoch: 1, State: orbit.AttemptSucceeded, AuthorizedAt: &authorized, EndedAt: &end, CreatedAt: item.CreatedAt}
|
||||
source := orbit.TaskSource{TaskID: taskID, Kind: "SCHEDULE", Schedule: &orbit.TaskScheduleSource{ScheduleID: id, ScheduleVersion: 1, OccurrenceID: item.ID, ScheduledAt: start, Decision: "FIRED"}}
|
||||
return o, &memoryAPI{task: task, history: orbit.AttemptPage{Items: []orbit.AttemptSummary{attempt}}, source: source}, orbit.ScheduleOccurrencePage{Items: []orbit.ScheduleOccurrence{item}}
|
||||
}
|
||||
func TestObserverWaitsAndCountsOnce(t *testing.T) {
|
||||
o, api, page := fixture()
|
||||
id := page.Items[0].ScheduleID
|
||||
api.task.State = orbit.TaskRunning
|
||||
if code := o.ingest(id, page); code != "" {
|
||||
t.Fatal(code)
|
||||
}
|
||||
if complete, code := o.verify(context.Background(), api, id); complete || code != "" {
|
||||
t.Fatalf("running counted as success %t %s", complete, code)
|
||||
}
|
||||
api.task.State = orbit.TaskSucceeded
|
||||
for i := 0; i < 3; i++ {
|
||||
if code := o.ingest(id, page); code != "" {
|
||||
t.Fatal(code)
|
||||
}
|
||||
if complete, code := o.verify(context.Background(), api, id); !complete || code != "" {
|
||||
t.Fatalf("verify %t %s", complete, code)
|
||||
}
|
||||
}
|
||||
if api.reads != 2 {
|
||||
t.Fatalf("verified task reread %d", api.reads)
|
||||
}
|
||||
r := newReport(o.cfg, "run")
|
||||
r.Plans = []planReport{{ID: id, Resumed: true, Exhausted: true, PauseConfirmed: true}}
|
||||
o.summarize(&r)
|
||||
r = r.finish()
|
||||
if r.Occurrences != 1 || r.Fired != 1 || r.VerifiedSuccess != 1 || r.Completion.Count != 1 || *r.Completion.Max != 3000 || *r.TriggerLag.Max != 1000 || *r.QueueWait.Max != 1000 || *r.Execution.Max != 1000 || r.Complete {
|
||||
t.Fatalf("incorrect report %#v", r)
|
||||
}
|
||||
log := newInvocationLog(1)
|
||||
_, err := log.handler(context.Background(), o.cfg, "run").Handle(context.Background(), orbit.WorkerExecution{Task: api.task, AttemptID: api.history.Items[0].ID, Epoch: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
log.summarize(&r, o)
|
||||
r.WorkerStarted = 1
|
||||
r = r.finish()
|
||||
if !r.Complete {
|
||||
t.Fatalf("full evidence not recognized: %#v", r)
|
||||
}
|
||||
r.Plans[0].PauseConfirmed = false
|
||||
if r.finish().Complete {
|
||||
t.Fatal("unconfirmed cleanup accepted")
|
||||
}
|
||||
}
|
||||
func TestObserverRejectsMalformedSuccess(t *testing.T) {
|
||||
tests := map[string]func(*memoryAPI){
|
||||
"wrong queue": func(m *memoryAPI) { m.task.QueueID = testID(90) },
|
||||
"wrong namespace": func(m *memoryAPI) { m.task.NamespaceID = testID(90) },
|
||||
"missing result field": func(m *memoryAPI) { m.task.Result = json.RawMessage(`{"run_id":"run"}`) },
|
||||
"wrong input": func(m *memoryAPI) { m.task.Payload = json.RawMessage(`{"run_id":"else","schedule":0}`) },
|
||||
"uncertain": func(m *memoryAPI) { m.task.EffectUncertain = true },
|
||||
"retry policy": func(m *memoryAPI) { m.task.Retry.MaxAttempts = 2 },
|
||||
"extra attempt": func(m *memoryAPI) { m.history.Items = append(m.history.Items, m.history.Items[0]) },
|
||||
"attempt cursor": func(m *memoryAPI) { m.history.NextCursor = "next" },
|
||||
"wrong attempt owner": func(m *memoryAPI) { m.history.Items[0].TaskID = testID(90) },
|
||||
"missing authorization": func(m *memoryAPI) { m.history.Items[0].AuthorizedAt = nil },
|
||||
"completion reversed": func(m *memoryAPI) { v := m.task.CreatedAt.Add(-time.Second); m.history.Items[0].EndedAt = &v },
|
||||
"source task": func(m *memoryAPI) { m.source.TaskID = testID(90) },
|
||||
"source occurrence": func(m *memoryAPI) { m.source.Schedule.OccurrenceID = testID(90) },
|
||||
"source version": func(m *memoryAPI) { m.source.Schedule.ScheduleVersion = 2 },
|
||||
"source time": func(m *memoryAPI) { m.source.Schedule.ScheduledAt = m.source.Schedule.ScheduledAt.Add(time.Second) },
|
||||
"source decision": func(m *memoryAPI) { m.source.Schedule.Decision = "COALESCED" },
|
||||
}
|
||||
for name, change := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
o, api, page := fixture()
|
||||
id := page.Items[0].ScheduleID
|
||||
if code := o.ingest(id, page); code != "" {
|
||||
t.Fatal(code)
|
||||
}
|
||||
change(api)
|
||||
if ok, code := o.verify(context.Background(), api, id); ok || code == "" {
|
||||
t.Fatal("malformed success accepted")
|
||||
}
|
||||
if o.plans[id].slots[0].verified {
|
||||
t.Fatal("invalid proof counted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestOccurrenceUniquenessAndHistory(t *testing.T) {
|
||||
for name, change := range map[string]func(*orbit.ScheduleOccurrencePage){
|
||||
"wrong parent": func(p *orbit.ScheduleOccurrencePage) { p.Items[0].ScheduleID = testID(90) },
|
||||
"wrong version": func(p *orbit.ScheduleOccurrencePage) { p.Items[0].ScheduleVersion = 2 },
|
||||
"off slot": func(p *orbit.ScheduleOccurrencePage) {
|
||||
p.Items[0].ScheduledAt = p.Items[0].ScheduledAt.Add(time.Millisecond)
|
||||
},
|
||||
"skipped": func(p *orbit.ScheduleOccurrencePage) { p.Items[0].Decision = "SKIPPED" },
|
||||
"cursor": func(p *orbit.ScheduleOccurrencePage) { p.NextCursor = "more" },
|
||||
"duplicate": func(p *orbit.ScheduleOccurrencePage) { p.Items = append(p.Items, p.Items[0]) },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
o, _, page := fixture()
|
||||
id := page.Items[0].ScheduleID
|
||||
change(&page)
|
||||
if o.ingest(id, page) == "" {
|
||||
t.Fatal("invalid page accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
o, _, page := fixture()
|
||||
id := page.Items[0].ScheduleID
|
||||
if code := o.ingest(id, page); code != "" {
|
||||
t.Fatal(code)
|
||||
}
|
||||
if o.ingest(id, orbit.ScheduleOccurrencePage{Items: []orbit.ScheduleOccurrence{}}) != "OCCURRENCE_DISAPPEARED" {
|
||||
t.Fatal("lost history accepted")
|
||||
}
|
||||
changed := page
|
||||
changed.Items = append([]orbit.ScheduleOccurrence(nil), page.Items...)
|
||||
changed.Items[0].ID = testID(91)
|
||||
if o.ingest(id, changed) != "OCCURRENCE_CHANGED" {
|
||||
t.Fatal("replaced slot accepted")
|
||||
}
|
||||
second := testID(92)
|
||||
o.addPlan(second, 1, o.plans[id].start, testID(4))
|
||||
changed.Items[0].ScheduleID = second
|
||||
if o.ingest(second, changed) != "DUPLICATE_TASK" {
|
||||
t.Fatal("task shared by two slots accepted")
|
||||
}
|
||||
}
|
||||
func TestHandlerDetectsDuplicateAndUnconfirmedCalls(t *testing.T) {
|
||||
o, api, page := fixture()
|
||||
id := page.Items[0].ScheduleID
|
||||
if code := o.ingest(id, page); code != "" {
|
||||
t.Fatal(code)
|
||||
}
|
||||
log := newInvocationLog(1)
|
||||
handler := log.handler(context.Background(), o.cfg, "run")
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 4; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, _ = handler.Handle(context.Background(), orbit.WorkerExecution{Task: api.task, AttemptID: testID(6), Epoch: 1})
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
r := newReport(o.cfg, "run")
|
||||
log.summarize(&r, o)
|
||||
if r.HandlerCalls != 4 || r.DuplicateInvocations != 3 || r.UnconfirmedInvocations != 4 {
|
||||
t.Fatalf("calls lost: %#v", r)
|
||||
}
|
||||
if ok, code := o.verify(context.Background(), api, id); !ok || code != "" {
|
||||
t.Fatal(code)
|
||||
}
|
||||
r = newReport(o.cfg, "run")
|
||||
log.summarize(&r, o)
|
||||
if r.UnconfirmedInvocations != 0 || r.DuplicateInvocations != 3 {
|
||||
t.Fatal("duplicate hidden after verification")
|
||||
}
|
||||
}
|
||||
func TestPauseOnlyRekeysExplicitRevisionConflict(t *testing.T) {
|
||||
cfg := testConfig()
|
||||
current := orbit.Schedule{ScheduleSummary: orbit.ScheduleSummary{ID: testID(2), QueueID: cfg.QueueID, Name: operationKey("run", 0, "create"), Revision: 7}}
|
||||
m := &memoryAPI{current: current, conflictOnce: true}
|
||||
p := planReport{ID: current.ID}
|
||||
if code := pauseSchedule(context.Background(), cfg, m, "run", &p); code != "" {
|
||||
t.Fatal(code)
|
||||
}
|
||||
if m.pauses != 2 || !p.PauseConfirmed || p.PauseRevision != 8 || !reflect.DeepEqual(m.keys, []string{operationKey("run", 0, "pause-0"), operationKey("run", 0, "pause-1")}) {
|
||||
t.Fatalf("conflict recovery lost identity %#v %#v", p, m)
|
||||
}
|
||||
for _, err := range []error{context.DeadlineExceeded, errors.New("transport lost"), &orbit.APIError{StatusCode: 409, Code: "IDEMPOTENCY_CONFLICT"}} {
|
||||
m = &memoryAPI{current: current, pauseError: err}
|
||||
p = planReport{ID: current.ID}
|
||||
if code := pauseSchedule(context.Background(), cfg, m, "run", &p); code == "" || m.pauses != 1 || p.PauseConfirmed || p.PauseKey == "" || p.PauseRevision != 7 {
|
||||
t.Fatalf("unknown outcome reissued: %#v %#v", p, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestScheduleDefinitionFrozenBoundedAndCompared(t *testing.T) {
|
||||
cfg := testConfig()
|
||||
start := time.Date(2026, 9, 20, 0, 0, 0, 0, time.UTC)
|
||||
def := scheduleDefinition(cfg, "run", 0, start)
|
||||
if def.Rule.EndAt == nil || !def.Rule.EndAt.Equal(start.Add(2*time.Second)) || def.Task.Retry.MaxAttempts != 1 || def.Policy.Overlap != "PARALLEL" {
|
||||
t.Fatal("schedule is not bounded as configured")
|
||||
}
|
||||
s := orbit.Schedule{ScheduleSummary: orbit.ScheduleSummary{NamespaceID: testID(4), QueueID: cfg.QueueID, Name: operationKey("run", 0, "create"), Revision: 1, CurrentVersion: 1}, Current: orbit.ScheduleVersion{Version: 1, EffectiveAt: start.Add(-time.Second), Definition: def}}
|
||||
s.Current.Definition.Rule.CalendarVersion = "go1.27.1-b2d18a7c8fa8"
|
||||
if !matchesSchedule(s, cfg, "run", 0, testID(4), start) {
|
||||
t.Fatal("normalized frozen schedule rejected")
|
||||
}
|
||||
s.Current.Definition.Task.Payload = json.RawMessage(`{"run_id":"run","schedule":1}`)
|
||||
if matchesSchedule(s, cfg, "run", 0, testID(4), start) {
|
||||
t.Fatal("changed template accepted")
|
||||
}
|
||||
}
|
||||
func TestPartialReportPreservesPlannedDenominator(t *testing.T) {
|
||||
c := testConfig()
|
||||
c.Schedules = 2
|
||||
r := newReport(c, "run")
|
||||
r.Occurrences = 3
|
||||
r.Fired = 2
|
||||
r.VerifiedSuccess = 1
|
||||
r = r.finish()
|
||||
if r.Expected != 4 || r.Unobserved != 1 || r.Unverified != 1 || r.Complete {
|
||||
t.Fatalf("bad denominators %#v", r)
|
||||
}
|
||||
if d := distributionOf(nil); d.P50 != nil || d.P99 != nil {
|
||||
t.Fatal("empty percentile is not null")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalendarSnapshotRemainsFixed(t *testing.T) {
|
||||
c := testConfig()
|
||||
start := time.Date(2026, 9, 20, 0, 0, 0, 0, time.UTC)
|
||||
p := planReport{CalendarVersion: "go1.27.1-b2d18a7c8fa8", EffectiveAt: start}
|
||||
s := orbit.Schedule{Current: orbit.ScheduleVersion{Definition: scheduleDefinition(c, "run", 0, start), EffectiveAt: start}}
|
||||
s.Current.Definition.Rule.CalendarVersion = p.CalendarVersion
|
||||
if !sameVersion(s, p) {
|
||||
t.Fatal("unchanged snapshot rejected")
|
||||
}
|
||||
s.Current.Definition.Rule.CalendarVersion = "replacement"
|
||||
if sameVersion(s, p) {
|
||||
t.Fatal("changed timezone snapshot accepted")
|
||||
}
|
||||
s.Current.Definition.Rule.CalendarVersion = p.CalendarVersion
|
||||
s.Current.EffectiveAt = start.Add(time.Microsecond)
|
||||
if sameVersion(s, p) {
|
||||
t.Fatal("changed effective time accepted")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user