feat: verify bounded simultaneous schedule execution

2026-09-20 13:56:44 +08:00
parent 71c33eae18
commit 29734596a3
21 changed files with 2371 additions and 2 deletions
--- a/sdk/go/cmd/orbit-schedule-load/measure.go
+++ b/sdk/go/cmd/orbit-schedule-load/measure.go
@@ -0,0 +1,351 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"reflect"
+	"sync"
+	"time"
+
+	orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
+)
+
+// schedulePayload 冻结运行和计划序号；精确字段集合用于检测错配输入。
+type schedulePayload struct {
+	RunID    string `json:"run_id"`   // RunID 本次测量身份，不是业务幂等保证。
+	Schedule int    `json:"schedule"` // Schedule 从零开始的计划序号。
+}
+
+func decodePayload(raw json.RawMessage) (schedulePayload, bool) {
+	// map中的RawMessage保留整数文本，不接受缺字段、null和非精确字段名。
+	var fields map[string]json.RawMessage
+	var out schedulePayload
+	if json.Unmarshal(raw, &fields) != nil || len(fields) != 2 {
+		return out, false
+	}
+	run, hasRun := fields["run_id"]
+	index, hasIndex := fields["schedule"]
+	if !hasRun || !hasIndex || string(run) == "null" || string(index) == "null" || json.Unmarshal(run, &out.RunID) != nil || json.Unmarshal(index, &out.Schedule) != nil {
+		return schedulePayload{}, false
+	}
+	// 按原文计数，拒绝重复键；字段次序可以不同。
+	var keys []string
+	decoder := json.NewDecoder(bytes.NewReader(raw))
+	if _, err := decoder.Token(); err != nil {
+		return schedulePayload{}, false
+	}
+	for decoder.More() {
+		k, err := decoder.Token()
+		if err != nil {
+			return schedulePayload{}, false
+		}
+		keys = append(keys, k.(string))
+		var value json.RawMessage
+		if decoder.Decode(&value) != nil {
+			return schedulePayload{}, false
+		}
+	}
+	return out, len(keys) == 2
+}
+
+func handlerName(runID string) string { return "load.schedule." + runID }
+func operationKey(runID string, index int, op string) string {
+	return fmt.Sprintf("schedule-%s-%02d-%s", runID, index, op)
+}
+func businessKey(runID string, index int) string { return operationKey(runID, index, "task") }
+
+type scheduleAPI interface {
+	GetQueue(context.Context, string) (orbit.Queue, error)
+	CreateSchedule(context.Context, string, orbit.CreateScheduleRequest) (orbit.ScheduleResult, error)
+	GetSchedule(context.Context, string) (orbit.Schedule, error)
+	ResumeSchedule(context.Context, string, string, orbit.ControlScheduleRequest) (orbit.ScheduleResult, error)
+	PauseSchedule(context.Context, string, string, orbit.ControlScheduleRequest) (orbit.ScheduleResult, error)
+	ListScheduleOccurrences(context.Context, string, orbit.SchedulePageRequest) (orbit.ScheduleOccurrencePage, error)
+	GetTask(context.Context, string) (orbit.Task, error)
+	ListTaskAttempts(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error)
+	GetTaskSource(context.Context, string) (orbit.TaskSource, error)
+}
+
+func scheduleDefinition(cfg config, runID string, index int, start time.Time) orbit.ScheduleDefinition {
+	end := start.Add(time.Duration(cfg.Requests) * cfg.Interval)
+	payload, _ := json.Marshal(schedulePayload{RunID: runID, Schedule: index})
+	return orbit.ScheduleDefinition{
+		Rule:   orbit.ScheduleRule{Kind: "FIXED_RATE", Timezone: "UTC", StartAt: start, EndAt: &end, IntervalMS: uint64(cfg.Interval.Milliseconds())},
+		Policy: orbit.SchedulePolicy{Misfire: "CATCH_UP", Overlap: "PARALLEL", CatchupLimit: 100, CatchupWindowMS: uint64((cfg.RunTimeout + time.Duration(cfg.Requests)*cfg.Interval + cfg.StartDelay).Milliseconds()), GraceMS: 1000},
+		Task:   orbit.SubmitTaskRequest{QueueID: cfg.QueueID, Handler: handlerName(runID), HandlerVersion: "v1", PayloadVersion: "v1", BusinessKey: businessKey(runID, index), Payload: payload, TaskTimeoutMS: uint64(cfg.RunTimeout.Milliseconds()), RecoverySafety: orbit.RecoveryReconcileOnly, Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: uint64(cfg.RunTimeout.Milliseconds())}},
+	}
+}
+
+func measure(parent context.Context, cfg config, api *orbit.Client, runID string) (r scheduleReport) {
+	r = newReport(cfg, runID)
+	ctx, cancel := context.WithTimeout(parent, cfg.RunTimeout)
+	defer cancel()
+	var workers []*orbit.Worker
+	var stopWorkers context.CancelFunc
+	var done chan error
+	log := newInvocationLog(cfg.Requests * cfg.Schedules)
+	observations := newObservations(cfg, runID)
+	// 所有返回路径都执行有界收尾；超时后的暂停使用独立上下文，结果不明不换键重试。
+	defer func() {
+		if stopWorkers != nil {
+			stopWorkers()
+			for range workers {
+				if err := <-done; err != nil {
+					r.Errors["WORKER_STOP"]++
+				}
+			}
+		}
+		for _, w := range workers {
+			s := w.Stats()
+			r.WorkerStarted += s.Started
+			r.Reconnects += s.Reconnects
+			r.Renewals += s.RenewalsConfirmed
+		}
+		cleanup, stop := context.WithTimeout(context.Background(), 5*time.Second)
+		var wg sync.WaitGroup
+		codes := make(chan string, len(r.Plans))
+		for i := range r.Plans {
+			wg.Add(1)
+			go func(p *planReport) {
+				defer wg.Done()
+				if validID(p.ID) {
+					codes <- pauseSchedule(cleanup, cfg, api, runID, p)
+				}
+			}(&r.Plans[i])
+		}
+		wg.Wait()
+		stop()
+		close(codes)
+		for code := range codes {
+			if code != "" {
+				r.Errors[code]++
+			}
+		}
+		observations.summarize(&r)
+		log.summarize(&r, observations)
+		if ctx.Err() != nil {
+			r.Errors["RUN_"+classify(ctx.Err())]++
+		}
+		r = r.finish()
+	}()
+	call, stop := context.WithTimeout(ctx, cfg.Timeout)
+	queue, err := api.GetQueue(call, cfg.QueueID)
+	stop()
+	if err != nil {
+		r.Errors["PREFLIGHT_"+classify(err)]++
+		return
+	}
+	r.Queue = &queue
+	if queue.ID != cfg.QueueID || !validID(queue.NamespaceID) || queue.Paused || queue.MaxConcurrency == 0 || queue.RatePerSecond == 0 {
+		r.Errors["QUEUE_UNAVAILABLE"]++
+		return
+	}
+	r.ScheduleStart = time.Now().UTC().Truncate(time.Microsecond).Add(cfg.StartDelay)
+	r.ScheduleEnd = r.ScheduleStart.Add(time.Duration(cfg.Requests) * cfg.Interval)
+	// 在恢复任何计划之前完成全部创建；计划有限终止，即使进程被强杀也不会无限触发。
+	seen := map[string]bool{}
+	for i := 0; i < cfg.Schedules; i++ {
+		if ctx.Err() != nil {
+			return
+		}
+		definition := scheduleDefinition(cfg, runID, i, r.ScheduleStart)
+		key := operationKey(runID, i, "create")
+		p := planReport{Index: i, CreateKey: key}
+		r.Plans = append(r.Plans, p)
+		call, stop = context.WithTimeout(ctx, cfg.Timeout)
+		res, e := api.CreateSchedule(call, key, orbit.CreateScheduleRequest{Name: key, Paused: true, Definition: definition})
+		stop()
+		if e != nil {
+			r.Errors["CREATE_"+classify(e)]++
+			return
+		}
+		p.ID = res.Schedule.ID
+		p.CalendarVersion = res.Schedule.Current.Definition.Rule.CalendarVersion
+		p.EffectiveAt = res.Schedule.Current.EffectiveAt
+		r.Plans[i] = p
+		if res.Deduplicated || !validID(p.ID) || seen[p.ID] || !matchesSchedule(res.Schedule, cfg, runID, i, queue.NamespaceID, r.ScheduleStart) || !res.Schedule.Paused || res.Schedule.Current.EffectiveAt.After(r.ScheduleStart) {
+			r.Errors["CREATE_MISMATCH"]++
+			return
+		}
+		seen[p.ID] = true
+		observations.addPlan(p.ID, i, r.ScheduleStart, queue.NamespaceID)
+	}
+	for i := 0; i < cfg.Workers; i++ {
+		w, e := orbit.NewWorker(api, orbit.WorkerOptions{QueueID: cfg.QueueID, Capacity: uint32(cfg.Capacity), ShutdownTimeout: 5 * time.Second, Handlers: []orbit.WorkerHandler{log.handler(ctx, cfg, runID)}})
+		if e != nil {
+			r.Errors["WORKER_CONFIG"]++
+			return
+		}
+		workers = append(workers, w)
+	}
+	workerCtx, workerCancel := context.WithCancel(ctx)
+	stopWorkers = workerCancel
+	done = make(chan error, len(workers))
+	for _, w := range workers {
+		go func(w *orbit.Worker) {
+			err := w.Run(workerCtx)
+			if workerCtx.Err() == nil {
+				cancel()
+			}
+			done <- err
+		}(w)
+	}
+	for i := range r.Plans {
+		p := &r.Plans[i]
+		call, stop = context.WithTimeout(ctx, cfg.Timeout)
+		current, e := api.GetSchedule(call, p.ID)
+		stop()
+		if e != nil {
+			r.Errors["RESUME_READ_"+classify(e)]++
+			return
+		}
+		if !time.Now().Before(r.ScheduleStart) || current.ID != p.ID || !matchesSchedule(current, cfg, runID, i, queue.NamespaceID, r.ScheduleStart) || !current.Paused || !sameVersion(current, *p) {
+			r.Errors["SETUP_LATE_OR_CHANGED"]++
+			return
+		}
+		p.ResumeKey = operationKey(runID, i, "resume")
+		p.ResumeRevision = current.Revision
+		call, stop = context.WithTimeout(ctx, cfg.Timeout)
+		res, e := api.ResumeSchedule(call, p.ID, p.ResumeKey, orbit.ControlScheduleRequest{ExpectedRevision: current.Revision, Reason: "start bounded schedule verification"})
+		stop()
+		if e != nil {
+			r.Errors["RESUME_"+classify(e)]++
+			return
+		}
+		if res.Deduplicated || res.Schedule.ID != p.ID || res.Schedule.Paused || !matchesSchedule(res.Schedule, cfg, runID, i, queue.NamespaceID, r.ScheduleStart) || !res.Schedule.UpdatedAt.Before(r.ScheduleStart) || !sameVersion(res.Schedule, *p) {
+			r.Errors["RESUME_MISMATCH_OR_LATE"]++
+			return
+		}
+		p.Resumed = true
+	}
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+		all := true
+		for i := range r.Plans {
+			p := &r.Plans[i]
+			call, stop = context.WithTimeout(ctx, cfg.Timeout)
+			page, e := api.ListScheduleOccurrences(call, p.ID, orbit.SchedulePageRequest{Limit: 100})
+			stop()
+			if e != nil {
+				r.Errors["OCCURRENCES_"+classify(e)]++
+				return
+			}
+			if code := observations.ingest(p.ID, page); code != "" {
+				r.Errors[code]++
+				return
+			}
+			complete, code := observations.verify(ctx, api, p.ID)
+			if code != "" {
+				r.Errors[code]++
+				return
+			}
+			if complete {
+				call, stop = context.WithTimeout(ctx, cfg.Timeout)
+				current, e := api.GetSchedule(call, p.ID)
+				stop()
+				if e != nil {
+					r.Errors["FINAL_READ_"+classify(e)]++
+					return
+				}
+				if current.ID != p.ID || !matchesSchedule(current, cfg, runID, i, queue.NamespaceID, r.ScheduleStart) || current.Paused || !sameVersion(current, *p) {
+					r.Errors["FINAL_SCHEDULE_MISMATCH"]++
+					return
+				}
+				p.Exhausted = current.CheckAt == nil && current.NextFireAt == nil && current.ProcessingVersion == 1
+			}
+			all = all && complete && p.Exhausted
+		}
+		if all {
+			return
+		}
+		timer := time.NewTimer(cfg.PollInterval)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return
+		case <-timer.C:
+		}
+	}
+}
+
+func matchesSchedule(s orbit.Schedule, cfg config, runID string, index int, namespace string, start time.Time) bool {
+	want := scheduleDefinition(cfg, runID, index, start)
+	got := s.Current.Definition
+	// 服务端冻结具体时区快照标识；不把API版本v1误当作时区数据版本。
+	if !validCalendarVersion(got.Rule.CalendarVersion) {
+		return false
+	}
+	got.Rule.CalendarVersion = ""
+	return s.NamespaceID == namespace && s.QueueID == cfg.QueueID && s.Name == operationKey(runID, index, "create") && s.CurrentVersion == 1 && s.Current.Version == 1 && s.Revision > 0 && !s.Current.EffectiveAt.IsZero() && sameDefinition(got, want)
+}
+
+func sameDefinition(a, b orbit.ScheduleDefinition) bool {
+	// 原始JSON可有空白差异；模板和结果保持精确整数，不经float64。
+	pa, ok := decodePayload(a.Task.Payload)
+	pb, ok2 := decodePayload(b.Task.Payload)
+	a.Task.Payload = nil
+	b.Task.Payload = nil
+	return ok && ok2 && pa == pb && reflect.DeepEqual(a, b)
+}
+
+func pauseSchedule(ctx context.Context, cfg config, api scheduleAPI, runID string, p *planReport) string {
+	for attempt := 0; attempt < 3; attempt++ {
+		call, stop := context.WithTimeout(ctx, cfg.Timeout)
+		current, err := api.GetSchedule(call, p.ID)
+		stop()
+		if err != nil {
+			return "PAUSE_READ_" + classify(err)
+		}
+		if current.ID != p.ID || current.Name != operationKey(runID, p.Index, "create") || current.QueueID != cfg.QueueID {
+			return "PAUSE_IDENTITY_MISMATCH"
+		}
+		if current.Paused {
+			p.PauseConfirmed = true
+			return ""
+		}
+		p.PauseKey = operationKey(runID, p.Index, fmt.Sprintf("pause-%d", attempt))
+		p.PauseRevision = current.Revision
+		call, stop = context.WithTimeout(ctx, cfg.Timeout)
+		res, err := api.PauseSchedule(call, p.ID, p.PauseKey, orbit.ControlScheduleRequest{ExpectedRevision: p.PauseRevision, Reason: "stop bounded schedule verification"})
+		stop()
+		if err == nil {
+			if res.Schedule.ID != p.ID || !res.Schedule.Paused {
+				return "PAUSE_RECEIPT_MISMATCH"
+			}
+			p.PauseConfirmed = true
+			return ""
+		}
+		var apiErr *orbit.APIError
+		if !errors.As(err, &apiErr) || apiErr.Code != "STATE_CONFLICT" || apiErr.StatusCode != 409 {
+			return "PAUSE_" + classify(err)
+		}
+		timer := time.NewTimer(25 * time.Millisecond)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return "PAUSE_" + classify(ctx.Err())
+		case <-timer.C:
+		}
+	}
+	return "PAUSE_CONFLICT_LIMIT"
+}
+
+func validCalendarVersion(v string) bool {
+	if len(v) == 0 || len(v) > 64 {
+		return false
+	}
+	for _, c := range v {
+		if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '.' || c == '-' || c == '_') {
+			return false
+		}
+	}
+	return true
+}
+func sameVersion(s orbit.Schedule, p planReport) bool {
+	return s.Current.Definition.Rule.CalendarVersion == p.CalendarVersion && s.Current.EffectiveAt.Equal(p.EffectiveAt)
+}