feat: verify bounded simultaneous schedule execution

sdk/go/cmd/orbit-schedule-load/handler.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
+	"sync"
+)
+
+// invocation 固定首次调用的任务/尝试身份，重复调用不得覆盖首条证据。
+type invocation struct {
+	index     int    // index 输入中的计划序号。
+	attemptID string // attemptID 首次调用的尝试。
+	calls     int    // calls 同一任务的调用次数。
+}
+
+// invocationLog 最多保存预期任务数个身份，异常输入不能无限增长内存。
+type invocationLog struct {
+	mu       sync.Mutex             // mu 保护多个Worker并发回调。
+	limit    int                    // limit 可存任务身份上限。
+	calls    map[string]*invocation // calls 以任务身份记录真实调用。
+	invalid  int                    // invalid 无法核验的输入次数。
+	overflow int                    // overflow 超出身份预算的调用次数。
+}
+
+func newInvocationLog(n int) *invocationLog {
+	return &invocationLog{limit: n, calls: map[string]*invocation{}}
+}
+func (l *invocationLog) handler(run context.Context, c config, runID string) orbit.WorkerHandler {
+	return orbit.WorkerHandler{Capability: orbit.WorkerCapability{Handler: handlerName(runID), HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(ctx context.Context, e orbit.WorkerExecution) (json.RawMessage, error) {
+		payload, ok := decodePayload(e.Task.Payload)
+		if !ok || payload.Schedule < 0 || payload.Schedule >= c.Schedules || !matchesTask(e.Task, c, runID, payload.Schedule) || !validID(e.AttemptID) || e.Epoch != 1 || e.ConcurrencyToken != 0 {
+			l.mu.Lock()
+			l.invalid++
+			l.mu.Unlock()
+			return nil, &orbit.WorkerFailure{Code: "SCHEDULE_INPUT_MISMATCH"}
+		}
+		l.mu.Lock()
+		existing := l.calls[e.Task.ID]
+		if existing != nil {
+			existing.calls++
+			l.mu.Unlock()
+			return nil, &orbit.WorkerFailure{Code: "DUPLICATE_INVOCATION"}
+		}
+		if len(l.calls) >= l.limit {
+			l.overflow++
+			l.mu.Unlock()
+			return nil, &orbit.WorkerFailure{Code: "INVOCATION_LIMIT"}
+		}
+		l.calls[e.Task.ID] = &invocation{index: payload.Schedule, attemptID: e.AttemptID, calls: 1}
+		l.mu.Unlock()
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-run.Done():
+			return nil, run.Err()
+		default:
+		}
+		return e.Task.Payload, nil
+	}}
+}
+func (l *invocationLog) summarize(r *scheduleReport, o *observations) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	r.Errors["INVALID_HANDLER_INPUT"] += l.invalid
+	r.Errors["HANDLER_IDENTITY_LIMIT"] += l.overflow
+	for taskID, call := range l.calls {
+		r.HandlerCalls += call.calls
+		if call.calls > 1 {
+			r.DuplicateInvocations += call.calls - 1
+		}
+		s := o.tasks[taskID]
+		if s == nil || !s.verified || s.attemptID != call.attemptID || o.plans[s.occurrence.ScheduleID].index != call.index {
+			r.UnconfirmedInvocations += call.calls
+		}
+	}
+	for taskID, s := range o.tasks {
+		if s.verified && l.calls[taskID] == nil {
+			r.Errors["MISSING_LOCAL_INVOCATION"]++
+		}
+	}
+}

sdk/go/cmd/orbit-schedule-load/main.go

@@ -0,0 +1,153 @@
+// orbit-schedule-load 在隔离环境测量固定间隔计划的持久化触发、真实Worker执行和来源核对。
+// 计划模板只包含固定JSON输入，处理器没有外部业务副作用。
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
+)
+
+type config struct {
+	URL, Token, QueueID                    string
+	Requests, Schedules, Workers, Capacity int
+	Interval, Timeout, RunTimeout          time.Duration
+	PollInterval, StartDelay               time.Duration
+}
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	cfg, err := parseConfig(os.Args[1:])
+	if err != nil {
+		return err
+	}
+	client, err := orbit.NewClient(cfg.URL, orbit.Options{Token: cfg.Token, HTTPClient: &http.Client{Timeout: cfg.Timeout}})
+	if err != nil {
+		return errors.New("invalid client configuration")
+	}
+	runID, err := orbit.NewWorkerMessageID()
+	if err != nil {
+		return errors.New("cannot generate run identity")
+	}
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	report := measure(ctx, cfg, client, runID)
+	if err := json.NewEncoder(os.Stdout).Encode(report); err != nil {
+		return err
+	}
+	if !report.Complete {
+		return errors.New("schedule sample incomplete; inspect report and original run identities")
+	}
+	return nil
+}
+
+func parseConfig(args []string) (config, error) {
+	cfg := config{URL: os.Getenv("ORBIT_URL"), Token: os.Getenv("ORBIT_TOKEN"), QueueID: os.Getenv("ORBIT_QUEUE_ID"), Requests: 6, Schedules: 1, StartDelay: 5 * time.Second, Workers: 2, Capacity: 2, Interval: time.Second, Timeout: 5 * time.Second, RunTimeout: 2 * time.Minute, PollInterval: 250 * time.Millisecond}
+	flags := flag.NewFlagSet("orbit-schedule-load", flag.ContinueOnError)
+	var credentials string
+	flags.StringVar(&cfg.URL, "url", cfg.URL, "Orbit HTTP(S) URL")
+	flags.StringVar(&credentials, "credentials", os.Getenv("ORBIT_CREDENTIALS_FILE"), "private credentials file for an isolated namespace")
+	flags.StringVar(&cfg.QueueID, "queue", cfg.QueueID, "existing isolated queue; configuration is not changed")
+	flags.IntVar(&cfg.Requests, "requests", cfg.Requests, "expected occurrences per schedule, 1..100")
+	flags.IntVar(&cfg.Schedules, "schedules", cfg.Schedules, "schedules sharing the same trigger times, 1..32")
+	flags.DurationVar(&cfg.StartDelay, "start-delay", cfg.StartDelay, "lead time for setup, 1s..1m; late setup fails the sample")
+	flags.IntVar(&cfg.Workers, "workers", cfg.Workers, "independent WSS Workers in this process, 1..16")
+	flags.IntVar(&cfg.Capacity, "capacity", cfg.Capacity, "concurrent handlers per Worker, 1..128; total at most 256")
+	flags.DurationVar(&cfg.Interval, "interval", cfg.Interval, "fixed-rate interval in whole milliseconds, 1s..1m")
+	flags.DurationVar(&cfg.Timeout, "timeout", cfg.Timeout, "per HTTP request timeout, 1ms..1m")
+	flags.DurationVar(&cfg.RunTimeout, "run-timeout", cfg.RunTimeout, "whole sample deadline, 5s..1h, plus at most 5s Worker shutdown and 5s schedule cleanup")
+	flags.DurationVar(&cfg.PollInterval, "poll-interval", cfg.PollInterval, "occurrence polling interval, 100ms..10s")
+	if err := flags.Parse(args); err != nil {
+		return config{}, err
+	}
+	if flags.NArg() != 0 {
+		return config{}, errors.New("positional arguments are not supported")
+	}
+	if credentials != "" {
+		f, err := os.Open(credentials)
+		if err != nil {
+			return config{}, errors.New("cannot open private credentials file")
+		}
+		raw, err := io.ReadAll(io.LimitReader(f, (16<<10)+1))
+		_ = f.Close()
+		var saved struct {
+			Token   string `json:"token"`
+			QueueID string `json:"queue_id"`
+		}
+		if err != nil || len(raw) > 16<<10 || json.Unmarshal(raw, &saved) != nil {
+			return config{}, errors.New("invalid credentials file (maximum 16 KiB)")
+		}
+		if cfg.Token == "" {
+			cfg.Token = saved.Token
+		}
+		if cfg.QueueID == "" {
+			cfg.QueueID = saved.QueueID
+		}
+	}
+	if cfg.URL == "" || cfg.Token == "" || !validID(cfg.QueueID) {
+		return config{}, errors.New("URL, token and a 32-character lowercase hexadecimal queue ID are required")
+	}
+	if cfg.Requests < 1 || cfg.Requests > 100 || cfg.Schedules < 1 || cfg.Schedules > 32 || cfg.Workers < 1 || cfg.Workers > 16 || cfg.Capacity < 1 || cfg.Capacity > 128 || cfg.Workers*cfg.Capacity > 256 {
+		return config{}, errors.New("request or Worker count is out of bounds")
+	}
+	if cfg.Interval < time.Second || cfg.Interval > time.Minute || cfg.Interval%time.Millisecond != 0 || cfg.StartDelay < time.Second || cfg.StartDelay > time.Minute || cfg.StartDelay%time.Millisecond != 0 || cfg.Timeout < time.Millisecond || cfg.Timeout > time.Minute || cfg.RunTimeout < 5*time.Second || cfg.RunTimeout > time.Hour || cfg.PollInterval < 100*time.Millisecond || cfg.PollInterval > 10*time.Second {
+		return config{}, errors.New("interval, timeout or poll interval is out of bounds")
+	}
+	return cfg, nil
+}
+
+func validID(id string) bool {
+	if len(id) != 32 {
+		return false
+	}
+	for _, c := range id {
+		if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') {
+			return false
+		}
+	}
+	return true
+}
+
+func classify(err error) string {
+	if errors.Is(err, context.DeadlineExceeded) {
+		return "TIMEOUT"
+	}
+	if errors.Is(err, context.Canceled) {
+		return "CANCELLED"
+	}
+	var api *orbit.APIError
+	if errors.As(err, &api) {
+		switch api.StatusCode {
+		case 401, 403:
+			return "AUTH"
+		case 409:
+			return "CONFLICT"
+		case 429:
+			return "BACKPRESSURE"
+		case 400, 422:
+			return "INVALID_REQUEST"
+		}
+		return "HTTP"
+	}
+	var h *orbit.HTTPError
+	if errors.As(err, &h) {
+		return "HTTP"
+	}
+	return "TRANSPORT"
+}

sdk/go/cmd/orbit-schedule-load/measure.go

@@ -0,0 +1,351 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"reflect"
+	"sync"
+	"time"
+
+	orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
+)
+
+// schedulePayload 冻结运行和计划序号；精确字段集合用于检测错配输入。
+type schedulePayload struct {
+	RunID    string `json:"run_id"`   // RunID 本次测量身份，不是业务幂等保证。
+	Schedule int    `json:"schedule"` // Schedule 从零开始的计划序号。
+}
+
+func decodePayload(raw json.RawMessage) (schedulePayload, bool) {
+	// map中的RawMessage保留整数文本，不接受缺字段、null和非精确字段名。
+	var fields map[string]json.RawMessage
+	var out schedulePayload
+	if json.Unmarshal(raw, &fields) != nil || len(fields) != 2 {
+		return out, false
+	}
+	run, hasRun := fields["run_id"]
+	index, hasIndex := fields["schedule"]
+	if !hasRun || !hasIndex || string(run) == "null" || string(index) == "null" || json.Unmarshal(run, &out.RunID) != nil || json.Unmarshal(index, &out.Schedule) != nil {
+		return schedulePayload{}, false
+	}
+	// 按原文计数，拒绝重复键；字段次序可以不同。
+	var keys []string
+	decoder := json.NewDecoder(bytes.NewReader(raw))
+	if _, err := decoder.Token(); err != nil {
+		return schedulePayload{}, false
+	}
+	for decoder.More() {
+		k, err := decoder.Token()
+		if err != nil {
+			return schedulePayload{}, false
+		}
+		keys = append(keys, k.(string))
+		var value json.RawMessage
+		if decoder.Decode(&value) != nil {
+			return schedulePayload{}, false
+		}
+	}
+	return out, len(keys) == 2
+}
+
+func handlerName(runID string) string { return "load.schedule." + runID }
+func operationKey(runID string, index int, op string) string {
+	return fmt.Sprintf("schedule-%s-%02d-%s", runID, index, op)
+}
+func businessKey(runID string, index int) string { return operationKey(runID, index, "task") }
+
+type scheduleAPI interface {
+	GetQueue(context.Context, string) (orbit.Queue, error)
+	CreateSchedule(context.Context, string, orbit.CreateScheduleRequest) (orbit.ScheduleResult, error)
+	GetSchedule(context.Context, string) (orbit.Schedule, error)
+	ResumeSchedule(context.Context, string, string, orbit.ControlScheduleRequest) (orbit.ScheduleResult, error)
+	PauseSchedule(context.Context, string, string, orbit.ControlScheduleRequest) (orbit.ScheduleResult, error)
+	ListScheduleOccurrences(context.Context, string, orbit.SchedulePageRequest) (orbit.ScheduleOccurrencePage, error)
+	GetTask(context.Context, string) (orbit.Task, error)
+	ListTaskAttempts(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error)
+	GetTaskSource(context.Context, string) (orbit.TaskSource, error)
+}
+
+func scheduleDefinition(cfg config, runID string, index int, start time.Time) orbit.ScheduleDefinition {
+	end := start.Add(time.Duration(cfg.Requests) * cfg.Interval)
+	payload, _ := json.Marshal(schedulePayload{RunID: runID, Schedule: index})
+	return orbit.ScheduleDefinition{
+		Rule:   orbit.ScheduleRule{Kind: "FIXED_RATE", Timezone: "UTC", StartAt: start, EndAt: &end, IntervalMS: uint64(cfg.Interval.Milliseconds())},
+		Policy: orbit.SchedulePolicy{Misfire: "CATCH_UP", Overlap: "PARALLEL", CatchupLimit: 100, CatchupWindowMS: uint64((cfg.RunTimeout + time.Duration(cfg.Requests)*cfg.Interval + cfg.StartDelay).Milliseconds()), GraceMS: 1000},
+		Task:   orbit.SubmitTaskRequest{QueueID: cfg.QueueID, Handler: handlerName(runID), HandlerVersion: "v1", PayloadVersion: "v1", BusinessKey: businessKey(runID, index), Payload: payload, TaskTimeoutMS: uint64(cfg.RunTimeout.Milliseconds()), RecoverySafety: orbit.RecoveryReconcileOnly, Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: uint64(cfg.RunTimeout.Milliseconds())}},
+	}
+}
+
+func measure(parent context.Context, cfg config, api *orbit.Client, runID string) (r scheduleReport) {
+	r = newReport(cfg, runID)
+	ctx, cancel := context.WithTimeout(parent, cfg.RunTimeout)
+	defer cancel()
+	var workers []*orbit.Worker
+	var stopWorkers context.CancelFunc
+	var done chan error
+	log := newInvocationLog(cfg.Requests * cfg.Schedules)
+	observations := newObservations(cfg, runID)
+	// 所有返回路径都执行有界收尾；超时后的暂停使用独立上下文，结果不明不换键重试。
+	defer func() {
+		if stopWorkers != nil {
+			stopWorkers()
+			for range workers {
+				if err := <-done; err != nil {
+					r.Errors["WORKER_STOP"]++
+				}
+			}
+		}
+		for _, w := range workers {
+			s := w.Stats()
+			r.WorkerStarted += s.Started
+			r.Reconnects += s.Reconnects
+			r.Renewals += s.RenewalsConfirmed
+		}
+		cleanup, stop := context.WithTimeout(context.Background(), 5*time.Second)
+		var wg sync.WaitGroup
+		codes := make(chan string, len(r.Plans))
+		for i := range r.Plans {
+			wg.Add(1)
+			go func(p *planReport) {
+				defer wg.Done()
+				if validID(p.ID) {
+					codes <- pauseSchedule(cleanup, cfg, api, runID, p)
+				}
+			}(&r.Plans[i])
+		}
+		wg.Wait()
+		stop()
+		close(codes)
+		for code := range codes {
+			if code != "" {
+				r.Errors[code]++
+			}
+		}
+		observations.summarize(&r)
+		log.summarize(&r, observations)
+		if ctx.Err() != nil {
+			r.Errors["RUN_"+classify(ctx.Err())]++
+		}
+		r = r.finish()
+	}()
+	call, stop := context.WithTimeout(ctx, cfg.Timeout)
+	queue, err := api.GetQueue(call, cfg.QueueID)
+	stop()
+	if err != nil {
+		r.Errors["PREFLIGHT_"+classify(err)]++
+		return
+	}
+	r.Queue = &queue
+	if queue.ID != cfg.QueueID || !validID(queue.NamespaceID) || queue.Paused || queue.MaxConcurrency == 0 || queue.RatePerSecond == 0 {
+		r.Errors["QUEUE_UNAVAILABLE"]++
+		return
+	}
+	r.ScheduleStart = time.Now().UTC().Truncate(time.Microsecond).Add(cfg.StartDelay)
+	r.ScheduleEnd = r.ScheduleStart.Add(time.Duration(cfg.Requests) * cfg.Interval)
+	// 在恢复任何计划之前完成全部创建；计划有限终止，即使进程被强杀也不会无限触发。
+	seen := map[string]bool{}
+	for i := 0; i < cfg.Schedules; i++ {
+		if ctx.Err() != nil {
+			return
+		}
+		definition := scheduleDefinition(cfg, runID, i, r.ScheduleStart)
+		key := operationKey(runID, i, "create")
+		p := planReport{Index: i, CreateKey: key}
+		r.Plans = append(r.Plans, p)
+		call, stop = context.WithTimeout(ctx, cfg.Timeout)
+		res, e := api.CreateSchedule(call, key, orbit.CreateScheduleRequest{Name: key, Paused: true, Definition: definition})
+		stop()
+		if e != nil {
+			r.Errors["CREATE_"+classify(e)]++
+			return
+		}
+		p.ID = res.Schedule.ID
+		p.CalendarVersion = res.Schedule.Current.Definition.Rule.CalendarVersion
+		p.EffectiveAt = res.Schedule.Current.EffectiveAt
+		r.Plans[i] = p
+		if res.Deduplicated || !validID(p.ID) || seen[p.ID] || !matchesSchedule(res.Schedule, cfg, runID, i, queue.NamespaceID, r.ScheduleStart) || !res.Schedule.Paused || res.Schedule.Current.EffectiveAt.After(r.ScheduleStart) {
+			r.Errors["CREATE_MISMATCH"]++
+			return
+		}
+		seen[p.ID] = true
+		observations.addPlan(p.ID, i, r.ScheduleStart, queue.NamespaceID)
+	}
+	for i := 0; i < cfg.Workers; i++ {
+		w, e := orbit.NewWorker(api, orbit.WorkerOptions{QueueID: cfg.QueueID, Capacity: uint32(cfg.Capacity), ShutdownTimeout: 5 * time.Second, Handlers: []orbit.WorkerHandler{log.handler(ctx, cfg, runID)}})
+		if e != nil {
+			r.Errors["WORKER_CONFIG"]++
+			return
+		}
+		workers = append(workers, w)
+	}
+	workerCtx, workerCancel := context.WithCancel(ctx)
+	stopWorkers = workerCancel
+	done = make(chan error, len(workers))
+	for _, w := range workers {
+		go func(w *orbit.Worker) {
+			err := w.Run(workerCtx)
+			if workerCtx.Err() == nil {
+				cancel()
+			}
+			done <- err
+		}(w)
+	}
+	for i := range r.Plans {
+		p := &r.Plans[i]
+		call, stop = context.WithTimeout(ctx, cfg.Timeout)
+		current, e := api.GetSchedule(call, p.ID)
+		stop()
+		if e != nil {
+			r.Errors["RESUME_READ_"+classify(e)]++
+			return
+		}
+		if !time.Now().Before(r.ScheduleStart) || current.ID != p.ID || !matchesSchedule(current, cfg, runID, i, queue.NamespaceID, r.ScheduleStart) || !current.Paused || !sameVersion(current, *p) {
+			r.Errors["SETUP_LATE_OR_CHANGED"]++
+			return
+		}
+		p.ResumeKey = operationKey(runID, i, "resume")
+		p.ResumeRevision = current.Revision
+		call, stop = context.WithTimeout(ctx, cfg.Timeout)
+		res, e := api.ResumeSchedule(call, p.ID, p.ResumeKey, orbit.ControlScheduleRequest{ExpectedRevision: current.Revision, Reason: "start bounded schedule verification"})
+		stop()
+		if e != nil {
+			r.Errors["RESUME_"+classify(e)]++
+			return
+		}
+		if res.Deduplicated || res.Schedule.ID != p.ID || res.Schedule.Paused || !matchesSchedule(res.Schedule, cfg, runID, i, queue.NamespaceID, r.ScheduleStart) || !res.Schedule.UpdatedAt.Before(r.ScheduleStart) || !sameVersion(res.Schedule, *p) {
+			r.Errors["RESUME_MISMATCH_OR_LATE"]++
+			return
+		}
+		p.Resumed = true
+	}
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+		all := true
+		for i := range r.Plans {
+			p := &r.Plans[i]
+			call, stop = context.WithTimeout(ctx, cfg.Timeout)
+			page, e := api.ListScheduleOccurrences(call, p.ID, orbit.SchedulePageRequest{Limit: 100})
+			stop()
+			if e != nil {
+				r.Errors["OCCURRENCES_"+classify(e)]++
+				return
+			}
+			if code := observations.ingest(p.ID, page); code != "" {
+				r.Errors[code]++
+				return
+			}
+			complete, code := observations.verify(ctx, api, p.ID)
+			if code != "" {
+				r.Errors[code]++
+				return
+			}
+			if complete {
+				call, stop = context.WithTimeout(ctx, cfg.Timeout)
+				current, e := api.GetSchedule(call, p.ID)
+				stop()
+				if e != nil {
+					r.Errors["FINAL_READ_"+classify(e)]++
+					return
+				}
+				if current.ID != p.ID || !matchesSchedule(current, cfg, runID, i, queue.NamespaceID, r.ScheduleStart) || current.Paused || !sameVersion(current, *p) {
+					r.Errors["FINAL_SCHEDULE_MISMATCH"]++
+					return
+				}
+				p.Exhausted = current.CheckAt == nil && current.NextFireAt == nil && current.ProcessingVersion == 1
+			}
+			all = all && complete && p.Exhausted
+		}
+		if all {
+			return
+		}
+		timer := time.NewTimer(cfg.PollInterval)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return
+		case <-timer.C:
+		}
+	}
+}
+
+func matchesSchedule(s orbit.Schedule, cfg config, runID string, index int, namespace string, start time.Time) bool {
+	want := scheduleDefinition(cfg, runID, index, start)
+	got := s.Current.Definition
+	// 服务端冻结具体时区快照标识；不把API版本v1误当作时区数据版本。
+	if !validCalendarVersion(got.Rule.CalendarVersion) {
+		return false
+	}
+	got.Rule.CalendarVersion = ""
+	return s.NamespaceID == namespace && s.QueueID == cfg.QueueID && s.Name == operationKey(runID, index, "create") && s.CurrentVersion == 1 && s.Current.Version == 1 && s.Revision > 0 && !s.Current.EffectiveAt.IsZero() && sameDefinition(got, want)
+}
+
+func sameDefinition(a, b orbit.ScheduleDefinition) bool {
+	// 原始JSON可有空白差异；模板和结果保持精确整数，不经float64。
+	pa, ok := decodePayload(a.Task.Payload)
+	pb, ok2 := decodePayload(b.Task.Payload)
+	a.Task.Payload = nil
+	b.Task.Payload = nil
+	return ok && ok2 && pa == pb && reflect.DeepEqual(a, b)
+}
+
+func pauseSchedule(ctx context.Context, cfg config, api scheduleAPI, runID string, p *planReport) string {
+	for attempt := 0; attempt < 3; attempt++ {
+		call, stop := context.WithTimeout(ctx, cfg.Timeout)
+		current, err := api.GetSchedule(call, p.ID)
+		stop()
+		if err != nil {
+			return "PAUSE_READ_" + classify(err)
+		}
+		if current.ID != p.ID || current.Name != operationKey(runID, p.Index, "create") || current.QueueID != cfg.QueueID {
+			return "PAUSE_IDENTITY_MISMATCH"
+		}
+		if current.Paused {
+			p.PauseConfirmed = true
+			return ""
+		}
+		p.PauseKey = operationKey(runID, p.Index, fmt.Sprintf("pause-%d", attempt))
+		p.PauseRevision = current.Revision
+		call, stop = context.WithTimeout(ctx, cfg.Timeout)
+		res, err := api.PauseSchedule(call, p.ID, p.PauseKey, orbit.ControlScheduleRequest{ExpectedRevision: p.PauseRevision, Reason: "stop bounded schedule verification"})
+		stop()
+		if err == nil {
+			if res.Schedule.ID != p.ID || !res.Schedule.Paused {
+				return "PAUSE_RECEIPT_MISMATCH"
+			}
+			p.PauseConfirmed = true
+			return ""
+		}
+		var apiErr *orbit.APIError
+		if !errors.As(err, &apiErr) || apiErr.Code != "STATE_CONFLICT" || apiErr.StatusCode != 409 {
+			return "PAUSE_" + classify(err)
+		}
+		timer := time.NewTimer(25 * time.Millisecond)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return "PAUSE_" + classify(ctx.Err())
+		case <-timer.C:
+		}
+	}
+	return "PAUSE_CONFLICT_LIMIT"
+}
+
+func validCalendarVersion(v string) bool {
+	if len(v) == 0 || len(v) > 64 {
+		return false
+	}
+	for _, c := range v {
+		if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '.' || c == '-' || c == '_') {
+			return false
+		}
+	}
+	return true
+}
+func sameVersion(s orbit.Schedule, p planReport) bool {
+	return s.Current.Definition.Rule.CalendarVersion == p.CalendarVersion && s.Current.EffectiveAt.Equal(p.EffectiveAt)
+}

sdk/go/cmd/orbit-schedule-load/measure_test.go

@@ -0,0 +1,313 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+)
+
+func testID(n int) string { return fmt.Sprintf("%032x", n) }
+func testConfig() config {
+	return config{QueueID: testID(1), Requests: 2, Schedules: 1, Workers: 1, Capacity: 2, StartDelay: 5 * time.Second, Interval: time.Second, RunTimeout: time.Minute, Timeout: time.Second, PollInterval: 100 * time.Millisecond}
+}
+func TestDecodePayloadStrict(t *testing.T) {
+	want := schedulePayload{RunID: "run", Schedule: 0}
+	for _, raw := range []string{`{"run_id":"run","schedule":0}`, ` {"schedule":0,"run_id":"run"} `} {
+		if got, ok := decodePayload(json.RawMessage(raw)); !ok || got != want {
+			t.Fatalf("valid payload rejected %s", raw)
+		}
+	}
+	for _, raw := range []string{`{}`, `null`, `[]`, `{"run_id":"run"}`, `{"schedule":0}`, `{"run_id":null,"schedule":0}`, `{"run_id":"run","schedule": null }`, `{"run_id":"run","schedule":0.0}`, `{"run_id":"run","schedule":0,"extra":1}`, `{"run_id":"run","Schedule":0}`, `{"run_id":"run","schedule":0,"schedule":0}`, `{"run_id":"run","schedule":0} {}`} {
+		if _, ok := decodePayload(json.RawMessage(raw)); ok {
+			t.Fatalf("invalid payload accepted %s", raw)
+		}
+	}
+}
+
+// memoryAPI 仅为观察/收尾提供持久快照替身；未使用方法通过嵌入接口留空。
+type memoryAPI struct {
+	scheduleAPI
+	task          orbit.Task
+	history       orbit.AttemptPage
+	source        orbit.TaskSource
+	current       orbit.Schedule
+	reads, pauses int
+	keys          []string
+	pauseError    error
+	conflictOnce  bool
+}
+
+func (m *memoryAPI) GetTask(context.Context, string) (orbit.Task, error) {
+	m.reads++
+	return m.task, nil
+}
+func (m *memoryAPI) ListTaskAttempts(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error) {
+	return m.history, nil
+}
+func (m *memoryAPI) GetTaskSource(context.Context, string) (orbit.TaskSource, error) {
+	return m.source, nil
+}
+func (m *memoryAPI) GetSchedule(context.Context, string) (orbit.Schedule, error) {
+	return m.current, nil
+}
+func (m *memoryAPI) PauseSchedule(_ context.Context, _ string, key string, in orbit.ControlScheduleRequest) (orbit.ScheduleResult, error) {
+	m.pauses++
+	m.keys = append(m.keys, key)
+	if m.pauseError != nil {
+		return orbit.ScheduleResult{}, m.pauseError
+	}
+	if m.conflictOnce && m.pauses == 1 {
+		m.current.Revision++
+		return orbit.ScheduleResult{}, &orbit.APIError{StatusCode: 409, Code: "STATE_CONFLICT"}
+	}
+	if in.ExpectedRevision != m.current.Revision {
+		return orbit.ScheduleResult{}, errors.New("stale revision")
+	}
+	m.current.Paused = true
+	return orbit.ScheduleResult{Schedule: m.current}, nil
+}
+func fixture() (*observations, *memoryAPI, orbit.ScheduleOccurrencePage) {
+	c := testConfig()
+	c.Requests = 1
+	start := time.Date(2026, 9, 20, 0, 0, 0, 0, time.UTC)
+	id, taskID := testID(2), testID(3)
+	o := newObservations(c, "run")
+	o.addPlan(id, 0, start, testID(4))
+	item := orbit.ScheduleOccurrence{ID: testID(5), ScheduleID: id, ScheduleVersion: 1, ScheduledAt: start, Decision: "FIRED", TaskID: &taskID, Reason: "SCHEDULED", CreatedAt: start.Add(time.Second)}
+	raw, _ := json.Marshal(schedulePayload{RunID: "run", Schedule: 0})
+	authorized, end := start.Add(2*time.Second), start.Add(3*time.Second)
+	task := orbit.Task{ID: taskID, NamespaceID: testID(4), QueueID: c.QueueID, State: orbit.TaskSucceeded, Handler: handlerName("run"), HandlerVersion: "v1", PayloadVersion: "v1", BusinessKey: businessKey("run", 0), Payload: raw, Result: raw, RecoverySafety: orbit.RecoveryReconcileOnly, Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: uint64(c.RunTimeout.Milliseconds())}, AttemptCount: 1, CreatedAt: item.CreatedAt, AvailableAt: item.CreatedAt, DeadlineAt: item.CreatedAt.Add(c.RunTimeout), CompletedAt: &end}
+	attempt := orbit.AttemptSummary{ID: testID(6), TaskID: taskID, Epoch: 1, State: orbit.AttemptSucceeded, AuthorizedAt: &authorized, EndedAt: &end, CreatedAt: item.CreatedAt}
+	source := orbit.TaskSource{TaskID: taskID, Kind: "SCHEDULE", Schedule: &orbit.TaskScheduleSource{ScheduleID: id, ScheduleVersion: 1, OccurrenceID: item.ID, ScheduledAt: start, Decision: "FIRED"}}
+	return o, &memoryAPI{task: task, history: orbit.AttemptPage{Items: []orbit.AttemptSummary{attempt}}, source: source}, orbit.ScheduleOccurrencePage{Items: []orbit.ScheduleOccurrence{item}}
+}
+func TestObserverWaitsAndCountsOnce(t *testing.T) {
+	o, api, page := fixture()
+	id := page.Items[0].ScheduleID
+	api.task.State = orbit.TaskRunning
+	if code := o.ingest(id, page); code != "" {
+		t.Fatal(code)
+	}
+	if complete, code := o.verify(context.Background(), api, id); complete || code != "" {
+		t.Fatalf("running counted as success %t %s", complete, code)
+	}
+	api.task.State = orbit.TaskSucceeded
+	for i := 0; i < 3; i++ {
+		if code := o.ingest(id, page); code != "" {
+			t.Fatal(code)
+		}
+		if complete, code := o.verify(context.Background(), api, id); !complete || code != "" {
+			t.Fatalf("verify %t %s", complete, code)
+		}
+	}
+	if api.reads != 2 {
+		t.Fatalf("verified task reread %d", api.reads)
+	}
+	r := newReport(o.cfg, "run")
+	r.Plans = []planReport{{ID: id, Resumed: true, Exhausted: true, PauseConfirmed: true}}
+	o.summarize(&r)
+	r = r.finish()
+	if r.Occurrences != 1 || r.Fired != 1 || r.VerifiedSuccess != 1 || r.Completion.Count != 1 || *r.Completion.Max != 3000 || *r.TriggerLag.Max != 1000 || *r.QueueWait.Max != 1000 || *r.Execution.Max != 1000 || r.Complete {
+		t.Fatalf("incorrect report %#v", r)
+	}
+	log := newInvocationLog(1)
+	_, err := log.handler(context.Background(), o.cfg, "run").Handle(context.Background(), orbit.WorkerExecution{Task: api.task, AttemptID: api.history.Items[0].ID, Epoch: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	log.summarize(&r, o)
+	r.WorkerStarted = 1
+	r = r.finish()
+	if !r.Complete {
+		t.Fatalf("full evidence not recognized: %#v", r)
+	}
+	r.Plans[0].PauseConfirmed = false
+	if r.finish().Complete {
+		t.Fatal("unconfirmed cleanup accepted")
+	}
+}
+func TestObserverRejectsMalformedSuccess(t *testing.T) {
+	tests := map[string]func(*memoryAPI){
+		"wrong queue":           func(m *memoryAPI) { m.task.QueueID = testID(90) },
+		"wrong namespace":       func(m *memoryAPI) { m.task.NamespaceID = testID(90) },
+		"missing result field":  func(m *memoryAPI) { m.task.Result = json.RawMessage(`{"run_id":"run"}`) },
+		"wrong input":           func(m *memoryAPI) { m.task.Payload = json.RawMessage(`{"run_id":"else","schedule":0}`) },
+		"uncertain":             func(m *memoryAPI) { m.task.EffectUncertain = true },
+		"retry policy":          func(m *memoryAPI) { m.task.Retry.MaxAttempts = 2 },
+		"extra attempt":         func(m *memoryAPI) { m.history.Items = append(m.history.Items, m.history.Items[0]) },
+		"attempt cursor":        func(m *memoryAPI) { m.history.NextCursor = "next" },
+		"wrong attempt owner":   func(m *memoryAPI) { m.history.Items[0].TaskID = testID(90) },
+		"missing authorization": func(m *memoryAPI) { m.history.Items[0].AuthorizedAt = nil },
+		"completion reversed":   func(m *memoryAPI) { v := m.task.CreatedAt.Add(-time.Second); m.history.Items[0].EndedAt = &v },
+		"source task":           func(m *memoryAPI) { m.source.TaskID = testID(90) },
+		"source occurrence":     func(m *memoryAPI) { m.source.Schedule.OccurrenceID = testID(90) },
+		"source version":        func(m *memoryAPI) { m.source.Schedule.ScheduleVersion = 2 },
+		"source time":           func(m *memoryAPI) { m.source.Schedule.ScheduledAt = m.source.Schedule.ScheduledAt.Add(time.Second) },
+		"source decision":       func(m *memoryAPI) { m.source.Schedule.Decision = "COALESCED" },
+	}
+	for name, change := range tests {
+		t.Run(name, func(t *testing.T) {
+			o, api, page := fixture()
+			id := page.Items[0].ScheduleID
+			if code := o.ingest(id, page); code != "" {
+				t.Fatal(code)
+			}
+			change(api)
+			if ok, code := o.verify(context.Background(), api, id); ok || code == "" {
+				t.Fatal("malformed success accepted")
+			}
+			if o.plans[id].slots[0].verified {
+				t.Fatal("invalid proof counted")
+			}
+		})
+	}
+}
+func TestOccurrenceUniquenessAndHistory(t *testing.T) {
+	for name, change := range map[string]func(*orbit.ScheduleOccurrencePage){
+		"wrong parent":  func(p *orbit.ScheduleOccurrencePage) { p.Items[0].ScheduleID = testID(90) },
+		"wrong version": func(p *orbit.ScheduleOccurrencePage) { p.Items[0].ScheduleVersion = 2 },
+		"off slot": func(p *orbit.ScheduleOccurrencePage) {
+			p.Items[0].ScheduledAt = p.Items[0].ScheduledAt.Add(time.Millisecond)
+		},
+		"skipped":   func(p *orbit.ScheduleOccurrencePage) { p.Items[0].Decision = "SKIPPED" },
+		"cursor":    func(p *orbit.ScheduleOccurrencePage) { p.NextCursor = "more" },
+		"duplicate": func(p *orbit.ScheduleOccurrencePage) { p.Items = append(p.Items, p.Items[0]) },
+	} {
+		t.Run(name, func(t *testing.T) {
+			o, _, page := fixture()
+			id := page.Items[0].ScheduleID
+			change(&page)
+			if o.ingest(id, page) == "" {
+				t.Fatal("invalid page accepted")
+			}
+		})
+	}
+	o, _, page := fixture()
+	id := page.Items[0].ScheduleID
+	if code := o.ingest(id, page); code != "" {
+		t.Fatal(code)
+	}
+	if o.ingest(id, orbit.ScheduleOccurrencePage{Items: []orbit.ScheduleOccurrence{}}) != "OCCURRENCE_DISAPPEARED" {
+		t.Fatal("lost history accepted")
+	}
+	changed := page
+	changed.Items = append([]orbit.ScheduleOccurrence(nil), page.Items...)
+	changed.Items[0].ID = testID(91)
+	if o.ingest(id, changed) != "OCCURRENCE_CHANGED" {
+		t.Fatal("replaced slot accepted")
+	}
+	second := testID(92)
+	o.addPlan(second, 1, o.plans[id].start, testID(4))
+	changed.Items[0].ScheduleID = second
+	if o.ingest(second, changed) != "DUPLICATE_TASK" {
+		t.Fatal("task shared by two slots accepted")
+	}
+}
+func TestHandlerDetectsDuplicateAndUnconfirmedCalls(t *testing.T) {
+	o, api, page := fixture()
+	id := page.Items[0].ScheduleID
+	if code := o.ingest(id, page); code != "" {
+		t.Fatal(code)
+	}
+	log := newInvocationLog(1)
+	handler := log.handler(context.Background(), o.cfg, "run")
+	var wg sync.WaitGroup
+	for i := 0; i < 4; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			_, _ = handler.Handle(context.Background(), orbit.WorkerExecution{Task: api.task, AttemptID: testID(6), Epoch: 1})
+		}()
+	}
+	wg.Wait()
+	r := newReport(o.cfg, "run")
+	log.summarize(&r, o)
+	if r.HandlerCalls != 4 || r.DuplicateInvocations != 3 || r.UnconfirmedInvocations != 4 {
+		t.Fatalf("calls lost: %#v", r)
+	}
+	if ok, code := o.verify(context.Background(), api, id); !ok || code != "" {
+		t.Fatal(code)
+	}
+	r = newReport(o.cfg, "run")
+	log.summarize(&r, o)
+	if r.UnconfirmedInvocations != 0 || r.DuplicateInvocations != 3 {
+		t.Fatal("duplicate hidden after verification")
+	}
+}
+func TestPauseOnlyRekeysExplicitRevisionConflict(t *testing.T) {
+	cfg := testConfig()
+	current := orbit.Schedule{ScheduleSummary: orbit.ScheduleSummary{ID: testID(2), QueueID: cfg.QueueID, Name: operationKey("run", 0, "create"), Revision: 7}}
+	m := &memoryAPI{current: current, conflictOnce: true}
+	p := planReport{ID: current.ID}
+	if code := pauseSchedule(context.Background(), cfg, m, "run", &p); code != "" {
+		t.Fatal(code)
+	}
+	if m.pauses != 2 || !p.PauseConfirmed || p.PauseRevision != 8 || !reflect.DeepEqual(m.keys, []string{operationKey("run", 0, "pause-0"), operationKey("run", 0, "pause-1")}) {
+		t.Fatalf("conflict recovery lost identity %#v %#v", p, m)
+	}
+	for _, err := range []error{context.DeadlineExceeded, errors.New("transport lost"), &orbit.APIError{StatusCode: 409, Code: "IDEMPOTENCY_CONFLICT"}} {
+		m = &memoryAPI{current: current, pauseError: err}
+		p = planReport{ID: current.ID}
+		if code := pauseSchedule(context.Background(), cfg, m, "run", &p); code == "" || m.pauses != 1 || p.PauseConfirmed || p.PauseKey == "" || p.PauseRevision != 7 {
+			t.Fatalf("unknown outcome reissued: %#v %#v", p, m)
+		}
+	}
+}
+func TestScheduleDefinitionFrozenBoundedAndCompared(t *testing.T) {
+	cfg := testConfig()
+	start := time.Date(2026, 9, 20, 0, 0, 0, 0, time.UTC)
+	def := scheduleDefinition(cfg, "run", 0, start)
+	if def.Rule.EndAt == nil || !def.Rule.EndAt.Equal(start.Add(2*time.Second)) || def.Task.Retry.MaxAttempts != 1 || def.Policy.Overlap != "PARALLEL" {
+		t.Fatal("schedule is not bounded as configured")
+	}
+	s := orbit.Schedule{ScheduleSummary: orbit.ScheduleSummary{NamespaceID: testID(4), QueueID: cfg.QueueID, Name: operationKey("run", 0, "create"), Revision: 1, CurrentVersion: 1}, Current: orbit.ScheduleVersion{Version: 1, EffectiveAt: start.Add(-time.Second), Definition: def}}
+	s.Current.Definition.Rule.CalendarVersion = "go1.27.1-b2d18a7c8fa8"
+	if !matchesSchedule(s, cfg, "run", 0, testID(4), start) {
+		t.Fatal("normalized frozen schedule rejected")
+	}
+	s.Current.Definition.Task.Payload = json.RawMessage(`{"run_id":"run","schedule":1}`)
+	if matchesSchedule(s, cfg, "run", 0, testID(4), start) {
+		t.Fatal("changed template accepted")
+	}
+}
+func TestPartialReportPreservesPlannedDenominator(t *testing.T) {
+	c := testConfig()
+	c.Schedules = 2
+	r := newReport(c, "run")
+	r.Occurrences = 3
+	r.Fired = 2
+	r.VerifiedSuccess = 1
+	r = r.finish()
+	if r.Expected != 4 || r.Unobserved != 1 || r.Unverified != 1 || r.Complete {
+		t.Fatalf("bad denominators %#v", r)
+	}
+	if d := distributionOf(nil); d.P50 != nil || d.P99 != nil {
+		t.Fatal("empty percentile is not null")
+	}
+}
+
+func TestCalendarSnapshotRemainsFixed(t *testing.T) {
+	c := testConfig()
+	start := time.Date(2026, 9, 20, 0, 0, 0, 0, time.UTC)
+	p := planReport{CalendarVersion: "go1.27.1-b2d18a7c8fa8", EffectiveAt: start}
+	s := orbit.Schedule{Current: orbit.ScheduleVersion{Definition: scheduleDefinition(c, "run", 0, start), EffectiveAt: start}}
+	s.Current.Definition.Rule.CalendarVersion = p.CalendarVersion
+	if !sameVersion(s, p) {
+		t.Fatal("unchanged snapshot rejected")
+	}
+	s.Current.Definition.Rule.CalendarVersion = "replacement"
+	if sameVersion(s, p) {
+		t.Fatal("changed timezone snapshot accepted")
+	}
+	s.Current.Definition.Rule.CalendarVersion = p.CalendarVersion
+	s.Current.EffectiveAt = start.Add(time.Microsecond)
+	if sameVersion(s, p) {
+		t.Fatal("changed effective time accepted")
+	}
+}

sdk/go/cmd/orbit-schedule-load/observe.go

@@ -0,0 +1,206 @@
+package main
+
+import (
+	"context"
+	orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
+	"reflect"
+	"time"
+)
+
+// slotObservation 只缓存一个固定时间槽的不可变触发和核验证据。
+type slotObservation struct {
+	occurrence                            *orbit.ScheduleOccurrence // occurrence 首次读到的快照；不得改写。
+	verified                              bool                      // verified 是否已完成全部持久化核验。
+	attemptID                             string                    // attemptID 唯一成功尝试，用于与本地调用关联。
+	trigger, queue, execution, completion time.Duration             // 各阶段只使用数据库时间。
+}
+
+// planObservation 预分配固定槽位，内存上限由配置限制。
+type planObservation struct {
+	index     int               // index 计划在本次配置中的序号。
+	start     time.Time         // start 固定触发基准。
+	namespace string            // namespace 预检认证队列命名空间。
+	slots     []slotObservation // slots 预期触发槽位，不随服务端输入增长。
+}
+
+// observations 合并轮询快照，跨计划检查唯一触发和任务身份。
+type observations struct {
+	cfg         config                      // cfg 固定样本配置。
+	runID       string                      // runID 固定测量身份。
+	plans       map[string]*planObservation // plans 创建时登记的有限计划集合。
+	occurrences map[string]string           // occurrences 触发ID到计划ID，避免跨计划重复。
+	tasks       map[string]*slotObservation // tasks 已观察任务到唯一槽位。
+}
+
+func newObservations(cfg config, runID string) *observations {
+	return &observations{cfg: cfg, runID: runID, plans: map[string]*planObservation{}, occurrences: map[string]string{}, tasks: map[string]*slotObservation{}}
+}
+func (o *observations) addPlan(id string, index int, start time.Time, namespace string) {
+	o.plans[id] = &planObservation{index: index, start: start, namespace: namespace, slots: make([]slotObservation, o.cfg.Requests)}
+}
+func (o *observations) ingest(id string, page orbit.ScheduleOccurrencePage) string {
+	p := o.plans[id]
+	if p == nil || page.Items == nil || page.NextCursor != "" || len(page.Items) > len(p.slots) {
+		return "OCCURRENCE_PAGE_MISMATCH"
+	}
+	seen := map[string]bool{}
+	for _, item := range page.Items {
+		delta := item.ScheduledAt.Sub(p.start)
+		if !validID(item.ID) || item.ScheduleID != id || item.ScheduleVersion != 1 || delta < 0 || delta%o.cfg.Interval != 0 || delta/o.cfg.Interval >= time.Duration(len(p.slots)) || item.CreatedAt.Before(item.ScheduledAt) {
+			return "OCCURRENCE_MISMATCH"
+		}
+		if seen[item.ID] {
+			return "DUPLICATE_OCCURRENCE"
+		}
+		seen[item.ID] = true
+		slot := &p.slots[int(delta/o.cfg.Interval)]
+		if slot.occurrence != nil {
+			if !reflect.DeepEqual(*slot.occurrence, item) {
+				return "OCCURRENCE_CHANGED"
+			}
+			continue
+		}
+		if _, exists := o.occurrences[item.ID]; exists {
+			return "DUPLICATE_OCCURRENCE"
+		}
+		// 保存已观察证据再判定策略异常；未验证的触发不会充当成功分母。
+		snapshot := item
+		slot.occurrence = &snapshot
+		o.occurrences[item.ID] = id
+		if item.Decision != "FIRED" || item.CoveredUntil != nil || item.Reason != "SCHEDULED" {
+			return "OCCURRENCE_DECISION_MISMATCH"
+		}
+		if item.TaskID == nil || !validID(*item.TaskID) {
+			return "OCCURRENCE_TASK_MISMATCH"
+		}
+		if _, exists := o.tasks[*item.TaskID]; exists {
+			return "DUPLICATE_TASK"
+		}
+		o.tasks[*item.TaskID] = slot
+	}
+	// 此样本总量不超过单页上限，历史不可消失；分页异常不降级继续。
+	known := 0
+	for _, s := range p.slots {
+		if s.occurrence != nil {
+			known++
+		}
+	}
+	if len(page.Items) != known {
+		return "OCCURRENCE_DISAPPEARED"
+	}
+	return ""
+}
+func (o *observations) verify(ctx context.Context, api scheduleAPI, id string) (bool, string) {
+	p := o.plans[id]
+	complete := true
+	for i := range p.slots {
+		slot := &p.slots[i]
+		if slot.occurrence == nil {
+			complete = false
+			continue
+		}
+		if slot.verified {
+			continue
+		}
+		if ctx.Err() != nil {
+			return false, "VERIFY_" + classify(ctx.Err())
+		}
+		item := slot.occurrence
+		if item.TaskID == nil || o.tasks[*item.TaskID] != slot {
+			return false, "OCCURRENCE_TASK_MISMATCH"
+		}
+		call, stop := context.WithTimeout(ctx, o.cfg.Timeout)
+		task, err := api.GetTask(call, *item.TaskID)
+		stop()
+		if err != nil {
+			return false, "TASK_" + classify(err)
+		}
+		if task.ID != *item.TaskID || task.NamespaceID != p.namespace || !matchesTask(task, o.cfg, o.runID, p.index) || !task.CreatedAt.Equal(item.CreatedAt) || !task.AvailableAt.Equal(task.CreatedAt) || !task.DeadlineAt.Equal(task.CreatedAt.Add(time.Duration(o.cfg.RunTimeout.Milliseconds())*time.Millisecond)) {
+			return false, "TASK_MISMATCH"
+		}
+		switch task.State {
+		case orbit.TaskWaiting, orbit.TaskLeased, orbit.TaskRunning:
+			complete = false
+			continue
+		case orbit.TaskFailed, orbit.TaskCancelled, orbit.TaskNeedsReview:
+			return false, "TASK_" + string(task.State)
+		case orbit.TaskSucceeded:
+		default:
+			return false, "UNKNOWN_TASK_STATE"
+		}
+		output, valid := decodePayload(task.Result)
+		if !valid || output != (schedulePayload{RunID: o.runID, Schedule: p.index}) || task.CompletedAt == nil || task.EffectUncertain || task.AttemptCount != 1 || task.CancelRequestedAt != nil {
+			return false, "COMPLETION_MISMATCH"
+		}
+		call, stop = context.WithTimeout(ctx, o.cfg.Timeout)
+		history, err := api.ListTaskAttempts(call, task.ID, orbit.HistoryPageRequest{Limit: 2})
+		stop()
+		if err != nil {
+			return false, "HISTORY_" + classify(err)
+		}
+		if len(history.Items) != 1 || history.NextCursor != "" {
+			return false, "ATTEMPT_COUNT_MISMATCH"
+		}
+		a := history.Items[0]
+		if !validID(a.ID) || a.TaskID != task.ID || a.Epoch != 1 || a.State != orbit.AttemptSucceeded || a.AuthorizedAt == nil || a.EndedAt == nil || a.ErrorCode != "" || a.Retryable || a.ConcurrencyToken != 0 {
+			return false, "ATTEMPT_MISMATCH"
+		}
+		if a.CreatedAt.Before(task.AvailableAt) || a.AuthorizedAt.Before(a.CreatedAt) || a.EndedAt.Before(*a.AuthorizedAt) || !a.EndedAt.Equal(*task.CompletedAt) {
+			return false, "TIMESTAMP_MISMATCH"
+		}
+		call, stop = context.WithTimeout(ctx, o.cfg.Timeout)
+		source, err := api.GetTaskSource(call, task.ID)
+		stop()
+		if err != nil {
+			return false, "SOURCE_" + classify(err)
+		}
+		s := source.Schedule
+		if source.TaskID != task.ID || source.Kind != "SCHEDULE" || source.Workflow != nil || source.RetryOf != nil || s == nil || s.ScheduleID != id || s.ScheduleVersion != 1 || s.OccurrenceID != item.ID || !s.ScheduledAt.Equal(item.ScheduledAt) || s.Decision != "FIRED" || s.CoveredUntil != nil {
+			return false, "SOURCE_MISMATCH"
+		}
+		slot.verified = true
+		slot.attemptID = a.ID
+		slot.trigger = task.CreatedAt.Sub(item.ScheduledAt)
+		slot.queue = a.AuthorizedAt.Sub(task.CreatedAt)
+		slot.execution = a.EndedAt.Sub(*a.AuthorizedAt)
+		slot.completion = task.CompletedAt.Sub(item.ScheduledAt)
+	}
+	return complete, ""
+}
+func matchesTask(t orbit.Task, c config, runID string, index int) bool {
+	payload, ok := decodePayload(t.Payload)
+	return ok && payload == (schedulePayload{RunID: runID, Schedule: index}) && validID(t.ID) && t.QueueID == c.QueueID && t.Handler == handlerName(runID) && t.HandlerVersion == "v1" && t.PayloadVersion == "v1" && t.BusinessKey == businessKey(runID, index) && t.ConcurrencyKey == "" && t.Priority == 0 && t.RetryOf == nil && t.StartDeadlineAt == nil && t.Retry == (orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: uint64(c.RunTimeout.Milliseconds())}) && t.RecoverySafety == orbit.RecoveryReconcileOnly
+}
+func (o *observations) summarize(r *scheduleReport) {
+	var trigger, queue, execution, completion []time.Duration
+	for i := range r.Plans {
+		p := &r.Plans[i]
+		data := o.plans[p.ID]
+		if data == nil {
+			continue
+		}
+		for j := range data.slots {
+			s := &data.slots[j]
+			if s.occurrence != nil {
+				p.Occurrences++
+			}
+			if s.occurrence != nil && s.occurrence.TaskID != nil && o.tasks[*s.occurrence.TaskID] == s {
+				p.Fired++
+			}
+			if s.verified {
+				p.VerifiedSuccess++
+				trigger = append(trigger, s.trigger)
+				queue = append(queue, s.queue)
+				execution = append(execution, s.execution)
+				completion = append(completion, s.completion)
+			}
+		}
+		r.Occurrences += p.Occurrences
+		r.Fired += p.Fired
+		r.VerifiedSuccess += p.VerifiedSuccess
+	}
+	r.TriggerLag = distributionOf(trigger)
+	r.QueueWait = distributionOf(queue)
+	r.Execution = distributionOf(execution)
+	r.Completion = distributionOf(completion)
+}

sdk/go/cmd/orbit-schedule-load/report.go

@@ -0,0 +1,111 @@
+package main
+
+import (
+	orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
+	"math"
+	"runtime"
+	"sort"
+	"time"
+)
+
+// distribution 仅统计已逐项核验的任务；空集合百分位为null，单位毫秒。
+type distribution struct {
+	Count int      `json:"count"`  // Count 统计分母。
+	P50   *float64 `json:"p50_ms"` // P50 最近秩中位数。
+	P95   *float64 `json:"p95_ms"` // P95 最近秩95百分位。
+	P99   *float64 `json:"p99_ms"` // P99 最近秩99百分位。
+	Max   *float64 `json:"max_ms"` // Max 最大耗时。
+}
+
+// planReport 保存原操作身份；请求结果不明时只能按原键和原请求核查。
+type planReport struct {
+	CalendarVersion string    `json:"calendar_version,omitempty"` // CalendarVersion 创建回执中的时区快照标识，后续读取不得变化。
+	EffectiveAt     time.Time `json:"effective_at"`               // EffectiveAt 固定计划版本生效时间。
+	Index           int       `json:"index"`                      // Index 从零开始的计划序号。
+	ID              string    `json:"id,omitempty"`               // ID 已取得的计划身份；空不证明创建未提交。
+	CreateKey       string    `json:"create_key"`                 // CreateKey 创建键，同时作为计划名称。
+	ResumeKey       string    `json:"resume_key,omitempty"`       // ResumeKey 原恢复键。
+	ResumeRevision  uint64    `json:"resume_revision,string"`     // ResumeRevision 原恢复请求版本，零表示尚未发送。
+	Resumed         bool      `json:"resumed"`                    // Resumed 已收到且核对恢复回执。
+	Exhausted       bool      `json:"exhausted"`                  // Exhausted 已观察到固定版本无下个触发点和扫描工作。
+	PauseKey        string    `json:"pause_key,omitempty"`        // PauseKey 最后发送的暂停键，仅明确版本冲突才换键。
+	PauseRevision   uint64    `json:"pause_revision,string"`      // PauseRevision 最后暂停请求的预期版本。
+	PauseConfirmed  bool      `json:"pause_confirmed"`            // PauseConfirmed 已确认计划暂停；不取消已生成任务。
+	Occurrences     int       `json:"occurrences"`                // Occurrences 已观察唯一触发点数。
+	Fired           int       `json:"fired"`                      // Fired 已关联唯一任务的单点触发数。
+	VerifiedSuccess int       `json:"verified_success"`           // VerifiedSuccess 完成任务/尝试/来源核对数。
+}
+
+// scheduleReport 是隔离环境有界样本，不作为生产容量或高可用承诺。
+type scheduleReport struct {
+	RunID                  string         `json:"run_id"`                    // RunID 计划、处理器和操作键的隔离身份。
+	GoVersion              string         `json:"go_version"`                // GoVersion 实际编译工具链。
+	GOOS                   string         `json:"goos"`                      // GOOS 运行系统。
+	GOARCH                 string         `json:"goarch"`                    // GOARCH 运行架构。
+	StartedAt              time.Time      `json:"started_at"`                // StartedAt 预检开始本机时间。
+	FinishedAt             time.Time      `json:"finished_at"`               // FinishedAt 全部有界收尾完成本机时间。
+	Requests               int            `json:"requests_per_schedule"`     // Requests 每计划预期触发数。
+	Schedules              int            `json:"schedules"`                 // Schedules 预期计划数。
+	Expected               int            `json:"expected_occurrences"`      // Expected 总计划触发数，包含未观察的触发点。
+	Workers                int            `json:"workers"`                   // Workers 本进程独立WSS Worker数。
+	Capacity               int            `json:"capacity_per_worker"`       // Capacity 每Worker并行槽数。
+	IntervalMS             int64          `json:"interval_ms"`               // IntervalMS 固定间隔毫秒。
+	StartDelayMS           int64          `json:"start_delay_ms"`            // StartDelayMS 创建/恢复的预留时间毫秒。
+	RunTimeoutMS           int64          `json:"run_timeout_ms"`            // RunTimeoutMS 整个测量截止毫秒。
+	PollIntervalMS         int64          `json:"poll_interval_ms"`          // PollIntervalMS 历史查询间隔毫秒。
+	HTTPTimeoutMS          int64          `json:"http_timeout_ms"`           // HTTPTimeoutMS 每HTTP请求截止毫秒。
+	Queue                  *orbit.Queue   `json:"queue,omitempty"`           // Queue 预检队列快照，只读配置。
+	ScheduleStart          time.Time      `json:"schedule_start"`            // ScheduleStart 所有计划共同起点，包含边界。
+	ScheduleEnd            time.Time      `json:"schedule_end"`              // ScheduleEnd 所有计划共同截止，排除边界。
+	Plans                  []planReport   `json:"plans"`                     // Plans 已尝试创建的固定身份和收尾证据。
+	Occurrences            int            `json:"occurrences"`               // Occurrences 已观察唯一触发数。
+	Fired                  int            `json:"fired"`                     // Fired 单点生成且关联唯一任务数，不代表成功。
+	VerifiedSuccess        int            `json:"verified_success"`          // VerifiedSuccess 持久化结果/唯一尝试/来源全部通过数。
+	Unobserved             int            `json:"unobserved"`                // Unobserved 预期但未观察到的触发点数。
+	Unverified             int            `json:"unverified"`                // Unverified 已触发但未核验成功的任务数。
+	HandlerCalls           int            `json:"handler_calls"`             // HandlerCalls 通过输入核对的处理器调用总数。
+	DuplicateInvocations   int            `json:"duplicate_invocations"`     // DuplicateInvocations 同一任务的额外调用数。
+	UnconfirmedInvocations int            `json:"unconfirmed_invocations"`   // UnconfirmedInvocations 未对应到已核验任务及尝试的调用数。
+	WorkerStarted          uint64         `json:"worker_started"`            // WorkerStarted SDK实际调用处理器数。
+	Reconnects             uint64         `json:"worker_reconnects"`         // Reconnects SDK连接重建次数。
+	Renewals               uint64         `json:"worker_renewals_confirmed"` // Renewals 已确认租约续期次数。
+	TriggerLag             distribution   `json:"trigger_lag"`               // TriggerLag 计划时间至任务受理数据库耗时。
+	QueueWait              distribution   `json:"queue_wait"`                // QueueWait 任务受理至开始授权数据库耗时。
+	Execution              distribution   `json:"authorized_execution"`      // Execution 授权至结果落库耗时，包含网络和事务。
+	Completion             distribution   `json:"completion"`                // Completion 计划时间至结果落库耗时。
+	Errors                 map[string]int `json:"errors"`                    // Errors 有限分类，不含凭据或原异常正文。
+	Complete               bool           `json:"complete"`                  // Complete 全部计数、证据和收尾满足时为真。
+}
+
+func newReport(c config, runID string) scheduleReport {
+	return scheduleReport{RunID: runID, GoVersion: runtime.Version(), GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, StartedAt: time.Now().UTC(), Requests: c.Requests, Schedules: c.Schedules, Expected: c.Requests * c.Schedules, Workers: c.Workers, Capacity: c.Capacity, IntervalMS: c.Interval.Milliseconds(), StartDelayMS: c.StartDelay.Milliseconds(), RunTimeoutMS: c.RunTimeout.Milliseconds(), PollIntervalMS: c.PollInterval.Milliseconds(), HTTPTimeoutMS: c.Timeout.Milliseconds(), Plans: []planReport{}, Errors: map[string]int{}}
+}
+func (r scheduleReport) finish() scheduleReport {
+	for code, n := range r.Errors {
+		if n == 0 {
+			delete(r.Errors, code)
+		}
+	}
+	r.FinishedAt = time.Now().UTC()
+	r.Unobserved = r.Expected - r.Occurrences
+	r.Unverified = r.Fired - r.VerifiedSuccess
+	plansReady := len(r.Plans) == r.Schedules
+	for _, p := range r.Plans {
+		plansReady = plansReady && p.Resumed && p.Exhausted && p.PauseConfirmed && p.VerifiedSuccess == r.Requests
+	}
+	r.Complete = r.Expected > 0 && plansReady && r.Occurrences == r.Expected && r.Fired == r.Expected && r.VerifiedSuccess == r.Expected && r.HandlerCalls == r.Expected && r.WorkerStarted == uint64(r.Expected) && r.DuplicateInvocations == 0 && r.UnconfirmedInvocations == 0 && len(r.Errors) == 0
+	return r
+}
+func distributionOf(values []time.Duration) distribution {
+	d := distribution{Count: len(values)}
+	if len(values) == 0 {
+		return d
+	}
+	sort.Slice(values, func(i, j int) bool { return values[i] < values[j] })
+	percentile := func(q float64) *float64 {
+		v := float64(values[int(math.Ceil(float64(len(values))*q))-1]) / float64(time.Millisecond)
+		return &v
+	}
+	d.P50, d.P95, d.P99, d.Max = percentile(.5), percentile(.95), percentile(.99), percentile(1)
+	return d
+}