feat: measure bounded worker execution with durable verification

This commit is contained in:
2026-09-20 02:28:53 +08:00
parent 3137bce8d5
commit 1dfc7e4234
19 changed files with 1132 additions and 2 deletions

View File

@@ -0,0 +1,139 @@
// orbit-exec-load 在隔离环境测量真实 HTTP 受理、WSS Worker 和持久化完成链路。
// 处理器只有有界等待及回显,不访问外部业务系统。
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net/http"
"os"
"os/signal"
"strings"
"syscall"
"time"
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
)
type config struct {
URL, Token, QueueID string
Requests, Concurrency, Workers, Capacity int
Timeout, RunTimeout, PollInterval time.Duration
Durations []time.Duration
}
func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run() error {
cfg, err := parseConfig(os.Args[1:])
if err != nil {
return err
}
client, err := orbit.NewClient(cfg.URL, orbit.Options{Token: cfg.Token, HTTPClient: &http.Client{Timeout: cfg.Timeout}})
if err != nil {
return errors.New("invalid client configuration")
}
runID, err := orbit.NewWorkerMessageID()
if err != nil {
return errors.New("cannot generate run identity")
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
report := measure(ctx, cfg, client, runID)
if err := json.NewEncoder(os.Stdout).Encode(report); err != nil {
return err
}
if !report.Complete {
return errors.New("execution sample incomplete; inspect report and original run identities")
}
return nil
}
func parseConfig(args []string) (config, error) {
cfg := config{URL: os.Getenv("ORBIT_URL"), Token: os.Getenv("ORBIT_TOKEN"), QueueID: os.Getenv("ORBIT_QUEUE_ID"), Requests: 100, Concurrency: 8, Workers: 2, Capacity: 4, Timeout: 5 * time.Second, RunTimeout: 2 * time.Minute, PollInterval: 250 * time.Millisecond}
flags := flag.NewFlagSet("orbit-exec-load", flag.ContinueOnError)
var credentials, durations string
flags.StringVar(&cfg.URL, "url", cfg.URL, "Orbit HTTP(S) URL")
flags.StringVar(&credentials, "credentials", os.Getenv("ORBIT_CREDENTIALS_FILE"), "private credentials file for an isolated namespace")
flags.StringVar(&cfg.QueueID, "queue", cfg.QueueID, "existing isolated queue; configuration is not changed")
flags.IntVar(&cfg.Requests, "requests", cfg.Requests, "tasks, 1..10000")
flags.IntVar(&cfg.Concurrency, "concurrency", cfg.Concurrency, "maximum concurrent submission/verification requests, 1..128")
flags.IntVar(&cfg.Workers, "workers", cfg.Workers, "independent WSS Workers in this process, 1..32")
flags.IntVar(&cfg.Capacity, "capacity", cfg.Capacity, "concurrent handlers per Worker, 1..256; total at most 1024")
flags.DurationVar(&cfg.Timeout, "timeout", cfg.Timeout, "per HTTP request timeout, 1ms..1m")
flags.DurationVar(&cfg.RunTimeout, "run-timeout", cfg.RunTimeout, "whole sample deadline, 1s..1h, plus at most 5s Worker shutdown")
flags.DurationVar(&cfg.PollInterval, "poll-interval", cfg.PollInterval, "per active verifier polling interval, 100ms..10s")
flags.StringVar(&durations, "durations", "0ms", "1..16 round-robin simulated handler durations, e.g. 0ms,100ms,2s; whole milliseconds, 0..60s")
if err := flags.Parse(args); err != nil {
return config{}, err
}
if flags.NArg() != 0 {
return config{}, errors.New("positional arguments are not supported")
}
if credentials != "" {
f, err := os.Open(credentials)
if err != nil {
return config{}, errors.New("cannot open private credentials file")
}
raw, err := io.ReadAll(io.LimitReader(f, (16<<10)+1))
f.Close()
var saved struct {
Token string `json:"token"`
QueueID string `json:"queue_id"`
}
if err != nil || len(raw) > 16<<10 || json.Unmarshal(raw, &saved) != nil {
return config{}, errors.New("invalid credentials file (maximum 16 KiB)")
}
if cfg.Token == "" {
cfg.Token = saved.Token
}
if cfg.QueueID == "" {
cfg.QueueID = saved.QueueID
}
}
if cfg.URL == "" || cfg.Token == "" || !validID(cfg.QueueID) {
return config{}, errors.New("URL, token and a 32-character lowercase hexadecimal queue ID are required")
}
if cfg.Requests < 1 || cfg.Requests > 10000 || cfg.Concurrency < 1 || cfg.Concurrency > 128 || cfg.Workers < 1 || cfg.Workers > 32 || cfg.Capacity < 1 || cfg.Capacity > 256 || cfg.Workers*cfg.Capacity > 1024 {
return config{}, errors.New("request, concurrency or Worker count is out of bounds")
}
if cfg.Timeout < time.Millisecond || cfg.Timeout > time.Minute || cfg.RunTimeout < time.Second || cfg.RunTimeout > time.Hour || cfg.PollInterval < 100*time.Millisecond || cfg.PollInterval > 10*time.Second {
return config{}, errors.New("timeout or poll interval is out of bounds")
}
parts := strings.Split(durations, ",")
if len(parts) > 16 {
return config{}, errors.New("at most 16 handler durations are allowed")
}
for _, part := range parts {
d, err := time.ParseDuration(strings.TrimSpace(part))
if err != nil || d < 0 || d > time.Minute || d%time.Millisecond != 0 || d+time.Second > cfg.RunTimeout {
return config{}, errors.New("handler duration must be whole milliseconds in 0..60s and allow at least 1s within run-timeout")
}
cfg.Durations = append(cfg.Durations, d)
}
if cfg.Concurrency > cfg.Requests {
cfg.Concurrency = cfg.Requests
}
return cfg, nil
}
func validID(id string) bool {
if len(id) != 32 {
return false
}
for _, c := range id {
if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') {
return false
}
}
return true
}

View File

@@ -0,0 +1,367 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"sync"
"time"
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
)
// loadPayload 的固定内容同时是处理器的输入和预期结果;不含凭据或外部地址。
type loadPayload struct {
RunID string `json:"run_id"` // RunID 隔离本次运行,不作为业务幂等保证。
Sequence int `json:"sequence"` // Sequence 从0开始的请求序号,用于发现重复执行及错配。
DurationMS int64 `json:"duration_ms"` // DurationMS 模拟等待毫秒数,按本次固定时长数组取模。
}
func decodePayload(raw json.RawMessage) (loadPayload, bool) {
var wire struct {
RunID *string `json:"run_id"`
Sequence *int `json:"sequence"`
DurationMS *int64 `json:"duration_ms"`
}
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.DisallowUnknownFields()
if decoder.Decode(&wire) != nil || wire.RunID == nil || wire.Sequence == nil || wire.DurationMS == nil {
return loadPayload{}, false
}
var trailing any
if decoder.Decode(&trailing) != io.EOF {
return loadPayload{}, false
}
return loadPayload{*wire.RunID, *wire.Sequence, *wire.DurationMS}, true
}
type admission struct {
sequence int
task orbit.Task
duration time.Duration
code string
}
type taskAPI interface {
SubmitTask(context.Context, string, orbit.SubmitTaskRequest) (orbit.SubmitTaskResult, error)
GetTask(context.Context, string) (orbit.Task, error)
ListTaskAttempts(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error)
}
type measurement struct {
queue, execution, completed time.Duration
}
func expected(cfg config, runID string, sequence int) loadPayload {
return loadPayload{runID, sequence, cfg.Durations[sequence%len(cfg.Durations)].Milliseconds()}
}
func handlerName(runID string) string { return "load.exec." + runID }
func businessKey(runID string) string { return "orbit-exec-" + runID }
func requestKey(runID string, sequence int) string {
return fmt.Sprintf("orbit-exec-%s-%06d", runID, sequence)
}
// invocationLog 有界保存本次每个序号的实际处理器调用,发现重复时不覆盖首条证据。
type invocationLog struct {
mu sync.Mutex
calls []int
taskIDs []string
invalid int
}
func newInvocationLog(n int) *invocationLog {
return &invocationLog{calls: make([]int, n), taskIDs: make([]string, n)}
}
func (log *invocationLog) handler(runCtx context.Context, cfg config, runID string) orbit.WorkerHandler {
return orbit.WorkerHandler{Capability: orbit.WorkerCapability{Handler: handlerName(runID), HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(ctx context.Context, execution orbit.WorkerExecution) (json.RawMessage, error) {
payload, valid := decodePayload(execution.Task.Payload)
if !valid || payload.RunID != runID || payload.Sequence < 0 || payload.Sequence >= cfg.Requests || payload != expected(cfg, runID, payload.Sequence) || execution.Task.QueueID != cfg.QueueID || execution.Task.BusinessKey != businessKey(runID) || execution.Task.Handler != handlerName(runID) {
log.mu.Lock()
log.invalid++
log.mu.Unlock()
return nil, &orbit.WorkerFailure{Code: "LOAD_INPUT_MISMATCH"}
}
log.mu.Lock()
log.calls[payload.Sequence]++
first := log.calls[payload.Sequence] == 1
if first {
log.taskIDs[payload.Sequence] = execution.Task.ID
}
log.mu.Unlock()
if !first {
return nil, &orbit.WorkerFailure{Code: "LOAD_DUPLICATE_EXECUTION"}
}
timer := time.NewTimer(time.Duration(payload.DurationMS) * time.Millisecond)
defer timer.Stop()
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-runCtx.Done():
return nil, runCtx.Err()
case <-timer.C:
return json.Marshal(payload)
}
}}
}
func measure(parent context.Context, cfg config, client *orbit.Client, runID string) executionReport {
r := newReport(cfg, runID)
ctx, cancel := context.WithTimeout(parent, cfg.RunTimeout)
defer cancel()
requestCtx, stop := context.WithTimeout(ctx, cfg.Timeout)
queue, err := client.GetQueue(requestCtx, cfg.QueueID)
stop()
if err != nil {
r.Errors["PREFLIGHT_"+classify(err)]++
return r.finish()
}
r.Queue = &queue
if queue.ID != cfg.QueueID || queue.Paused || queue.MaxConcurrency == 0 || queue.RatePerSecond == 0 {
r.Errors["QUEUE_UNAVAILABLE"]++
return r.finish()
}
log := newInvocationLog(cfg.Requests)
var workers []*orbit.Worker
for i := 0; i < cfg.Workers; i++ {
w, err := orbit.NewWorker(client, orbit.WorkerOptions{QueueID: cfg.QueueID, Capacity: uint32(cfg.Capacity), ShutdownTimeout: 5 * time.Second, Handlers: []orbit.WorkerHandler{log.handler(ctx, cfg, runID)}})
if err != nil {
r.Errors["WORKER_CONFIG"]++
return r.finish()
}
workers = append(workers, w)
}
workerCtx, stopWorkers := context.WithCancel(ctx)
done := make(chan error, len(workers))
for _, w := range workers {
go func(w *orbit.Worker) {
err := w.Run(workerCtx)
if workerCtx.Err() == nil {
cancel()
} // 不可恢复连接错误立即停止继续提交。
done <- err
}(w)
}
admissions := submit(ctx, cfg, client, runID)
var successful []admission
var admissionDurations []time.Duration
seen := map[string]bool{}
for _, a := range admissions {
admissionDurations = append(admissionDurations, a.duration)
if a.code != "" {
r.AdmissionErrors++
r.Errors["ADMISSION_"+a.code]++
continue
}
if seen[a.task.ID] {
r.AdmissionErrors++
r.Errors["DUPLICATE_TASK_ID"]++
continue
}
seen[a.task.ID] = true
successful = append(successful, a)
}
r.Attempted = len(admissions)
r.Accepted = len(successful)
r.NotSubmitted = cfg.Requests - r.Attempted
r.Admission = distributionOf(admissionDurations)
r.AdmissionFinishedAt = time.Now().UTC()
verified := verifyAll(ctx, cfg, client, runID, successful)
var queued, executed, completed []time.Duration
for _, v := range verified {
if v.code != "" {
r.Errors[v.code]++
continue
}
r.VerifiedSuccess++
queued = append(queued, v.measurement.queue)
executed = append(executed, v.measurement.execution)
completed = append(completed, v.measurement.completed)
}
if ctx.Err() != nil {
r.Errors["RUN_"+classify(ctx.Err())]++
}
stopWorkers()
for range workers {
if err := <-done; err != nil {
r.Errors["WORKER_STOP"]++
}
}
for _, w := range workers {
s := w.Stats()
r.WorkerStarted += s.Started
r.Reconnects += s.Reconnects
r.Renewals += s.RenewalsConfirmed
}
log.mu.Lock()
for i, calls := range log.calls {
r.HandlerCalls += calls
if calls > 1 {
r.DuplicateInvocations += calls - 1
}
if calls > 0 && !seen[log.taskIDs[i]] {
r.UnconfirmedHandlerCalls += calls
}
}
for _, a := range successful {
if log.calls[a.sequence] != 1 || log.taskIDs[a.sequence] != a.task.ID {
r.Errors["INVOCATION_MISMATCH"]++
}
}
r.Errors["INVALID_HANDLER_INPUT"] += log.invalid
log.mu.Unlock()
r.QueueWait = distributionOf(queued)
r.AuthorizedExecution = distributionOf(executed)
r.TaskCompletion = distributionOf(completed)
return r.finish()
}
func submit(ctx context.Context, cfg config, api taskAPI, runID string) []admission {
jobs := make(chan int)
out := make(chan admission, cfg.Requests)
var wg sync.WaitGroup
for i := 0; i < cfg.Concurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for sequence := range jobs {
if ctx.Err() != nil {
continue
}
payload, _ := json.Marshal(expected(cfg, runID, sequence))
start := time.Now()
call, stop := context.WithTimeout(ctx, cfg.Timeout)
res, err := api.SubmitTask(call, requestKey(runID, sequence), orbit.SubmitTaskRequest{QueueID: cfg.QueueID, Handler: handlerName(runID), HandlerVersion: "v1", PayloadVersion: "v1", BusinessKey: businessKey(runID), Payload: payload, TaskTimeoutMS: uint64(cfg.RunTimeout.Milliseconds()), RecoverySafety: orbit.RecoveryReconcileOnly, Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: uint64(cfg.RunTimeout.Milliseconds())}})
stop()
a := admission{sequence: sequence, task: res.Task, duration: time.Since(start)}
if err != nil {
a.code = classify(err)
} else if res.Deduplicated || !validID(res.Task.ID) || !matches(res.Task, cfg, runID, sequence, false) {
a.code = "INVALID_RECEIPT"
}
out <- a
}
}()
}
feed:
for i := 0; i < cfg.Requests; i++ {
select {
case <-ctx.Done():
break feed
case jobs <- i:
}
}
close(jobs)
wg.Wait()
close(out)
results := make([]admission, 0, cfg.Requests)
for a := range out {
results = append(results, a)
}
return results
}
type verification struct {
measurement measurement
code string
}
func verifyAll(ctx context.Context, cfg config, api taskAPI, runID string, admissions []admission) []verification {
jobs := make(chan admission, len(admissions))
for _, a := range admissions {
jobs <- a
}
close(jobs)
out := make(chan verification, len(admissions))
var wg sync.WaitGroup
for i := 0; i < cfg.Concurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for a := range jobs {
m, code := verify(ctx, cfg, api, runID, a)
out <- verification{m, code}
}
}()
}
wg.Wait()
close(out)
results := make([]verification, 0, len(admissions))
for v := range out {
results = append(results, v)
}
return results
}
func verify(ctx context.Context, cfg config, api taskAPI, runID string, a admission) (measurement, string) {
for {
if ctx.Err() != nil {
return measurement{}, "VERIFY_" + classify(ctx.Err())
}
call, stop := context.WithTimeout(ctx, cfg.Timeout)
task, err := api.GetTask(call, a.task.ID)
stop()
if err != nil {
return measurement{}, "READ_" + classify(err)
}
if task.ID != a.task.ID || !matches(task, cfg, runID, a.sequence, false) {
return measurement{}, "TASK_MISMATCH"
}
switch task.State {
case orbit.TaskSucceeded:
call, stop = context.WithTimeout(ctx, cfg.Timeout)
history, err := api.ListTaskAttempts(call, task.ID, orbit.HistoryPageRequest{Limit: 2})
stop()
if err != nil {
return measurement{}, "HISTORY_" + classify(err)
}
return checkCompletion(cfg, runID, a.sequence, task, history)
case orbit.TaskFailed, orbit.TaskCancelled, orbit.TaskNeedsReview:
return measurement{}, "TASK_" + string(task.State)
case orbit.TaskWaiting, orbit.TaskLeased, orbit.TaskRunning:
default:
return measurement{}, "UNKNOWN_TASK_STATE"
}
timer := time.NewTimer(cfg.PollInterval)
select {
case <-ctx.Done():
timer.Stop()
return measurement{}, "VERIFY_" + classify(ctx.Err())
case <-timer.C:
}
}
}
func matches(task orbit.Task, cfg config, runID string, sequence int, result bool) bool {
payload, valid := decodePayload(task.Payload)
if task.QueueID != cfg.QueueID || task.Handler != handlerName(runID) || task.HandlerVersion != "v1" || task.PayloadVersion != "v1" || task.BusinessKey != businessKey(runID) || task.Retry.MaxAttempts != 1 || task.RecoverySafety != orbit.RecoveryReconcileOnly || !valid || payload != expected(cfg, runID, sequence) {
return false
}
if result {
output, valid := decodePayload(task.Result)
if !valid || output != expected(cfg, runID, sequence) {
return false
}
}
return true
}
func checkCompletion(cfg config, runID string, sequence int, task orbit.Task, history orbit.AttemptPage) (measurement, string) {
if task.State != orbit.TaskSucceeded || task.EffectUncertain || task.AttemptCount != 1 || task.CompletedAt == nil || !matches(task, cfg, runID, sequence, true) {
return measurement{}, "COMPLETION_MISMATCH"
}
if len(history.Items) != 1 || history.NextCursor != "" {
return measurement{}, "ATTEMPT_COUNT_MISMATCH"
}
a := history.Items[0]
if !validID(a.ID) || a.TaskID != task.ID || a.Epoch != 1 || a.State != orbit.AttemptSucceeded || a.AuthorizedAt == nil || a.EndedAt == nil {
return measurement{}, "ATTEMPT_MISMATCH"
}
if task.CreatedAt.IsZero() || task.AvailableAt.Before(task.CreatedAt) || a.CreatedAt.Before(task.AvailableAt) || a.AuthorizedAt.Before(a.CreatedAt) || a.EndedAt.Before(*a.AuthorizedAt) || !a.EndedAt.Equal(*task.CompletedAt) {
return measurement{}, "TIMESTAMP_MISMATCH"
}
return measurement{queue: a.AuthorizedAt.Sub(task.AvailableAt), execution: a.EndedAt.Sub(*a.AuthorizedAt), completed: task.CompletedAt.Sub(task.CreatedAt)}, ""
}

View File

@@ -0,0 +1,268 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"sync"
"testing"
"time"
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
)
const testRunID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
const testQueueID = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
func testConfig() config {
return config{QueueID: testQueueID, Requests: 6, Concurrency: 3, Workers: 2, Capacity: 2, Timeout: time.Second, RunTimeout: time.Minute, PollInterval: 100 * time.Millisecond, Durations: []time.Duration{0, 10 * time.Millisecond}}
}
func completedFixture(cfg config) (orbit.Task, orbit.AttemptPage) {
t0 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
leasing, auth, end := t0.Add(time.Second), t0.Add(2*time.Second), t0.Add(3*time.Second)
payload, _ := json.Marshal(expected(cfg, testRunID, 0))
task := orbit.Task{ID: "cccccccccccccccccccccccccccccccc", QueueID: cfg.QueueID, Handler: handlerName(testRunID), HandlerVersion: "v1", PayloadVersion: "v1", BusinessKey: businessKey(testRunID), Payload: payload, Result: payload, State: orbit.TaskSucceeded, AttemptCount: 1, CreatedAt: t0, AvailableAt: t0, CompletedAt: &end, RecoverySafety: orbit.RecoveryReconcileOnly, Retry: orbit.RetryPolicy{MaxAttempts: 1}}
history := orbit.AttemptPage{Items: []orbit.AttemptSummary{{ID: "dddddddddddddddddddddddddddddddd", TaskID: task.ID, Epoch: 1, State: orbit.AttemptSucceeded, CreatedAt: leasing, AuthorizedAt: &auth, EndedAt: &end}}}
return task, history
}
func TestCompletionRequiresResultSingleAttemptAndOrderedServerTimestamps(t *testing.T) {
cfg := testConfig()
task, history := completedFixture(cfg)
m, code := checkCompletion(cfg, testRunID, 0, task, history)
if code != "" || m.queue != 2*time.Second || m.execution != time.Second || m.completed != 3*time.Second {
t.Fatal(m, code)
}
for _, tc := range []struct {
name string
mutate func(*orbit.Task, *orbit.AttemptPage)
}{
{"wrong-result", func(v *orbit.Task, _ *orbit.AttemptPage) { v.Result = json.RawMessage(`{}`) }},
{"missing-zero-valued-fields", func(v *orbit.Task, _ *orbit.AttemptPage) {
v.Result = json.RawMessage(`{"run_id":"` + testRunID + `"}`)
}},
{"null-zero-valued-fields", func(v *orbit.Task, _ *orbit.AttemptPage) {
v.Result = json.RawMessage(`{"run_id":"` + testRunID + `","sequence":null,"duration_ms":null}`)
}},
{"result-other-run", func(v *orbit.Task, _ *orbit.AttemptPage) {
v.Result = json.RawMessage(`{"run_id":"other","sequence":0,"duration_ms":0}`)
}},
{"unknown-effect", func(v *orbit.Task, _ *orbit.AttemptPage) { v.EffectUncertain = true }},
{"wrong-task", func(_ *orbit.Task, h *orbit.AttemptPage) { h.Items[0].TaskID = testRunID }},
{"second-attempt", func(_ *orbit.Task, h *orbit.AttemptPage) { h.Items = append(h.Items, h.Items[0]) }},
{"more-attempt-pages", func(_ *orbit.Task, h *orbit.AttemptPage) { h.NextCursor = "more" }},
{"missing-authorization", func(_ *orbit.Task, h *orbit.AttemptPage) { h.Items[0].AuthorizedAt = nil }},
{"reversed-time", func(v *orbit.Task, h *orbit.AttemptPage) { h.Items[0].EndedAt = &v.CreatedAt }},
{"completion-not-settled", func(v *orbit.Task, _ *orbit.AttemptPage) { v.CompletedAt = nil }},
{"wrong-budget", func(v *orbit.Task, _ *orbit.AttemptPage) { v.Retry.MaxAttempts = 2 }},
} {
t.Run(tc.name, func(t *testing.T) {
v, h := completedFixture(cfg)
tc.mutate(&v, &h)
if _, code := checkCompletion(cfg, testRunID, 0, v, h); code == "" {
t.Fatal("invalid completion accepted")
}
})
}
}
type stubAPI struct {
submit func(context.Context, string, orbit.SubmitTaskRequest) (orbit.SubmitTaskResult, error)
get func(context.Context, string) (orbit.Task, error)
history func(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error)
}
func (s stubAPI) SubmitTask(c context.Context, k string, v orbit.SubmitTaskRequest) (orbit.SubmitTaskResult, error) {
return s.submit(c, k, v)
}
func (s stubAPI) GetTask(c context.Context, k string) (orbit.Task, error) { return s.get(c, k) }
func (s stubAPI) ListTaskAttempts(c context.Context, k string, v orbit.HistoryPageRequest) (orbit.AttemptPage, error) {
return s.history(c, k, v)
}
func TestSubmissionIsBoundedDoesNotRetryOrCountAmbiguousReceipts(t *testing.T) {
cfg := testConfig()
var mu sync.Mutex
keys := map[string]int{}
active, peak := 0, 0
api := stubAPI{submit: func(ctx context.Context, key string, in orbit.SubmitTaskRequest) (orbit.SubmitTaskResult, error) {
mu.Lock()
keys[key]++
active++
if active > peak {
peak = active
}
mu.Unlock()
defer func() { mu.Lock(); active--; mu.Unlock() }()
var payload loadPayload
if json.Unmarshal(in.Payload, &payload) != nil {
t.Error("invalid sample payload")
}
if key != requestKey(testRunID, payload.Sequence) || in.Retry.MaxAttempts != 1 {
t.Error("identity or retry budget changed")
}
if payload.Sequence == 0 {
<-ctx.Done()
return orbit.SubmitTaskResult{}, ctx.Err()
}
if payload.Sequence == 1 {
return orbit.SubmitTaskResult{}, &orbit.APIError{StatusCode: 429, Code: "QUEUE_FULL"}
}
task := orbit.Task{ID: fmt.Sprintf("%032x", payload.Sequence), QueueID: in.QueueID, Handler: in.Handler, HandlerVersion: in.HandlerVersion, PayloadVersion: in.PayloadVersion, BusinessKey: in.BusinessKey, Payload: in.Payload, Retry: in.Retry, RecoverySafety: in.RecoverySafety}
return orbit.SubmitTaskResult{Task: task}, nil
}}
cfg.Timeout = 30 * time.Millisecond
results := submit(context.Background(), cfg, api, testRunID)
if len(results) != cfg.Requests || len(keys) != cfg.Requests || peak > cfg.Concurrency {
t.Fatal("lost requests or exceeded bound", len(results), len(keys), peak)
}
counts := map[string]int{}
for _, a := range results {
counts[a.code]++
}
if !reflect.DeepEqual(counts, map[string]int{"TIMEOUT": 1, "BACKPRESSURE": 1, "": 4}) {
t.Fatal(counts)
}
for _, n := range keys {
if n != 1 {
t.Fatal("submission retried")
}
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if len(submit(ctx, cfg, api, testRunID)) != 0 {
t.Fatal("submitted after cancellation")
}
}
func TestVerificationCannotPromoteReviewTimeoutOrMissingHistory(t *testing.T) {
cfg := testConfig()
task, history := completedFixture(cfg)
for _, state := range []orbit.TaskState{orbit.TaskNeedsReview, orbit.TaskFailed, orbit.TaskCancelled} {
v := task
v.State = state
api := stubAPI{get: func(context.Context, string) (orbit.Task, error) { return v, nil }}
if _, code := verify(context.Background(), cfg, api, testRunID, admission{task: task}); code != "TASK_"+string(state) {
t.Fatal(state, code)
}
}
waiting := task
waiting.State = orbit.TaskRunning
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer cancel()
api := stubAPI{get: func(context.Context, string) (orbit.Task, error) { return waiting, nil }}
if _, code := verify(ctx, cfg, api, testRunID, admission{task: task}); code != "VERIFY_TIMEOUT" {
t.Fatal(code)
}
api.get = func(context.Context, string) (orbit.Task, error) { return task, nil }
api.history = func(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error) {
return orbit.AttemptPage{}, errors.New("private history error")
}
if _, code := verify(context.Background(), cfg, api, testRunID, admission{task: task}); code != "HISTORY_TRANSPORT" {
t.Fatal(code)
}
api.history = func(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error) {
return history, nil
}
if _, code := verify(context.Background(), cfg, api, testRunID, admission{task: task}); code != "" {
t.Fatal(code)
}
}
func TestHandlerDetectsDuplicatesAndForeignPayloadWithoutExternalEffects(t *testing.T) {
cfg := testConfig()
task, _ := completedFixture(cfg)
log := newInvocationLog(cfg.Requests)
h := log.handler(context.Background(), cfg, testRunID)
result, err := h.Handle(context.Background(), orbit.WorkerExecution{Task: task})
if err != nil || string(result) != string(task.Payload) {
t.Fatal(err, string(result))
}
if _, err = h.Handle(context.Background(), orbit.WorkerExecution{Task: task}); err == nil {
t.Fatal("duplicate execution accepted")
}
task.Payload = json.RawMessage(`{"run_id":"other","sequence":0,"duration_ms":0}`)
if _, err = h.Handle(context.Background(), orbit.WorkerExecution{Task: task}); err == nil {
t.Fatal("foreign sample accepted")
}
if log.calls[0] != 2 || log.invalid != 1 {
t.Fatal("lost invocation evidence")
}
}
func TestReportDenominatorsAndRedaction(t *testing.T) {
r := newReport(testConfig(), testRunID)
r.Accepted = 5
r.VerifiedSuccess = 4
r.HandlerCalls = 6
r = r.finish()
if r.Complete || r.Unverified != 1 {
t.Fatal("incomplete sample reported complete")
}
raw, _ := json.Marshal(r)
if !strings.Contains(string(raw), `"p99_ms":null`) {
t.Fatal("empty distribution not explicit")
}
d := distributionOf([]time.Duration{3 * time.Millisecond, time.Millisecond, 2 * time.Millisecond})
if d.Count != 3 || *d.P50 != 2 || *d.P99 != 3 {
t.Fatal(d)
}
for _, err := range []error{errors.New("secret"), &orbit.APIError{StatusCode: 503, Code: "secret", Message: "token"}} {
if strings.Contains(classify(err), "secret") {
t.Fatal("error leaked")
}
}
}
func TestConfigBoundsAndDeterministicMix(t *testing.T) {
t.Setenv("ORBIT_URL", "http://127.0.0.1:8080")
t.Setenv("ORBIT_TOKEN", "test-token")
t.Setenv("ORBIT_QUEUE_ID", testQueueID)
t.Setenv("ORBIT_CREDENTIALS_FILE", "")
cfg, err := parseConfig([]string{"-durations", "0ms,10ms,2s", "-requests", "3", "-concurrency", "8"})
if err != nil || cfg.Concurrency != 3 || expected(cfg, testRunID, 5).DurationMS != 2000 {
t.Fatal(cfg, err)
}
for _, args := range [][]string{{"-requests", "0"}, {"-workers", "32", "-capacity", "256"}, {"-durations", "-1s"}, {"-durations", "1us"}, {"-durations", "60s", "-run-timeout", "60s"}, {"-poll-interval", "1ms"}, {"-timeout", "2h"}, {"-run-timeout", "2h"}} {
if _, err := parseConfig(args); err == nil {
t.Fatal("invalid bounds accepted", args)
}
}
}
func TestPreflightDenialNeverSubmitsOrStartsWorkers(t *testing.T) {
var mu sync.Mutex
var paths []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
paths = append(paths, r.URL.Path)
mu.Unlock()
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(403)
fmt.Fprint(w, `{"request_id":"`+testRunID+`","error":{"code":"FORBIDDEN","message":"private data"}}`)
}))
defer server.Close()
client, err := orbit.NewClient(server.URL, orbit.Options{Token: "test-token"})
if err != nil {
t.Fatal(err)
}
r := measure(context.Background(), testConfig(), client, testRunID)
if r.Complete || r.Attempted != 0 || r.NotSubmitted != 6 || r.Errors["PREFLIGHT_AUTH"] != 1 {
t.Fatal(r)
}
mu.Lock()
defer mu.Unlock()
if !reflect.DeepEqual(paths, []string{"/v1/queues/" + testQueueID}) {
t.Fatal(paths)
}
raw, _ := json.Marshal(r)
if strings.Contains(string(raw), "private data") || strings.Contains(string(raw), "test-token") {
t.Fatal("private data leaked")
}
}

View File

@@ -0,0 +1,121 @@
package main
import (
"context"
"errors"
"math"
"runtime"
"sort"
"time"
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
)
// distribution 只计算明确纳入的样本;空集合的百分位返回null,不能冒充零延迟。
type distribution struct {
Count int `json:"count"` // Count 样本数;完成阶段仅包含通过持久化核验的成功任务。
P50 *float64 `json:"p50_ms"` // P50 最近秩中位数,毫秒。
P95 *float64 `json:"p95_ms"` // P95 最近秩95百分位,毫秒。
P99 *float64 `json:"p99_ms"` // P99 最近秩99百分位,毫秒。
Max *float64 `json:"max_ms"` // Max 样本最大耗时,毫秒。
}
// executionReport 是单次有界测量结果,不是容量承诺或持久化审计。
type executionReport struct {
RunID string `json:"run_id"` // RunID 固定提交键和处理器身份的组成部分。
GoVersion string `json:"go_version"` // GoVersion 实际编译工具链版本,不从仓库要求推断。
GOOS string `json:"goos"` // GOOS 负载进程操作系统。
GOARCH string `json:"goarch"` // GOARCH 负载进程架构。
StartedAt time.Time `json:"started_at"` // StartedAt 本机开始时间,包含预检。
AdmissionFinishedAt time.Time `json:"admission_finished_at"` // AdmissionFinishedAt 本机受理组结束时间;预检失败时为零值。
FinishedAt time.Time `json:"finished_at"` // FinishedAt 核验及Worker收尾结束时间。
Requests int `json:"requests"` // Requests 计划独立任务数。
Concurrency int `json:"concurrency"` // Concurrency 提交及核验各自最大并发。
Workers int `json:"workers"` // Workers 同一负载进程内独立WSS运行器数。
Capacity int `json:"capacity_per_worker"` // Capacity 每个运行器物理处理器并发上限。
DurationsMS []int64 `json:"durations_ms"` // DurationsMS 按序号循环选取的模拟等待时长。
RunTimeoutMS int64 `json:"run_timeout_ms"` // RunTimeoutMS 整体期限,收尾最多另加5秒。
PollIntervalMS int64 `json:"poll_interval_ms"` // PollIntervalMS 每个活跃核验器两次读取之间的间隔。
HTTPTimeoutMS int64 `json:"http_timeout_ms"` // HTTPTimeoutMS 单次HTTP请求时限,不是任务时限。
Queue *orbit.Queue `json:"queue,omitempty"` // Queue 预检时队列额度快照;工具不会修改配置。
Attempted int `json:"attempted"` // Attempted 实际发起的提交调用数,不自动重试。
NotSubmitted int `json:"not_submitted"` // NotSubmitted 截止/错误前未发起的项数。
Accepted int `json:"accepted"` // Accepted 收到并核验唯一回执的项数。
AdmissionErrors int `json:"admission_errors"` // AdmissionErrors 明确拒绝或提交结果无法确认的项数,不能当作全部未落库。
VerifiedSuccess int `json:"verified_success"` // VerifiedSuccess 最终任务、尝试及结果均通过核验的任务数。
Unverified int `json:"unverified"` // Unverified 已受理但未获核验成功的数量,包括失败/核查/仍待执行。
HandlerCalls int `json:"handler_calls"` // HandlerCalls 本进程有效输入处理器调用数。
DuplicateInvocations int `json:"duplicate_invocations"` // DuplicateInvocations 同一序号超过一次的调用数。
UnconfirmedHandlerCalls int `json:"unconfirmed_handler_calls"` // UnconfirmedHandlerCalls 没有已确认受理回执的处理器调用数。
WorkerStarted uint64 `json:"worker_started"` // WorkerStarted SDK本进程统计,不替代持久化成功数。
Reconnects uint64 `json:"worker_reconnects"` // Reconnects SDK重连计数。
Renewals uint64 `json:"worker_renewals_confirmed"` // Renewals SDK续租确认数。
Admission distribution `json:"admission"` // Admission 全部提交HTTP交换的本机单调时钟耗时,含失败。
QueueWait distribution `json:"queue_wait"` // QueueWait 数据库available_at至authorized_at,含领取及开始授权。
AuthorizedExecution distribution `json:"authorized_execution"` // AuthorizedExecution 数据库authorized_at至ended_at,止于结束事务取时点,不包含其后提交确认。
TaskCompletion distribution `json:"task_completion"` // TaskCompletion 数据库created_at至completed_at,不受客户端轮询顺序影响。
Errors map[string]int `json:"errors"` // Errors 固定分类,不保存内部异常、SQL、凭据或请求原文。
Complete bool `json:"complete"` // Complete 仅所有计划任务和本地调用逐项核验且无错误时为true。
}
func newReport(cfg config, runID string) executionReport {
r := executionReport{RunID: runID, GoVersion: runtime.Version(), GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, StartedAt: time.Now().UTC(), Requests: cfg.Requests, NotSubmitted: cfg.Requests, Concurrency: cfg.Concurrency, Workers: cfg.Workers, Capacity: cfg.Capacity, RunTimeoutMS: cfg.RunTimeout.Milliseconds(), PollIntervalMS: cfg.PollInterval.Milliseconds(), HTTPTimeoutMS: cfg.Timeout.Milliseconds(), Errors: map[string]int{}}
for _, d := range cfg.Durations {
r.DurationsMS = append(r.DurationsMS, d.Milliseconds())
}
return r
}
func (r executionReport) finish() executionReport {
for code, count := range r.Errors {
if count == 0 {
delete(r.Errors, code)
}
}
r.FinishedAt = time.Now().UTC()
r.Unverified = r.Accepted - r.VerifiedSuccess
r.Complete = r.Attempted == r.Requests && r.NotSubmitted == 0 && r.AdmissionErrors == 0 && r.Accepted == r.Requests && r.VerifiedSuccess == r.Requests && r.HandlerCalls == r.Requests && r.WorkerStarted == uint64(r.Requests) && r.DuplicateInvocations == 0 && r.UnconfirmedHandlerCalls == 0 && len(r.Errors) == 0
return r
}
func distributionOf(values []time.Duration) distribution {
d := distribution{Count: len(values)}
if len(values) == 0 {
return d
}
sort.Slice(values, func(i, j int) bool { return values[i] < values[j] })
percentile := func(q float64) *float64 {
v := float64(values[int(math.Ceil(float64(len(values))*q))-1]) / float64(time.Millisecond)
return &v
}
d.P50 = percentile(.5)
d.P95 = percentile(.95)
d.P99 = percentile(.99)
d.Max = percentile(1)
return d
}
func classify(err error) string {
if errors.Is(err, context.DeadlineExceeded) {
return "TIMEOUT"
}
if errors.Is(err, context.Canceled) {
return "CANCELLED"
}
var api *orbit.APIError
if errors.As(err, &api) {
switch api.StatusCode {
case 401, 403:
return "AUTH"
case 409:
return "CONFLICT"
case 429:
return "BACKPRESSURE"
case 400, 422:
return "INVALID_REQUEST"
}
return "HTTP"
}
var h *orbit.HTTPError
if errors.As(err, &h) {
return "HTTP"
}
return "TRANSPORT"
}