feat: measure bounded worker execution with durable verification
This commit is contained in:
139
sdk/go/cmd/orbit-exec-load/main.go
Normal file
139
sdk/go/cmd/orbit-exec-load/main.go
Normal file
@@ -0,0 +1,139 @@
|
||||
// orbit-exec-load 在隔离环境测量真实 HTTP 受理、WSS Worker 和持久化完成链路。
|
||||
// 处理器只有有界等待及回显,不访问外部业务系统。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
type config struct {
|
||||
URL, Token, QueueID string
|
||||
Requests, Concurrency, Workers, Capacity int
|
||||
Timeout, RunTimeout, PollInterval time.Duration
|
||||
Durations []time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
cfg, err := parseConfig(os.Args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client, err := orbit.NewClient(cfg.URL, orbit.Options{Token: cfg.Token, HTTPClient: &http.Client{Timeout: cfg.Timeout}})
|
||||
if err != nil {
|
||||
return errors.New("invalid client configuration")
|
||||
}
|
||||
runID, err := orbit.NewWorkerMessageID()
|
||||
if err != nil {
|
||||
return errors.New("cannot generate run identity")
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
report := measure(ctx, cfg, client, runID)
|
||||
if err := json.NewEncoder(os.Stdout).Encode(report); err != nil {
|
||||
return err
|
||||
}
|
||||
if !report.Complete {
|
||||
return errors.New("execution sample incomplete; inspect report and original run identities")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseConfig(args []string) (config, error) {
|
||||
cfg := config{URL: os.Getenv("ORBIT_URL"), Token: os.Getenv("ORBIT_TOKEN"), QueueID: os.Getenv("ORBIT_QUEUE_ID"), Requests: 100, Concurrency: 8, Workers: 2, Capacity: 4, Timeout: 5 * time.Second, RunTimeout: 2 * time.Minute, PollInterval: 250 * time.Millisecond}
|
||||
flags := flag.NewFlagSet("orbit-exec-load", flag.ContinueOnError)
|
||||
var credentials, durations string
|
||||
flags.StringVar(&cfg.URL, "url", cfg.URL, "Orbit HTTP(S) URL")
|
||||
flags.StringVar(&credentials, "credentials", os.Getenv("ORBIT_CREDENTIALS_FILE"), "private credentials file for an isolated namespace")
|
||||
flags.StringVar(&cfg.QueueID, "queue", cfg.QueueID, "existing isolated queue; configuration is not changed")
|
||||
flags.IntVar(&cfg.Requests, "requests", cfg.Requests, "tasks, 1..10000")
|
||||
flags.IntVar(&cfg.Concurrency, "concurrency", cfg.Concurrency, "maximum concurrent submission/verification requests, 1..128")
|
||||
flags.IntVar(&cfg.Workers, "workers", cfg.Workers, "independent WSS Workers in this process, 1..32")
|
||||
flags.IntVar(&cfg.Capacity, "capacity", cfg.Capacity, "concurrent handlers per Worker, 1..256; total at most 1024")
|
||||
flags.DurationVar(&cfg.Timeout, "timeout", cfg.Timeout, "per HTTP request timeout, 1ms..1m")
|
||||
flags.DurationVar(&cfg.RunTimeout, "run-timeout", cfg.RunTimeout, "whole sample deadline, 1s..1h, plus at most 5s Worker shutdown")
|
||||
flags.DurationVar(&cfg.PollInterval, "poll-interval", cfg.PollInterval, "per active verifier polling interval, 100ms..10s")
|
||||
flags.StringVar(&durations, "durations", "0ms", "1..16 round-robin simulated handler durations, e.g. 0ms,100ms,2s; whole milliseconds, 0..60s")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
if flags.NArg() != 0 {
|
||||
return config{}, errors.New("positional arguments are not supported")
|
||||
}
|
||||
if credentials != "" {
|
||||
f, err := os.Open(credentials)
|
||||
if err != nil {
|
||||
return config{}, errors.New("cannot open private credentials file")
|
||||
}
|
||||
raw, err := io.ReadAll(io.LimitReader(f, (16<<10)+1))
|
||||
f.Close()
|
||||
var saved struct {
|
||||
Token string `json:"token"`
|
||||
QueueID string `json:"queue_id"`
|
||||
}
|
||||
if err != nil || len(raw) > 16<<10 || json.Unmarshal(raw, &saved) != nil {
|
||||
return config{}, errors.New("invalid credentials file (maximum 16 KiB)")
|
||||
}
|
||||
if cfg.Token == "" {
|
||||
cfg.Token = saved.Token
|
||||
}
|
||||
if cfg.QueueID == "" {
|
||||
cfg.QueueID = saved.QueueID
|
||||
}
|
||||
}
|
||||
if cfg.URL == "" || cfg.Token == "" || !validID(cfg.QueueID) {
|
||||
return config{}, errors.New("URL, token and a 32-character lowercase hexadecimal queue ID are required")
|
||||
}
|
||||
if cfg.Requests < 1 || cfg.Requests > 10000 || cfg.Concurrency < 1 || cfg.Concurrency > 128 || cfg.Workers < 1 || cfg.Workers > 32 || cfg.Capacity < 1 || cfg.Capacity > 256 || cfg.Workers*cfg.Capacity > 1024 {
|
||||
return config{}, errors.New("request, concurrency or Worker count is out of bounds")
|
||||
}
|
||||
if cfg.Timeout < time.Millisecond || cfg.Timeout > time.Minute || cfg.RunTimeout < time.Second || cfg.RunTimeout > time.Hour || cfg.PollInterval < 100*time.Millisecond || cfg.PollInterval > 10*time.Second {
|
||||
return config{}, errors.New("timeout or poll interval is out of bounds")
|
||||
}
|
||||
parts := strings.Split(durations, ",")
|
||||
if len(parts) > 16 {
|
||||
return config{}, errors.New("at most 16 handler durations are allowed")
|
||||
}
|
||||
for _, part := range parts {
|
||||
d, err := time.ParseDuration(strings.TrimSpace(part))
|
||||
if err != nil || d < 0 || d > time.Minute || d%time.Millisecond != 0 || d+time.Second > cfg.RunTimeout {
|
||||
return config{}, errors.New("handler duration must be whole milliseconds in 0..60s and allow at least 1s within run-timeout")
|
||||
}
|
||||
cfg.Durations = append(cfg.Durations, d)
|
||||
}
|
||||
if cfg.Concurrency > cfg.Requests {
|
||||
cfg.Concurrency = cfg.Requests
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func validID(id string) bool {
|
||||
if len(id) != 32 {
|
||||
return false
|
||||
}
|
||||
for _, c := range id {
|
||||
if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
367
sdk/go/cmd/orbit-exec-load/measure.go
Normal file
367
sdk/go/cmd/orbit-exec-load/measure.go
Normal file
@@ -0,0 +1,367 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
// loadPayload 的固定内容同时是处理器的输入和预期结果;不含凭据或外部地址。
|
||||
type loadPayload struct {
|
||||
RunID string `json:"run_id"` // RunID 隔离本次运行,不作为业务幂等保证。
|
||||
Sequence int `json:"sequence"` // Sequence 从0开始的请求序号,用于发现重复执行及错配。
|
||||
DurationMS int64 `json:"duration_ms"` // DurationMS 模拟等待毫秒数,按本次固定时长数组取模。
|
||||
}
|
||||
|
||||
func decodePayload(raw json.RawMessage) (loadPayload, bool) {
|
||||
var wire struct {
|
||||
RunID *string `json:"run_id"`
|
||||
Sequence *int `json:"sequence"`
|
||||
DurationMS *int64 `json:"duration_ms"`
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.DisallowUnknownFields()
|
||||
if decoder.Decode(&wire) != nil || wire.RunID == nil || wire.Sequence == nil || wire.DurationMS == nil {
|
||||
return loadPayload{}, false
|
||||
}
|
||||
var trailing any
|
||||
if decoder.Decode(&trailing) != io.EOF {
|
||||
return loadPayload{}, false
|
||||
}
|
||||
return loadPayload{*wire.RunID, *wire.Sequence, *wire.DurationMS}, true
|
||||
}
|
||||
|
||||
type admission struct {
|
||||
sequence int
|
||||
task orbit.Task
|
||||
duration time.Duration
|
||||
code string
|
||||
}
|
||||
|
||||
type taskAPI interface {
|
||||
SubmitTask(context.Context, string, orbit.SubmitTaskRequest) (orbit.SubmitTaskResult, error)
|
||||
GetTask(context.Context, string) (orbit.Task, error)
|
||||
ListTaskAttempts(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error)
|
||||
}
|
||||
|
||||
type measurement struct {
|
||||
queue, execution, completed time.Duration
|
||||
}
|
||||
|
||||
func expected(cfg config, runID string, sequence int) loadPayload {
|
||||
return loadPayload{runID, sequence, cfg.Durations[sequence%len(cfg.Durations)].Milliseconds()}
|
||||
}
|
||||
|
||||
func handlerName(runID string) string { return "load.exec." + runID }
|
||||
func businessKey(runID string) string { return "orbit-exec-" + runID }
|
||||
func requestKey(runID string, sequence int) string {
|
||||
return fmt.Sprintf("orbit-exec-%s-%06d", runID, sequence)
|
||||
}
|
||||
|
||||
// invocationLog 有界保存本次每个序号的实际处理器调用,发现重复时不覆盖首条证据。
|
||||
type invocationLog struct {
|
||||
mu sync.Mutex
|
||||
calls []int
|
||||
taskIDs []string
|
||||
invalid int
|
||||
}
|
||||
|
||||
func newInvocationLog(n int) *invocationLog {
|
||||
return &invocationLog{calls: make([]int, n), taskIDs: make([]string, n)}
|
||||
}
|
||||
|
||||
func (log *invocationLog) handler(runCtx context.Context, cfg config, runID string) orbit.WorkerHandler {
|
||||
return orbit.WorkerHandler{Capability: orbit.WorkerCapability{Handler: handlerName(runID), HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(ctx context.Context, execution orbit.WorkerExecution) (json.RawMessage, error) {
|
||||
payload, valid := decodePayload(execution.Task.Payload)
|
||||
if !valid || payload.RunID != runID || payload.Sequence < 0 || payload.Sequence >= cfg.Requests || payload != expected(cfg, runID, payload.Sequence) || execution.Task.QueueID != cfg.QueueID || execution.Task.BusinessKey != businessKey(runID) || execution.Task.Handler != handlerName(runID) {
|
||||
log.mu.Lock()
|
||||
log.invalid++
|
||||
log.mu.Unlock()
|
||||
return nil, &orbit.WorkerFailure{Code: "LOAD_INPUT_MISMATCH"}
|
||||
}
|
||||
log.mu.Lock()
|
||||
log.calls[payload.Sequence]++
|
||||
first := log.calls[payload.Sequence] == 1
|
||||
if first {
|
||||
log.taskIDs[payload.Sequence] = execution.Task.ID
|
||||
}
|
||||
log.mu.Unlock()
|
||||
if !first {
|
||||
return nil, &orbit.WorkerFailure{Code: "LOAD_DUPLICATE_EXECUTION"}
|
||||
}
|
||||
timer := time.NewTimer(time.Duration(payload.DurationMS) * time.Millisecond)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-runCtx.Done():
|
||||
return nil, runCtx.Err()
|
||||
case <-timer.C:
|
||||
return json.Marshal(payload)
|
||||
}
|
||||
}}
|
||||
}
|
||||
|
||||
func measure(parent context.Context, cfg config, client *orbit.Client, runID string) executionReport {
|
||||
r := newReport(cfg, runID)
|
||||
ctx, cancel := context.WithTimeout(parent, cfg.RunTimeout)
|
||||
defer cancel()
|
||||
requestCtx, stop := context.WithTimeout(ctx, cfg.Timeout)
|
||||
queue, err := client.GetQueue(requestCtx, cfg.QueueID)
|
||||
stop()
|
||||
if err != nil {
|
||||
r.Errors["PREFLIGHT_"+classify(err)]++
|
||||
return r.finish()
|
||||
}
|
||||
r.Queue = &queue
|
||||
if queue.ID != cfg.QueueID || queue.Paused || queue.MaxConcurrency == 0 || queue.RatePerSecond == 0 {
|
||||
r.Errors["QUEUE_UNAVAILABLE"]++
|
||||
return r.finish()
|
||||
}
|
||||
log := newInvocationLog(cfg.Requests)
|
||||
var workers []*orbit.Worker
|
||||
for i := 0; i < cfg.Workers; i++ {
|
||||
w, err := orbit.NewWorker(client, orbit.WorkerOptions{QueueID: cfg.QueueID, Capacity: uint32(cfg.Capacity), ShutdownTimeout: 5 * time.Second, Handlers: []orbit.WorkerHandler{log.handler(ctx, cfg, runID)}})
|
||||
if err != nil {
|
||||
r.Errors["WORKER_CONFIG"]++
|
||||
return r.finish()
|
||||
}
|
||||
workers = append(workers, w)
|
||||
}
|
||||
workerCtx, stopWorkers := context.WithCancel(ctx)
|
||||
done := make(chan error, len(workers))
|
||||
for _, w := range workers {
|
||||
go func(w *orbit.Worker) {
|
||||
err := w.Run(workerCtx)
|
||||
if workerCtx.Err() == nil {
|
||||
cancel()
|
||||
} // 不可恢复连接错误立即停止继续提交。
|
||||
done <- err
|
||||
}(w)
|
||||
}
|
||||
admissions := submit(ctx, cfg, client, runID)
|
||||
var successful []admission
|
||||
var admissionDurations []time.Duration
|
||||
seen := map[string]bool{}
|
||||
for _, a := range admissions {
|
||||
admissionDurations = append(admissionDurations, a.duration)
|
||||
if a.code != "" {
|
||||
r.AdmissionErrors++
|
||||
r.Errors["ADMISSION_"+a.code]++
|
||||
continue
|
||||
}
|
||||
if seen[a.task.ID] {
|
||||
r.AdmissionErrors++
|
||||
r.Errors["DUPLICATE_TASK_ID"]++
|
||||
continue
|
||||
}
|
||||
seen[a.task.ID] = true
|
||||
successful = append(successful, a)
|
||||
}
|
||||
r.Attempted = len(admissions)
|
||||
r.Accepted = len(successful)
|
||||
r.NotSubmitted = cfg.Requests - r.Attempted
|
||||
r.Admission = distributionOf(admissionDurations)
|
||||
r.AdmissionFinishedAt = time.Now().UTC()
|
||||
verified := verifyAll(ctx, cfg, client, runID, successful)
|
||||
var queued, executed, completed []time.Duration
|
||||
for _, v := range verified {
|
||||
if v.code != "" {
|
||||
r.Errors[v.code]++
|
||||
continue
|
||||
}
|
||||
r.VerifiedSuccess++
|
||||
queued = append(queued, v.measurement.queue)
|
||||
executed = append(executed, v.measurement.execution)
|
||||
completed = append(completed, v.measurement.completed)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
r.Errors["RUN_"+classify(ctx.Err())]++
|
||||
}
|
||||
stopWorkers()
|
||||
for range workers {
|
||||
if err := <-done; err != nil {
|
||||
r.Errors["WORKER_STOP"]++
|
||||
}
|
||||
}
|
||||
for _, w := range workers {
|
||||
s := w.Stats()
|
||||
r.WorkerStarted += s.Started
|
||||
r.Reconnects += s.Reconnects
|
||||
r.Renewals += s.RenewalsConfirmed
|
||||
}
|
||||
log.mu.Lock()
|
||||
for i, calls := range log.calls {
|
||||
r.HandlerCalls += calls
|
||||
if calls > 1 {
|
||||
r.DuplicateInvocations += calls - 1
|
||||
}
|
||||
if calls > 0 && !seen[log.taskIDs[i]] {
|
||||
r.UnconfirmedHandlerCalls += calls
|
||||
}
|
||||
}
|
||||
for _, a := range successful {
|
||||
if log.calls[a.sequence] != 1 || log.taskIDs[a.sequence] != a.task.ID {
|
||||
r.Errors["INVOCATION_MISMATCH"]++
|
||||
}
|
||||
}
|
||||
r.Errors["INVALID_HANDLER_INPUT"] += log.invalid
|
||||
log.mu.Unlock()
|
||||
r.QueueWait = distributionOf(queued)
|
||||
r.AuthorizedExecution = distributionOf(executed)
|
||||
r.TaskCompletion = distributionOf(completed)
|
||||
return r.finish()
|
||||
}
|
||||
|
||||
func submit(ctx context.Context, cfg config, api taskAPI, runID string) []admission {
|
||||
jobs := make(chan int)
|
||||
out := make(chan admission, cfg.Requests)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < cfg.Concurrency; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for sequence := range jobs {
|
||||
if ctx.Err() != nil {
|
||||
continue
|
||||
}
|
||||
payload, _ := json.Marshal(expected(cfg, runID, sequence))
|
||||
start := time.Now()
|
||||
call, stop := context.WithTimeout(ctx, cfg.Timeout)
|
||||
res, err := api.SubmitTask(call, requestKey(runID, sequence), orbit.SubmitTaskRequest{QueueID: cfg.QueueID, Handler: handlerName(runID), HandlerVersion: "v1", PayloadVersion: "v1", BusinessKey: businessKey(runID), Payload: payload, TaskTimeoutMS: uint64(cfg.RunTimeout.Milliseconds()), RecoverySafety: orbit.RecoveryReconcileOnly, Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: uint64(cfg.RunTimeout.Milliseconds())}})
|
||||
stop()
|
||||
a := admission{sequence: sequence, task: res.Task, duration: time.Since(start)}
|
||||
if err != nil {
|
||||
a.code = classify(err)
|
||||
} else if res.Deduplicated || !validID(res.Task.ID) || !matches(res.Task, cfg, runID, sequence, false) {
|
||||
a.code = "INVALID_RECEIPT"
|
||||
}
|
||||
out <- a
|
||||
}
|
||||
}()
|
||||
}
|
||||
feed:
|
||||
for i := 0; i < cfg.Requests; i++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break feed
|
||||
case jobs <- i:
|
||||
}
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
close(out)
|
||||
results := make([]admission, 0, cfg.Requests)
|
||||
for a := range out {
|
||||
results = append(results, a)
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
type verification struct {
|
||||
measurement measurement
|
||||
code string
|
||||
}
|
||||
|
||||
func verifyAll(ctx context.Context, cfg config, api taskAPI, runID string, admissions []admission) []verification {
|
||||
jobs := make(chan admission, len(admissions))
|
||||
for _, a := range admissions {
|
||||
jobs <- a
|
||||
}
|
||||
close(jobs)
|
||||
out := make(chan verification, len(admissions))
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < cfg.Concurrency; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for a := range jobs {
|
||||
m, code := verify(ctx, cfg, api, runID, a)
|
||||
out <- verification{m, code}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(out)
|
||||
results := make([]verification, 0, len(admissions))
|
||||
for v := range out {
|
||||
results = append(results, v)
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
func verify(ctx context.Context, cfg config, api taskAPI, runID string, a admission) (measurement, string) {
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return measurement{}, "VERIFY_" + classify(ctx.Err())
|
||||
}
|
||||
call, stop := context.WithTimeout(ctx, cfg.Timeout)
|
||||
task, err := api.GetTask(call, a.task.ID)
|
||||
stop()
|
||||
if err != nil {
|
||||
return measurement{}, "READ_" + classify(err)
|
||||
}
|
||||
if task.ID != a.task.ID || !matches(task, cfg, runID, a.sequence, false) {
|
||||
return measurement{}, "TASK_MISMATCH"
|
||||
}
|
||||
switch task.State {
|
||||
case orbit.TaskSucceeded:
|
||||
call, stop = context.WithTimeout(ctx, cfg.Timeout)
|
||||
history, err := api.ListTaskAttempts(call, task.ID, orbit.HistoryPageRequest{Limit: 2})
|
||||
stop()
|
||||
if err != nil {
|
||||
return measurement{}, "HISTORY_" + classify(err)
|
||||
}
|
||||
return checkCompletion(cfg, runID, a.sequence, task, history)
|
||||
case orbit.TaskFailed, orbit.TaskCancelled, orbit.TaskNeedsReview:
|
||||
return measurement{}, "TASK_" + string(task.State)
|
||||
case orbit.TaskWaiting, orbit.TaskLeased, orbit.TaskRunning:
|
||||
default:
|
||||
return measurement{}, "UNKNOWN_TASK_STATE"
|
||||
}
|
||||
timer := time.NewTimer(cfg.PollInterval)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return measurement{}, "VERIFY_" + classify(ctx.Err())
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func matches(task orbit.Task, cfg config, runID string, sequence int, result bool) bool {
|
||||
payload, valid := decodePayload(task.Payload)
|
||||
if task.QueueID != cfg.QueueID || task.Handler != handlerName(runID) || task.HandlerVersion != "v1" || task.PayloadVersion != "v1" || task.BusinessKey != businessKey(runID) || task.Retry.MaxAttempts != 1 || task.RecoverySafety != orbit.RecoveryReconcileOnly || !valid || payload != expected(cfg, runID, sequence) {
|
||||
return false
|
||||
}
|
||||
if result {
|
||||
output, valid := decodePayload(task.Result)
|
||||
if !valid || output != expected(cfg, runID, sequence) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func checkCompletion(cfg config, runID string, sequence int, task orbit.Task, history orbit.AttemptPage) (measurement, string) {
|
||||
if task.State != orbit.TaskSucceeded || task.EffectUncertain || task.AttemptCount != 1 || task.CompletedAt == nil || !matches(task, cfg, runID, sequence, true) {
|
||||
return measurement{}, "COMPLETION_MISMATCH"
|
||||
}
|
||||
if len(history.Items) != 1 || history.NextCursor != "" {
|
||||
return measurement{}, "ATTEMPT_COUNT_MISMATCH"
|
||||
}
|
||||
a := history.Items[0]
|
||||
if !validID(a.ID) || a.TaskID != task.ID || a.Epoch != 1 || a.State != orbit.AttemptSucceeded || a.AuthorizedAt == nil || a.EndedAt == nil {
|
||||
return measurement{}, "ATTEMPT_MISMATCH"
|
||||
}
|
||||
if task.CreatedAt.IsZero() || task.AvailableAt.Before(task.CreatedAt) || a.CreatedAt.Before(task.AvailableAt) || a.AuthorizedAt.Before(a.CreatedAt) || a.EndedAt.Before(*a.AuthorizedAt) || !a.EndedAt.Equal(*task.CompletedAt) {
|
||||
return measurement{}, "TIMESTAMP_MISMATCH"
|
||||
}
|
||||
return measurement{queue: a.AuthorizedAt.Sub(task.AvailableAt), execution: a.EndedAt.Sub(*a.AuthorizedAt), completed: task.CompletedAt.Sub(task.CreatedAt)}, ""
|
||||
}
|
||||
268
sdk/go/cmd/orbit-exec-load/measure_test.go
Normal file
268
sdk/go/cmd/orbit-exec-load/measure_test.go
Normal file
@@ -0,0 +1,268 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
const testRunID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
const testQueueID = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
|
||||
func testConfig() config {
|
||||
return config{QueueID: testQueueID, Requests: 6, Concurrency: 3, Workers: 2, Capacity: 2, Timeout: time.Second, RunTimeout: time.Minute, PollInterval: 100 * time.Millisecond, Durations: []time.Duration{0, 10 * time.Millisecond}}
|
||||
}
|
||||
|
||||
func completedFixture(cfg config) (orbit.Task, orbit.AttemptPage) {
|
||||
t0 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
leasing, auth, end := t0.Add(time.Second), t0.Add(2*time.Second), t0.Add(3*time.Second)
|
||||
payload, _ := json.Marshal(expected(cfg, testRunID, 0))
|
||||
task := orbit.Task{ID: "cccccccccccccccccccccccccccccccc", QueueID: cfg.QueueID, Handler: handlerName(testRunID), HandlerVersion: "v1", PayloadVersion: "v1", BusinessKey: businessKey(testRunID), Payload: payload, Result: payload, State: orbit.TaskSucceeded, AttemptCount: 1, CreatedAt: t0, AvailableAt: t0, CompletedAt: &end, RecoverySafety: orbit.RecoveryReconcileOnly, Retry: orbit.RetryPolicy{MaxAttempts: 1}}
|
||||
history := orbit.AttemptPage{Items: []orbit.AttemptSummary{{ID: "dddddddddddddddddddddddddddddddd", TaskID: task.ID, Epoch: 1, State: orbit.AttemptSucceeded, CreatedAt: leasing, AuthorizedAt: &auth, EndedAt: &end}}}
|
||||
return task, history
|
||||
}
|
||||
|
||||
func TestCompletionRequiresResultSingleAttemptAndOrderedServerTimestamps(t *testing.T) {
|
||||
cfg := testConfig()
|
||||
task, history := completedFixture(cfg)
|
||||
m, code := checkCompletion(cfg, testRunID, 0, task, history)
|
||||
if code != "" || m.queue != 2*time.Second || m.execution != time.Second || m.completed != 3*time.Second {
|
||||
t.Fatal(m, code)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
mutate func(*orbit.Task, *orbit.AttemptPage)
|
||||
}{
|
||||
{"wrong-result", func(v *orbit.Task, _ *orbit.AttemptPage) { v.Result = json.RawMessage(`{}`) }},
|
||||
{"missing-zero-valued-fields", func(v *orbit.Task, _ *orbit.AttemptPage) {
|
||||
v.Result = json.RawMessage(`{"run_id":"` + testRunID + `"}`)
|
||||
}},
|
||||
{"null-zero-valued-fields", func(v *orbit.Task, _ *orbit.AttemptPage) {
|
||||
v.Result = json.RawMessage(`{"run_id":"` + testRunID + `","sequence":null,"duration_ms":null}`)
|
||||
}},
|
||||
{"result-other-run", func(v *orbit.Task, _ *orbit.AttemptPage) {
|
||||
v.Result = json.RawMessage(`{"run_id":"other","sequence":0,"duration_ms":0}`)
|
||||
}},
|
||||
{"unknown-effect", func(v *orbit.Task, _ *orbit.AttemptPage) { v.EffectUncertain = true }},
|
||||
{"wrong-task", func(_ *orbit.Task, h *orbit.AttemptPage) { h.Items[0].TaskID = testRunID }},
|
||||
{"second-attempt", func(_ *orbit.Task, h *orbit.AttemptPage) { h.Items = append(h.Items, h.Items[0]) }},
|
||||
{"more-attempt-pages", func(_ *orbit.Task, h *orbit.AttemptPage) { h.NextCursor = "more" }},
|
||||
{"missing-authorization", func(_ *orbit.Task, h *orbit.AttemptPage) { h.Items[0].AuthorizedAt = nil }},
|
||||
{"reversed-time", func(v *orbit.Task, h *orbit.AttemptPage) { h.Items[0].EndedAt = &v.CreatedAt }},
|
||||
{"completion-not-settled", func(v *orbit.Task, _ *orbit.AttemptPage) { v.CompletedAt = nil }},
|
||||
{"wrong-budget", func(v *orbit.Task, _ *orbit.AttemptPage) { v.Retry.MaxAttempts = 2 }},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
v, h := completedFixture(cfg)
|
||||
tc.mutate(&v, &h)
|
||||
if _, code := checkCompletion(cfg, testRunID, 0, v, h); code == "" {
|
||||
t.Fatal("invalid completion accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type stubAPI struct {
|
||||
submit func(context.Context, string, orbit.SubmitTaskRequest) (orbit.SubmitTaskResult, error)
|
||||
get func(context.Context, string) (orbit.Task, error)
|
||||
history func(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error)
|
||||
}
|
||||
|
||||
func (s stubAPI) SubmitTask(c context.Context, k string, v orbit.SubmitTaskRequest) (orbit.SubmitTaskResult, error) {
|
||||
return s.submit(c, k, v)
|
||||
}
|
||||
func (s stubAPI) GetTask(c context.Context, k string) (orbit.Task, error) { return s.get(c, k) }
|
||||
func (s stubAPI) ListTaskAttempts(c context.Context, k string, v orbit.HistoryPageRequest) (orbit.AttemptPage, error) {
|
||||
return s.history(c, k, v)
|
||||
}
|
||||
|
||||
func TestSubmissionIsBoundedDoesNotRetryOrCountAmbiguousReceipts(t *testing.T) {
|
||||
cfg := testConfig()
|
||||
var mu sync.Mutex
|
||||
keys := map[string]int{}
|
||||
active, peak := 0, 0
|
||||
api := stubAPI{submit: func(ctx context.Context, key string, in orbit.SubmitTaskRequest) (orbit.SubmitTaskResult, error) {
|
||||
mu.Lock()
|
||||
keys[key]++
|
||||
active++
|
||||
if active > peak {
|
||||
peak = active
|
||||
}
|
||||
mu.Unlock()
|
||||
defer func() { mu.Lock(); active--; mu.Unlock() }()
|
||||
var payload loadPayload
|
||||
if json.Unmarshal(in.Payload, &payload) != nil {
|
||||
t.Error("invalid sample payload")
|
||||
}
|
||||
if key != requestKey(testRunID, payload.Sequence) || in.Retry.MaxAttempts != 1 {
|
||||
t.Error("identity or retry budget changed")
|
||||
}
|
||||
if payload.Sequence == 0 {
|
||||
<-ctx.Done()
|
||||
return orbit.SubmitTaskResult{}, ctx.Err()
|
||||
}
|
||||
if payload.Sequence == 1 {
|
||||
return orbit.SubmitTaskResult{}, &orbit.APIError{StatusCode: 429, Code: "QUEUE_FULL"}
|
||||
}
|
||||
task := orbit.Task{ID: fmt.Sprintf("%032x", payload.Sequence), QueueID: in.QueueID, Handler: in.Handler, HandlerVersion: in.HandlerVersion, PayloadVersion: in.PayloadVersion, BusinessKey: in.BusinessKey, Payload: in.Payload, Retry: in.Retry, RecoverySafety: in.RecoverySafety}
|
||||
return orbit.SubmitTaskResult{Task: task}, nil
|
||||
}}
|
||||
cfg.Timeout = 30 * time.Millisecond
|
||||
results := submit(context.Background(), cfg, api, testRunID)
|
||||
if len(results) != cfg.Requests || len(keys) != cfg.Requests || peak > cfg.Concurrency {
|
||||
t.Fatal("lost requests or exceeded bound", len(results), len(keys), peak)
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for _, a := range results {
|
||||
counts[a.code]++
|
||||
}
|
||||
if !reflect.DeepEqual(counts, map[string]int{"TIMEOUT": 1, "BACKPRESSURE": 1, "": 4}) {
|
||||
t.Fatal(counts)
|
||||
}
|
||||
for _, n := range keys {
|
||||
if n != 1 {
|
||||
t.Fatal("submission retried")
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if len(submit(ctx, cfg, api, testRunID)) != 0 {
|
||||
t.Fatal("submitted after cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerificationCannotPromoteReviewTimeoutOrMissingHistory(t *testing.T) {
|
||||
cfg := testConfig()
|
||||
task, history := completedFixture(cfg)
|
||||
for _, state := range []orbit.TaskState{orbit.TaskNeedsReview, orbit.TaskFailed, orbit.TaskCancelled} {
|
||||
v := task
|
||||
v.State = state
|
||||
api := stubAPI{get: func(context.Context, string) (orbit.Task, error) { return v, nil }}
|
||||
if _, code := verify(context.Background(), cfg, api, testRunID, admission{task: task}); code != "TASK_"+string(state) {
|
||||
t.Fatal(state, code)
|
||||
}
|
||||
}
|
||||
waiting := task
|
||||
waiting.State = orbit.TaskRunning
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
api := stubAPI{get: func(context.Context, string) (orbit.Task, error) { return waiting, nil }}
|
||||
if _, code := verify(ctx, cfg, api, testRunID, admission{task: task}); code != "VERIFY_TIMEOUT" {
|
||||
t.Fatal(code)
|
||||
}
|
||||
api.get = func(context.Context, string) (orbit.Task, error) { return task, nil }
|
||||
api.history = func(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error) {
|
||||
return orbit.AttemptPage{}, errors.New("private history error")
|
||||
}
|
||||
if _, code := verify(context.Background(), cfg, api, testRunID, admission{task: task}); code != "HISTORY_TRANSPORT" {
|
||||
t.Fatal(code)
|
||||
}
|
||||
api.history = func(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error) {
|
||||
return history, nil
|
||||
}
|
||||
if _, code := verify(context.Background(), cfg, api, testRunID, admission{task: task}); code != "" {
|
||||
t.Fatal(code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerDetectsDuplicatesAndForeignPayloadWithoutExternalEffects(t *testing.T) {
|
||||
cfg := testConfig()
|
||||
task, _ := completedFixture(cfg)
|
||||
log := newInvocationLog(cfg.Requests)
|
||||
h := log.handler(context.Background(), cfg, testRunID)
|
||||
result, err := h.Handle(context.Background(), orbit.WorkerExecution{Task: task})
|
||||
if err != nil || string(result) != string(task.Payload) {
|
||||
t.Fatal(err, string(result))
|
||||
}
|
||||
if _, err = h.Handle(context.Background(), orbit.WorkerExecution{Task: task}); err == nil {
|
||||
t.Fatal("duplicate execution accepted")
|
||||
}
|
||||
task.Payload = json.RawMessage(`{"run_id":"other","sequence":0,"duration_ms":0}`)
|
||||
if _, err = h.Handle(context.Background(), orbit.WorkerExecution{Task: task}); err == nil {
|
||||
t.Fatal("foreign sample accepted")
|
||||
}
|
||||
if log.calls[0] != 2 || log.invalid != 1 {
|
||||
t.Fatal("lost invocation evidence")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportDenominatorsAndRedaction(t *testing.T) {
|
||||
r := newReport(testConfig(), testRunID)
|
||||
r.Accepted = 5
|
||||
r.VerifiedSuccess = 4
|
||||
r.HandlerCalls = 6
|
||||
r = r.finish()
|
||||
if r.Complete || r.Unverified != 1 {
|
||||
t.Fatal("incomplete sample reported complete")
|
||||
}
|
||||
raw, _ := json.Marshal(r)
|
||||
if !strings.Contains(string(raw), `"p99_ms":null`) {
|
||||
t.Fatal("empty distribution not explicit")
|
||||
}
|
||||
d := distributionOf([]time.Duration{3 * time.Millisecond, time.Millisecond, 2 * time.Millisecond})
|
||||
if d.Count != 3 || *d.P50 != 2 || *d.P99 != 3 {
|
||||
t.Fatal(d)
|
||||
}
|
||||
for _, err := range []error{errors.New("secret"), &orbit.APIError{StatusCode: 503, Code: "secret", Message: "token"}} {
|
||||
if strings.Contains(classify(err), "secret") {
|
||||
t.Fatal("error leaked")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigBoundsAndDeterministicMix(t *testing.T) {
|
||||
t.Setenv("ORBIT_URL", "http://127.0.0.1:8080")
|
||||
t.Setenv("ORBIT_TOKEN", "test-token")
|
||||
t.Setenv("ORBIT_QUEUE_ID", testQueueID)
|
||||
t.Setenv("ORBIT_CREDENTIALS_FILE", "")
|
||||
cfg, err := parseConfig([]string{"-durations", "0ms,10ms,2s", "-requests", "3", "-concurrency", "8"})
|
||||
if err != nil || cfg.Concurrency != 3 || expected(cfg, testRunID, 5).DurationMS != 2000 {
|
||||
t.Fatal(cfg, err)
|
||||
}
|
||||
for _, args := range [][]string{{"-requests", "0"}, {"-workers", "32", "-capacity", "256"}, {"-durations", "-1s"}, {"-durations", "1us"}, {"-durations", "60s", "-run-timeout", "60s"}, {"-poll-interval", "1ms"}, {"-timeout", "2h"}, {"-run-timeout", "2h"}} {
|
||||
if _, err := parseConfig(args); err == nil {
|
||||
t.Fatal("invalid bounds accepted", args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreflightDenialNeverSubmitsOrStartsWorkers(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var paths []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
paths = append(paths, r.URL.Path)
|
||||
mu.Unlock()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(403)
|
||||
fmt.Fprint(w, `{"request_id":"`+testRunID+`","error":{"code":"FORBIDDEN","message":"private data"}}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := orbit.NewClient(server.URL, orbit.Options{Token: "test-token"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := measure(context.Background(), testConfig(), client, testRunID)
|
||||
if r.Complete || r.Attempted != 0 || r.NotSubmitted != 6 || r.Errors["PREFLIGHT_AUTH"] != 1 {
|
||||
t.Fatal(r)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !reflect.DeepEqual(paths, []string{"/v1/queues/" + testQueueID}) {
|
||||
t.Fatal(paths)
|
||||
}
|
||||
raw, _ := json.Marshal(r)
|
||||
if strings.Contains(string(raw), "private data") || strings.Contains(string(raw), "test-token") {
|
||||
t.Fatal("private data leaked")
|
||||
}
|
||||
}
|
||||
121
sdk/go/cmd/orbit-exec-load/report.go
Normal file
121
sdk/go/cmd/orbit-exec-load/report.go
Normal file
@@ -0,0 +1,121 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"runtime"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
// distribution 只计算明确纳入的样本;空集合的百分位返回null,不能冒充零延迟。
|
||||
type distribution struct {
|
||||
Count int `json:"count"` // Count 样本数;完成阶段仅包含通过持久化核验的成功任务。
|
||||
P50 *float64 `json:"p50_ms"` // P50 最近秩中位数,毫秒。
|
||||
P95 *float64 `json:"p95_ms"` // P95 最近秩95百分位,毫秒。
|
||||
P99 *float64 `json:"p99_ms"` // P99 最近秩99百分位,毫秒。
|
||||
Max *float64 `json:"max_ms"` // Max 样本最大耗时,毫秒。
|
||||
}
|
||||
|
||||
// executionReport 是单次有界测量结果,不是容量承诺或持久化审计。
|
||||
type executionReport struct {
|
||||
RunID string `json:"run_id"` // RunID 固定提交键和处理器身份的组成部分。
|
||||
GoVersion string `json:"go_version"` // GoVersion 实际编译工具链版本,不从仓库要求推断。
|
||||
GOOS string `json:"goos"` // GOOS 负载进程操作系统。
|
||||
GOARCH string `json:"goarch"` // GOARCH 负载进程架构。
|
||||
StartedAt time.Time `json:"started_at"` // StartedAt 本机开始时间,包含预检。
|
||||
AdmissionFinishedAt time.Time `json:"admission_finished_at"` // AdmissionFinishedAt 本机受理组结束时间;预检失败时为零值。
|
||||
FinishedAt time.Time `json:"finished_at"` // FinishedAt 核验及Worker收尾结束时间。
|
||||
Requests int `json:"requests"` // Requests 计划独立任务数。
|
||||
Concurrency int `json:"concurrency"` // Concurrency 提交及核验各自最大并发。
|
||||
Workers int `json:"workers"` // Workers 同一负载进程内独立WSS运行器数。
|
||||
Capacity int `json:"capacity_per_worker"` // Capacity 每个运行器物理处理器并发上限。
|
||||
DurationsMS []int64 `json:"durations_ms"` // DurationsMS 按序号循环选取的模拟等待时长。
|
||||
RunTimeoutMS int64 `json:"run_timeout_ms"` // RunTimeoutMS 整体期限,收尾最多另加5秒。
|
||||
PollIntervalMS int64 `json:"poll_interval_ms"` // PollIntervalMS 每个活跃核验器两次读取之间的间隔。
|
||||
HTTPTimeoutMS int64 `json:"http_timeout_ms"` // HTTPTimeoutMS 单次HTTP请求时限,不是任务时限。
|
||||
Queue *orbit.Queue `json:"queue,omitempty"` // Queue 预检时队列额度快照;工具不会修改配置。
|
||||
Attempted int `json:"attempted"` // Attempted 实际发起的提交调用数,不自动重试。
|
||||
NotSubmitted int `json:"not_submitted"` // NotSubmitted 截止/错误前未发起的项数。
|
||||
Accepted int `json:"accepted"` // Accepted 收到并核验唯一回执的项数。
|
||||
AdmissionErrors int `json:"admission_errors"` // AdmissionErrors 明确拒绝或提交结果无法确认的项数,不能当作全部未落库。
|
||||
VerifiedSuccess int `json:"verified_success"` // VerifiedSuccess 最终任务、尝试及结果均通过核验的任务数。
|
||||
Unverified int `json:"unverified"` // Unverified 已受理但未获核验成功的数量,包括失败/核查/仍待执行。
|
||||
HandlerCalls int `json:"handler_calls"` // HandlerCalls 本进程有效输入处理器调用数。
|
||||
DuplicateInvocations int `json:"duplicate_invocations"` // DuplicateInvocations 同一序号超过一次的调用数。
|
||||
UnconfirmedHandlerCalls int `json:"unconfirmed_handler_calls"` // UnconfirmedHandlerCalls 没有已确认受理回执的处理器调用数。
|
||||
WorkerStarted uint64 `json:"worker_started"` // WorkerStarted SDK本进程统计,不替代持久化成功数。
|
||||
Reconnects uint64 `json:"worker_reconnects"` // Reconnects SDK重连计数。
|
||||
Renewals uint64 `json:"worker_renewals_confirmed"` // Renewals SDK续租确认数。
|
||||
Admission distribution `json:"admission"` // Admission 全部提交HTTP交换的本机单调时钟耗时,含失败。
|
||||
QueueWait distribution `json:"queue_wait"` // QueueWait 数据库available_at至authorized_at,含领取及开始授权。
|
||||
AuthorizedExecution distribution `json:"authorized_execution"` // AuthorizedExecution 数据库authorized_at至ended_at,止于结束事务取时点,不包含其后提交确认。
|
||||
TaskCompletion distribution `json:"task_completion"` // TaskCompletion 数据库created_at至completed_at,不受客户端轮询顺序影响。
|
||||
Errors map[string]int `json:"errors"` // Errors 固定分类,不保存内部异常、SQL、凭据或请求原文。
|
||||
Complete bool `json:"complete"` // Complete 仅所有计划任务和本地调用逐项核验且无错误时为true。
|
||||
}
|
||||
|
||||
func newReport(cfg config, runID string) executionReport {
|
||||
r := executionReport{RunID: runID, GoVersion: runtime.Version(), GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, StartedAt: time.Now().UTC(), Requests: cfg.Requests, NotSubmitted: cfg.Requests, Concurrency: cfg.Concurrency, Workers: cfg.Workers, Capacity: cfg.Capacity, RunTimeoutMS: cfg.RunTimeout.Milliseconds(), PollIntervalMS: cfg.PollInterval.Milliseconds(), HTTPTimeoutMS: cfg.Timeout.Milliseconds(), Errors: map[string]int{}}
|
||||
for _, d := range cfg.Durations {
|
||||
r.DurationsMS = append(r.DurationsMS, d.Milliseconds())
|
||||
}
|
||||
return r
|
||||
}
|
||||
func (r executionReport) finish() executionReport {
|
||||
for code, count := range r.Errors {
|
||||
if count == 0 {
|
||||
delete(r.Errors, code)
|
||||
}
|
||||
}
|
||||
r.FinishedAt = time.Now().UTC()
|
||||
r.Unverified = r.Accepted - r.VerifiedSuccess
|
||||
r.Complete = r.Attempted == r.Requests && r.NotSubmitted == 0 && r.AdmissionErrors == 0 && r.Accepted == r.Requests && r.VerifiedSuccess == r.Requests && r.HandlerCalls == r.Requests && r.WorkerStarted == uint64(r.Requests) && r.DuplicateInvocations == 0 && r.UnconfirmedHandlerCalls == 0 && len(r.Errors) == 0
|
||||
return r
|
||||
}
|
||||
func distributionOf(values []time.Duration) distribution {
|
||||
d := distribution{Count: len(values)}
|
||||
if len(values) == 0 {
|
||||
return d
|
||||
}
|
||||
sort.Slice(values, func(i, j int) bool { return values[i] < values[j] })
|
||||
percentile := func(q float64) *float64 {
|
||||
v := float64(values[int(math.Ceil(float64(len(values))*q))-1]) / float64(time.Millisecond)
|
||||
return &v
|
||||
}
|
||||
d.P50 = percentile(.5)
|
||||
d.P95 = percentile(.95)
|
||||
d.P99 = percentile(.99)
|
||||
d.Max = percentile(1)
|
||||
return d
|
||||
}
|
||||
func classify(err error) string {
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return "TIMEOUT"
|
||||
}
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return "CANCELLED"
|
||||
}
|
||||
var api *orbit.APIError
|
||||
if errors.As(err, &api) {
|
||||
switch api.StatusCode {
|
||||
case 401, 403:
|
||||
return "AUTH"
|
||||
case 409:
|
||||
return "CONFLICT"
|
||||
case 429:
|
||||
return "BACKPRESSURE"
|
||||
case 400, 422:
|
||||
return "INVALID_REQUEST"
|
||||
}
|
||||
return "HTTP"
|
||||
}
|
||||
var h *orbit.HTTPError
|
||||
if errors.As(err, &h) {
|
||||
return "HTTP"
|
||||
}
|
||||
return "TRANSPORT"
|
||||
}
|
||||
Reference in New Issue
Block a user