269 lines
11 KiB
Go
269 lines
11 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"reflect"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
|
)
|
|
|
|
const testRunID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
|
const testQueueID = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
|
|
|
func testConfig() config {
|
|
return config{QueueID: testQueueID, Requests: 6, Concurrency: 3, Workers: 2, Capacity: 2, Timeout: time.Second, RunTimeout: time.Minute, PollInterval: 100 * time.Millisecond, Durations: []time.Duration{0, 10 * time.Millisecond}}
|
|
}
|
|
|
|
func completedFixture(cfg config) (orbit.Task, orbit.AttemptPage) {
|
|
t0 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
leasing, auth, end := t0.Add(time.Second), t0.Add(2*time.Second), t0.Add(3*time.Second)
|
|
payload, _ := json.Marshal(expected(cfg, testRunID, 0))
|
|
task := orbit.Task{ID: "cccccccccccccccccccccccccccccccc", QueueID: cfg.QueueID, Handler: handlerName(testRunID), HandlerVersion: "v1", PayloadVersion: "v1", BusinessKey: businessKey(testRunID), Payload: payload, Result: payload, State: orbit.TaskSucceeded, AttemptCount: 1, CreatedAt: t0, AvailableAt: t0, CompletedAt: &end, RecoverySafety: orbit.RecoveryReconcileOnly, Retry: orbit.RetryPolicy{MaxAttempts: 1}}
|
|
history := orbit.AttemptPage{Items: []orbit.AttemptSummary{{ID: "dddddddddddddddddddddddddddddddd", TaskID: task.ID, Epoch: 1, State: orbit.AttemptSucceeded, CreatedAt: leasing, AuthorizedAt: &auth, EndedAt: &end}}}
|
|
return task, history
|
|
}
|
|
|
|
func TestCompletionRequiresResultSingleAttemptAndOrderedServerTimestamps(t *testing.T) {
|
|
cfg := testConfig()
|
|
task, history := completedFixture(cfg)
|
|
m, code := checkCompletion(cfg, testRunID, 0, task, history)
|
|
if code != "" || m.queue != 2*time.Second || m.execution != time.Second || m.completed != 3*time.Second {
|
|
t.Fatal(m, code)
|
|
}
|
|
for _, tc := range []struct {
|
|
name string
|
|
mutate func(*orbit.Task, *orbit.AttemptPage)
|
|
}{
|
|
{"wrong-result", func(v *orbit.Task, _ *orbit.AttemptPage) { v.Result = json.RawMessage(`{}`) }},
|
|
{"missing-zero-valued-fields", func(v *orbit.Task, _ *orbit.AttemptPage) {
|
|
v.Result = json.RawMessage(`{"run_id":"` + testRunID + `"}`)
|
|
}},
|
|
{"null-zero-valued-fields", func(v *orbit.Task, _ *orbit.AttemptPage) {
|
|
v.Result = json.RawMessage(`{"run_id":"` + testRunID + `","sequence":null,"duration_ms":null}`)
|
|
}},
|
|
{"result-other-run", func(v *orbit.Task, _ *orbit.AttemptPage) {
|
|
v.Result = json.RawMessage(`{"run_id":"other","sequence":0,"duration_ms":0}`)
|
|
}},
|
|
{"unknown-effect", func(v *orbit.Task, _ *orbit.AttemptPage) { v.EffectUncertain = true }},
|
|
{"wrong-task", func(_ *orbit.Task, h *orbit.AttemptPage) { h.Items[0].TaskID = testRunID }},
|
|
{"second-attempt", func(_ *orbit.Task, h *orbit.AttemptPage) { h.Items = append(h.Items, h.Items[0]) }},
|
|
{"more-attempt-pages", func(_ *orbit.Task, h *orbit.AttemptPage) { h.NextCursor = "more" }},
|
|
{"missing-authorization", func(_ *orbit.Task, h *orbit.AttemptPage) { h.Items[0].AuthorizedAt = nil }},
|
|
{"reversed-time", func(v *orbit.Task, h *orbit.AttemptPage) { h.Items[0].EndedAt = &v.CreatedAt }},
|
|
{"completion-not-settled", func(v *orbit.Task, _ *orbit.AttemptPage) { v.CompletedAt = nil }},
|
|
{"wrong-budget", func(v *orbit.Task, _ *orbit.AttemptPage) { v.Retry.MaxAttempts = 2 }},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
v, h := completedFixture(cfg)
|
|
tc.mutate(&v, &h)
|
|
if _, code := checkCompletion(cfg, testRunID, 0, v, h); code == "" {
|
|
t.Fatal("invalid completion accepted")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
type stubAPI struct {
|
|
submit func(context.Context, string, orbit.SubmitTaskRequest) (orbit.SubmitTaskResult, error)
|
|
get func(context.Context, string) (orbit.Task, error)
|
|
history func(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error)
|
|
}
|
|
|
|
func (s stubAPI) SubmitTask(c context.Context, k string, v orbit.SubmitTaskRequest) (orbit.SubmitTaskResult, error) {
|
|
return s.submit(c, k, v)
|
|
}
|
|
func (s stubAPI) GetTask(c context.Context, k string) (orbit.Task, error) { return s.get(c, k) }
|
|
func (s stubAPI) ListTaskAttempts(c context.Context, k string, v orbit.HistoryPageRequest) (orbit.AttemptPage, error) {
|
|
return s.history(c, k, v)
|
|
}
|
|
|
|
func TestSubmissionIsBoundedDoesNotRetryOrCountAmbiguousReceipts(t *testing.T) {
|
|
cfg := testConfig()
|
|
var mu sync.Mutex
|
|
keys := map[string]int{}
|
|
active, peak := 0, 0
|
|
api := stubAPI{submit: func(ctx context.Context, key string, in orbit.SubmitTaskRequest) (orbit.SubmitTaskResult, error) {
|
|
mu.Lock()
|
|
keys[key]++
|
|
active++
|
|
if active > peak {
|
|
peak = active
|
|
}
|
|
mu.Unlock()
|
|
defer func() { mu.Lock(); active--; mu.Unlock() }()
|
|
var payload loadPayload
|
|
if json.Unmarshal(in.Payload, &payload) != nil {
|
|
t.Error("invalid sample payload")
|
|
}
|
|
if key != requestKey(testRunID, payload.Sequence) || in.Retry.MaxAttempts != 1 {
|
|
t.Error("identity or retry budget changed")
|
|
}
|
|
if payload.Sequence == 0 {
|
|
<-ctx.Done()
|
|
return orbit.SubmitTaskResult{}, ctx.Err()
|
|
}
|
|
if payload.Sequence == 1 {
|
|
return orbit.SubmitTaskResult{}, &orbit.APIError{StatusCode: 429, Code: "QUEUE_FULL"}
|
|
}
|
|
task := orbit.Task{ID: fmt.Sprintf("%032x", payload.Sequence), QueueID: in.QueueID, Handler: in.Handler, HandlerVersion: in.HandlerVersion, PayloadVersion: in.PayloadVersion, BusinessKey: in.BusinessKey, Payload: in.Payload, Retry: in.Retry, RecoverySafety: in.RecoverySafety}
|
|
return orbit.SubmitTaskResult{Task: task}, nil
|
|
}}
|
|
cfg.Timeout = 30 * time.Millisecond
|
|
results := submit(context.Background(), cfg, api, testRunID)
|
|
if len(results) != cfg.Requests || len(keys) != cfg.Requests || peak > cfg.Concurrency {
|
|
t.Fatal("lost requests or exceeded bound", len(results), len(keys), peak)
|
|
}
|
|
counts := map[string]int{}
|
|
for _, a := range results {
|
|
counts[a.code]++
|
|
}
|
|
if !reflect.DeepEqual(counts, map[string]int{"TIMEOUT": 1, "BACKPRESSURE": 1, "": 4}) {
|
|
t.Fatal(counts)
|
|
}
|
|
for _, n := range keys {
|
|
if n != 1 {
|
|
t.Fatal("submission retried")
|
|
}
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if len(submit(ctx, cfg, api, testRunID)) != 0 {
|
|
t.Fatal("submitted after cancellation")
|
|
}
|
|
}
|
|
|
|
func TestVerificationCannotPromoteReviewTimeoutOrMissingHistory(t *testing.T) {
|
|
cfg := testConfig()
|
|
task, history := completedFixture(cfg)
|
|
for _, state := range []orbit.TaskState{orbit.TaskNeedsReview, orbit.TaskFailed, orbit.TaskCancelled} {
|
|
v := task
|
|
v.State = state
|
|
api := stubAPI{get: func(context.Context, string) (orbit.Task, error) { return v, nil }}
|
|
if _, code := verify(context.Background(), cfg, api, testRunID, admission{task: task}); code != "TASK_"+string(state) {
|
|
t.Fatal(state, code)
|
|
}
|
|
}
|
|
waiting := task
|
|
waiting.State = orbit.TaskRunning
|
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
|
defer cancel()
|
|
api := stubAPI{get: func(context.Context, string) (orbit.Task, error) { return waiting, nil }}
|
|
if _, code := verify(ctx, cfg, api, testRunID, admission{task: task}); code != "VERIFY_TIMEOUT" {
|
|
t.Fatal(code)
|
|
}
|
|
api.get = func(context.Context, string) (orbit.Task, error) { return task, nil }
|
|
api.history = func(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error) {
|
|
return orbit.AttemptPage{}, errors.New("private history error")
|
|
}
|
|
if _, code := verify(context.Background(), cfg, api, testRunID, admission{task: task}); code != "HISTORY_TRANSPORT" {
|
|
t.Fatal(code)
|
|
}
|
|
api.history = func(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error) {
|
|
return history, nil
|
|
}
|
|
if _, code := verify(context.Background(), cfg, api, testRunID, admission{task: task}); code != "" {
|
|
t.Fatal(code)
|
|
}
|
|
}
|
|
|
|
func TestHandlerDetectsDuplicatesAndForeignPayloadWithoutExternalEffects(t *testing.T) {
|
|
cfg := testConfig()
|
|
task, _ := completedFixture(cfg)
|
|
log := newInvocationLog(cfg.Requests)
|
|
h := log.handler(context.Background(), cfg, testRunID)
|
|
result, err := h.Handle(context.Background(), orbit.WorkerExecution{Task: task})
|
|
if err != nil || string(result) != string(task.Payload) {
|
|
t.Fatal(err, string(result))
|
|
}
|
|
if _, err = h.Handle(context.Background(), orbit.WorkerExecution{Task: task}); err == nil {
|
|
t.Fatal("duplicate execution accepted")
|
|
}
|
|
task.Payload = json.RawMessage(`{"run_id":"other","sequence":0,"duration_ms":0}`)
|
|
if _, err = h.Handle(context.Background(), orbit.WorkerExecution{Task: task}); err == nil {
|
|
t.Fatal("foreign sample accepted")
|
|
}
|
|
if log.calls[0] != 2 || log.invalid != 1 {
|
|
t.Fatal("lost invocation evidence")
|
|
}
|
|
}
|
|
|
|
func TestReportDenominatorsAndRedaction(t *testing.T) {
|
|
r := newReport(testConfig(), testRunID)
|
|
r.Accepted = 5
|
|
r.VerifiedSuccess = 4
|
|
r.HandlerCalls = 6
|
|
r = r.finish()
|
|
if r.Complete || r.Unverified != 1 {
|
|
t.Fatal("incomplete sample reported complete")
|
|
}
|
|
raw, _ := json.Marshal(r)
|
|
if !strings.Contains(string(raw), `"p99_ms":null`) {
|
|
t.Fatal("empty distribution not explicit")
|
|
}
|
|
d := distributionOf([]time.Duration{3 * time.Millisecond, time.Millisecond, 2 * time.Millisecond})
|
|
if d.Count != 3 || *d.P50 != 2 || *d.P99 != 3 {
|
|
t.Fatal(d)
|
|
}
|
|
for _, err := range []error{errors.New("secret"), &orbit.APIError{StatusCode: 503, Code: "secret", Message: "token"}} {
|
|
if strings.Contains(classify(err), "secret") {
|
|
t.Fatal("error leaked")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestConfigBoundsAndDeterministicMix(t *testing.T) {
|
|
t.Setenv("ORBIT_URL", "http://127.0.0.1:8080")
|
|
t.Setenv("ORBIT_TOKEN", "test-token")
|
|
t.Setenv("ORBIT_QUEUE_ID", testQueueID)
|
|
t.Setenv("ORBIT_CREDENTIALS_FILE", "")
|
|
cfg, err := parseConfig([]string{"-durations", "0ms,10ms,2s", "-requests", "3", "-concurrency", "8"})
|
|
if err != nil || cfg.Concurrency != 3 || expected(cfg, testRunID, 5).DurationMS != 2000 {
|
|
t.Fatal(cfg, err)
|
|
}
|
|
for _, args := range [][]string{{"-requests", "0"}, {"-workers", "32", "-capacity", "256"}, {"-durations", "-1s"}, {"-durations", "1us"}, {"-durations", "60s", "-run-timeout", "60s"}, {"-poll-interval", "1ms"}, {"-timeout", "2h"}, {"-run-timeout", "2h"}} {
|
|
if _, err := parseConfig(args); err == nil {
|
|
t.Fatal("invalid bounds accepted", args)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPreflightDenialNeverSubmitsOrStartsWorkers(t *testing.T) {
|
|
var mu sync.Mutex
|
|
var paths []string
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
mu.Lock()
|
|
paths = append(paths, r.URL.Path)
|
|
mu.Unlock()
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(403)
|
|
fmt.Fprint(w, `{"request_id":"`+testRunID+`","error":{"code":"FORBIDDEN","message":"private data"}}`)
|
|
}))
|
|
defer server.Close()
|
|
client, err := orbit.NewClient(server.URL, orbit.Options{Token: "test-token"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r := measure(context.Background(), testConfig(), client, testRunID)
|
|
if r.Complete || r.Attempted != 0 || r.NotSubmitted != 6 || r.Errors["PREFLIGHT_AUTH"] != 1 {
|
|
t.Fatal(r)
|
|
}
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if !reflect.DeepEqual(paths, []string{"/v1/queues/" + testQueueID}) {
|
|
t.Fatal(paths)
|
|
}
|
|
raw, _ := json.Marshal(r)
|
|
if strings.Contains(string(raw), "private data") || strings.Contains(string(raw), "test-token") {
|
|
t.Fatal("private data leaked")
|
|
}
|
|
}
|