639 lines
25 KiB
Go
639 lines
25 KiB
Go
package mysql
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
|
core "git.yuntuo.hk/k2safe/orbit/model"
|
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
|
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
|
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
|
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
|
)
|
|
|
|
type executionFixture struct {
|
|
t *testing.T
|
|
ctx context.Context
|
|
db *sql.DB
|
|
resources BootstrapResult
|
|
token auth.Token
|
|
tasks *service.Service
|
|
storeA, storeB *ExecutionStore
|
|
a, b *worker.Service
|
|
hello worker.ConnectInput
|
|
binding worker.Binding
|
|
}
|
|
|
|
func executionID(t *testing.T) core.ID {
|
|
t.Helper()
|
|
id, err := core.NewID()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return id
|
|
}
|
|
func newExecutionFixture(t *testing.T, ctx context.Context, db *sql.DB) *executionFixture {
|
|
t.Helper()
|
|
f := &executionFixture{t: t, ctx: ctx, db: db, tasks: service.New(NewTaskStore(db))}
|
|
err := Bootstrap(ctx, db, BootstrapInput{Application: "worker-" + string(executionID(t)), Namespace: "default", Environment: "test", Queue: "work"}, func(v BootstrapResult) error { f.resources = v; return nil })
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, _ := json.Marshal(auth.Permissions{Actions: []string{"tasks:submit", "tasks:read", "tasks:cancel", "queues:read", "queues:manage", "workers:connect", "workers:consume"}, Queues: []string{"*"}})
|
|
f.exec("UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?", string(raw), f.resources.NamespaceID, f.resources.CredentialID)
|
|
f.token, err = auth.Parse(f.resources.Token)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.storeA, err = NewExecutionStore(db, executionID(t), ExecutionOptions{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.storeB, err = NewExecutionStore(db, executionID(t), ExecutionOptions{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.a, f.b = worker.New(f.storeA), worker.New(f.storeB)
|
|
secret, _, err := worker.NewSecret()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.hello = worker.ConnectInput{InstanceID: executionID(t), InstanceSecret: secret, QueueID: f.resources.QueueID, Capacity: 4, Capabilities: []worker.Capability{{Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1"}}}
|
|
f.binding, _, err = f.a.Connect(ctx, f.token, f.hello)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return f
|
|
}
|
|
func (f *executionFixture) exec(q string, args ...any) {
|
|
f.t.Helper()
|
|
if _, err := f.db.ExecContext(f.ctx, q, args...); err != nil {
|
|
f.t.Fatal(err)
|
|
}
|
|
}
|
|
func (f *executionFixture) submit(max uint32) service.TaskView {
|
|
f.t.Helper()
|
|
out, err := f.tasks.Submit(f.ctx, f.token, string(executionID(f.t)), service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`), Retry: pulse.RetryPolicy{MaxAttempts: max, AttemptTimeoutMS: 120000}})
|
|
if err != nil {
|
|
f.t.Fatal(err)
|
|
}
|
|
return out.Task
|
|
}
|
|
func (f *executionFixture) claim() (*worker.Assignment, worker.AttemptRef) {
|
|
f.t.Helper()
|
|
out, err := f.a.Claim(f.ctx, f.token, f.binding)
|
|
if err != nil {
|
|
f.t.Fatal(err)
|
|
}
|
|
if out == nil {
|
|
f.t.Fatal("expected assignment")
|
|
}
|
|
return out, worker.AttemptRef{TaskID: out.Task.ID, AttemptID: out.AttemptID, Epoch: out.Epoch, LeaseSecret: out.LeaseSecret}
|
|
}
|
|
func (f *executionFixture) get(id core.ID) service.TaskView {
|
|
f.t.Helper()
|
|
out, err := f.tasks.GetTask(f.ctx, f.token, id)
|
|
if err != nil {
|
|
f.t.Fatal(err)
|
|
}
|
|
return out
|
|
}
|
|
func (f *executionFixture) expire(ref worker.AttemptRef) {
|
|
f.t.Helper()
|
|
f.exec("UPDATE pulse_tasks SET lease_until=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, ref.TaskID)
|
|
f.exec("UPDATE pulse_attempts SET lease_until=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, ref.AttemptID)
|
|
}
|
|
func (f *executionFixture) count(table, where string, args ...any) int {
|
|
f.t.Helper()
|
|
var n int
|
|
if err := f.db.QueryRowContext(f.ctx, "SELECT COUNT(*) FROM "+table+" WHERE namespace_id=? AND "+where, append([]any{f.resources.NamespaceID}, args...)...).Scan(&n); err != nil {
|
|
f.t.Fatal(err)
|
|
}
|
|
return n
|
|
}
|
|
func (f *executionFixture) used() uint32 {
|
|
f.t.Helper()
|
|
var n uint32
|
|
if err := f.db.QueryRowContext(f.ctx, "SELECT COALESCE(SUM(used_permits),0) FROM pulse_concurrency_limits WHERE namespace_id=?", f.resources.NamespaceID).Scan(&n); err != nil {
|
|
f.t.Fatal(err)
|
|
}
|
|
return n
|
|
}
|
|
func expectWorkerCode(t *testing.T, err error, code string) {
|
|
t.Helper()
|
|
if fault.Code(err) != code {
|
|
t.Fatalf("wanted %s got %v", code, err)
|
|
}
|
|
}
|
|
func TestIntegrationWorkerExecution(t *testing.T) {
|
|
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
|
if dsn == "" {
|
|
t.Skip("requires disposable MySQL")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
|
defer cancel()
|
|
db, err := Open(ctx, dsn, 32, 8)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
if err := Migrate(ctx, db); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Run("durable_start_renew_and_exact_result_confirmation", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
task := f.submit(3)
|
|
assignment, ref := f.claim()
|
|
if assignment.Task.ID != task.ID || assignment.Task.State != pulse.TaskLeased || assignment.Task.AttemptCount != 1 || f.used() != 1 {
|
|
t.Fatal("claim state/permit invariant")
|
|
}
|
|
startID := executionID(t)
|
|
started, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
|
|
if err != nil || !started.Lease.Active || started.Receipt.TaskState != pulse.TaskRunning {
|
|
t.Fatalf("start: %v", err)
|
|
}
|
|
duplicate, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
|
|
if err != nil || !duplicate.Deduplicated || duplicate.Receipt != started.Receipt {
|
|
t.Fatalf("start replay: %v", err)
|
|
}
|
|
_, err = f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
|
expectWorkerCode(t, err, "STATE_CONFLICT")
|
|
renewID := executionID(t)
|
|
renewed, err := f.a.Renew(ctx, f.token, f.binding, renewID, ref)
|
|
if err != nil || !renewed.Lease.Active || renewed.Lease.LeaseUntil.Before(started.Lease.LeaseUntil) {
|
|
t.Fatalf("renew: %v", err)
|
|
}
|
|
repeated, err := f.a.Renew(ctx, f.token, f.binding, renewID, ref)
|
|
if err != nil || !repeated.Deduplicated || repeated.Receipt.LeaseUntil != renewed.Receipt.LeaseUntil {
|
|
t.Fatalf("renew replay: %v", err)
|
|
}
|
|
result := worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"amount":12345678901234567890.1234567890123456789,"version":9007199254740993}`)}
|
|
message := executionID(t)
|
|
finished, err := f.a.Finish(ctx, f.token, f.binding, message, result)
|
|
if err != nil || finished.Lease.Active || finished.Receipt.Disposition != "APPLIED" {
|
|
t.Fatalf("finish: %v", err)
|
|
}
|
|
duplicate, err = f.a.Finish(ctx, f.token, f.binding, message, result)
|
|
if err != nil || !duplicate.Deduplicated {
|
|
t.Fatalf("finish replay %v", err)
|
|
}
|
|
duplicate, err = f.a.Finish(ctx, f.token, f.binding, executionID(t), result)
|
|
if err != nil || !duplicate.Deduplicated {
|
|
t.Fatalf("new message equivalent result %v", err)
|
|
}
|
|
current := f.get(task.ID)
|
|
if current.State != pulse.TaskSucceeded || current.CompletedAt == nil || !strings.Contains(string(current.Result), "12345678901234567890.1234567890123456789") || f.used() != 0 {
|
|
t.Fatal("finish lost state, precision or permit")
|
|
}
|
|
if f.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 1 || f.count("orbit_audit_events", "resource_id=? AND action='task.result'", task.ID) != 1 {
|
|
t.Fatal("duplicated completion history")
|
|
}
|
|
_, err = f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
|
expectWorkerCode(t, err, "LEASE_LOST")
|
|
oldAck, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
|
|
if err != nil || !oldAck.Deduplicated || oldAck.Lease.Active {
|
|
t.Fatalf("old start acknowledgement resurrected execution %v", err)
|
|
}
|
|
changed := result
|
|
changed.Result = json.RawMessage(`{"changed":true}`)
|
|
_, err = f.a.Finish(ctx, f.token, f.binding, message, changed)
|
|
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
|
|
conflict, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), changed)
|
|
if err != nil || conflict.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(task.ID).State != pulse.TaskSucceeded {
|
|
t.Fatalf("conflict rewrote outcome %v", err)
|
|
}
|
|
if f.count("pulse_attempt_evidence", "attempt_id=? AND reason='RESULT_CONFLICT'", ref.AttemptID) != 1 {
|
|
t.Fatal("conflict evidence missing")
|
|
}
|
|
})
|
|
t.Run("cross_node_resume_fences_old_session_and_requires_secrets", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
f.submit(3)
|
|
_, ref := f.claim()
|
|
start := executionID(t)
|
|
if _, err := f.a.Start(ctx, f.token, f.binding, start, ref); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := f.get(ref.TaskID)
|
|
newBinding, welcome, err := f.b.Connect(ctx, f.token, f.hello)
|
|
if err != nil || welcome.Epoch != 2 {
|
|
t.Fatalf("reconnect %v", err)
|
|
}
|
|
_, err = f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
|
expectWorkerCode(t, err, "STALE_SESSION")
|
|
_, err = f.b.Renew(ctx, f.token, newBinding, executionID(t), ref)
|
|
expectWorkerCode(t, err, "LEASE_LOST")
|
|
wrong := ref
|
|
wrong.LeaseSecret = strings.Repeat("A", 43)
|
|
_, err = f.b.Resume(ctx, f.token, newBinding, wrong)
|
|
expectWorkerCode(t, err, "PERMISSION_DENIED")
|
|
resumed, err := f.b.Resume(ctx, f.token, newBinding, ref)
|
|
if err != nil || !resumed.Active || resumed.State != pulse.AttemptRunning {
|
|
t.Fatalf("resume %v", err)
|
|
}
|
|
duplicate, err := f.b.Start(ctx, f.token, newBinding, start, ref)
|
|
if err != nil || !duplicate.Deduplicated || !duplicate.Lease.Active {
|
|
t.Fatalf("start ack recovery %v", err)
|
|
}
|
|
if f.get(ref.TaskID).AttemptCount != before.AttemptCount || f.used() != 1 {
|
|
t.Fatal("resume created attempt or duplicated permit")
|
|
}
|
|
if _, err := f.b.Resume(ctx, f.token, newBinding, ref); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if f.count("orbit_events", "aggregate_id=? AND type='attempt.resumed'", ref.AttemptID) != 1 {
|
|
t.Fatal("resume event duplicated")
|
|
}
|
|
badHello := f.hello
|
|
badHello.InstanceSecret = strings.Repeat("A", 43)
|
|
_, _, err = f.b.Connect(ctx, f.token, badHello)
|
|
expectWorkerCode(t, err, "PERMISSION_DENIED")
|
|
badHello = f.hello
|
|
badHello.Capabilities = []worker.Capability{{Handler: "another", HandlerVersion: "v1", PayloadVersion: "v1"}}
|
|
_, _, err = f.b.Connect(ctx, f.token, badHello)
|
|
expectWorkerCode(t, err, "STATE_CONFLICT")
|
|
f.expire(ref)
|
|
expired, err := f.b.Resume(ctx, f.token, newBinding, ref)
|
|
if err != nil || expired.Active {
|
|
t.Fatalf("expired resume %v", err)
|
|
}
|
|
})
|
|
t.Run("multi_node_claim_obeys_queue_and_process_capacity", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
f.exec("UPDATE pulse_queues SET max_concurrency=2 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.QueueID)
|
|
for i := 0; i < 8; i++ {
|
|
f.submit(3)
|
|
}
|
|
secret, _, err := worker.NewSecret()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
hello := f.hello
|
|
hello.InstanceID = executionID(t)
|
|
hello.InstanceSecret = secret
|
|
hello.Capacity = 1
|
|
bindingB, _, err := f.b.Connect(ctx, f.token, hello)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.a.Capacity(ctx, f.token, f.binding, worker.CapacityInput{Sequence: 1, Capacity: 1}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assignments := make(chan *worker.Assignment, 16)
|
|
errs := make(chan error, 16)
|
|
var wg sync.WaitGroup
|
|
start := make(chan struct{})
|
|
for i := 0; i < 16; i++ {
|
|
wg.Add(1)
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
<-start
|
|
var a *worker.Assignment
|
|
var err error
|
|
if i%2 == 0 {
|
|
a, err = f.a.Claim(ctx, f.token, f.binding)
|
|
} else {
|
|
a, err = f.b.Claim(ctx, f.token, bindingB)
|
|
}
|
|
assignments <- a
|
|
errs <- err
|
|
}(i)
|
|
}
|
|
close(start)
|
|
wg.Wait()
|
|
close(assignments)
|
|
close(errs)
|
|
for err := range errs {
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
ids := map[core.ID]bool{}
|
|
for a := range assignments {
|
|
if a == nil {
|
|
continue
|
|
}
|
|
if ids[a.Task.ID] {
|
|
t.Fatal("task dispatched twice")
|
|
}
|
|
ids[a.Task.ID] = true
|
|
}
|
|
if len(ids) != 2 || f.used() != 2 {
|
|
t.Fatalf("concurrency overrun: assignments=%d used=%d", len(ids), f.used())
|
|
}
|
|
b2, _, err := f.b.Connect(ctx, f.token, hello)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a, err := f.b.Claim(ctx, f.token, b2)
|
|
if err != nil || a != nil {
|
|
t.Fatalf("reconnect reset capacity %v", err)
|
|
}
|
|
})
|
|
t.Run("drain_is_durable_and_capacity_sequence_is_not_additive", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
f.submit(3)
|
|
_, ref := f.claim()
|
|
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
input := worker.CapacityInput{Sequence: 1, Capacity: 0, Drain: true}
|
|
if _, err := f.a.Capacity(ctx, f.token, f.binding, input); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.a.Capacity(ctx, f.token, f.binding, input); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
input.Capacity = 2
|
|
_, err := f.a.Capacity(ctx, f.token, f.binding, input)
|
|
expectWorkerCode(t, err, "STATE_CONFLICT")
|
|
input.Sequence = 2
|
|
input.Drain = false
|
|
_, err = f.a.Capacity(ctx, f.token, f.binding, input)
|
|
expectWorkerCode(t, err, "STATE_CONFLICT")
|
|
f.submit(3)
|
|
a, err := f.a.Claim(ctx, f.token, f.binding)
|
|
if err != nil || a != nil {
|
|
t.Fatalf("draining worker received work %v", err)
|
|
}
|
|
if f.used() != 1 {
|
|
t.Fatal("capacity zero released in-flight permit")
|
|
}
|
|
newBinding, welcome, err := f.b.Connect(ctx, f.token, f.hello)
|
|
if err != nil || !welcome.Draining {
|
|
t.Fatalf("drain lost on reconnect %v", err)
|
|
}
|
|
if _, err := f.b.Resume(ctx, f.token, newBinding, ref); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.b.Finish(ctx, f.token, newBinding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if f.used() != 0 {
|
|
t.Fatal("drained execution did not release capacity")
|
|
}
|
|
})
|
|
t.Run("paused_gate_blocks_start_but_not_saved_result", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
gate := executionID(t)
|
|
f.exec("INSERT INTO pulse_execution_gates(namespace_id,id,paused) VALUES (?,?,TRUE)", f.resources.NamespaceID, gate)
|
|
// 超过一次候选页的暂停任务不能饿死后面的普通任务。
|
|
for i := 0; i < 18; i++ {
|
|
task := f.submit(3)
|
|
f.exec("UPDATE pulse_tasks SET gate_id=? WHERE namespace_id=? AND id=?", gate, f.resources.NamespaceID, task.ID)
|
|
}
|
|
ready := f.submit(3)
|
|
a, ref := f.claim()
|
|
if a.Task.ID != ready.ID {
|
|
t.Fatal("paused candidate starvation")
|
|
}
|
|
f.exec("UPDATE pulse_tasks SET gate_id=? WHERE namespace_id=? AND id=?", gate, f.resources.NamespaceID, ready.ID)
|
|
start := executionID(t)
|
|
_, err := f.a.Start(ctx, f.token, f.binding, start, ref)
|
|
expectWorkerCode(t, err, "EXECUTION_PAUSED")
|
|
f.exec("UPDATE pulse_execution_gates SET paused=FALSE WHERE namespace_id=? AND id=?", f.resources.NamespaceID, gate)
|
|
if _, err := f.a.Start(ctx, f.token, f.binding, start, ref); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.exec("UPDATE pulse_execution_gates SET paused=TRUE WHERE namespace_id=? AND id=?", f.resources.NamespaceID, gate)
|
|
duplicate, err := f.a.Start(ctx, f.token, f.binding, start, ref)
|
|
if err != nil || !duplicate.Deduplicated || !duplicate.Lease.Active {
|
|
t.Fatalf("pause blocked original start receipt %v", err)
|
|
}
|
|
if _, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
t.Run("recovery_is_bounded_and_unknown_effects_are_not_replayed", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
task := f.submit(2)
|
|
_, ref := f.claim()
|
|
f.expire(ref)
|
|
stats, err := f.storeB.Recover(ctx, 100)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
current := f.get(task.ID)
|
|
if current.State != pulse.TaskWaiting || current.AttemptCount != 1 || f.used() != 0 || stats.Requeued < 1 {
|
|
t.Fatal("pre-start lease recovery failed")
|
|
}
|
|
f.exec("UPDATE pulse_tasks SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID)
|
|
_, newRef := f.claim()
|
|
if newRef.Epoch != 2 || newRef.AttemptID == ref.AttemptID {
|
|
t.Fatal("attempt epoch not advanced")
|
|
}
|
|
_, err = f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
|
expectWorkerCode(t, err, "LEASE_LOST")
|
|
start := executionID(t)
|
|
if _, err := f.a.Start(ctx, f.token, f.binding, start, newRef); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.expire(newRef)
|
|
var wg sync.WaitGroup
|
|
errs := make(chan error, 2)
|
|
for _, store := range []*ExecutionStore{f.storeA, f.storeB} {
|
|
wg.Add(1)
|
|
go func(store *ExecutionStore) { defer wg.Done(); _, err := store.Recover(ctx, 100); errs <- err }(store)
|
|
}
|
|
wg.Wait()
|
|
close(errs)
|
|
for err := range errs {
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
current = f.get(task.ID)
|
|
if current.State != pulse.TaskNeedsReview || current.AttemptCount != 2 || current.CompletedAt != nil || f.used() != 0 {
|
|
t.Fatal("running expiry must become review without replay")
|
|
}
|
|
if f.count("orbit_events", "aggregate_id=? AND type='task.needs_review'", task.ID) != 1 {
|
|
t.Fatal("recovery event duplicated")
|
|
}
|
|
late := worker.FinishInput{Ref: newRef, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"external_ref":"late-result"}`)}
|
|
message := executionID(t)
|
|
evidence, err := f.a.Finish(ctx, f.token, f.binding, message, late)
|
|
if err != nil || evidence.Receipt.Disposition != "EVIDENCE_ONLY" || evidence.Lease.Active {
|
|
t.Fatalf("late evidence %v", err)
|
|
}
|
|
evidence, err = f.a.Finish(ctx, f.token, f.binding, message, late)
|
|
if err != nil || !evidence.Deduplicated {
|
|
t.Fatalf("late replay %v", err)
|
|
}
|
|
if f.get(task.ID).State != pulse.TaskNeedsReview || f.count("pulse_attempt_evidence", "attempt_id=?", newRef.AttemptID) != 1 {
|
|
t.Fatal("late result rewrote authority or duplicated evidence")
|
|
}
|
|
duplicate, err := f.a.Start(ctx, f.token, f.binding, start, newRef)
|
|
if err != nil || !duplicate.Deduplicated || duplicate.Lease.Active {
|
|
t.Fatalf("old acknowledgement restored lease %v", err)
|
|
}
|
|
})
|
|
t.Run("attempt_timeout_and_session_expiry_cap_renewal", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
task := f.submit(3)
|
|
_, ref := f.claim()
|
|
f.exec("UPDATE pulse_tasks SET attempt_timeout_ms=5000 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID)
|
|
begun, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
renewed, err := f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
|
if err != nil || renewed.Lease.LeaseUntil.After(begun.Lease.LeaseUntil) {
|
|
t.Fatalf("renew bypassed absolute attempt deadline %v", err)
|
|
}
|
|
f.exec("UPDATE pulse_worker_sessions SET expires_at=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.binding.SessionID)
|
|
renewed, err = f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
|
if err != nil || renewed.Lease.LeaseUntil.Sub(renewed.Lease.ServerTime) > time.Second {
|
|
t.Fatalf("renew exceeded session expiration %v", err)
|
|
}
|
|
f.exec("UPDATE pulse_worker_sessions SET expires_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.binding.SessionID)
|
|
_, err = f.a.Capacity(ctx, f.token, f.binding, worker.CapacityInput{Sequence: 1, Capacity: 4})
|
|
expectWorkerCode(t, err, "STALE_SESSION")
|
|
})
|
|
t.Run("result_event_failure_rolls_back_task_receipt_and_permit", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
task := f.submit(3)
|
|
_, ref := f.claim()
|
|
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.exec("CREATE TRIGGER orbit_test_reject_worker_audit BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='worker audit failure'")
|
|
message := executionID(t)
|
|
in := worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}
|
|
_, err := f.a.Finish(ctx, f.token, f.binding, message, in)
|
|
f.exec("DROP TRIGGER orbit_test_reject_worker_audit")
|
|
if err == nil {
|
|
t.Fatal("expected injected failure")
|
|
}
|
|
if f.get(task.ID).State != pulse.TaskRunning || f.used() != 1 || f.count("pulse_attempt_receipts", "attempt_id=? AND operation='finish'", ref.AttemptID) != 0 || f.count("orbit_events", "aggregate_id=? AND type='task.succeeded'", task.ID) != 0 {
|
|
t.Fatal("partial result transaction committed")
|
|
}
|
|
if _, err := f.a.Finish(ctx, f.token, f.binding, message, in); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
t.Run("finish_without_start_is_only_evidence_and_cannot_free_capacity", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
task := f.submit(1)
|
|
_, ref := f.claim()
|
|
result, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)})
|
|
if err != nil || result.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(task.ID).State != pulse.TaskLeased || f.used() != 1 {
|
|
t.Fatalf("unstarted result accepted %v", err)
|
|
}
|
|
f.expire(ref)
|
|
if _, err := f.storeA.Recover(ctx, 100); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if f.get(task.ID).State != pulse.TaskFailed || f.used() != 0 {
|
|
t.Fatal("exhausted pre-start attempt did not terminate")
|
|
}
|
|
})
|
|
t.Run("known_failure_unknown_and_waiting_deadlines", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
for _, outcome := range []string{"FAILED", "UNKNOWN"} {
|
|
task := f.submit(3)
|
|
_, ref := f.claim()
|
|
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
out, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: outcome, ErrorCode: "EXTERNAL_RESULT", Result: json.RawMessage(`{"evidence_ref":"object/result"}`)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := pulse.TaskFailed
|
|
if outcome == "UNKNOWN" {
|
|
want = pulse.TaskNeedsReview
|
|
}
|
|
if out.Receipt.TaskState != want || f.get(task.ID).State != want {
|
|
t.Fatal("outcome classification lost")
|
|
}
|
|
}
|
|
task := f.submit(3)
|
|
f.exec("UPDATE pulse_tasks SET available_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 2 HOUR),deadline_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 HOUR) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, task.ID)
|
|
if _, err := f.storeA.Recover(ctx, 100); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if f.get(task.ID).State != pulse.TaskFailed {
|
|
t.Fatal("waiting deadline not enforced")
|
|
}
|
|
})
|
|
t.Run("authority_is_checked_on_every_execution_command", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
f.submit(3)
|
|
_, ref := f.claim()
|
|
f.exec("UPDATE orbit_credentials SET revoked_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.CredentialID)
|
|
_, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
|
expectWorkerCode(t, err, "UNAUTHENTICATED")
|
|
f.exec("UPDATE orbit_credentials SET revoked_at=NULL WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.CredentialID)
|
|
other := newExecutionFixture(t, ctx, db)
|
|
_, err = other.a.Resume(ctx, other.token, other.binding, ref)
|
|
expectWorkerCode(t, err, "NOT_FOUND")
|
|
copied := f.binding
|
|
copied.Proof = core.Digest{}
|
|
_, err = f.a.Claim(ctx, f.token, copied)
|
|
expectWorkerCode(t, err, "STALE_SESSION")
|
|
cancelled, stop := context.WithCancel(ctx)
|
|
stop()
|
|
_, err = f.a.Start(cancelled, f.token, f.binding, executionID(t), ref)
|
|
if !errors.Is(err, context.Canceled) {
|
|
t.Fatal(fmt.Sprint(err))
|
|
}
|
|
})
|
|
t.Run("claim_event_failure_does_not_leave_an_invisible_lease", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
task := f.submit(3)
|
|
f.exec("CREATE TRIGGER orbit_test_reject_claim_event BEFORE INSERT ON orbit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='claim event failure'")
|
|
assignment, err := f.a.Claim(ctx, f.token, f.binding)
|
|
f.exec("DROP TRIGGER orbit_test_reject_claim_event")
|
|
if err == nil || assignment != nil {
|
|
t.Fatal("expected failed claim without usable assignment")
|
|
}
|
|
if f.get(task.ID).State != pulse.TaskWaiting || f.used() != 0 || f.count("pulse_attempts", "task_id=?", task.ID) != 0 || f.count("pulse_permits", "TRUE") != 0 {
|
|
t.Fatal("claim partially committed")
|
|
}
|
|
if _, err := f.a.Claim(ctx, f.token, f.binding); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
t.Run("lock_wait_does_not_reuse_an_earlier_clock_sample", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
f.submit(3)
|
|
_, ref := f.claim()
|
|
holder, err := db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer holder.Rollback()
|
|
if _, err := holder.ExecContext(ctx, "UPDATE pulse_queues SET revision=revision+1 WHERE namespace_id=? AND id=?", f.resources.NamespaceID, f.resources.QueueID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, table := range []string{"pulse_tasks", "pulse_attempts"} {
|
|
id := ref.TaskID
|
|
if table == "pulse_attempts" {
|
|
id = ref.AttemptID
|
|
}
|
|
if _, err := holder.ExecContext(ctx, "UPDATE "+table+" SET lease_until=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 150000 MICROSECOND) WHERE namespace_id=? AND id=?", f.resources.NamespaceID, id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
result := make(chan error, 1)
|
|
go func() { _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); result <- err }()
|
|
time.Sleep(250 * time.Millisecond)
|
|
if err := holder.Commit(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
expectWorkerCode(t, <-result, "LEASE_LOST")
|
|
if f.get(ref.TaskID).State != pulse.TaskLeased {
|
|
t.Fatal("stale clock authorized start")
|
|
}
|
|
})
|
|
}
|