242 lines
11 KiB
Go
242 lines
11 KiB
Go
package mysql
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
|
"git.yuntuo.hk/k2safe/orbit/platform/events"
|
|
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
|
|
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
|
"git.yuntuo.hk/k2safe/orbit/weave/definition"
|
|
"git.yuntuo.hk/k2safe/orbit/weave/instance"
|
|
)
|
|
|
|
func addRetirementPermission(t *testing.T, f *executionFixture) {
|
|
t.Helper()
|
|
var raw []byte
|
|
var p auth.Permissions
|
|
if err := f.db.QueryRowContext(f.ctx, `SELECT permissions FROM orbit_credentials WHERE namespace_id=? AND id=?`, f.token.NamespaceID, f.token.CredentialID).Scan(&raw); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := json.Unmarshal(raw, &p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p.Actions = append(p.Actions, "queues:manage", "queues:read")
|
|
raw, _ = json.Marshal(p)
|
|
f.exec(`UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?`, string(raw), f.token.NamespaceID, f.token.CredentialID)
|
|
}
|
|
func TestIntegrationResourceRetirement(t *testing.T) {
|
|
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
|
if dsn == "" {
|
|
t.Skip("requires disposable MySQL")
|
|
}
|
|
ctx, stop := context.WithTimeout(context.Background(), 90*time.Second)
|
|
defer stop()
|
|
db, err := Open(ctx, dsn, 32, 8)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
if err = Migrate(ctx, db); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Run("no_lost_backlog_leases_or_history", func(t *testing.T) {
|
|
f := newEventFixture(t, ctx, db)
|
|
sub := f.create(events.CreateInput{MaxAttempts: 1})
|
|
task := f.cancelResult()
|
|
in := events.RetireInput{ExpectedRevision: sub.Revision, Reason: "completed temporary resources"}
|
|
_, err := f.events.Retire(ctx, f.token, sub.ID, "retire", in)
|
|
expectWorkerCode(t, err, "RESOURCE_BUSY")
|
|
g := f.grant(sub.ID)
|
|
_, err = f.events.Retire(ctx, f.token, sub.ID, "retire", in)
|
|
expectWorkerCode(t, err, "RESOURCE_BUSY")
|
|
f.expireDelivery(g)
|
|
noEventGrant(t, f, sub.ID)
|
|
_, err = f.events.Retire(ctx, f.token, sub.ID, "retire", in)
|
|
expectWorkerCode(t, err, "RESOURCE_BUSY")
|
|
f.ack(sub.ID, g)
|
|
done, err := f.events.Retire(ctx, f.token, sub.ID, "retire", in)
|
|
if err != nil || done.Subscription.RetiredAt == nil || done.Subscription.RetiredSequence == nil || done.Subscription.Enabled {
|
|
t.Fatalf("retire: %+v %v", done, err)
|
|
}
|
|
again, err := f.events.Retire(ctx, f.token, sub.ID, "retire", in)
|
|
if err != nil || !again.Deduplicated || again.Subscription.Revision != done.Subscription.Revision {
|
|
t.Fatal(again, err)
|
|
}
|
|
changed := in
|
|
changed.Reason = "other"
|
|
_, err = f.events.Retire(ctx, f.token, sub.ID, "retire", changed)
|
|
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
|
|
_, err = f.events.Control(ctx, f.token, sub.ID, "enable", events.ControlInput{ExpectedRevision: done.Subscription.Revision, Enabled: true, Reason: "attempt resume"})
|
|
expectWorkerCode(t, err, "RESOURCE_RETIRED")
|
|
detail, err := f.events.Detail(ctx, f.token, sub.ID, g.Delivery.ID)
|
|
if err != nil || detail.Event.AggregateID != task.ID {
|
|
t.Fatal("history unavailable", err)
|
|
}
|
|
f.cancelResult()
|
|
noEventGrant(t, f, sub.ID)
|
|
if f.count("orbit_audit_events", "action='subscription.retire'") != 1 {
|
|
t.Fatal("retirement audit missing or duplicated")
|
|
}
|
|
})
|
|
t.Run("paused_counts_retired_releases_capacity", func(t *testing.T) {
|
|
f := newEventFixture(t, ctx, db)
|
|
sub := f.create(events.CreateInput{})
|
|
paused, err := f.events.Control(ctx, f.token, sub.ID, "pause", events.ControlInput{ExpectedRevision: sub.Revision, Enabled: false, Reason: "pause"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 1; i < events.MaxSubscriptions; i++ {
|
|
f.create(events.CreateInput{Name: fmt.Sprintf("capacity-%d", i)})
|
|
}
|
|
_, err = f.events.Create(ctx, f.token, "next", events.CreateInput{Name: "next", QueueID: f.resources.QueueID})
|
|
expectWorkerCode(t, err, "SUBSCRIPTION_LIMIT")
|
|
if _, err = f.events.Retire(ctx, f.token, sub.ID, "retire", events.RetireInput{ExpectedRevision: paused.Subscription.Revision, Reason: "retire paused"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = f.events.Create(ctx, f.token, "next", events.CreateInput{Name: "next", QueueID: f.resources.QueueID}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if f.count("orbit_subscriptions", "1=1") != 65 {
|
|
t.Fatal("history was removed")
|
|
}
|
|
})
|
|
t.Run("queue_protects_tasks_subscriptions_and_original_receipts", func(t *testing.T) {
|
|
f := newEventFixture(t, ctx, db)
|
|
q := f.resources.QueueID
|
|
sub := f.create(events.CreateInput{})
|
|
task := f.submit(1)
|
|
in := service.RetireQueueInput{ExpectedRevision: 1, Reason: "queue lifecycle"}
|
|
_, err := f.tasks.RetireQueue(ctx, f.token, q, "retire", in)
|
|
expectWorkerCode(t, err, "RESOURCE_BUSY")
|
|
if _, err = f.tasks.CancelTask(ctx, f.token, task.ID, "cancel", service.CancelInput{ExpectedRevision: task.Revision, Reason: "finish test"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = f.tasks.RetireQueue(ctx, f.token, q, "retire", in)
|
|
expectWorkerCode(t, err, "RESOURCE_BUSY")
|
|
g := f.grant(sub.ID)
|
|
f.ack(sub.ID, g)
|
|
if _, err = f.events.Retire(ctx, f.token, sub.ID, "retire", events.RetireInput{ExpectedRevision: sub.Revision, Reason: "retire"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
retired, err := f.tasks.RetireQueue(ctx, f.token, q, "retire", in)
|
|
if err != nil || retired.Queue.RetiredAt == nil || !retired.Queue.Paused {
|
|
t.Fatal(retired, err)
|
|
}
|
|
replay, err := f.tasks.RetireQueue(ctx, f.token, q, "retire", in)
|
|
if err != nil || !replay.Deduplicated {
|
|
t.Fatal(replay, err)
|
|
}
|
|
_, err = f.tasks.ConfigureQueue(ctx, f.token, q, "resume", service.QueueConfigInput{ExpectedRevision: retired.Queue.Revision, Reason: "resume", MaxConcurrency: 1, RatePerSecond: 1, MaxUnfinishedTasks: 1})
|
|
expectWorkerCode(t, err, "RESOURCE_RETIRED")
|
|
_, err = f.tasks.Submit(ctx, f.token, "new-task", service.SubmitInput{QueueID: q, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)})
|
|
expectWorkerCode(t, err, "RESOURCE_RETIRED")
|
|
_, err = f.events.Create(ctx, f.token, "new-sub", events.CreateInput{Name: "new", QueueID: q})
|
|
expectWorkerCode(t, err, "RESOURCE_RETIRED")
|
|
if _, err = f.tasks.GetTask(ctx, f.token, task.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
other := newEventFixture(t, ctx, db)
|
|
_, err = other.events.Retire(ctx, other.token, sub.ID, "foreign", events.RetireInput{ExpectedRevision: 1, Reason: "foreign"})
|
|
expectWorkerCode(t, err, "NOT_FOUND")
|
|
})
|
|
t.Run("schedule_resume_cannot_revive_retired_queue", func(t *testing.T) {
|
|
f := newScheduleFixture(t, newExecutionFixture(t, ctx, db))
|
|
addRetirementPermission(t, f.executionFixture)
|
|
v := f.create(f.definition())
|
|
_, err := f.tasks.RetireQueue(ctx, f.token, f.resources.QueueID, "retire", service.RetireQueueInput{ExpectedRevision: 1, Reason: "retire"})
|
|
expectWorkerCode(t, err, "RESOURCE_BUSY")
|
|
pause, err := f.api.Control(ctx, f.token, v.ID, "pause", schedule.ControlInput{ExpectedRevision: v.Revision, Reason: "pause before retire"}, true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = f.tasks.RetireQueue(ctx, f.token, f.resources.QueueID, "retire", service.RetireQueueInput{ExpectedRevision: 1, Reason: "retire"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = f.api.Control(ctx, f.token, v.ID, "resume", schedule.ControlInput{ExpectedRevision: pause.Schedule.Revision, Reason: "resume"}, false)
|
|
expectWorkerCode(t, err, "RESOURCE_RETIRED")
|
|
if _, err = f.api.Get(ctx, f.token, v.ID); err != nil {
|
|
t.Fatal("schedule history", err)
|
|
}
|
|
})
|
|
t.Run("waiting_workflow_with_future_queue_reference_blocks_retirement", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
spec := runtimeSpec(f.resources.QueueID, 1)
|
|
spec.Body.Steps = append([]definition.Step{{Key: "wait", Kind: "TIMER", Timer: &definition.Timer{DelayMS: 60000}}}, spec.Body.Steps...)
|
|
runtimeStart(t, f, spec)
|
|
addRetirementPermission(t, f)
|
|
_, err := f.tasks.RetireQueue(ctx, f.token, f.resources.QueueID, "retire", service.RetireQueueInput{ExpectedRevision: 1, Reason: "retire"})
|
|
expectWorkerCode(t, err, "RESOURCE_BUSY")
|
|
})
|
|
t.Run("completed_workflow_receipt_survives_queue_retirement", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
grantWorkflowInstance(t, f)
|
|
input := publishInstanceSpec(t, f, runtimeSpec(f.resources.QueueID, 1))
|
|
api := instance.New(NewWorkflowInstanceStore(db))
|
|
key := string(executionID(t))
|
|
first, err := api.Start(ctx, f.token, key, input)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ref := retryStart(t, f)
|
|
retryFinish(t, f, ref, "SUCCEEDED", false)
|
|
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
|
runtimeView(t, f, first.Instance.ID, "SUCCEEDED")
|
|
addRetirementPermission(t, f)
|
|
if _, err = f.tasks.RetireQueue(ctx, f.token, f.resources.QueueID, "retire", service.RetireQueueInput{ExpectedRevision: 1, Reason: "completed workflow"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
replay, err := api.Start(ctx, f.token, key, input)
|
|
if err != nil || !replay.Deduplicated || replay.Instance.ID != first.Instance.ID {
|
|
t.Fatal("receipt after retirement", err)
|
|
}
|
|
if _, err = api.Get(ctx, f.token, first.Instance.ID); err != nil {
|
|
t.Fatal("workflow history", err)
|
|
}
|
|
_, err = api.Start(ctx, f.token, "new-instance", input)
|
|
expectWorkerCode(t, err, "RESOURCE_RETIRED")
|
|
})
|
|
t.Run("submit_and_retire_serialize_under_real_database_contention", func(t *testing.T) {
|
|
f := newExecutionFixture(t, ctx, db)
|
|
key := string(executionID(t))
|
|
q := f.resources.QueueID
|
|
results := competeWorkflow(t, f, nil, func() error {
|
|
_, e := f.tasks.Submit(ctx, f.token, key, service.SubmitInput{QueueID: q, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)})
|
|
return e
|
|
}, func() error {
|
|
_, e := f.tasks.RetireQueue(ctx, f.token, q, "retire", service.RetireQueueInput{ExpectedRevision: 1, Reason: "concurrent retirement"})
|
|
return e
|
|
})
|
|
if results[0] == nil {
|
|
expectWorkerCode(t, results[1], "RESOURCE_BUSY")
|
|
} else {
|
|
expectWorkerCode(t, results[0], "RESOURCE_RETIRED")
|
|
if results[1] != nil {
|
|
t.Fatal(results[1])
|
|
}
|
|
}
|
|
})
|
|
t.Run("audit_failure_rolls_back_retirement_and_quota", func(t *testing.T) {
|
|
f := newEventFixture(t, ctx, db)
|
|
sub := f.create(events.CreateInput{})
|
|
f.exec(`CREATE TRIGGER retirement_audit_reject BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='retirement audit unavailable'`)
|
|
defer db.ExecContext(ctx, `DROP TRIGGER IF EXISTS retirement_audit_reject`)
|
|
_, err := f.events.Retire(ctx, f.token, sub.ID, "retire", events.RetireInput{ExpectedRevision: sub.Revision, Reason: "audit"})
|
|
if err == nil {
|
|
t.Fatal("audit failure accepted")
|
|
}
|
|
if f.subscription(sub.ID).RetiredAt != nil {
|
|
t.Fatal("partial retirement committed")
|
|
}
|
|
})
|
|
var foreignKeys int
|
|
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&foreignKeys); err != nil || foreignKeys != 0 {
|
|
t.Fatal("foreign keys", foreignKeys, err)
|
|
}
|
|
}
|