package mysql import ( "context" "encoding/json" "fmt" "os" "testing" "time" "git.yuntuo.hk/k2safe/orbit/platform/auth" "git.yuntuo.hk/k2safe/orbit/platform/events" "git.yuntuo.hk/k2safe/orbit/pulse/schedule" "git.yuntuo.hk/k2safe/orbit/pulse/service" "git.yuntuo.hk/k2safe/orbit/weave/definition" "git.yuntuo.hk/k2safe/orbit/weave/instance" ) func addRetirementPermission(t *testing.T, f *executionFixture) { t.Helper() var raw []byte var p auth.Permissions if err := f.db.QueryRowContext(f.ctx, `SELECT permissions FROM orbit_credentials WHERE namespace_id=? AND id=?`, f.token.NamespaceID, f.token.CredentialID).Scan(&raw); err != nil { t.Fatal(err) } if err := json.Unmarshal(raw, &p); err != nil { t.Fatal(err) } p.Actions = append(p.Actions, "queues:manage", "queues:read") raw, _ = json.Marshal(p) f.exec(`UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?`, string(raw), f.token.NamespaceID, f.token.CredentialID) } func TestIntegrationResourceRetirement(t *testing.T) { dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN") if dsn == "" { t.Skip("requires disposable MySQL") } ctx, stop := context.WithTimeout(context.Background(), 90*time.Second) defer stop() db, err := Open(ctx, dsn, 32, 8) if err != nil { t.Fatal(err) } defer db.Close() if err = Migrate(ctx, db); err != nil { t.Fatal(err) } t.Run("no_lost_backlog_leases_or_history", func(t *testing.T) { f := newEventFixture(t, ctx, db) sub := f.create(events.CreateInput{MaxAttempts: 1}) task := f.cancelResult() in := events.RetireInput{ExpectedRevision: sub.Revision, Reason: "completed temporary resources"} _, err := f.events.Retire(ctx, f.token, sub.ID, "retire", in) expectWorkerCode(t, err, "RESOURCE_BUSY") g := f.grant(sub.ID) _, err = f.events.Retire(ctx, f.token, sub.ID, "retire", in) expectWorkerCode(t, err, "RESOURCE_BUSY") f.expireDelivery(g) noEventGrant(t, f, sub.ID) _, err = f.events.Retire(ctx, f.token, sub.ID, "retire", in) expectWorkerCode(t, err, "RESOURCE_BUSY") f.ack(sub.ID, g) done, err := f.events.Retire(ctx, f.token, sub.ID, "retire", in) if err != nil || done.Subscription.RetiredAt == nil || done.Subscription.RetiredSequence == nil || done.Subscription.Enabled { t.Fatalf("retire: %+v %v", done, err) } again, err := f.events.Retire(ctx, f.token, sub.ID, "retire", in) if err != nil || !again.Deduplicated || again.Subscription.Revision != done.Subscription.Revision { t.Fatal(again, err) } changed := in changed.Reason = "other" _, err = f.events.Retire(ctx, f.token, sub.ID, "retire", changed) expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT") _, err = f.events.Control(ctx, f.token, sub.ID, "enable", events.ControlInput{ExpectedRevision: done.Subscription.Revision, Enabled: true, Reason: "attempt resume"}) expectWorkerCode(t, err, "RESOURCE_RETIRED") detail, err := f.events.Detail(ctx, f.token, sub.ID, g.Delivery.ID) if err != nil || detail.Event.AggregateID != task.ID { t.Fatal("history unavailable", err) } f.cancelResult() noEventGrant(t, f, sub.ID) if f.count("orbit_audit_events", "action='subscription.retire'") != 1 { t.Fatal("retirement audit missing or duplicated") } }) t.Run("paused_counts_retired_releases_capacity", func(t *testing.T) { f := newEventFixture(t, ctx, db) sub := f.create(events.CreateInput{}) paused, err := f.events.Control(ctx, f.token, sub.ID, "pause", events.ControlInput{ExpectedRevision: sub.Revision, Enabled: false, Reason: "pause"}) if err != nil { t.Fatal(err) } for i := 1; i < events.MaxSubscriptions; i++ { f.create(events.CreateInput{Name: fmt.Sprintf("capacity-%d", i)}) } _, err = f.events.Create(ctx, f.token, "next", events.CreateInput{Name: "next", QueueID: f.resources.QueueID}) expectWorkerCode(t, err, "SUBSCRIPTION_LIMIT") if _, err = f.events.Retire(ctx, f.token, sub.ID, "retire", events.RetireInput{ExpectedRevision: paused.Subscription.Revision, Reason: "retire paused"}); err != nil { t.Fatal(err) } if _, err = f.events.Create(ctx, f.token, "next", events.CreateInput{Name: "next", QueueID: f.resources.QueueID}); err != nil { t.Fatal(err) } if f.count("orbit_subscriptions", "1=1") != 65 { t.Fatal("history was removed") } }) t.Run("queue_protects_tasks_subscriptions_and_original_receipts", func(t *testing.T) { f := newEventFixture(t, ctx, db) q := f.resources.QueueID sub := f.create(events.CreateInput{}) task := f.submit(1) in := service.RetireQueueInput{ExpectedRevision: 1, Reason: "queue lifecycle"} _, err := f.tasks.RetireQueue(ctx, f.token, q, "retire", in) expectWorkerCode(t, err, "RESOURCE_BUSY") if _, err = f.tasks.CancelTask(ctx, f.token, task.ID, "cancel", service.CancelInput{ExpectedRevision: task.Revision, Reason: "finish test"}); err != nil { t.Fatal(err) } _, err = f.tasks.RetireQueue(ctx, f.token, q, "retire", in) expectWorkerCode(t, err, "RESOURCE_BUSY") g := f.grant(sub.ID) f.ack(sub.ID, g) if _, err = f.events.Retire(ctx, f.token, sub.ID, "retire", events.RetireInput{ExpectedRevision: sub.Revision, Reason: "retire"}); err != nil { t.Fatal(err) } retired, err := f.tasks.RetireQueue(ctx, f.token, q, "retire", in) if err != nil || retired.Queue.RetiredAt == nil || !retired.Queue.Paused { t.Fatal(retired, err) } replay, err := f.tasks.RetireQueue(ctx, f.token, q, "retire", in) if err != nil || !replay.Deduplicated { t.Fatal(replay, err) } _, err = f.tasks.ConfigureQueue(ctx, f.token, q, "resume", service.QueueConfigInput{ExpectedRevision: retired.Queue.Revision, Reason: "resume", MaxConcurrency: 1, RatePerSecond: 1, MaxUnfinishedTasks: 1}) expectWorkerCode(t, err, "RESOURCE_RETIRED") _, err = f.tasks.Submit(ctx, f.token, "new-task", service.SubmitInput{QueueID: q, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)}) expectWorkerCode(t, err, "RESOURCE_RETIRED") _, err = f.events.Create(ctx, f.token, "new-sub", events.CreateInput{Name: "new", QueueID: q}) expectWorkerCode(t, err, "RESOURCE_RETIRED") if _, err = f.tasks.GetTask(ctx, f.token, task.ID); err != nil { t.Fatal(err) } other := newEventFixture(t, ctx, db) _, err = other.events.Retire(ctx, other.token, sub.ID, "foreign", events.RetireInput{ExpectedRevision: 1, Reason: "foreign"}) expectWorkerCode(t, err, "NOT_FOUND") }) t.Run("schedule_resume_cannot_revive_retired_queue", func(t *testing.T) { f := newScheduleFixture(t, newExecutionFixture(t, ctx, db)) addRetirementPermission(t, f.executionFixture) v := f.create(f.definition()) _, err := f.tasks.RetireQueue(ctx, f.token, f.resources.QueueID, "retire", service.RetireQueueInput{ExpectedRevision: 1, Reason: "retire"}) expectWorkerCode(t, err, "RESOURCE_BUSY") pause, err := f.api.Control(ctx, f.token, v.ID, "pause", schedule.ControlInput{ExpectedRevision: v.Revision, Reason: "pause before retire"}, true) if err != nil { t.Fatal(err) } if _, err = f.tasks.RetireQueue(ctx, f.token, f.resources.QueueID, "retire", service.RetireQueueInput{ExpectedRevision: 1, Reason: "retire"}); err != nil { t.Fatal(err) } _, err = f.api.Control(ctx, f.token, v.ID, "resume", schedule.ControlInput{ExpectedRevision: pause.Schedule.Revision, Reason: "resume"}, false) expectWorkerCode(t, err, "RESOURCE_RETIRED") if _, err = f.api.Get(ctx, f.token, v.ID); err != nil { t.Fatal("schedule history", err) } }) t.Run("waiting_workflow_with_future_queue_reference_blocks_retirement", func(t *testing.T) { f := newExecutionFixture(t, ctx, db) spec := runtimeSpec(f.resources.QueueID, 1) spec.Body.Steps = append([]definition.Step{{Key: "wait", Kind: "TIMER", Timer: &definition.Timer{DelayMS: 60000}}}, spec.Body.Steps...) runtimeStart(t, f, spec) addRetirementPermission(t, f) _, err := f.tasks.RetireQueue(ctx, f.token, f.resources.QueueID, "retire", service.RetireQueueInput{ExpectedRevision: 1, Reason: "retire"}) expectWorkerCode(t, err, "RESOURCE_BUSY") }) t.Run("completed_workflow_receipt_survives_queue_retirement", func(t *testing.T) { f := newExecutionFixture(t, ctx, db) grantWorkflowInstance(t, f) input := publishInstanceSpec(t, f, runtimeSpec(f.resources.QueueID, 1)) api := instance.New(NewWorkflowInstanceStore(db)) key := string(executionID(t)) first, err := api.Start(ctx, f.token, key, input) if err != nil { t.Fatal(err) } ref := retryStart(t, f) retryFinish(t, f, ref, "SUCCEEDED", false) runtimeConsume(t, f, newRuntime(t, db), ref.TaskID) runtimeView(t, f, first.Instance.ID, "SUCCEEDED") addRetirementPermission(t, f) if _, err = f.tasks.RetireQueue(ctx, f.token, f.resources.QueueID, "retire", service.RetireQueueInput{ExpectedRevision: 1, Reason: "completed workflow"}); err != nil { t.Fatal(err) } replay, err := api.Start(ctx, f.token, key, input) if err != nil || !replay.Deduplicated || replay.Instance.ID != first.Instance.ID { t.Fatal("receipt after retirement", err) } if _, err = api.Get(ctx, f.token, first.Instance.ID); err != nil { t.Fatal("workflow history", err) } _, err = api.Start(ctx, f.token, "new-instance", input) expectWorkerCode(t, err, "RESOURCE_RETIRED") }) t.Run("submit_and_retire_serialize_under_real_database_contention", func(t *testing.T) { f := newExecutionFixture(t, ctx, db) key := string(executionID(t)) q := f.resources.QueueID results := competeWorkflow(t, f, nil, func() error { _, e := f.tasks.Submit(ctx, f.token, key, service.SubmitInput{QueueID: q, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)}) return e }, func() error { _, e := f.tasks.RetireQueue(ctx, f.token, q, "retire", service.RetireQueueInput{ExpectedRevision: 1, Reason: "concurrent retirement"}) return e }) if results[0] == nil { expectWorkerCode(t, results[1], "RESOURCE_BUSY") } else { expectWorkerCode(t, results[0], "RESOURCE_RETIRED") if results[1] != nil { t.Fatal(results[1]) } } }) t.Run("audit_failure_rolls_back_retirement_and_quota", func(t *testing.T) { f := newEventFixture(t, ctx, db) sub := f.create(events.CreateInput{}) f.exec(`CREATE TRIGGER retirement_audit_reject BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='retirement audit unavailable'`) defer db.ExecContext(ctx, `DROP TRIGGER IF EXISTS retirement_audit_reject`) _, err := f.events.Retire(ctx, f.token, sub.ID, "retire", events.RetireInput{ExpectedRevision: sub.Revision, Reason: "audit"}) if err == nil { t.Fatal("audit failure accepted") } if f.subscription(sub.ID).RetiredAt != nil { t.Fatal("partial retirement committed") } }) var foreignKeys int if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&foreignKeys); err != nil || foreignKeys != 0 { t.Fatal("foreign keys", foreignKeys, err) } }