feat: add generic queue and subscription retirement

This commit is contained in:
2026-09-27 14:39:49 +08:00
parent d54bdd9847
commit c9e877c3f7
53 changed files with 1013 additions and 88 deletions

View File

@@ -63,6 +63,8 @@ func TestIntegrationApplicationResourcesUpgrade(t *testing.T) {
finished := retryStart(t, g)
retryFinish(t, g, finished, "SUCCEEDED", false)
assertApplicationCounts(t, f, 4, 2, 1)
f.exec(`ALTER TABLE orbit_subscriptions DROP CHECK ck_subscription_retired,DROP INDEX ix_subscription_active,DROP COLUMN retired_at,DROP COLUMN retired_sequence`)
f.exec(`ALTER TABLE pulse_queues DROP CHECK ck_queue_retired,DROP COLUMN retired_at`)
// 还原后续新增表、字段和索引,得到真实 v9 结构;现有任务内容保持不变。
f.exec(`ALTER TABLE orbit_events DROP INDEX ix_event_workflow_history,DROP COLUMN workflow_id`)
f.exec(`ALTER TABLE pulse_tasks DROP INDEX ix_task_observe_state_available`)

View File

@@ -933,3 +933,13 @@ CREATE TABLE orbit_archive_jobs (
CONSTRAINT ck_archive_job_config CHECK(revision>0 AND plan_revision>0 AND policy_revision>0 AND batch_size BETWEEN 1 AND 100 AND max_bytes BETWEEN 1048576 AND 1073741824),
CONSTRAINT ck_archive_job_lease CHECK((state='RUNNING' AND lease_id IS NOT NULL AND lease_until IS NOT NULL) OR (state<>'RUNNING' AND lease_id IS NULL AND lease_until IS NULL))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='持久归档作业及租约,只读源数据,无外键';
-- 通用资源退役,只关闭未来工作并释放活动订阅名额,不删除历史或审计;禁止外键。
ALTER TABLE pulse_queues
ADD retired_at DATETIME(6) NULL COMMENT '永久退役时间,UTC;NULL表示未退役,退役后禁止新工作且不可恢复',
ADD CONSTRAINT ck_queue_retired CHECK (retired_at IS NULL OR paused=TRUE);
ALTER TABLE orbit_subscriptions
ADD retired_at DATETIME(6) NULL COMMENT '永久退役时间,UTC;NULL表示仍占用订阅名额,退役保留历史交付',
ADD retired_sequence BIGINT UNSIGNED NULL COMMENT '退役事务锁定的结果流提交水位;NULL表示未退役,之后的事件不再订阅',
ADD KEY ix_subscription_active(namespace_id,retired_at),
ADD CONSTRAINT ck_subscription_retired CHECK ((retired_at IS NULL AND retired_sequence IS NULL) OR (retired_at IS NOT NULL AND retired_sequence IS NOT NULL AND enabled=FALSE AND pending_count=0));

View File

@@ -13,6 +13,7 @@ import (
"git.yuntuo.hk/k2safe/orbit/internal/fault"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/platform/auth"
"git.yuntuo.hk/k2safe/orbit/platform/events"
"git.yuntuo.hk/k2safe/orbit/pulse/service"
driver "github.com/go-sql-driver/mysql"
)
@@ -106,13 +107,13 @@ func (s *ConsoleApplicationStore) GetApplication(ctx context.Context, token auth
if err != nil {
return err
}
rows, err := tx.QueryContext(ctx, `SELECT id,name FROM pulse_queues WHERE namespace_id=? ORDER BY id LIMIT 101`, namespace)
rows, err := tx.QueryContext(ctx, `SELECT id,name,paused,retired_at FROM pulse_queues WHERE namespace_id=? ORDER BY (retired_at IS NOT NULL),paused,created_at,id LIMIT 101`, namespace)
if err != nil {
return err
}
for rows.Next() {
var q consoleauth.AccessQueue
if err := rows.Scan(&q.ID, &q.Name); err != nil {
if err := rows.Scan(&q.ID, &q.Name, &q.Paused, &q.RetiredAt); err != nil {
rows.Close()
return err
}
@@ -123,6 +124,10 @@ func (s *ConsoleApplicationStore) GetApplication(ctx context.Context, token auth
if err != nil {
return err
}
out.SubscriptionLimit = events.MaxSubscriptions
if err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM orbit_subscriptions WHERE namespace_id=? AND retired_at IS NULL`, namespace).Scan(&out.ActiveSubscriptions); err != nil {
return err
}
out.TokenVaultEnabled = s.vault != nil
rows, err = tx.QueryContext(ctx, `SELECT id,name,revision,expires_at,revoked_at,previous_valid_until,permissions,secret_ciphertext IS NOT NULL FROM orbit_credentials WHERE namespace_id=? ORDER BY id LIMIT 101`, namespace)
if err != nil {

View File

@@ -90,7 +90,7 @@ func (s *ConsoleApplicationStore) GetScope(ctx context.Context, t auth.Token, af
if err != nil {
return err
}
query := `SELECT id,name FROM pulse_queues WHERE namespace_id=?`
query := `SELECT id,name,paused,retired_at FROM pulse_queues WHERE namespace_id=?`
args := []any{t.NamespaceID}
if after != "" {
query += ` AND id>?`
@@ -120,7 +120,7 @@ func (s *ConsoleApplicationStore) GetScope(ctx context.Context, t auth.Token, af
defer rows.Close()
for rows.Next() {
var q consoleauth.AccessQueue
if err := rows.Scan(&q.ID, &q.Name); err != nil {
if err := rows.Scan(&q.ID, &q.Name, &q.Paused, &q.RetiredAt); err != nil {
return err
}
out.Queues = append(out.Queues, q)

View File

@@ -24,7 +24,7 @@ func NewEventStore(db *sql.DB) *EventStore { return &EventStore{NewTaskStore(db)
var _ events.Store = (*EventStore)(nil)
const subscriptionColumns = `id,queue_id,credential_id,name,revision,enabled,filter,cursor_sequence,pending_count,max_pending,max_attempts,created_at,updated_at`
const subscriptionColumns = `id,queue_id,credential_id,name,revision,enabled,filter,cursor_sequence,pending_count,max_pending,max_attempts,created_at,updated_at,retired_at,retired_sequence`
func readSubscription(ctx context.Context, tx *sql.Tx, t auth.Token, p auth.Permissions, id core.ID, lock bool) (events.Subscription, error) {
q := `SELECT ` + subscriptionColumns + ` FROM orbit_subscriptions WHERE namespace_id=? AND id=? AND credential_id=? AND queue_id IS NOT NULL`
@@ -52,7 +52,7 @@ func readSubscription(ctx context.Context, tx *sql.Tx, t auth.Token, p auth.Perm
func scanSubscription(row scanner) (events.Subscription, error) {
var out events.Subscription
var raw []byte
err := row.Scan(&out.ID, &out.QueueID, &out.CredentialID, &out.Name, &out.Revision, &out.Enabled, &raw, &out.CursorSequence, &out.PendingCount, &out.MaxPending, &out.MaxAttempts, &out.CreatedAt, &out.UpdatedAt)
err := row.Scan(&out.ID, &out.QueueID, &out.CredentialID, &out.Name, &out.Revision, &out.Enabled, &raw, &out.CursorSequence, &out.PendingCount, &out.MaxPending, &out.MaxAttempts, &out.CreatedAt, &out.UpdatedAt, &out.RetiredAt, &out.RetiredSequence)
if err != nil {
return out, err
}
@@ -87,6 +87,9 @@ func (s *EventStore) CreateSubscription(ctx context.Context, t auth.Token, key s
} else if err != nil {
return err
}
if err := requireActiveQueue(ctx, tx, t.NamespaceID, in.QueueID); err != nil {
return err
}
head, retained, err := lockEventStream(ctx, tx, t.NamespaceID)
if err != nil {
return err
@@ -102,7 +105,7 @@ func (s *EventStore) CreateSubscription(ctx context.Context, t auth.Token, key s
return events.HistoryExpired
}
var count int
if err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM orbit_subscriptions WHERE namespace_id=?`, t.NamespaceID).Scan(&count); err != nil {
if err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM orbit_subscriptions WHERE namespace_id=? AND retired_at IS NULL`, t.NamespaceID).Scan(&count); err != nil {
return err
}
if count >= events.MaxSubscriptions {
@@ -158,6 +161,9 @@ func (s *EventStore) ControlSubscription(ctx context.Context, t auth.Token, id c
out.Deduplicated = true
return nil
}
if before.RetiredAt != nil {
return fault.Retired
}
if before.Revision != in.ExpectedRevision || before.Revision == math.MaxUint64 {
return fault.Conflict
}

View File

@@ -26,7 +26,7 @@ var migrationFiles embed.FS
//go:embed baseline.sql
var freshInstallBaseline string
const freshInstallBaselineVersion uint64 = 33
const freshInstallBaselineVersion uint64 = 34
type Migration struct {
Version uint64

View File

@@ -0,0 +1,9 @@
-- 通用资源退役,只关闭未来工作并释放活动订阅名额,不删除历史或审计;禁止外键。
ALTER TABLE pulse_queues
ADD retired_at DATETIME(6) NULL COMMENT '永久退役时间,UTC;NULL表示未退役,退役后禁止新工作且不可恢复',
ADD CONSTRAINT ck_queue_retired CHECK (retired_at IS NULL OR paused=TRUE);
ALTER TABLE orbit_subscriptions
ADD retired_at DATETIME(6) NULL COMMENT '永久退役时间,UTC;NULL表示仍占用订阅名额,退役保留历史交付',
ADD retired_sequence BIGINT UNSIGNED NULL COMMENT '退役事务锁定的结果流提交水位;NULL表示未退役,之后的事件不再订阅',
ADD KEY ix_subscription_active(namespace_id,retired_at),
ADD CONSTRAINT ck_subscription_retired CHECK ((retired_at IS NULL AND retired_sequence IS NULL) OR (retired_at IS NOT NULL AND retired_sequence IS NOT NULL AND enabled=FALSE AND pending_count=0));

View File

@@ -147,7 +147,7 @@ func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
indexes := func() []string {
t.Helper()
// 排除后续新增/替换索引;原有项在此逐项比较,v24/v25新增项由对应Query/Timeline升级测试核对。
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events','orbit_console_users','orbit_console_sessions','orbit_retention_policies','orbit_archive_jobs','orbit_archive_plans','weave_version_scopes') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result','ix_signal_expiry','ix_signal_activation','uq_wait_winner_signal','ix_task_workflow_owner','ix_signal_history','ix_event_workflow_history','ix_workflow_created','ix_workflow_state_created','ix_workflow_business_state_created','ix_workflow_definition_created','ix_workflow_definition_state_created','ix_workflow_definition_business_created','ix_workflow_definition_business_state_created','ix_task_observe_state_available','ix_delivery_observe_state_available','ix_workflow_observe_state_updated','ix_event_observe_type_occurred','ix_task_observe_terminal_completed','ix_workflow_observe_terminal_completed') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events','orbit_console_users','orbit_console_sessions','orbit_retention_policies','orbit_archive_jobs','orbit_archive_plans','weave_version_scopes') AND INDEX_NAME NOT IN ('ix_subscription_active','ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result','ix_signal_expiry','ix_signal_activation','uq_wait_winner_signal','ix_task_workflow_owner','ix_signal_history','ix_event_workflow_history','ix_workflow_created','ix_workflow_state_created','ix_workflow_business_state_created','ix_workflow_definition_created','ix_workflow_definition_state_created','ix_workflow_definition_business_created','ix_workflow_definition_business_state_created','ix_task_observe_state_available','ix_delivery_observe_state_available','ix_workflow_observe_state_updated','ix_event_observe_type_occurred','ix_task_observe_terminal_completed','ix_workflow_observe_terminal_completed') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
if err != nil {
t.Fatal(err)
}

View File

@@ -0,0 +1,175 @@
package mysql
import (
"context"
"database/sql"
"encoding/json"
"errors"
"math"
"time"
"git.yuntuo.hk/k2safe/orbit/internal/fault"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/platform/auth"
"git.yuntuo.hk/k2safe/orbit/platform/events"
"git.yuntuo.hk/k2safe/orbit/pulse/service"
)
// requireActiveQueue 对创建/恢复工作取得共享锁,与退役排他锁串行;历史读取不使用此检查。
func requireActiveQueue(ctx context.Context, tx *sql.Tx, ns, id core.ID) error {
var retired *time.Time
err := tx.QueryRowContext(ctx, `SELECT retired_at FROM pulse_queues WHERE namespace_id=? AND id=? FOR SHARE`, ns, id).Scan(&retired)
if errors.Is(err, sql.ErrNoRows) {
return fault.NotFound
}
if err != nil {
return err
}
if retired != nil {
return fault.Retired
}
return nil
}
func retirementBusy(message string) error { return fault.New("RESOURCE_BUSY", message) }
// RetireSubscription 锁定提交水位后锁订阅,既不丢未物化结果,也不丢过期交付。
// 与创建共用流锁串行化活动名额;消费只锁订阅,不反向锁流/队列。
func (s *EventStore) RetireSubscription(ctx context.Context, t auth.Token, id core.ID, key string, in events.RetireInput) (events.SubscriptionResult, error) {
var out events.SubscriptionResult
err := s.transaction(ctx, t, "events:manage", func(tx *sql.Tx, p auth.Permissions) error {
if _, err := readSubscription(ctx, tx, t, p, id, false); err != nil {
return err
}
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "subscription.retire", eventScope(t.CredentialID, id), key, "subscription", id, in)
if err != nil {
return err
}
if cached != nil {
if err = json.Unmarshal(cached, &out); err != nil {
return err
}
out.Deduplicated = true
return nil
}
head, retained, err := lockEventStream(ctx, tx, t.NamespaceID)
if err != nil {
return err
}
before, err := readSubscription(ctx, tx, t, p, id, true)
if err != nil {
return err
}
if before.RetiredAt != nil {
return fault.Retired
}
if before.Revision != in.ExpectedRevision || before.Revision == math.MaxUint64 {
return fault.Conflict
}
if before.PendingCount != 0 {
return retirementBusy("subscription has unacknowledged deliveries; drain or resolve expired deliveries first")
}
var pending bool
if err = tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM orbit_deliveries WHERE namespace_id=? AND subscription_id=? AND state IN ('PENDING','LEASED','EXPIRED'))`, t.NamespaceID, id).Scan(&pending); err != nil {
return err
}
if pending {
return retirementBusy("subscription has unacknowledged deliveries")
}
if before.CursorSequence < retained-1 {
return events.HistoryExpired
}
filter, err := json.Marshal(before.Types)
if err != nil {
return err
}
// 缺失来源事件也拒绝退役,不能将损坏记录误当成不匹配过滤器。
if err = tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM orbit_result_events r LEFT JOIN orbit_events e ON e.namespace_id=r.namespace_id AND e.id=r.id WHERE r.namespace_id=? AND r.queue_id=? AND r.sequence>? AND r.sequence<=? AND (e.id IS NULL OR JSON_CONTAINS(?,JSON_QUOTE(e.type))))`, t.NamespaceID, before.QueueID, before.CursorSequence, head, string(filter)).Scan(&pending); err != nil {
return err
}
if pending {
return retirementBusy("subscription has unconsumed results; resume consumption and acknowledge them before retirement")
}
if _, err = tx.ExecContext(ctx, `UPDATE orbit_subscriptions SET enabled=FALSE,retired_at=UTC_TIMESTAMP(6),retired_sequence=?,cursor_sequence=?,revision=revision+1,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, head, head, t.NamespaceID, id); err != nil {
return err
}
out.Subscription, err = readSubscription(ctx, tx, t, p, id, false)
if err != nil {
return err
}
if err = appendAudit(ctx, tx, t, identity, id, "subscription", "subscription.retire", in.Reason, before, out.Subscription); err != nil {
return err
}
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
})
return out, err
}
// RetireQueue 按应用额度→身份→队列持锁,拒绝尚未结清的工作和未来工作来源。
// 暂停计划可以保留;其恢复/发布与本操作共用队列锁,退役后无法重新激活。
func (s *TaskStore) RetireQueue(ctx context.Context, t auth.Token, id core.ID, key string, in service.RetireQueueInput) (service.QueueResult, error) {
var out service.QueueResult
err := s.transaction(ctx, t, "queues:manage", func(tx *sql.Tx, p auth.Permissions) error {
if !p.AllowsQueue(id) {
return fault.NotFound
}
if _, err := lockApplicationResources(ctx, tx, t.NamespaceID, false); err != nil {
return err
}
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "queue.retire", string(id), key, "queue", id, in)
if err != nil {
return err
}
if cached != nil {
if err = json.Unmarshal(cached, &out); err != nil {
return err
}
out.Deduplicated = true
return nil
}
if _, _, err = lockQueueAfterApplication(ctx, tx, t.NamespaceID, id); err != nil {
return err
}
before, err := readQueue(ctx, tx, t.NamespaceID, id)
if err != nil {
return err
}
if before.Revision != in.ExpectedRevision || before.Revision == math.MaxUint64 {
return fault.Conflict
}
if before.UnfinishedTasks != 0 {
return retirementBusy("queue has unfinished tasks, including tasks needing review")
}
checks := []struct {
query, message string
args []any
}{
{`SELECT EXISTS(SELECT 1 FROM pulse_tasks WHERE namespace_id=? AND queue_id=? AND state IN ('WAITING','LEASED','RUNNING','NEEDS_REVIEW'))`, "queue has unfinished or uncertain tasks", []any{t.NamespaceID, id}},
{`SELECT EXISTS(SELECT 1 FROM pulse_attempts a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id WHERE t.namespace_id=? AND t.queue_id=? AND a.state IN ('LEASED','RUNNING'))`, "queue has active execution attempts", []any{t.NamespaceID, id}},
{`SELECT EXISTS(SELECT 1 FROM pulse_permits p JOIN pulse_attempts a ON a.namespace_id=p.namespace_id AND a.id=p.attempt_id JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id WHERE t.namespace_id=? AND t.queue_id=? AND p.released_at IS NULL)`, "queue has unreleased execution permits", []any{t.NamespaceID, id}},
{`SELECT EXISTS(SELECT 1 FROM pulse_schedules WHERE namespace_id=? AND queue_id=? AND paused=FALSE)`, "queue has enabled schedules; pause them first", []any{t.NamespaceID, id}},
{`SELECT EXISTS(SELECT 1 FROM weave_instances i LEFT JOIN weave_version_scopes v ON v.namespace_id=i.namespace_id AND v.definition_version_id=i.definition_version_id WHERE i.namespace_id=? AND i.state IN ('RUNNING','WAITING','CANCELLING','NEEDS_REVIEW') AND (v.definition_version_id IS NULL OR JSON_CONTAINS(v.queue_ids,JSON_QUOTE(?))))`, "queue is referenced by unfinished workflows", []any{t.NamespaceID, string(id)}},
{`SELECT EXISTS(SELECT 1 FROM orbit_subscriptions WHERE namespace_id=? AND queue_id=? AND retired_at IS NULL)`, "queue has subscriptions; retire them first", []any{t.NamespaceID, id}},
}
for _, check := range checks {
var busy bool
if err = tx.QueryRowContext(ctx, check.query, check.args...).Scan(&busy); err != nil {
return err
}
if busy {
return retirementBusy(check.message)
}
}
if _, err = tx.ExecContext(ctx, `UPDATE pulse_queues SET paused=TRUE,retired_at=UTC_TIMESTAMP(6),revision=revision+1,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, t.NamespaceID, id); err != nil {
return err
}
out.Queue, err = readQueue(ctx, tx, t.NamespaceID, id)
if err != nil {
return err
}
if err = appendAudit(ctx, tx, t, identity, id, "queue", "queue.retire", in.Reason, before, out.Queue); err != nil {
return err
}
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
})
return out, err
}

View File

@@ -0,0 +1,241 @@
package mysql
import (
"context"
"encoding/json"
"fmt"
"os"
"testing"
"time"
"git.yuntuo.hk/k2safe/orbit/platform/auth"
"git.yuntuo.hk/k2safe/orbit/platform/events"
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
"git.yuntuo.hk/k2safe/orbit/pulse/service"
"git.yuntuo.hk/k2safe/orbit/weave/definition"
"git.yuntuo.hk/k2safe/orbit/weave/instance"
)
func addRetirementPermission(t *testing.T, f *executionFixture) {
t.Helper()
var raw []byte
var p auth.Permissions
if err := f.db.QueryRowContext(f.ctx, `SELECT permissions FROM orbit_credentials WHERE namespace_id=? AND id=?`, f.token.NamespaceID, f.token.CredentialID).Scan(&raw); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(raw, &p); err != nil {
t.Fatal(err)
}
p.Actions = append(p.Actions, "queues:manage", "queues:read")
raw, _ = json.Marshal(p)
f.exec(`UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?`, string(raw), f.token.NamespaceID, f.token.CredentialID)
}
func TestIntegrationResourceRetirement(t *testing.T) {
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("requires disposable MySQL")
}
ctx, stop := context.WithTimeout(context.Background(), 90*time.Second)
defer stop()
db, err := Open(ctx, dsn, 32, 8)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if err = Migrate(ctx, db); err != nil {
t.Fatal(err)
}
t.Run("no_lost_backlog_leases_or_history", func(t *testing.T) {
f := newEventFixture(t, ctx, db)
sub := f.create(events.CreateInput{MaxAttempts: 1})
task := f.cancelResult()
in := events.RetireInput{ExpectedRevision: sub.Revision, Reason: "completed temporary resources"}
_, err := f.events.Retire(ctx, f.token, sub.ID, "retire", in)
expectWorkerCode(t, err, "RESOURCE_BUSY")
g := f.grant(sub.ID)
_, err = f.events.Retire(ctx, f.token, sub.ID, "retire", in)
expectWorkerCode(t, err, "RESOURCE_BUSY")
f.expireDelivery(g)
noEventGrant(t, f, sub.ID)
_, err = f.events.Retire(ctx, f.token, sub.ID, "retire", in)
expectWorkerCode(t, err, "RESOURCE_BUSY")
f.ack(sub.ID, g)
done, err := f.events.Retire(ctx, f.token, sub.ID, "retire", in)
if err != nil || done.Subscription.RetiredAt == nil || done.Subscription.RetiredSequence == nil || done.Subscription.Enabled {
t.Fatalf("retire: %+v %v", done, err)
}
again, err := f.events.Retire(ctx, f.token, sub.ID, "retire", in)
if err != nil || !again.Deduplicated || again.Subscription.Revision != done.Subscription.Revision {
t.Fatal(again, err)
}
changed := in
changed.Reason = "other"
_, err = f.events.Retire(ctx, f.token, sub.ID, "retire", changed)
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
_, err = f.events.Control(ctx, f.token, sub.ID, "enable", events.ControlInput{ExpectedRevision: done.Subscription.Revision, Enabled: true, Reason: "attempt resume"})
expectWorkerCode(t, err, "RESOURCE_RETIRED")
detail, err := f.events.Detail(ctx, f.token, sub.ID, g.Delivery.ID)
if err != nil || detail.Event.AggregateID != task.ID {
t.Fatal("history unavailable", err)
}
f.cancelResult()
noEventGrant(t, f, sub.ID)
if f.count("orbit_audit_events", "action='subscription.retire'") != 1 {
t.Fatal("retirement audit missing or duplicated")
}
})
t.Run("paused_counts_retired_releases_capacity", func(t *testing.T) {
f := newEventFixture(t, ctx, db)
sub := f.create(events.CreateInput{})
paused, err := f.events.Control(ctx, f.token, sub.ID, "pause", events.ControlInput{ExpectedRevision: sub.Revision, Enabled: false, Reason: "pause"})
if err != nil {
t.Fatal(err)
}
for i := 1; i < events.MaxSubscriptions; i++ {
f.create(events.CreateInput{Name: fmt.Sprintf("capacity-%d", i)})
}
_, err = f.events.Create(ctx, f.token, "next", events.CreateInput{Name: "next", QueueID: f.resources.QueueID})
expectWorkerCode(t, err, "SUBSCRIPTION_LIMIT")
if _, err = f.events.Retire(ctx, f.token, sub.ID, "retire", events.RetireInput{ExpectedRevision: paused.Subscription.Revision, Reason: "retire paused"}); err != nil {
t.Fatal(err)
}
if _, err = f.events.Create(ctx, f.token, "next", events.CreateInput{Name: "next", QueueID: f.resources.QueueID}); err != nil {
t.Fatal(err)
}
if f.count("orbit_subscriptions", "1=1") != 65 {
t.Fatal("history was removed")
}
})
t.Run("queue_protects_tasks_subscriptions_and_original_receipts", func(t *testing.T) {
f := newEventFixture(t, ctx, db)
q := f.resources.QueueID
sub := f.create(events.CreateInput{})
task := f.submit(1)
in := service.RetireQueueInput{ExpectedRevision: 1, Reason: "queue lifecycle"}
_, err := f.tasks.RetireQueue(ctx, f.token, q, "retire", in)
expectWorkerCode(t, err, "RESOURCE_BUSY")
if _, err = f.tasks.CancelTask(ctx, f.token, task.ID, "cancel", service.CancelInput{ExpectedRevision: task.Revision, Reason: "finish test"}); err != nil {
t.Fatal(err)
}
_, err = f.tasks.RetireQueue(ctx, f.token, q, "retire", in)
expectWorkerCode(t, err, "RESOURCE_BUSY")
g := f.grant(sub.ID)
f.ack(sub.ID, g)
if _, err = f.events.Retire(ctx, f.token, sub.ID, "retire", events.RetireInput{ExpectedRevision: sub.Revision, Reason: "retire"}); err != nil {
t.Fatal(err)
}
retired, err := f.tasks.RetireQueue(ctx, f.token, q, "retire", in)
if err != nil || retired.Queue.RetiredAt == nil || !retired.Queue.Paused {
t.Fatal(retired, err)
}
replay, err := f.tasks.RetireQueue(ctx, f.token, q, "retire", in)
if err != nil || !replay.Deduplicated {
t.Fatal(replay, err)
}
_, err = f.tasks.ConfigureQueue(ctx, f.token, q, "resume", service.QueueConfigInput{ExpectedRevision: retired.Queue.Revision, Reason: "resume", MaxConcurrency: 1, RatePerSecond: 1, MaxUnfinishedTasks: 1})
expectWorkerCode(t, err, "RESOURCE_RETIRED")
_, err = f.tasks.Submit(ctx, f.token, "new-task", service.SubmitInput{QueueID: q, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)})
expectWorkerCode(t, err, "RESOURCE_RETIRED")
_, err = f.events.Create(ctx, f.token, "new-sub", events.CreateInput{Name: "new", QueueID: q})
expectWorkerCode(t, err, "RESOURCE_RETIRED")
if _, err = f.tasks.GetTask(ctx, f.token, task.ID); err != nil {
t.Fatal(err)
}
other := newEventFixture(t, ctx, db)
_, err = other.events.Retire(ctx, other.token, sub.ID, "foreign", events.RetireInput{ExpectedRevision: 1, Reason: "foreign"})
expectWorkerCode(t, err, "NOT_FOUND")
})
t.Run("schedule_resume_cannot_revive_retired_queue", func(t *testing.T) {
f := newScheduleFixture(t, newExecutionFixture(t, ctx, db))
addRetirementPermission(t, f.executionFixture)
v := f.create(f.definition())
_, err := f.tasks.RetireQueue(ctx, f.token, f.resources.QueueID, "retire", service.RetireQueueInput{ExpectedRevision: 1, Reason: "retire"})
expectWorkerCode(t, err, "RESOURCE_BUSY")
pause, err := f.api.Control(ctx, f.token, v.ID, "pause", schedule.ControlInput{ExpectedRevision: v.Revision, Reason: "pause before retire"}, true)
if err != nil {
t.Fatal(err)
}
if _, err = f.tasks.RetireQueue(ctx, f.token, f.resources.QueueID, "retire", service.RetireQueueInput{ExpectedRevision: 1, Reason: "retire"}); err != nil {
t.Fatal(err)
}
_, err = f.api.Control(ctx, f.token, v.ID, "resume", schedule.ControlInput{ExpectedRevision: pause.Schedule.Revision, Reason: "resume"}, false)
expectWorkerCode(t, err, "RESOURCE_RETIRED")
if _, err = f.api.Get(ctx, f.token, v.ID); err != nil {
t.Fatal("schedule history", err)
}
})
t.Run("waiting_workflow_with_future_queue_reference_blocks_retirement", func(t *testing.T) {
f := newExecutionFixture(t, ctx, db)
spec := runtimeSpec(f.resources.QueueID, 1)
spec.Body.Steps = append([]definition.Step{{Key: "wait", Kind: "TIMER", Timer: &definition.Timer{DelayMS: 60000}}}, spec.Body.Steps...)
runtimeStart(t, f, spec)
addRetirementPermission(t, f)
_, err := f.tasks.RetireQueue(ctx, f.token, f.resources.QueueID, "retire", service.RetireQueueInput{ExpectedRevision: 1, Reason: "retire"})
expectWorkerCode(t, err, "RESOURCE_BUSY")
})
t.Run("completed_workflow_receipt_survives_queue_retirement", func(t *testing.T) {
f := newExecutionFixture(t, ctx, db)
grantWorkflowInstance(t, f)
input := publishInstanceSpec(t, f, runtimeSpec(f.resources.QueueID, 1))
api := instance.New(NewWorkflowInstanceStore(db))
key := string(executionID(t))
first, err := api.Start(ctx, f.token, key, input)
if err != nil {
t.Fatal(err)
}
ref := retryStart(t, f)
retryFinish(t, f, ref, "SUCCEEDED", false)
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
runtimeView(t, f, first.Instance.ID, "SUCCEEDED")
addRetirementPermission(t, f)
if _, err = f.tasks.RetireQueue(ctx, f.token, f.resources.QueueID, "retire", service.RetireQueueInput{ExpectedRevision: 1, Reason: "completed workflow"}); err != nil {
t.Fatal(err)
}
replay, err := api.Start(ctx, f.token, key, input)
if err != nil || !replay.Deduplicated || replay.Instance.ID != first.Instance.ID {
t.Fatal("receipt after retirement", err)
}
if _, err = api.Get(ctx, f.token, first.Instance.ID); err != nil {
t.Fatal("workflow history", err)
}
_, err = api.Start(ctx, f.token, "new-instance", input)
expectWorkerCode(t, err, "RESOURCE_RETIRED")
})
t.Run("submit_and_retire_serialize_under_real_database_contention", func(t *testing.T) {
f := newExecutionFixture(t, ctx, db)
key := string(executionID(t))
q := f.resources.QueueID
results := competeWorkflow(t, f, nil, func() error {
_, e := f.tasks.Submit(ctx, f.token, key, service.SubmitInput{QueueID: q, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)})
return e
}, func() error {
_, e := f.tasks.RetireQueue(ctx, f.token, q, "retire", service.RetireQueueInput{ExpectedRevision: 1, Reason: "concurrent retirement"})
return e
})
if results[0] == nil {
expectWorkerCode(t, results[1], "RESOURCE_BUSY")
} else {
expectWorkerCode(t, results[0], "RESOURCE_RETIRED")
if results[1] != nil {
t.Fatal(results[1])
}
}
})
t.Run("audit_failure_rolls_back_retirement_and_quota", func(t *testing.T) {
f := newEventFixture(t, ctx, db)
sub := f.create(events.CreateInput{})
f.exec(`CREATE TRIGGER retirement_audit_reject BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='retirement audit unavailable'`)
defer db.ExecContext(ctx, `DROP TRIGGER IF EXISTS retirement_audit_reject`)
_, err := f.events.Retire(ctx, f.token, sub.ID, "retire", events.RetireInput{ExpectedRevision: sub.Revision, Reason: "audit"})
if err == nil {
t.Fatal("audit failure accepted")
}
if f.subscription(sub.ID).RetiredAt != nil {
t.Fatal("partial retirement committed")
}
})
var foreignKeys int
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&foreignKeys); err != nil || foreignKeys != 0 {
t.Fatal("foreign keys", foreignKeys, err)
}
}

View File

@@ -83,6 +83,13 @@ func TestIntegrationResultSubscriptionsUpgrade(t *testing.T) {
if err = CheckSchema(ctx, db); err != nil {
t.Fatal(err)
}
var retired *time.Time
if err = db.QueryRowContext(ctx, `SELECT retired_at FROM pulse_queues WHERE namespace_id=? AND id=?`, ns, queue).Scan(&retired); err != nil || retired != nil {
t.Fatal("migration unexpectedly retired legacy queue", err)
}
if err = db.QueryRowContext(ctx, `SELECT retired_at FROM orbit_subscriptions WHERE namespace_id=? AND id=?`, ns, sub).Scan(&retired); err != nil || retired != nil {
t.Fatal("migration unexpectedly retired legacy subscription", err)
}
var enabled bool
var queueID *core.ID
var filter string

View File

@@ -129,6 +129,9 @@ func (s *ScheduleStore) Create(ctx context.Context, t auth.Token, key string, in
if err != nil {
return err
}
if err := requireActiveQueue(ctx, tx, t.NamespaceID, in.Definition.Task.QueueID); err != nil {
return err
}
now, err := dbNow(ctx, tx)
if err != nil {
return err
@@ -252,6 +255,13 @@ func (s *ScheduleStore) mutate(ctx context.Context, t auth.Token, id core.ID, ke
out.Deduplicated = true
return nil
}
current, err := readSchedule(ctx, tx, t.NamespaceID, id, false)
if err != nil {
return err
}
if err = requireActiveQueue(ctx, tx, t.NamespaceID, current.QueueID); err != nil {
return err
}
v, err := readSchedule(ctx, tx, t.NamespaceID, id, true)
if err != nil {
return err

View File

@@ -374,7 +374,7 @@ func (s *TaskStore) CreateQueue(ctx context.Context, token auth.Token, key strin
}
func readQueue(ctx context.Context, tx *sql.Tx, ns, id core.ID) (service.QueueView, error) {
var q service.QueueView
err := tx.QueryRowContext(ctx, "SELECT id,namespace_id,name,paused,max_concurrency,rate_per_second,max_unfinished_tasks,unfinished_tasks,revision FROM pulse_queues WHERE namespace_id=? AND id=?", ns, id).Scan(&q.ID, &q.NamespaceID, &q.Name, &q.Paused, &q.MaxConcurrency, &q.RatePerSecond, &q.MaxUnfinishedTasks, &q.UnfinishedTasks, &q.Revision)
err := tx.QueryRowContext(ctx, "SELECT id,namespace_id,name,paused,max_concurrency,rate_per_second,max_unfinished_tasks,unfinished_tasks,revision,retired_at FROM pulse_queues WHERE namespace_id=? AND id=?", ns, id).Scan(&q.ID, &q.NamespaceID, &q.Name, &q.Paused, &q.MaxConcurrency, &q.RatePerSecond, &q.MaxUnfinishedTasks, &q.UnfinishedTasks, &q.Revision, &q.RetiredAt)
if errors.Is(err, sql.ErrNoRows) {
err = fault.NotFound
}

View File

@@ -54,7 +54,11 @@ func lockQueue(ctx context.Context, tx *sql.Tx, ns, id core.ID) (paused bool, ma
// lockQueueAfterApplication 仅在同一事务已持有应用额度排他锁时调用。
func lockQueueAfterApplication(ctx context.Context, tx *sql.Tx, ns, id core.ID) (paused bool, max uint32, err error) {
err = tx.QueryRowContext(ctx, "SELECT paused,max_concurrency FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE", ns, id).Scan(&paused, &max)
var retired *time.Time
err = tx.QueryRowContext(ctx, "SELECT paused,max_concurrency,retired_at FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE", ns, id).Scan(&paused, &max, &retired)
if err == nil && retired != nil {
err = fault.Retired
}
if errors.Is(err, sql.ErrNoRows) {
err = fault.NotFound
}

View File

@@ -103,6 +103,9 @@ func (s *ExecutionStore) Connect(ctx context.Context, token auth.Token, in worke
// 不依赖数据库外键:创建会话前验证队列存在且属于当前命名空间。
// 进程头在队列之前加锁,与执行命令保持一致;失败时回滚新建的进程头。
var queue core.ID
if err := requireActiveQueue(ctx, tx, token.NamespaceID, in.QueueID); err != nil {
return err
}
if err := tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR SHARE`, token.NamespaceID, in.QueueID).Scan(&queue); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return fault.NotFound

View File

@@ -129,10 +129,11 @@ func (s *WorkflowInstanceStore) Start(ctx context.Context, t auth.Token, key str
if len(version.Analysis.Queues) > 0 && !p.Allows("tasks:submit") {
return fault.Forbidden
}
if err := lockWorkflowQueues(ctx, tx, t.NamespaceID, p, version.Analysis.Queues); err != nil {
return err
}
if cached != nil {
// 已完成流程所用队列可以退役;原受理回执仍核对当前读取范围,但不重新申请执行权。
if err := authorizeWorkflowQueues(ctx, tx, t.NamespaceID, p, version.Analysis.Queues); err != nil {
return err
}
if err := json.Unmarshal(cached, &out); err != nil {
return err
}
@@ -146,6 +147,9 @@ func (s *WorkflowInstanceStore) Start(ctx context.Context, t auth.Token, key str
out.Deduplicated = true
return nil
}
if err := lockWorkflowQueues(ctx, tx, t.NamespaceID, p, version.Analysis.Queues); err != nil {
return err
}
if err := definition.ValidateData(version.Specification.InputType, in.Input); err != nil {
return fault.Invalid("workflow input does not match fixed version")
}