feat: add durable result journal and subscription delivery core
This commit is contained in:
@@ -65,6 +65,9 @@ func TestIntegrationApplicationResourcesUpgrade(t *testing.T) {
|
||||
assertApplicationCounts(t, f, 4, 2, 1)
|
||||
// 还原后续新增表和字段，得到真实 v9 结构；现有任务内容保持不变。
|
||||
f.exec(`DROP TABLE orbit_application_resources`)
|
||||
f.exec(`DROP TABLE orbit_result_events,orbit_event_streams`)
|
||||
f.exec(`ALTER TABLE orbit_subscriptions DROP CHECK ck_subscription_window, DROP COLUMN queue_id, DROP COLUMN cursor_sequence, DROP COLUMN pending_count, DROP COLUMN max_pending, DROP COLUMN max_attempts`)
|
||||
f.exec(`ALTER TABLE orbit_deliveries DROP INDEX ix_delivery_subscription_due, DROP INDEX ix_delivery_subscription_lease, DROP CHECK ck_delivery_budget, DROP COLUMN attempt_budget, DROP COLUMN token_digest`)
|
||||
f.exec(`ALTER TABLE pulse_tasks DROP INDEX ix_task_priority_dispatch, DROP CHECK ck_task_priority, DROP COLUMN dispatch_at`)
|
||||
f.exec(`ALTER TABLE pulse_tasks DROP CHECK ck_task_concurrency_key, DROP COLUMN concurrency_key, DROP COLUMN concurrency_key_digest`)
|
||||
f.exec(`ALTER TABLE pulse_attempts DROP COLUMN concurrency_token`)
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
// BootstrapInput 仅供持有数据库管理权限的初始化命令使用，不暴露为公共 HTTP。
|
||||
type BootstrapInput struct {
|
||||
EventAccess bool // EventAccess 显式增加结果订阅查询、管理和消费权限。
|
||||
Application string // Application 新应用名称，与 ExistingApplicationID 二选一。
|
||||
Namespace string // Namespace 命名空间名称。
|
||||
Environment string // Environment 环境名称，必须显式指定。
|
||||
@@ -80,6 +81,9 @@ func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func(
|
||||
if in.ResourceAccess {
|
||||
permissions.Actions = append(permissions.Actions, "application:resources:read", "application:resources:manage")
|
||||
}
|
||||
if in.EventAccess {
|
||||
permissions.Actions = append(permissions.Actions, "events:read", "events:manage", "events:consume")
|
||||
}
|
||||
raw, err := json.Marshal(permissions)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
311
internal/storage/mysql/event_deliveries.go
Normal file
311
internal/storage/mysql/event_deliveries.go
Normal file
@@ -0,0 +1,311 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/events"
|
||||
platform "git.yuntuo.hk/k2safe/orbit/platform/model"
|
||||
)
|
||||
|
||||
// materializeResults 持有订阅锁，流和日志只做已提交读，不反向锁生产者。
|
||||
// 先保存独立交付记录再推进扫描序号；未确认窗口满时停止扫描，保留来源日志。
|
||||
func materializeResults(ctx context.Context, tx *sql.Tx, ns core.ID, sub *events.Subscription) error {
|
||||
if sub.PendingCount >= sub.MaxPending {
|
||||
return nil
|
||||
}
|
||||
var head, retained uint64
|
||||
if err := tx.QueryRowContext(ctx, `SELECT last_sequence,retained_from_sequence FROM orbit_event_streams WHERE namespace_id=?`, ns).Scan(&head, &retained); err != nil {
|
||||
return err
|
||||
}
|
||||
if sub.CursorSequence < retained-1 {
|
||||
return events.HistoryExpired
|
||||
}
|
||||
limit := min(uint32(100), sub.MaxPending-sub.PendingCount)
|
||||
rows, err := tx.QueryContext(ctx, `SELECT r.id,r.sequence,e.type FROM orbit_result_events r LEFT JOIN orbit_events e ON e.namespace_id=r.namespace_id AND e.id=r.id WHERE r.namespace_id=? AND r.queue_id=? AND r.sequence>? AND r.sequence<=? ORDER BY r.sequence LIMIT ?`, ns, sub.QueueID, sub.CursorSequence, head, limit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type candidate struct {
|
||||
id core.ID
|
||||
sequence uint64
|
||||
eventType string
|
||||
}
|
||||
batch := make([]candidate, 0, limit)
|
||||
for rows.Next() {
|
||||
var c candidate
|
||||
if err = rows.Scan(&c.id, &c.sequence, &c.eventType); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
batch = append(batch, c)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cursor := sub.CursorSequence
|
||||
pending := sub.PendingCount
|
||||
for _, c := range batch {
|
||||
cursor = c.sequence
|
||||
if !slices.Contains(sub.Types, c.eventType) {
|
||||
continue
|
||||
}
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO orbit_deliveries(namespace_id,id,subscription_id,event_id,available_at,attempt_budget) VALUES (?,?,?,?,UTC_TIMESTAMP(6),?)`, ns, id, sub.ID, c.id, sub.MaxAttempts); err != nil {
|
||||
return err
|
||||
}
|
||||
pending++
|
||||
}
|
||||
if len(batch) < int(limit) {
|
||||
cursor = head
|
||||
}
|
||||
if cursor == sub.CursorSequence && pending == sub.PendingCount {
|
||||
return nil
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE orbit_subscriptions SET cursor_sequence=?,pending_count=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, cursor, pending, ns, sub.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
sub.CursorSequence, sub.PendingCount = cursor, pending
|
||||
return nil
|
||||
}
|
||||
|
||||
// recoverDeliveries 在新领取前有界回收过期租约。耗尽预算仍占积压，禁止静默丢弃。
|
||||
// 最后一次证明保留；迟到确认只在还没有新交付代次时有效。
|
||||
func recoverDeliveries(ctx context.Context, tx *sql.Tx, ns, sub core.ID, now time.Time) error {
|
||||
_, err := tx.ExecContext(ctx, `UPDATE orbit_deliveries SET state=IF(attempts>=attempt_budget,'EXPIRED','PENDING'),available_at=TIMESTAMPADD(SECOND,LEAST(30,POW(2,LEAST(attempts-1,5))),?),lease_owner=NULL,lease_until=NULL,revision=revision+1,updated_at=? WHERE namespace_id=? AND subscription_id=? AND state='LEASED' AND lease_until<=?`, now, now, ns, sub, now)
|
||||
return err
|
||||
}
|
||||
|
||||
type storedDelivery struct {
|
||||
view events.Delivery
|
||||
digest []byte
|
||||
}
|
||||
|
||||
func readDelivery(ctx context.Context, tx *sql.Tx, ns, sub, id core.ID, lock bool) (storedDelivery, error) {
|
||||
var out storedDelivery
|
||||
q := `SELECT id,subscription_id,event_id,revision,state,available_at,lease_epoch,lease_until,attempts,attempt_budget,acknowledged_at,token_digest FROM orbit_deliveries WHERE namespace_id=? AND subscription_id=? AND id=?`
|
||||
if lock {
|
||||
q += ` FOR UPDATE`
|
||||
}
|
||||
v := &out.view
|
||||
err := tx.QueryRowContext(ctx, q, ns, sub, id).Scan(&v.ID, &v.SubscriptionID, &v.EventID, &v.Revision, &v.State, &v.AvailableAt, &v.LeaseEpoch, &v.LeaseUntil, &v.Attempts, &v.AttemptBudget, &v.AcknowledgedAt, &out.digest)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return out, fault.NotFound
|
||||
}
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
err = tx.QueryRowContext(ctx, `SELECT sequence FROM orbit_result_events WHERE namespace_id=? AND id=?`, ns, v.EventID).Scan(&v.Sequence)
|
||||
return out, err
|
||||
}
|
||||
func readResultEvent(ctx context.Context, tx *sql.Tx, ns, id core.ID) (events.Event, error) {
|
||||
var out events.Event
|
||||
var details, snapshot []byte
|
||||
err := tx.QueryRowContext(ctx, `SELECT r.id,r.sequence,e.type,e.aggregate_kind,e.aggregate_id,e.aggregate_revision,e.occurred_at,e.payload,r.snapshot FROM orbit_result_events r LEFT JOIN orbit_events e ON e.namespace_id=r.namespace_id AND e.id=r.id WHERE r.namespace_id=? AND r.id=?`, ns, id).Scan(&out.ID, &out.Sequence, &out.Type, &out.AggregateKind, &out.AggregateID, &out.AggregateRevision, &out.OccurredAt, &details, &snapshot)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Details = details
|
||||
err = json.Unmarshal(snapshot, &out.Snapshot)
|
||||
return out, err
|
||||
}
|
||||
func (s *EventStore) GetDelivery(ctx context.Context, t auth.Token, sub, id core.ID) (events.Delivery, error) {
|
||||
var out events.Delivery
|
||||
err := s.transaction(ctx, t, "events:read", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
if _, err := readSubscription(ctx, tx, t, p, sub, false); err != nil {
|
||||
return err
|
||||
}
|
||||
d, err := readDelivery(ctx, tx, t.NamespaceID, sub, id, false)
|
||||
out = d.view
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
func (s *EventStore) ClaimDelivery(ctx context.Context, t auth.Token, id core.ID, in events.ClaimInput) (*events.Grant, error) {
|
||||
var out *events.Grant
|
||||
err := s.transaction(ctx, t, "events:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
sub, err := readSubscription(ctx, tx, t, p, id, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !sub.Enabled {
|
||||
return nil
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = recoverDeliveries(ctx, tx, t.NamespaceID, id, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = materializeResults(ctx, tx, t.NamespaceID, &sub); err != nil {
|
||||
return err
|
||||
}
|
||||
// 物化记录的 available_at 来自写入时的数据库时钟；不能继续用物化前的时间领取。
|
||||
now, err = dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var active uint32
|
||||
if err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM orbit_deliveries WHERE namespace_id=? AND subscription_id=? AND state='LEASED' AND lease_owner=?`, t.NamespaceID, id, in.ConnectionID).Scan(&active); err != nil {
|
||||
return err
|
||||
}
|
||||
if active >= in.Capacity {
|
||||
return nil
|
||||
}
|
||||
var deliveryID core.ID
|
||||
err = tx.QueryRowContext(ctx, `SELECT id FROM orbit_deliveries WHERE namespace_id=? AND subscription_id=? AND state='PENDING' AND available_at<=? ORDER BY available_at,id LIMIT 1 FOR UPDATE`, t.NamespaceID, id, now).Scan(&deliveryID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stored, err := readDelivery(ctx, tx, t.NamespaceID, id, deliveryID, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
v := stored.view
|
||||
if v.Attempts >= v.AttemptBudget || v.LeaseEpoch == math.MaxUint64 || v.Revision == math.MaxUint64 {
|
||||
return fault.Conflict
|
||||
}
|
||||
secret, digest, err := events.NewSecret()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
until := now.Add(events.LeaseDuration)
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE orbit_deliveries SET state='LEASED',lease_owner=?,lease_epoch=lease_epoch+1,lease_until=?,token_digest=?,attempts=attempts+1,revision=revision+1,updated_at=? WHERE namespace_id=? AND subscription_id=? AND id=?`, in.ConnectionID, until, digest, now, t.NamespaceID, id, deliveryID); err != nil {
|
||||
return err
|
||||
}
|
||||
v.State = platform.DeliveryLeased
|
||||
v.LeaseEpoch++
|
||||
v.LeaseUntil = &until
|
||||
v.Attempts++
|
||||
v.Revision++
|
||||
event, err := readResultEvent(ctx, tx, t.NamespaceID, v.EventID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
operation, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = appendAudit(ctx, tx, t, operation, v.ID, "delivery", "delivery.lease", "grant durable result delivery", nil, map[string]any{"event_id": v.EventID, "epoch": fmt.Sprint(v.LeaseEpoch), "attempts": v.Attempts, "lease_until": until, "connection_id": in.ConnectionID}); err != nil {
|
||||
return err
|
||||
}
|
||||
out = &events.Grant{Delivery: v, Event: event, LeaseSecret: secret}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (s *EventStore) AckDelivery(ctx context.Context, t auth.Token, id core.ID, in events.AckInput) (events.AckResult, error) {
|
||||
var out events.AckResult
|
||||
digest, err := events.SecretDigest(in.LeaseSecret)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
err = s.transaction(ctx, t, "events:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
sub, err := readSubscription(ctx, tx, t, p, id, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d, err := readDelivery(ctx, tx, t.NamespaceID, id, in.DeliveryID, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
v := d.view
|
||||
if v.LeaseEpoch != in.Epoch || subtle.ConstantTimeCompare(d.digest, digest[:]) != 1 {
|
||||
return events.LeaseLost
|
||||
}
|
||||
out = events.AckResult{DeliveryID: v.ID, EventID: v.EventID, Epoch: in.Epoch}
|
||||
if v.State == platform.DeliveryAcked {
|
||||
if v.AcknowledgedAt == nil {
|
||||
return errors.New("acked delivery lacks confirmation timestamp")
|
||||
}
|
||||
out.AcknowledgedAt = *v.AcknowledgedAt
|
||||
out.Deduplicated = true
|
||||
return nil
|
||||
}
|
||||
if sub.PendingCount == 0 || v.Revision == math.MaxUint64 {
|
||||
return errors.New("delivery confirmation invariant failed")
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE orbit_deliveries SET state='ACKED',lease_owner=NULL,lease_until=NULL,acknowledged_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND subscription_id=? AND id=?`, now, now, t.NamespaceID, id, v.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE orbit_subscriptions SET pending_count=pending_count-1,updated_at=? WHERE namespace_id=? AND id=?`, now, t.NamespaceID, id); err != nil {
|
||||
return err
|
||||
}
|
||||
operation, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = appendAudit(ctx, tx, t, operation, v.ID, "delivery", "delivery.ack", "consumer confirmed durable processing", map[string]any{"state": v.State}, map[string]any{"state": platform.DeliveryAcked, "event_id": v.EventID, "epoch": fmt.Sprint(in.Epoch)}); err != nil {
|
||||
return err
|
||||
}
|
||||
out.AcknowledgedAt = now
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
func (s *EventStore) RetryDelivery(ctx context.Context, t auth.Token, sub, id core.ID, key string, in events.RetryInput) (events.RetryResult, error) {
|
||||
var out events.RetryResult
|
||||
err := s.transaction(ctx, t, "events:manage", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "delivery.retry", eventScope(t.CredentialID, sub, id), key, "delivery", id, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = readSubscription(ctx, tx, t, p, sub, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if cached != nil {
|
||||
if err = json.Unmarshal(cached, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Deduplicated = true
|
||||
return nil
|
||||
}
|
||||
d, err := readDelivery(ctx, tx, t.NamespaceID, sub, id, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
v := d.view
|
||||
if v.State != platform.DeliveryExpired || v.Revision != in.ExpectedRevision || v.Revision == math.MaxUint64 || uint64(v.AttemptBudget)+uint64(in.AdditionalAttempts) > 1000000 {
|
||||
return fault.Conflict
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE orbit_deliveries SET state='PENDING',attempt_budget=attempt_budget+?,available_at=UTC_TIMESTAMP(6),revision=revision+1,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND subscription_id=? AND id=?`, in.AdditionalAttempts, t.NamespaceID, sub, id); err != nil {
|
||||
return err
|
||||
}
|
||||
d, err = readDelivery(ctx, tx, t.NamespaceID, sub, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out.Delivery = d.view
|
||||
if err = appendAudit(ctx, tx, t, identity, id, "delivery", "delivery.retry", in.Reason, v, out.Delivery); err != nil {
|
||||
return err
|
||||
}
|
||||
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
177
internal/storage/mysql/event_subscriptions.go
Normal file
177
internal/storage/mysql/event_subscriptions.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/events"
|
||||
driver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// EventStore 管理独立结果订阅；消费者事务不锁任务、队列、应用额度或结果流行。
|
||||
// 创建时按认证→请求身份→队列共享锁→结果流锁顺序确定提交边界。
|
||||
type EventStore struct{ *TaskStore }
|
||||
|
||||
func NewEventStore(db *sql.DB) *EventStore { return &EventStore{NewTaskStore(db)} }
|
||||
|
||||
var _ events.Store = (*EventStore)(nil)
|
||||
|
||||
const subscriptionColumns = `id,queue_id,credential_id,name,revision,enabled,filter,cursor_sequence,pending_count,max_pending,max_attempts,created_at,updated_at`
|
||||
|
||||
func readSubscription(ctx context.Context, tx *sql.Tx, t auth.Token, p auth.Permissions, id core.ID, lock bool) (events.Subscription, error) {
|
||||
var out events.Subscription
|
||||
var raw []byte
|
||||
q := `SELECT ` + subscriptionColumns + ` FROM orbit_subscriptions WHERE namespace_id=? AND id=? AND credential_id=? AND queue_id IS NOT NULL`
|
||||
if lock {
|
||||
q += ` FOR UPDATE`
|
||||
}
|
||||
err := tx.QueryRowContext(ctx, q, t.NamespaceID, id, t.CredentialID).Scan(&out.ID, &out.QueueID, &out.CredentialID, &out.Name, &out.Revision, &out.Enabled, &raw, &out.CursorSequence, &out.PendingCount, &out.MaxPending, &out.MaxAttempts, &out.CreatedAt, &out.UpdatedAt)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return out, fault.NotFound
|
||||
}
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if !p.AllowsQueue(out.QueueID) {
|
||||
return events.Subscription{}, fault.NotFound
|
||||
}
|
||||
if err = json.Unmarshal(raw, &out.Types); err != nil {
|
||||
return out, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *EventStore) CreateSubscription(ctx context.Context, t auth.Token, key string, in events.CreateInput) (events.SubscriptionResult, error) {
|
||||
var out events.SubscriptionResult
|
||||
err := s.transaction(ctx, t, "events:manage", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
if !p.AllowsQueue(in.QueueID) {
|
||||
return fault.Forbidden
|
||||
}
|
||||
id, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "subscription.create", string(t.CredentialID), key, "subscription", id, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cached != nil {
|
||||
if err = json.Unmarshal(cached, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Deduplicated = true
|
||||
return nil
|
||||
}
|
||||
var queue core.ID
|
||||
if err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR SHARE`, t.NamespaceID, in.QueueID).Scan(&queue); errors.Is(err, sql.ErrNoRows) {
|
||||
return fault.NotFound
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
head, retained, err := lockEventStream(ctx, tx, t.NamespaceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cursor := head
|
||||
if in.FromSequence != nil {
|
||||
cursor = *in.FromSequence
|
||||
}
|
||||
if cursor > head {
|
||||
return fault.Invalid("from_sequence is ahead of committed history")
|
||||
}
|
||||
if cursor < retained-1 {
|
||||
return events.HistoryExpired
|
||||
}
|
||||
var count int
|
||||
if err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM orbit_subscriptions WHERE namespace_id=?`, t.NamespaceID).Scan(&count); err != nil {
|
||||
return err
|
||||
}
|
||||
if count >= events.MaxSubscriptions {
|
||||
return events.SubscriptionLimit
|
||||
}
|
||||
filter, err := json.Marshal(in.Types)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO orbit_subscriptions(namespace_id,id,name,credential_id,filter,queue_id,cursor_sequence,max_pending,max_attempts) VALUES (?,?,?,?,?,?,?,?,?)`, t.NamespaceID, id, in.Name, t.CredentialID, string(filter), in.QueueID, cursor, in.MaxPending, in.MaxAttempts)
|
||||
var me *driver.MySQLError
|
||||
if errors.As(err, &me) && me.Number == 1062 {
|
||||
return fault.Conflict
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out.Subscription, err = readSubscription(ctx, tx, t, p, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = appendAudit(ctx, tx, t, identity, id, "subscription", "subscription.create", "create durable result subscription", nil, out.Subscription); err != nil {
|
||||
return err
|
||||
}
|
||||
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
func (s *EventStore) GetSubscription(ctx context.Context, t auth.Token, id core.ID) (events.Subscription, error) {
|
||||
var out events.Subscription
|
||||
err := s.transaction(ctx, t, "events:read", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
var err error
|
||||
out, err = readSubscription(ctx, tx, t, p, id, false)
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
func (s *EventStore) ControlSubscription(ctx context.Context, t auth.Token, id core.ID, key string, in events.ControlInput) (events.SubscriptionResult, error) {
|
||||
var out events.SubscriptionResult
|
||||
err := s.transaction(ctx, t, "events:manage", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
identity, cached, err := reserveIdentity(ctx, tx, t.NamespaceID, "subscription.control", eventScope(t.CredentialID, id), key, "subscription", id, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
before, err := readSubscription(ctx, tx, t, p, id, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cached != nil {
|
||||
if err = json.Unmarshal(cached, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Deduplicated = true
|
||||
return nil
|
||||
}
|
||||
if before.Revision != in.ExpectedRevision || before.Revision == math.MaxUint64 {
|
||||
return fault.Conflict
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE orbit_subscriptions SET enabled=?,revision=revision+1,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, in.Enabled, t.NamespaceID, id); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Subscription, err = readSubscription(ctx, tx, t, p, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = appendAudit(ctx, tx, t, identity, id, "subscription", "subscription.control", in.Reason, before, out.Subscription); err != nil {
|
||||
return err
|
||||
}
|
||||
return saveIdentity(ctx, tx, t.NamespaceID, identity, out)
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
// eventScope 将带域隔离的复合资源身份压缩到已有64字符幂等作用域，避免截断或串联歧义。
|
||||
func eventScope(ids ...core.ID) string {
|
||||
h := sha256.New()
|
||||
h.Write([]byte("orbit-event-scope-v1"))
|
||||
for _, id := range ids {
|
||||
h.Write([]byte{0})
|
||||
h.Write([]byte(id))
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
709
internal/storage/mysql/event_subscriptions_integration_test.go
Normal file
709
internal/storage/mysql/event_subscriptions_integration_test.go
Normal file
@@ -0,0 +1,709 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/events"
|
||||
platform "git.yuntuo.hk/k2safe/orbit/platform/model"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
type eventFixture struct {
|
||||
*executionFixture
|
||||
events *events.Service
|
||||
store *EventStore
|
||||
}
|
||||
|
||||
func newEventFixture(t *testing.T, ctx context.Context, db *sql.DB) *eventFixture {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
var p auth.Permissions
|
||||
var raw []byte
|
||||
if err := db.QueryRowContext(ctx, `SELECT permissions FROM orbit_credentials WHERE namespace_id=? AND id=?`, f.token.NamespaceID, f.token.CredentialID).Scan(&raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(raw, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.Actions = append(p.Actions, "events:manage", "events:read", "events:consume")
|
||||
raw, _ = json.Marshal(p)
|
||||
f.exec(`UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?`, string(raw), f.token.NamespaceID, f.token.CredentialID)
|
||||
store := NewEventStore(db)
|
||||
return &eventFixture{f, events.New(store), store}
|
||||
}
|
||||
func (f *eventFixture) create(in events.CreateInput) events.Subscription {
|
||||
f.t.Helper()
|
||||
if in.Name == "" {
|
||||
in.Name = "result-" + string(executionID(f.t))
|
||||
}
|
||||
if in.QueueID == "" {
|
||||
in.QueueID = f.resources.QueueID
|
||||
}
|
||||
out, err := f.events.Create(f.ctx, f.token, string(executionID(f.t)), in)
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return out.Subscription
|
||||
}
|
||||
func (f *eventFixture) cancelResult() service.TaskView {
|
||||
f.t.Helper()
|
||||
task := f.submit(1)
|
||||
out, err := f.tasks.CancelTask(f.ctx, f.token, task.ID, string(executionID(f.t)), service.CancelInput{ExpectedRevision: task.Revision, Reason: "create result notification"})
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return out.Task
|
||||
}
|
||||
func (f *eventFixture) grant(sub core.ID) *events.Grant {
|
||||
f.t.Helper()
|
||||
v, err := f.events.Claim(f.ctx, f.token, sub, events.ClaimInput{ConnectionID: executionID(f.t), Capacity: 1})
|
||||
if err != nil || v == nil {
|
||||
f.t.Fatalf("grant: %v", err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
func (f *eventFixture) ack(sub core.ID, g *events.Grant) events.AckResult {
|
||||
f.t.Helper()
|
||||
v, err := f.events.Ack(f.ctx, f.token, sub, events.AckInput{DeliveryID: g.Delivery.ID, Epoch: g.Delivery.LeaseEpoch, LeaseSecret: g.LeaseSecret})
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
func (f *eventFixture) subscription(id core.ID) events.Subscription {
|
||||
f.t.Helper()
|
||||
v, err := f.events.Get(f.ctx, f.token, id)
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
func (f *eventFixture) expireDelivery(g *events.Grant) {
|
||||
f.exec(`UPDATE orbit_deliveries SET lease_until=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?`, f.token.NamespaceID, g.Delivery.ID)
|
||||
}
|
||||
func noEventGrant(t *testing.T, f *eventFixture, sub core.ID) {
|
||||
t.Helper()
|
||||
g, err := f.events.Claim(f.ctx, f.token, sub, events.ClaimInput{ConnectionID: executionID(t), Capacity: 1})
|
||||
if err != nil || g != nil {
|
||||
t.Fatalf("unexpected grant: %v %v", g, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationResultSubscriptions(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
db, err := Open(ctx, dsn, 32, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err = Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zero := uint64(0)
|
||||
t.Run("explicit_boundary_filters_and_frozen_result_precision", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
task := f.submit(1)
|
||||
ref := retryStart(t, f.executionFixture)
|
||||
raw := json.RawMessage(`{"n":9007199254740993,"text":"<&>"}`)
|
||||
in := worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: raw}
|
||||
message := executionID(t)
|
||||
if _, err := f.a.Finish(ctx, f.token, f.binding, message, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.a.Finish(ctx, f.token, f.binding, message, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.count("orbit_result_events", "1=1") != 1 {
|
||||
t.Fatal("duplicate result journal entry")
|
||||
}
|
||||
recent := f.create(events.CreateInput{})
|
||||
if recent.CursorSequence != 1 {
|
||||
t.Fatal("new subscription did not capture committed head")
|
||||
}
|
||||
noEventGrant(t, f, recent.ID)
|
||||
history := f.create(events.CreateInput{FromSequence: &zero, Types: []string{"task.succeeded"}})
|
||||
g := f.grant(history.ID)
|
||||
if g.Event.ID != g.Delivery.EventID || g.Event.Sequence != 1 || g.Event.Snapshot.TaskID != task.ID || string(g.Event.Snapshot.Result) != string(raw) || g.Event.Snapshot.CurrentAttemptID == nil || *g.Event.Snapshot.CurrentAttemptID != ref.AttemptID {
|
||||
t.Fatalf("bad snapshot: %+v", g.Event)
|
||||
}
|
||||
f.exec(`UPDATE pulse_tasks SET result='{"changed":true}' WHERE namespace_id=? AND id=?`, f.token.NamespaceID, task.ID)
|
||||
f.ack(history.ID, g)
|
||||
replay := f.create(events.CreateInput{FromSequence: &zero})
|
||||
g = f.grant(replay.ID)
|
||||
if string(g.Event.Snapshot.Result) != string(raw) {
|
||||
t.Fatal("snapshot read mutable task result")
|
||||
}
|
||||
f.ack(replay.ID, g)
|
||||
f.cancelResult()
|
||||
noEventGrant(t, f, history.ID)
|
||||
if f.subscription(history.ID).CursorSequence != 2 {
|
||||
t.Fatal("filtered event did not advance cursor")
|
||||
}
|
||||
g = f.grant(recent.ID)
|
||||
if g.Event.Type != "task.cancelled" || g.Event.Sequence != 2 {
|
||||
t.Fatal("new subscription missed future result")
|
||||
}
|
||||
_, err := db.ExecContext(ctx, `UPDATE orbit_result_events SET snapshot='{}' WHERE namespace_id=? AND id=?`, f.token.NamespaceID, g.Event.ID)
|
||||
if err == nil {
|
||||
t.Fatal("journal is mutable")
|
||||
}
|
||||
})
|
||||
t.Run("create_idempotency_queue_ownership_and_permission_isolation", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
other := newEventFixture(t, ctx, db)
|
||||
in := events.CreateInput{Name: "stable-result", QueueID: f.resources.QueueID, FromSequence: &zero}
|
||||
a, err := f.events.Create(ctx, f.token, "stable", in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := f.events.Create(ctx, f.token, "stable", in)
|
||||
if err != nil || !b.Deduplicated || a.Subscription.ID != b.Subscription.ID {
|
||||
t.Fatal("unstable create receipt", err)
|
||||
}
|
||||
in.MaxPending = 2
|
||||
_, err = f.events.Create(ctx, f.token, "stable", in)
|
||||
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
|
||||
_, err = f.events.Get(ctx, other.token, a.Subscription.ID)
|
||||
expectWorkerCode(t, err, "NOT_FOUND")
|
||||
_, err = f.events.Create(ctx, f.token, "cross-queue", events.CreateInput{Name: "cross", QueueID: other.resources.QueueID})
|
||||
expectWorkerCode(t, err, "NOT_FOUND")
|
||||
secondID := executionID(t)
|
||||
secret, digest, err := auth.Issue(f.token.NamespaceID, secondID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, _ := json.Marshal(auth.Permissions{Actions: []string{"events:read", "events:consume", "events:manage"}, Queues: []string{"*"}})
|
||||
f.exec(`INSERT INTO orbit_credentials(namespace_id,id,name,secret_digest,permissions) VALUES (?,?,?, ?,?)`, f.token.NamespaceID, secondID, "second", digest, string(p))
|
||||
second, err := auth.Parse(secret)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = f.events.Get(ctx, second, a.Subscription.ID)
|
||||
expectWorkerCode(t, err, "NOT_FOUND")
|
||||
own, err := f.events.Create(ctx, second, "stable", events.CreateInput{Name: "second", QueueID: f.resources.QueueID, FromSequence: &zero})
|
||||
if err != nil || own.Subscription.ID == a.Subscription.ID {
|
||||
t.Fatal("create receipt leaked across owners", err)
|
||||
}
|
||||
p, _ = json.Marshal(auth.Permissions{Actions: []string{"events:read", "events:consume", "events:manage"}, Queues: []string{string(executionID(t))}})
|
||||
f.exec(`UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?`, string(p), f.token.NamespaceID, secondID)
|
||||
_, err = f.events.Get(ctx, second, own.Subscription.ID)
|
||||
expectWorkerCode(t, err, "NOT_FOUND")
|
||||
p, _ = json.Marshal(auth.Permissions{Actions: []string{"tasks:read"}, Queues: []string{"*"}})
|
||||
f.exec(`UPDATE orbit_credentials SET permissions=? WHERE namespace_id=? AND id=?`, string(p), f.token.NamespaceID, f.token.CredentialID)
|
||||
_, err = f.events.Claim(ctx, f.token, a.Subscription.ID, events.ClaimInput{ConnectionID: executionID(t), Capacity: 1})
|
||||
expectWorkerCode(t, err, "PERMISSION_DENIED")
|
||||
f.exec(`UPDATE orbit_credentials SET revoked_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.token.NamespaceID, f.token.CredentialID)
|
||||
_, err = f.events.Get(ctx, f.token, a.Subscription.ID)
|
||||
expectWorkerCode(t, err, "UNAUTHENTICATED")
|
||||
})
|
||||
t.Run("backpressure_does_not_block_result_commit_or_skip_history", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
sub := f.create(events.CreateInput{MaxPending: 1})
|
||||
f.cancelResult()
|
||||
first := f.grant(sub.ID)
|
||||
for range 3 {
|
||||
f.cancelResult()
|
||||
}
|
||||
noEventGrant(t, f, sub.ID)
|
||||
got := f.subscription(sub.ID)
|
||||
if got.PendingCount != 1 || got.CursorSequence != 1 || f.count("orbit_result_events", "1=1") != 4 {
|
||||
t.Fatal("window violated or producer blocked")
|
||||
}
|
||||
f.ack(sub.ID, first)
|
||||
for seq := uint64(2); seq <= 4; seq++ {
|
||||
g := f.grant(sub.ID)
|
||||
if g.Event.Sequence != seq {
|
||||
t.Fatal("backlog skipped")
|
||||
}
|
||||
f.ack(sub.ID, g)
|
||||
}
|
||||
if got = f.subscription(sub.ID); got.PendingCount != 0 || got.CursorSequence != 4 || got.Revision != 1 {
|
||||
t.Fatal("invalid drained window or configuration revision changed")
|
||||
}
|
||||
})
|
||||
t.Run("competing_consumers_and_duplicate_acks", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
sub := f.create(events.CreateInput{})
|
||||
f.cancelResult()
|
||||
type answer struct {
|
||||
g *events.Grant
|
||||
err error
|
||||
}
|
||||
results := make(chan answer, 12)
|
||||
for range 12 {
|
||||
connection := executionID(t)
|
||||
go func() {
|
||||
g, err := f.events.Claim(ctx, f.token, sub.ID, events.ClaimInput{ConnectionID: connection, Capacity: 1})
|
||||
results <- answer{g, err}
|
||||
}()
|
||||
}
|
||||
var winner *events.Grant
|
||||
for range 12 {
|
||||
r := <-results
|
||||
if r.err != nil {
|
||||
t.Fatal(r.err)
|
||||
}
|
||||
if r.g != nil {
|
||||
if winner != nil {
|
||||
t.Fatal("two grants for one unexpired delivery")
|
||||
}
|
||||
winner = r.g
|
||||
}
|
||||
}
|
||||
if winner == nil {
|
||||
t.Fatal("no consumer won")
|
||||
}
|
||||
acks := make(chan events.AckResult, 12)
|
||||
errs := make(chan error, 12)
|
||||
for range 12 {
|
||||
go func() {
|
||||
v, err := f.events.Ack(ctx, f.token, sub.ID, events.AckInput{DeliveryID: winner.Delivery.ID, Epoch: winner.Delivery.LeaseEpoch, LeaseSecret: winner.LeaseSecret})
|
||||
acks <- v
|
||||
errs <- err
|
||||
}()
|
||||
}
|
||||
fresh := 0
|
||||
var ackTime time.Time
|
||||
for range 12 {
|
||||
v := <-acks
|
||||
if err := <-errs; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !v.Deduplicated {
|
||||
fresh++
|
||||
}
|
||||
if ackTime.IsZero() {
|
||||
ackTime = v.AcknowledgedAt
|
||||
} else if !ackTime.Equal(v.AcknowledgedAt) {
|
||||
t.Fatal("ack receipt changed")
|
||||
}
|
||||
}
|
||||
if fresh != 1 || f.subscription(sub.ID).PendingCount != 0 || f.count("orbit_audit_events", "resource_id=? AND action='delivery.ack'", winner.Delivery.ID) != 1 {
|
||||
t.Fatal("duplicate ack side effects")
|
||||
}
|
||||
})
|
||||
t.Run("lease_expiry_backoff_latest_proof_and_node_restart", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
sub := f.create(events.CreateInput{MaxAttempts: 2})
|
||||
f.cancelResult()
|
||||
first := f.grant(sub.ID)
|
||||
f.expireDelivery(first)
|
||||
// 新建存储实例模拟所有连接内存丢失；权威交付依然来自数据库。
|
||||
f.events = events.New(NewEventStore(db))
|
||||
noEventGrant(t, f, sub.ID)
|
||||
v, err := f.events.Delivery(ctx, f.token, sub.ID, first.Delivery.ID)
|
||||
if err != nil || v.State != platform.DeliveryPending || v.Attempts != 1 {
|
||||
t.Fatal("lease was not recovered", err)
|
||||
}
|
||||
f.exec(`UPDATE orbit_deliveries SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.token.NamespaceID, v.ID)
|
||||
second := f.grant(sub.ID)
|
||||
if second.Event.ID != first.Event.ID || second.Delivery.ID != first.Delivery.ID || second.Delivery.LeaseEpoch != 2 || second.Delivery.Attempts != 2 || second.LeaseSecret == first.LeaseSecret {
|
||||
t.Fatal("redelivery identity/proof incorrect")
|
||||
}
|
||||
_, err = f.events.Ack(ctx, f.token, sub.ID, events.AckInput{DeliveryID: first.Delivery.ID, Epoch: first.Delivery.LeaseEpoch, LeaseSecret: first.LeaseSecret})
|
||||
expectWorkerCode(t, err, "DELIVERY_LEASE_LOST")
|
||||
f.expireDelivery(second)
|
||||
noEventGrant(t, f, sub.ID)
|
||||
v, err = f.events.Delivery(ctx, f.token, sub.ID, second.Delivery.ID)
|
||||
if err != nil || v.State != platform.DeliveryExpired || f.subscription(sub.ID).PendingCount != 1 {
|
||||
t.Fatal("exhaustion discarded event", err)
|
||||
}
|
||||
// 确认本身不是新的执行授权；最近证明在未被新代次替代时允许迟到确认。
|
||||
f.ack(sub.ID, second)
|
||||
if !f.ack(sub.ID, second).Deduplicated {
|
||||
t.Fatal("late ack replay lost")
|
||||
}
|
||||
})
|
||||
t.Run("manual_retry_adds_budget_without_resetting_attempt_or_epoch", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
sub := f.create(events.CreateInput{MaxAttempts: 1})
|
||||
f.cancelResult()
|
||||
first := f.grant(sub.ID)
|
||||
f.expireDelivery(first)
|
||||
noEventGrant(t, f, sub.ID)
|
||||
v, err := f.events.Delivery(ctx, f.token, sub.ID, first.Delivery.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in := events.RetryInput{ExpectedRevision: v.Revision, Reason: "receiver fixed", AdditionalAttempts: 2}
|
||||
out, err := f.events.Retry(ctx, f.token, sub.ID, v.ID, "retry", in)
|
||||
if err != nil || out.Delivery.Attempts != 1 || out.Delivery.AttemptBudget != 3 || out.Delivery.LeaseEpoch != 1 {
|
||||
t.Fatal("budget reset", err)
|
||||
}
|
||||
second := f.grant(sub.ID)
|
||||
if second.Delivery.Attempts != 2 || second.Delivery.LeaseEpoch != 2 {
|
||||
t.Fatal("retry reused attempt")
|
||||
}
|
||||
again, err := f.events.Retry(ctx, f.token, sub.ID, v.ID, "retry", in)
|
||||
if err != nil || !again.Deduplicated || again.Delivery.Revision != out.Delivery.Revision {
|
||||
t.Fatal("retry replay changed receipt", err)
|
||||
}
|
||||
_, err = f.events.Retry(ctx, f.token, sub.ID, v.ID, "another", in)
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
_, err = f.events.Ack(ctx, f.token, sub.ID, events.AckInput{DeliveryID: first.Delivery.ID, Epoch: 1, LeaseSecret: first.LeaseSecret})
|
||||
expectWorkerCode(t, err, "DELIVERY_LEASE_LOST")
|
||||
f.ack(sub.ID, second)
|
||||
if f.count("orbit_audit_events", "resource_id=? AND action='delivery.retry'", v.ID) != 1 {
|
||||
t.Fatal("manual retry audit duplicated")
|
||||
}
|
||||
})
|
||||
t.Run("pause_blocks_new_delivery_but_allows_inflight_ack", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
sub := f.create(events.CreateInput{})
|
||||
f.cancelResult()
|
||||
g := f.grant(sub.ID)
|
||||
paused, err := f.events.Control(ctx, f.token, sub.ID, "pause", events.ControlInput{ExpectedRevision: 1, Enabled: false, Reason: "receiver maintenance"})
|
||||
if err != nil || paused.Subscription.Enabled {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.cancelResult()
|
||||
noEventGrant(t, f, sub.ID)
|
||||
f.ack(sub.ID, g)
|
||||
_, err = f.events.Control(ctx, f.token, sub.ID, "resume", events.ControlInput{ExpectedRevision: 2, Enabled: true, Reason: "receiver ready"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resumed := f.grant(sub.ID)
|
||||
if resumed.Event.Sequence != 2 {
|
||||
t.Fatal("resume lost result")
|
||||
}
|
||||
})
|
||||
t.Run("ack_and_materialization_are_transactional_with_audit", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
sub := f.create(events.CreateInput{})
|
||||
f.cancelResult()
|
||||
f.exec(`CREATE TRIGGER test_delivery_reject BEFORE INSERT ON orbit_deliveries FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected delivery failure'`)
|
||||
t.Cleanup(func() {
|
||||
f.exec(`DROP TRIGGER IF EXISTS test_delivery_reject`)
|
||||
f.exec(`DROP TRIGGER IF EXISTS test_delivery_ack_reject`)
|
||||
})
|
||||
_, err := f.events.Claim(ctx, f.token, sub.ID, events.ClaimInput{ConnectionID: executionID(t), Capacity: 1})
|
||||
if err == nil {
|
||||
t.Fatal("expected delivery insertion failure")
|
||||
}
|
||||
f.exec(`DROP TRIGGER test_delivery_reject`)
|
||||
v := f.subscription(sub.ID)
|
||||
if v.CursorSequence != 0 || v.PendingCount != 0 || f.count("orbit_deliveries", "subscription_id=?", sub.ID) != 0 {
|
||||
t.Fatal("partial materialization escaped rollback")
|
||||
}
|
||||
g := f.grant(sub.ID)
|
||||
f.exec(`CREATE TRIGGER test_delivery_ack_reject BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected audit failure'`)
|
||||
_, err = f.events.Ack(ctx, f.token, sub.ID, events.AckInput{DeliveryID: g.Delivery.ID, Epoch: g.Delivery.LeaseEpoch, LeaseSecret: g.LeaseSecret})
|
||||
if err == nil {
|
||||
t.Fatal("ack escaped audit failure")
|
||||
}
|
||||
f.exec(`DROP TRIGGER test_delivery_ack_reject`)
|
||||
d, err := f.events.Delivery(ctx, f.token, sub.ID, g.Delivery.ID)
|
||||
if err != nil || d.State != platform.DeliveryLeased || d.AcknowledgedAt != nil || f.subscription(sub.ID).PendingCount != 1 {
|
||||
t.Fatal("ack partial commit", err)
|
||||
}
|
||||
f.ack(sub.ID, g)
|
||||
})
|
||||
t.Run("producer_journal_failure_rolls_back_task_event_and_quota", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
task := f.submit(1)
|
||||
f.exec(`CREATE TRIGGER test_result_reject BEFORE INSERT ON orbit_result_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected journal failure'`)
|
||||
t.Cleanup(func() { f.exec(`DROP TRIGGER IF EXISTS test_result_reject`) })
|
||||
_, err := f.tasks.CancelTask(ctx, f.token, task.ID, "cancel", service.CancelInput{ExpectedRevision: task.Revision, Reason: "verify atomic result"})
|
||||
if err == nil {
|
||||
t.Fatal("expected journal failure")
|
||||
}
|
||||
f.exec(`DROP TRIGGER test_result_reject`)
|
||||
if f.get(task.ID).State != pulse.TaskWaiting || f.count("orbit_events", "aggregate_id=?", task.ID) != 1 || f.count("orbit_event_streams", "1=1") != 0 {
|
||||
t.Fatal("result partial commit")
|
||||
}
|
||||
assertApplicationCounts(t, f.executionFixture, 1, 0, 0)
|
||||
if _, err = f.tasks.CancelTask(ctx, f.token, task.ID, "cancel", service.CancelInput{ExpectedRevision: task.Revision, Reason: "verify atomic result"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
t.Run("all_terminal_and_unknown_paths_emit_but_safe_requeue_does_not", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
sub := f.create(events.CreateInput{})
|
||||
f.submit(2)
|
||||
first := retryStart(t, f.executionFixture)
|
||||
retryFinish(t, f.executionFixture, first, "FAILED", true)
|
||||
if f.count("orbit_result_events", "1=1") != 0 {
|
||||
t.Fatal("safe retry published final result")
|
||||
}
|
||||
makeRetryDue(f.executionFixture, first.TaskID)
|
||||
second := retryStart(t, f.executionFixture)
|
||||
retryFinish(t, f.executionFixture, second, "FAILED", false)
|
||||
g := f.grant(sub.ID)
|
||||
if g.Event.Type != "task.failed" {
|
||||
t.Fatal("missing failure result")
|
||||
}
|
||||
f.ack(sub.ID, g)
|
||||
f.submit(1)
|
||||
unknown := retryStart(t, f.executionFixture)
|
||||
retryFinish(t, f.executionFixture, unknown, "UNKNOWN", false)
|
||||
g = f.grant(sub.ID)
|
||||
if g.Event.Type != "task.needs_review" || !g.Event.Snapshot.EffectUncertain {
|
||||
t.Fatal("missing unknown result")
|
||||
}
|
||||
f.ack(sub.ID, g)
|
||||
f.submit(1)
|
||||
lost := retryStart(t, f.executionFixture)
|
||||
f.expire(lost)
|
||||
if _, err := f.storeB.Recover(ctx, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g = f.grant(sub.ID)
|
||||
if g.Event.Type != "task.needs_review" {
|
||||
t.Fatal("recovery failed to publish result")
|
||||
}
|
||||
})
|
||||
t.Run("namespace_limit_future_cursor_and_history_gap_are_explicit", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
future := uint64(1)
|
||||
_, err := f.events.Create(ctx, f.token, "future", events.CreateInput{Name: "future", QueueID: f.resources.QueueID, FromSequence: &future})
|
||||
expectWorkerCode(t, err, "INVALID_ARGUMENT")
|
||||
sub := f.create(events.CreateInput{})
|
||||
f.cancelResult()
|
||||
f.exec(`UPDATE orbit_event_streams SET retained_from_sequence=2 WHERE namespace_id=?`, f.token.NamespaceID)
|
||||
_, err = f.events.Claim(ctx, f.token, sub.ID, events.ClaimInput{ConnectionID: executionID(t), Capacity: 1})
|
||||
expectWorkerCode(t, err, "EVENT_HISTORY_EXPIRED")
|
||||
_, err = f.events.Create(ctx, f.token, "expired", events.CreateInput{Name: "expired", QueueID: f.resources.QueueID, FromSequence: &zero})
|
||||
expectWorkerCode(t, err, "EVENT_HISTORY_EXPIRED")
|
||||
for i := 1; i < events.MaxSubscriptions; i++ {
|
||||
f.create(events.CreateInput{Name: fmt.Sprintf("limit-%d", i)})
|
||||
}
|
||||
_, err = f.events.Create(ctx, f.token, "overflow", events.CreateInput{Name: "overflow", QueueID: f.resources.QueueID})
|
||||
expectWorkerCode(t, err, "SUBSCRIPTION_LIMIT")
|
||||
})
|
||||
|
||||
t.Run("locked_subscriber_cannot_block_task_result_commit", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
sub := f.create(events.CreateInput{})
|
||||
task := f.submit(1)
|
||||
tx, err := db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var id core.ID
|
||||
if err = tx.QueryRowContext(ctx, `SELECT id FROM orbit_subscriptions WHERE namespace_id=? AND id=? FOR UPDATE`, f.token.NamespaceID, sub.ID).Scan(&id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bounded, stop := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer stop()
|
||||
if _, err = f.tasks.CancelTask(bounded, f.token, task.ID, "while-locked", service.CancelInput{ExpectedRevision: task.Revision, Reason: "subscriber is slow"}); err != nil {
|
||||
t.Fatal("producer waited for subscriber lock", err)
|
||||
}
|
||||
if err = tx.Rollback(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := f.grant(sub.ID)
|
||||
if g.Event.Snapshot.TaskID != task.ID {
|
||||
t.Fatal("result lost while subscriber blocked")
|
||||
}
|
||||
})
|
||||
t.Run("connection_capacity_is_database_enforced", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
sub := f.create(events.CreateInput{})
|
||||
f.cancelResult()
|
||||
f.cancelResult()
|
||||
claim := events.ClaimInput{ConnectionID: executionID(t), Capacity: 1}
|
||||
first, err := f.events.Claim(ctx, f.token, sub.ID, claim)
|
||||
if err != nil || first == nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, err := f.events.Claim(ctx, f.token, sub.ID, claim); err != nil || g != nil {
|
||||
t.Fatal("connection capacity exceeded", err)
|
||||
}
|
||||
f.ack(sub.ID, first)
|
||||
next, err := f.events.Claim(ctx, f.token, sub.ID, claim)
|
||||
if err != nil || next == nil || next.Event.ID == first.Event.ID {
|
||||
t.Fatal("capacity not released on confirmation", err)
|
||||
}
|
||||
})
|
||||
t.Run("late_ack_races_new_delivery_without_double_confirmation", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
sub := f.create(events.CreateInput{MaxPending: 1, MaxAttempts: 2})
|
||||
for range 12 {
|
||||
f.cancelResult()
|
||||
first := f.grant(sub.ID)
|
||||
f.expireDelivery(first)
|
||||
noEventGrant(t, f, sub.ID)
|
||||
f.exec(`UPDATE orbit_deliveries SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.token.NamespaceID, first.Delivery.ID)
|
||||
type claimResult struct {
|
||||
grant *events.Grant
|
||||
err error
|
||||
}
|
||||
claims := make(chan claimResult, 1)
|
||||
acks := make(chan error, 1)
|
||||
start := make(chan struct{})
|
||||
connection := executionID(t)
|
||||
go func() {
|
||||
<-start
|
||||
g, err := f.events.Claim(ctx, f.token, sub.ID, events.ClaimInput{ConnectionID: connection, Capacity: 1})
|
||||
claims <- claimResult{g, err}
|
||||
}()
|
||||
go func() {
|
||||
<-start
|
||||
_, err := f.events.Ack(ctx, f.token, sub.ID, events.AckInput{DeliveryID: first.Delivery.ID, Epoch: first.Delivery.LeaseEpoch, LeaseSecret: first.LeaseSecret})
|
||||
acks <- err
|
||||
}()
|
||||
close(start)
|
||||
c := <-claims
|
||||
ackErr := <-acks
|
||||
if c.err != nil {
|
||||
t.Fatal(c.err)
|
||||
}
|
||||
if c.grant == nil {
|
||||
if ackErr != nil {
|
||||
t.Fatal("neither ack nor redelivery won", ackErr)
|
||||
}
|
||||
} else {
|
||||
expectWorkerCode(t, ackErr, "DELIVERY_LEASE_LOST")
|
||||
f.ack(sub.ID, c.grant)
|
||||
}
|
||||
if f.subscription(sub.ID).PendingCount != 0 || f.count("orbit_audit_events", "resource_id=? AND action='delivery.ack'", first.Delivery.ID) != 1 {
|
||||
t.Fatal("race duplicated confirmation or leaked window")
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("proof_is_not_stored_in_audit_or_plaintext_column", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
sub := f.create(events.CreateInput{})
|
||||
f.cancelResult()
|
||||
g := f.grant(sub.ID)
|
||||
f.ack(sub.ID, g)
|
||||
var before, after string
|
||||
if err := db.QueryRowContext(ctx, "SELECT `before`,`after` FROM orbit_audit_events WHERE namespace_id=? AND resource_id=? AND action='delivery.ack'", f.token.NamespaceID, g.Delivery.ID).Scan(&before, &after); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(before+after, g.LeaseSecret) {
|
||||
t.Fatal("delivery proof leaked to audit")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestIntegrationResultCommitOrder(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
db, err := Open(ctx, dsn, 16, 4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err = Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, rollback := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("first_rollback_%t", rollback), func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
sub := f.create(events.CreateInput{})
|
||||
first, second := f.submit(1), f.submit(1)
|
||||
finish := func(tx *sql.Tx, task service.TaskView) error {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='CANCELLED',completed_at=UTC_TIMESTAMP(6),revision=revision+1 WHERE namespace_id=? AND id=?`, f.token.NamespaceID, task.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
return emitEvent(ctx, tx, f.token.NamespaceID, task.ID, task.Revision+1, "task", "task.cancelled", map[string]any{"state": "CANCELLED"})
|
||||
}
|
||||
tx, err := db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err = finish(tx, first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started := make(chan struct{})
|
||||
result := make(chan error, 1)
|
||||
go func() {
|
||||
result <- WithTx(ctx, db, func(other *sql.Tx) error { close(started); return finish(other, second) })
|
||||
}()
|
||||
<-started
|
||||
select {
|
||||
case err := <-result:
|
||||
t.Fatalf("later transaction crossed uncommitted stream: %v", err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
// 已提交读在第一事务未提交时看不到较大的序号；消费也不等待生产者流锁。
|
||||
noEventGrant(t, f, sub.ID)
|
||||
if rollback {
|
||||
err = tx.Rollback()
|
||||
} else {
|
||||
err = tx.Commit()
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = <-result; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := uint64(2)
|
||||
if rollback {
|
||||
want = 1
|
||||
}
|
||||
var head uint64
|
||||
if err = db.QueryRowContext(ctx, `SELECT last_sequence FROM orbit_event_streams WHERE namespace_id=?`, f.token.NamespaceID).Scan(&head); err != nil || head != want {
|
||||
t.Fatal("nontransactional sequence", err)
|
||||
}
|
||||
for seq := uint64(1); seq <= want; seq++ {
|
||||
g := f.grant(sub.ID)
|
||||
if g.Event.Sequence != seq {
|
||||
t.Fatal("commit order skipped result")
|
||||
}
|
||||
f.ack(sub.ID, g)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("create_new_boundary_waits_for_earlier_producer", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
task := f.submit(1)
|
||||
tx, err := db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state='CANCELLED',completed_at=UTC_TIMESTAMP(6),revision=2 WHERE namespace_id=? AND id=?`, f.token.NamespaceID, task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = emitEvent(ctx, tx, f.token.NamespaceID, task.ID, 2, "task", "task.cancelled", map[string]any{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
type result struct {
|
||||
v events.SubscriptionResult
|
||||
err error
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
v, err := f.events.Create(ctx, f.token, "boundary", events.CreateInput{Name: "boundary", QueueID: f.resources.QueueID})
|
||||
ch <- result{v, err}
|
||||
}()
|
||||
select {
|
||||
case <-ch:
|
||||
t.Fatal("subscription creation crossed stream boundary")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := <-ch
|
||||
if r.err != nil || r.v.Subscription.CursorSequence != 1 {
|
||||
t.Fatal("boundary did not use committed head", r.err)
|
||||
}
|
||||
noEventGrant(t, f, r.v.Subscription.ID)
|
||||
})
|
||||
}
|
||||
@@ -189,6 +189,8 @@ func checkModelColumns(t *testing.T, ctx context.Context, db *sql.DB) {
|
||||
platform.Credential{},
|
||||
platform.IdempotencyKey{},
|
||||
platform.Event{},
|
||||
platform.EventStream{},
|
||||
platform.ResultEvent{},
|
||||
platform.Subscription{},
|
||||
platform.Delivery{},
|
||||
platform.AuditEvent{},
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
-- 结果日志按命名空间串行分配序号，与来源状态事件同事务提交；不使用自增 ID 推断提交顺序。
|
||||
-- 禁止外键；来源事件、队列和订阅归属由服务端事务校验。
|
||||
CREATE TABLE orbit_event_streams (
|
||||
namespace_id BINARY(16) NOT NULL COMMENT '结果流所属命名空间；主键同时作为该流序号分配的事务锁',
|
||||
last_sequence BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '已分配结果序号；从1开始，与结果日志同事务提交，回滚不消耗序号',
|
||||
retained_from_sequence BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '最早保留序号；首版不清理结果日志，固定为1，供以后安全清理校验',
|
||||
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '创建时间，统一UTC，精度微秒',
|
||||
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '最后分配序号时间，统一UTC，不代替序号',
|
||||
PRIMARY KEY(namespace_id),
|
||||
CONSTRAINT ck_event_stream_range CHECK (retained_from_sequence>0 AND retained_from_sequence-1<=last_sequence)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='命名空间结果流的提交序号和保留边界';
|
||||
|
||||
CREATE TABLE orbit_result_events (
|
||||
namespace_id BINARY(16) NOT NULL COMMENT '结果所属命名空间，参与主键与服务端关联校验',
|
||||
id BINARY(16) NOT NULL COMMENT '与来源orbit_events记录使用相同标识；跨订阅消费去重依据',
|
||||
revision BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '不可变记录版本，固定为1',
|
||||
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '创建时间，统一UTC，精度微秒',
|
||||
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '不可变记录的初始写入时间，统一UTC',
|
||||
sequence BIGINT UNSIGNED NOT NULL COMMENT '命名空间内事务分配的结果序号；较小序号必先提交或整体回滚',
|
||||
queue_id BINARY(16) NOT NULL COMMENT '来源任务的队列标识；订阅过滤和授权按该队列检查',
|
||||
snapshot MEDIUMTEXT NOT NULL COMMENT '版本化结果快照JSON文本，保留数值精度，不含任务输入和租约秘密',
|
||||
PRIMARY KEY(namespace_id,id),
|
||||
UNIQUE KEY uq_result_sequence(namespace_id,sequence),
|
||||
KEY ix_result_queue_sequence(namespace_id,queue_id,sequence),
|
||||
CONSTRAINT ck_result_event CHECK (revision=1 AND sequence>0 AND JSON_VALID(snapshot))
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='不可变任务结果日志；与任务状态及来源事件一起提交';
|
||||
|
||||
CREATE TRIGGER orbit_result_events_immutable BEFORE UPDATE ON orbit_result_events FOR EACH ROW
|
||||
SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='result events are immutable';
|
||||
|
||||
ALTER TABLE orbit_subscriptions
|
||||
MODIFY COLUMN filter JSON NOT NULL COMMENT '创建时冻结的结果事件类型数组JSON，不允许修改以免漏消费',
|
||||
MODIFY COLUMN enabled BOOLEAN NOT NULL DEFAULT TRUE COMMENT '是否启用新交付；暂停保留积压和在途结果，仍可接受有效确认',
|
||||
ADD COLUMN queue_id BINARY(16) NULL COMMENT '冻结订阅队列；历史占位配置为空且不可消费，须通过新接口创建有效订阅',
|
||||
ADD COLUMN cursor_sequence BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '已扫描并可靠创建交付记录的结果序号；不是消费者确认位置',
|
||||
ADD COLUMN pending_count INT UNSIGNED NOT NULL DEFAULT 0 COMMENT '已物化但未确认数量，包含等待、租约中和耗尽次数的交付',
|
||||
ADD COLUMN max_pending INT UNSIGNED NOT NULL DEFAULT 100 COMMENT '未确认交付上限，1至1000；达到后暂停物化但保留来源结果',
|
||||
ADD COLUMN max_attempts INT UNSIGNED NOT NULL DEFAULT 20 COMMENT '每条新交付的初始发送次数上限，1至100，不包含任务执行业务次数',
|
||||
ADD CONSTRAINT ck_subscription_window CHECK (max_pending BETWEEN 1 AND 1000 AND max_attempts BETWEEN 1 AND 100 AND pending_count<=max_pending);
|
||||
|
||||
-- 旧配置没有公开消费协议和队列授权绑定，保留原记录但不自动开启消费。
|
||||
UPDATE orbit_subscriptions SET enabled=FALSE;
|
||||
|
||||
ALTER TABLE orbit_deliveries
|
||||
ADD COLUMN attempt_budget INT UNSIGNED NOT NULL DEFAULT 20 COMMENT '累计允许交付次数，人工追加预算不重置已使用次数和租约代次',
|
||||
ADD COLUMN token_digest BINARY(32) NULL COMMENT '最近一次交付秘密的SHA256摘要；从未发出或历史占位记录为空，确认后保留以核对重复确认',
|
||||
ADD KEY ix_delivery_subscription_due(namespace_id,subscription_id,state,available_at,id),
|
||||
ADD KEY ix_delivery_subscription_lease(namespace_id,subscription_id,state,lease_until,id);
|
||||
|
||||
UPDATE orbit_deliveries SET attempt_budget=GREATEST(attempt_budget,attempts);
|
||||
ALTER TABLE orbit_deliveries ADD CONSTRAINT ck_delivery_budget CHECK (attempts<=attempt_budget AND attempt_budget>0);
|
||||
@@ -88,7 +88,7 @@ func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
|
||||
}
|
||||
indexes := func() []string {
|
||||
t.Helper()
|
||||
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME<>'orbit_application_resources' AND INDEX_NAME<>'ix_task_priority_dispatch' ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
|
||||
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
52
internal/storage/mysql/result_journal.go
Normal file
52
internal/storage/mysql/result_journal.go
Normal file
@@ -0,0 +1,52 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"math"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/jsonwire"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/events"
|
||||
)
|
||||
|
||||
// appendResultEvent 必须在来源任务更新和事件写入之后、原事务提交之前调用。
|
||||
// 流行一直锁到提交；后来的事务无法提交更大序号并越过尚未提交的较小序号。
|
||||
// 不访问订阅或交付行：慢消费者不能反向阻塞任务的结果持久化。
|
||||
func appendResultEvent(ctx context.Context, tx *sql.Tx, ns, eventID, taskID core.ID, revision uint64, eventType string) error {
|
||||
var snapshot events.TaskResult
|
||||
var raw []byte
|
||||
snapshot.SchemaVersion = 1
|
||||
err := tx.QueryRowContext(ctx, `SELECT id,queue_id,state,revision,result,effect_uncertain,current_attempt_id FROM pulse_tasks WHERE namespace_id=? AND id=?`, ns, taskID).Scan(&snapshot.TaskID, &snapshot.QueueID, &snapshot.State, &snapshot.Revision, &raw, &snapshot.EffectUncertain, &snapshot.CurrentAttemptID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if snapshot.Revision != revision || eventType != "task."+map[string]string{"SUCCEEDED": "succeeded", "FAILED": "failed", "CANCELLED": "cancelled", "NEEDS_REVIEW": "needs_review"}[snapshot.State] {
|
||||
return errors.New("task result event does not match persisted task state")
|
||||
}
|
||||
snapshot.Result = raw
|
||||
body, err := jsonwire.Marshal(snapshot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
head, _, err := lockEventStream(ctx, tx, ns)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if head == math.MaxUint64 {
|
||||
return errors.New("event sequence exhausted")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO orbit_result_events(namespace_id,id,sequence,queue_id,snapshot) VALUES (?,?,?,?,?)`, ns, eventID, head+1, snapshot.QueueID, string(body)); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE orbit_event_streams SET last_sequence=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=?`, head+1, ns)
|
||||
return err
|
||||
}
|
||||
func lockEventStream(ctx context.Context, tx *sql.Tx, ns core.ID) (head, retained uint64, err error) {
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO orbit_event_streams(namespace_id) VALUES (?) ON DUPLICATE KEY UPDATE namespace_id=orbit_event_streams.namespace_id`, ns); err != nil {
|
||||
return
|
||||
}
|
||||
err = tx.QueryRowContext(ctx, `SELECT last_sequence,retained_from_sequence FROM orbit_event_streams WHERE namespace_id=? FOR UPDATE`, ns).Scan(&head, &retained)
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
mysqldriver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// 从真实v12结构升级，历史占位订阅不被错误地当成已获队列授权的有效消费端。
|
||||
func TestIntegrationResultSubscriptionsUpgrade(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
admin, err := Open(ctx, dsn, 2, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer admin.Close()
|
||||
name := "orbit_result_upgrade_" + string(executionID(t))
|
||||
if _, err = admin.ExecContext(ctx, "CREATE DATABASE `"+name+"`"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
cleanup, stop := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer stop()
|
||||
if _, err := admin.ExecContext(cleanup, "DROP DATABASE `"+name+"`"); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
cfg, err := mysqldriver.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
t.Fatal("invalid test DSN")
|
||||
}
|
||||
cfg.DBName = name
|
||||
db, err := Open(ctx, cfg.FormatDSN(), 4, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
exec := func(q string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := db.ExecContext(ctx, q, args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
exec(`CREATE TABLE orbit_schema_migrations (version BIGINT UNSIGNED PRIMARY KEY,name VARCHAR(191) NOT NULL,checksum CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,dirty BOOLEAN NOT NULL,applied_at DATETIME(6) NULL) ENGINE=InnoDB`)
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range migrations {
|
||||
if m.Version > 12 {
|
||||
break
|
||||
}
|
||||
statements, err := splitSQL(m.SQL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, q := range statements {
|
||||
exec(q)
|
||||
}
|
||||
exec(`INSERT INTO orbit_schema_migrations(version,name,checksum,dirty,applied_at) VALUES (?,?,?,FALSE,UTC_TIMESTAMP(6))`, m.Version, m.Name, m.Checksum)
|
||||
}
|
||||
app, ns, queue, credential, task, event, sub, delivery := executionID(t), executionID(t), executionID(t), executionID(t), executionID(t), executionID(t), executionID(t), executionID(t)
|
||||
exec(`INSERT INTO orbit_applications(id,name) VALUES (?,?)`, app, string(app))
|
||||
exec(`INSERT INTO orbit_namespaces(id,application_id,name,environment) VALUES (?,?,'default','test')`, ns, app)
|
||||
exec(`INSERT INTO orbit_credentials(namespace_id,id,name,secret_digest,permissions) VALUES (?,?,'legacy',?, '{}')`, ns, credential, core.Digest{})
|
||||
exec(`INSERT INTO pulse_queues(namespace_id,id,name,max_concurrency,rate_per_second,policy) VALUES (?,?,'legacy',1,1,'{}')`, ns, queue)
|
||||
exec(`INSERT INTO pulse_tasks(namespace_id,id,queue_id,state,handler,handler_version,payload_version,payload,policy_snapshot,available_at,deadline_at,attempt_timeout_ms,max_attempts,completed_at) VALUES (?,?,?,'CANCELLED','object.prepare','v1','v1','{}','{}',UTC_TIMESTAMP(6),DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 1 HOUR),5000,1,UTC_TIMESTAMP(6))`, ns, task, queue)
|
||||
exec(`INSERT INTO orbit_events(namespace_id,id,aggregate_kind,aggregate_id,aggregate_revision,type,payload,occurred_at) VALUES (?,?,'task',?,1,'task.cancelled','{"n":9007199254740993}',UTC_TIMESTAMP(6))`, ns, event, task)
|
||||
exec(`INSERT INTO orbit_subscriptions(namespace_id,id,name,credential_id,filter) VALUES (?,?,'legacy',?,'{}')`, ns, sub, credential)
|
||||
exec(`INSERT INTO orbit_deliveries(namespace_id,id,subscription_id,event_id,state,available_at,attempts,lease_epoch) VALUES (?,?,?,?,'EXPIRED',UTC_TIMESTAMP(6),25,25)`, ns, delivery, sub, event)
|
||||
if err = Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = CheckSchema(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var enabled bool
|
||||
var queueID *core.ID
|
||||
var filter string
|
||||
if err = db.QueryRowContext(ctx, `SELECT enabled,queue_id,filter FROM orbit_subscriptions WHERE namespace_id=? AND id=?`, ns, sub).Scan(&enabled, &queueID, &filter); err != nil || enabled || queueID != nil || filter != "{}" {
|
||||
t.Fatal("legacy subscription acquired unintended authority", err)
|
||||
}
|
||||
var attempts, budget, epoch uint64
|
||||
var secret []byte
|
||||
if err = db.QueryRowContext(ctx, `SELECT attempts,attempt_budget,lease_epoch,token_digest FROM orbit_deliveries WHERE namespace_id=? AND id=?`, ns, delivery).Scan(&attempts, &budget, &epoch, &secret); err != nil || attempts != 25 || budget != 25 || epoch != 25 || secret != nil {
|
||||
t.Fatal("legacy delivery history changed", err)
|
||||
}
|
||||
var count int
|
||||
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM orbit_result_events`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatal("migration synthesized unverified historical snapshots", err)
|
||||
}
|
||||
var payload string
|
||||
if err = db.QueryRowContext(ctx, `SELECT payload FROM orbit_events WHERE namespace_id=? AND id=?`, ns, event).Scan(&payload); err != nil || payload != `{"n":9007199254740993}` {
|
||||
t.Fatal("original event changed", err)
|
||||
}
|
||||
if err = Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("v12 upgrade preserved source history and cumulative attempts, disabled unscoped legacy subscriptions, introduced no foreign keys")
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/jsonwire"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/events"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
driver "github.com/go-sql-driver/mysql"
|
||||
@@ -125,6 +126,9 @@ func emitEvent(ctx context.Context, tx *sql.Tx, ns, resource core.ID, revision u
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO orbit_events(namespace_id,id,aggregate_kind,aggregate_id,aggregate_revision,type,payload,occurred_at) VALUES (?,?,?,?,?,?,?,UTC_TIMESTAMP(6))`, ns, id, kind, resource, revision, event, string(raw))
|
||||
if err == nil && kind == "task" && events.ResultType(event) {
|
||||
return appendResultEvent(ctx, tx, ns, id, resource, revision, event)
|
||||
}
|
||||
return err
|
||||
}
|
||||
func appendAudit(ctx context.Context, tx *sql.Tx, token auth.Token, operation, resource core.ID, kind, action, reason string, before, after any) error {
|
||||
|
||||
Reference in New Issue
Block a user