feat: add durable result journal and subscription delivery core
This commit is contained in:
@@ -58,7 +58,7 @@ type Permissions struct {
|
||||
Queues []string `json:"queues"`
|
||||
}
|
||||
|
||||
var supported = map[string]bool{"application:resources:read": true, "application:resources:manage": true, "tasks:submit": true, "tasks:read": true, "tasks:cancel": true, "queues:read": true, "queues:manage": true, "workers:connect": true, "workers:consume": true, "schedules:read": true, "schedules:manage": true}
|
||||
var supported = map[string]bool{"events:read": true, "events:manage": true, "events:consume": true, "application:resources:read": true, "application:resources:manage": true, "tasks:submit": true, "tasks:read": true, "tasks:cancel": true, "queues:read": true, "queues:manage": true, "workers:connect": true, "workers:consume": true, "schedules:read": true, "schedules:manage": true}
|
||||
|
||||
func (p Permissions) Validate() error {
|
||||
if len(p.Actions) == 0 || len(p.Actions) > 32 || len(p.Queues) == 0 || len(p.Queues) > 128 {
|
||||
|
||||
277
platform/events/service.go
Normal file
277
platform/events/service.go
Normal file
@@ -0,0 +1,277 @@
|
||||
// Package events 定义可靠结果订阅契约；交付采用至少一次语义，接收方按事件ID去重。
|
||||
package events
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
platform "git.yuntuo.hk/k2safe/orbit/platform/model"
|
||||
)
|
||||
|
||||
const LeaseDuration = 30 * time.Second
|
||||
const MaxSubscriptions = 64
|
||||
|
||||
var (
|
||||
LeaseLost = fault.New("DELIVERY_LEASE_LOST", "delivery proof was superseded or is invalid")
|
||||
HistoryExpired = fault.New("EVENT_HISTORY_EXPIRED", "requested event history is no longer retained")
|
||||
SubscriptionLimit = fault.New("SUBSCRIPTION_LIMIT", "namespace subscription limit reached")
|
||||
)
|
||||
|
||||
func ResultType(s string) bool {
|
||||
switch s {
|
||||
case "task.succeeded", "task.failed", "task.cancelled", "task.needs_review":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// TaskResult 是事件发生时冻结的快照，不从后续可变任务状态重新生成。
|
||||
type TaskResult struct {
|
||||
SchemaVersion uint32 `json:"schema_version"`
|
||||
TaskID core.ID `json:"task_id"`
|
||||
QueueID core.ID `json:"queue_id"`
|
||||
State string `json:"state"`
|
||||
Revision uint64 `json:"revision,string"`
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
EffectUncertain bool `json:"effect_uncertain"`
|
||||
CurrentAttemptID *core.ID `json:"current_attempt_id,omitempty"`
|
||||
}
|
||||
|
||||
type Event struct {
|
||||
ID core.ID `json:"id"`
|
||||
Sequence uint64 `json:"sequence,string"`
|
||||
Type string `json:"type"`
|
||||
AggregateKind string `json:"aggregate_kind"`
|
||||
AggregateID core.ID `json:"aggregate_id"`
|
||||
AggregateRevision uint64 `json:"aggregate_revision,string"`
|
||||
OccurredAt time.Time `json:"occurred_at"`
|
||||
// Details 是来源事件元信息，不包含租约秘密和任务输入。
|
||||
Details json.RawMessage `json:"details"`
|
||||
Snapshot TaskResult `json:"snapshot"`
|
||||
}
|
||||
|
||||
type CreateInput struct {
|
||||
Name string `json:"name"`
|
||||
QueueID core.ID `json:"queue_id"`
|
||||
Types []string `json:"types,omitempty"`
|
||||
// FromSequence 之后的结果纳入订阅；省略为创建时已提交的最新序号，0为从现存日志开始。
|
||||
FromSequence *uint64 `json:"from_sequence,string,omitempty"`
|
||||
MaxPending uint32 `json:"max_pending,omitempty"`
|
||||
MaxAttempts uint32 `json:"max_attempts,omitempty"`
|
||||
}
|
||||
|
||||
type Subscription struct {
|
||||
ID core.ID `json:"id"`
|
||||
QueueID core.ID `json:"queue_id"`
|
||||
CredentialID core.ID `json:"credential_id"`
|
||||
Name string `json:"name"`
|
||||
Revision uint64 `json:"revision,string"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Types []string `json:"types"`
|
||||
CursorSequence uint64 `json:"cursor_sequence,string"`
|
||||
PendingCount uint32 `json:"pending_count"`
|
||||
MaxPending uint32 `json:"max_pending"`
|
||||
MaxAttempts uint32 `json:"max_attempts"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type SubscriptionResult struct {
|
||||
Subscription Subscription `json:"subscription"`
|
||||
Deduplicated bool `json:"deduplicated"`
|
||||
}
|
||||
|
||||
type ControlInput struct {
|
||||
ExpectedRevision uint64 `json:"expected_revision,string"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type Delivery struct {
|
||||
ID core.ID `json:"id"`
|
||||
SubscriptionID core.ID `json:"subscription_id"`
|
||||
EventID core.ID `json:"event_id"`
|
||||
Sequence uint64 `json:"sequence,string"`
|
||||
Revision uint64 `json:"revision,string"`
|
||||
State platform.DeliveryState `json:"state"`
|
||||
AvailableAt time.Time `json:"available_at"`
|
||||
LeaseEpoch uint64 `json:"lease_epoch,string"`
|
||||
LeaseUntil *time.Time `json:"lease_until,omitempty"`
|
||||
Attempts uint32 `json:"attempts"`
|
||||
AttemptBudget uint32 `json:"attempt_budget"`
|
||||
AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"`
|
||||
}
|
||||
|
||||
// ClaimInput 由服务端已认证的连接生成，ConnectionID不得直接使用客户端自报值。
|
||||
type ClaimInput struct {
|
||||
ConnectionID core.ID
|
||||
Capacity uint32
|
||||
}
|
||||
|
||||
type Grant struct {
|
||||
Delivery Delivery `json:"delivery"`
|
||||
Event Event `json:"event"`
|
||||
// LeaseSecret 仅交付给当前连接；持久层、日志和审计仅保存摘要。
|
||||
LeaseSecret string `json:"lease_secret"`
|
||||
}
|
||||
|
||||
type AckInput struct {
|
||||
DeliveryID core.ID `json:"delivery_id"`
|
||||
Epoch uint64 `json:"epoch,string"`
|
||||
LeaseSecret string `json:"lease_secret"`
|
||||
}
|
||||
|
||||
type AckResult struct {
|
||||
DeliveryID core.ID `json:"delivery_id"`
|
||||
EventID core.ID `json:"event_id"`
|
||||
Epoch uint64 `json:"epoch,string"`
|
||||
AcknowledgedAt time.Time `json:"acknowledged_at"`
|
||||
Deduplicated bool `json:"deduplicated"`
|
||||
}
|
||||
|
||||
type RetryInput struct {
|
||||
ExpectedRevision uint64 `json:"expected_revision,string"`
|
||||
Reason string `json:"reason"`
|
||||
AdditionalAttempts uint32 `json:"additional_attempts"`
|
||||
}
|
||||
|
||||
type RetryResult struct {
|
||||
Delivery Delivery `json:"delivery"`
|
||||
Deduplicated bool `json:"deduplicated"`
|
||||
}
|
||||
|
||||
type Store interface {
|
||||
CreateSubscription(context.Context, auth.Token, string, CreateInput) (SubscriptionResult, error)
|
||||
GetSubscription(context.Context, auth.Token, core.ID) (Subscription, error)
|
||||
ControlSubscription(context.Context, auth.Token, core.ID, string, ControlInput) (SubscriptionResult, error)
|
||||
GetDelivery(context.Context, auth.Token, core.ID, core.ID) (Delivery, error)
|
||||
ClaimDelivery(context.Context, auth.Token, core.ID, ClaimInput) (*Grant, error)
|
||||
AckDelivery(context.Context, auth.Token, core.ID, AckInput) (AckResult, error)
|
||||
RetryDelivery(context.Context, auth.Token, core.ID, core.ID, string, RetryInput) (RetryResult, error)
|
||||
}
|
||||
|
||||
type Service struct{ store Store }
|
||||
|
||||
func New(s Store) *Service { return &Service{store: s} }
|
||||
|
||||
var validName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
|
||||
func keyValid(key string) error {
|
||||
if len(key) < 1 || len(key) > 128 {
|
||||
return fault.Invalid("invalid Idempotency-Key")
|
||||
}
|
||||
for _, c := range key {
|
||||
if c < 33 || c > 126 {
|
||||
return fault.Invalid("invalid Idempotency-Key")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func reasonValid(revision uint64, reason string) bool {
|
||||
return revision > 0 && utf8.ValidString(reason) && strings.TrimSpace(reason) != "" && utf8.RuneCountInString(reason) <= 1024
|
||||
}
|
||||
func (s *Service) Create(ctx context.Context, t auth.Token, key string, in CreateInput) (SubscriptionResult, error) {
|
||||
if err := keyValid(key); err != nil {
|
||||
return SubscriptionResult{}, err
|
||||
}
|
||||
if len(in.Name) > 128 || !validName.MatchString(in.Name) || in.QueueID.Validate() != nil {
|
||||
return SubscriptionResult{}, fault.Invalid("invalid subscription name or queue_id")
|
||||
}
|
||||
if in.MaxPending == 0 {
|
||||
in.MaxPending = 100
|
||||
}
|
||||
if in.MaxAttempts == 0 {
|
||||
in.MaxAttempts = 20
|
||||
}
|
||||
if in.MaxPending > 1000 || in.MaxAttempts > 100 {
|
||||
return SubscriptionResult{}, fault.Invalid("subscription limits are invalid")
|
||||
}
|
||||
if len(in.Types) == 0 {
|
||||
in.Types = []string{"task.succeeded", "task.failed", "task.cancelled", "task.needs_review"}
|
||||
}
|
||||
if len(in.Types) > 4 {
|
||||
return SubscriptionResult{}, fault.Invalid("too many event types")
|
||||
}
|
||||
in.Types = slices.Clone(in.Types)
|
||||
slices.Sort(in.Types)
|
||||
for i, v := range in.Types {
|
||||
if !ResultType(v) || i > 0 && v == in.Types[i-1] {
|
||||
return SubscriptionResult{}, fault.Invalid("unsupported or repeated event type")
|
||||
}
|
||||
}
|
||||
return s.store.CreateSubscription(ctx, t, key, in)
|
||||
}
|
||||
func (s *Service) Get(ctx context.Context, t auth.Token, id core.ID) (Subscription, error) {
|
||||
if id.Validate() != nil {
|
||||
return Subscription{}, fault.Invalid("invalid subscription_id")
|
||||
}
|
||||
return s.store.GetSubscription(ctx, t, id)
|
||||
}
|
||||
func (s *Service) Control(ctx context.Context, t auth.Token, id core.ID, key string, in ControlInput) (SubscriptionResult, error) {
|
||||
if err := keyValid(key); err != nil {
|
||||
return SubscriptionResult{}, err
|
||||
}
|
||||
if id.Validate() != nil || !reasonValid(in.ExpectedRevision, in.Reason) {
|
||||
return SubscriptionResult{}, fault.Invalid("subscription_id, revision and reason are required")
|
||||
}
|
||||
return s.store.ControlSubscription(ctx, t, id, key, in)
|
||||
}
|
||||
func (s *Service) Delivery(ctx context.Context, t auth.Token, id, delivery core.ID) (Delivery, error) {
|
||||
if id.Validate() != nil || delivery.Validate() != nil {
|
||||
return Delivery{}, fault.Invalid("invalid subscription_id or delivery_id")
|
||||
}
|
||||
return s.store.GetDelivery(ctx, t, id, delivery)
|
||||
}
|
||||
func (s *Service) Claim(ctx context.Context, t auth.Token, id core.ID, in ClaimInput) (*Grant, error) {
|
||||
if id.Validate() != nil || in.ConnectionID.Validate() != nil || in.Capacity < 1 || in.Capacity > 32 {
|
||||
return nil, fault.Invalid("invalid delivery claim")
|
||||
}
|
||||
return s.store.ClaimDelivery(ctx, t, id, in)
|
||||
}
|
||||
func (s *Service) Ack(ctx context.Context, t auth.Token, id core.ID, in AckInput) (AckResult, error) {
|
||||
if id.Validate() != nil || in.DeliveryID.Validate() != nil || in.Epoch == 0 {
|
||||
return AckResult{}, fault.Invalid("invalid delivery confirmation")
|
||||
}
|
||||
if _, err := SecretDigest(in.LeaseSecret); err != nil {
|
||||
return AckResult{}, err
|
||||
}
|
||||
return s.store.AckDelivery(ctx, t, id, in)
|
||||
}
|
||||
func (s *Service) Retry(ctx context.Context, t auth.Token, id, delivery core.ID, key string, in RetryInput) (RetryResult, error) {
|
||||
if err := keyValid(key); err != nil {
|
||||
return RetryResult{}, err
|
||||
}
|
||||
if id.Validate() != nil || delivery.Validate() != nil || !reasonValid(in.ExpectedRevision, in.Reason) || in.AdditionalAttempts < 1 || in.AdditionalAttempts > 100 {
|
||||
return RetryResult{}, fault.Invalid("invalid delivery retry")
|
||||
}
|
||||
return s.store.RetryDelivery(ctx, t, id, delivery, key, in)
|
||||
}
|
||||
|
||||
func SecretDigest(secret string) (core.Digest, error) {
|
||||
if len(secret) != 43 {
|
||||
return core.Digest{}, fault.Invalid("invalid delivery secret")
|
||||
}
|
||||
raw, err := base64.RawURLEncoding.Strict().DecodeString(secret)
|
||||
if err != nil || len(raw) != 32 {
|
||||
return core.Digest{}, fault.Invalid("invalid delivery secret")
|
||||
}
|
||||
return core.Digest(sha256.Sum256(raw)), nil
|
||||
}
|
||||
func NewSecret() (string, core.Digest, error) {
|
||||
var raw [32]byte
|
||||
if _, err := rand.Read(raw[:]); err != nil {
|
||||
return "", core.Digest{}, err
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(raw[:]), core.Digest(sha256.Sum256(raw[:])), nil
|
||||
}
|
||||
84
platform/events/service_test.go
Normal file
84
platform/events/service_test.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
)
|
||||
|
||||
type recordingStore struct {
|
||||
Store
|
||||
input CreateInput
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *recordingStore) CreateSubscription(_ context.Context, _ auth.Token, _ string, in CreateInput) (SubscriptionResult, error) {
|
||||
s.input = in
|
||||
s.calls++
|
||||
return SubscriptionResult{}, nil
|
||||
}
|
||||
|
||||
func TestSubscriptionNormalizationAndValidation(t *testing.T) {
|
||||
store := &recordingStore{}
|
||||
svc := New(store)
|
||||
good := CreateInput{Name: "object-results", QueueID: core.ID(strings.Repeat("1", 32))}
|
||||
if _, err := svc.Create(context.Background(), auth.Token{}, "create", good); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if store.input.MaxPending != 100 || store.input.MaxAttempts != 20 || len(store.input.Types) != 4 || store.input.FromSequence != nil {
|
||||
t.Fatal("invalid defaults")
|
||||
}
|
||||
// 排序仅作用于内部副本，不修改调用方重报保存的原输入。
|
||||
types := []string{"task.succeeded", "task.cancelled"}
|
||||
good.Types = types
|
||||
if _, err := svc.Create(context.Background(), auth.Token{}, "create", good); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if types[0] != "task.succeeded" || store.input.Types[0] != "task.cancelled" {
|
||||
t.Fatal("normalization modified caller")
|
||||
}
|
||||
cases := []func(*CreateInput){
|
||||
func(in *CreateInput) { in.Name = " spaced " },
|
||||
func(in *CreateInput) { in.QueueID = "missing" },
|
||||
func(in *CreateInput) { in.MaxPending = 1001 },
|
||||
func(in *CreateInput) { in.MaxAttempts = 101 },
|
||||
func(in *CreateInput) { in.Types = []string{"task.accepted"} },
|
||||
func(in *CreateInput) { in.Types = []string{"task.failed", "task.failed"} },
|
||||
}
|
||||
for i, mutate := range cases {
|
||||
in := good
|
||||
mutate(&in)
|
||||
if _, err := svc.Create(context.Background(), auth.Token{}, "create", in); err == nil {
|
||||
t.Fatalf("accepted invalid case %d", i)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"", "with space", strings.Repeat("a", 129)} {
|
||||
if _, err := svc.Create(context.Background(), auth.Token{}, key, good); err == nil {
|
||||
t.Fatal("accepted invalid identity")
|
||||
}
|
||||
}
|
||||
if store.calls != 2 {
|
||||
t.Fatal("invalid input reached storage")
|
||||
}
|
||||
}
|
||||
func TestDeliveryProofStrictness(t *testing.T) {
|
||||
secret, digest, err := NewSecret()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
actual, err := SecretDigest(secret)
|
||||
if err != nil || actual != digest {
|
||||
t.Fatal("proof roundtrip", err)
|
||||
}
|
||||
for _, s := range []string{"", secret + "=", secret + "\n", secret[:42], strings.Repeat("!", 43)} {
|
||||
if _, err := SecretDigest(s); err == nil {
|
||||
t.Fatal("accepted noncanonical proof")
|
||||
}
|
||||
}
|
||||
if ResultType("task.requeued") || ResultType("task.accepted") || !ResultType("task.needs_review") {
|
||||
t.Fatal("invalid result event scope")
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,10 @@ type Delivery struct {
|
||||
LeaseUntil *time.Time `db:"lease_until"`
|
||||
// Attempts 已产生的交付尝试次数，不等同于业务执行次数。
|
||||
Attempts uint32 `db:"attempts"`
|
||||
// AttemptBudget 累计允许发送次数；人工追加预算，不重置Attempts或LeaseEpoch。
|
||||
AttemptBudget uint32 `db:"attempt_budget"`
|
||||
// TokenDigest 最近交付秘密的摘要；首次发送前为空，确认后保留以验证重复确认。
|
||||
TokenDigest *core.Digest `db:"token_digest"`
|
||||
// AcknowledgedAt 接收方可靠处理后的确认时间；未确认时为空。
|
||||
AcknowledgedAt *time.Time `db:"acknowledged_at"`
|
||||
}
|
||||
|
||||
38
platform/model/event_stream.go
Normal file
38
platform/model/event_stream.go
Normal file
@@ -0,0 +1,38 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"time"
|
||||
)
|
||||
|
||||
// EventStream 表示命名空间结果流；独立锁行避免升级认证身份共享锁。
|
||||
// 对应 orbit_event_streams；序号分配必须与任务状态和结果日志同事务提交。
|
||||
type EventStream struct {
|
||||
// NamespaceID 结果流所属命名空间，也是序号分配锁的主键。
|
||||
NamespaceID core.ID `db:"namespace_id"`
|
||||
// LastSequence 已分配序号，从1开始；回滚不消耗序号，0表示尚无结果。
|
||||
LastSequence uint64 `db:"last_sequence"`
|
||||
// RetainedFromSequence 最早保留序号；首版不清理，固定为1，防止以后静默跨过缺失历史。
|
||||
RetainedFromSequence uint64 `db:"retained_from_sequence"`
|
||||
// CreatedAt 结果流创建时间，统一UTC，精度微秒。
|
||||
CreatedAt time.Time `db:"created_at"`
|
||||
// UpdatedAt 最后分配序号时间，不能代替结果序号。
|
||||
UpdatedAt time.Time `db:"updated_at"`
|
||||
}
|
||||
|
||||
func (EventStream) TableName() string { return "orbit_event_streams" }
|
||||
|
||||
// ResultEvent 表示从迁移13启用的不可变任务结果快照；不补造历史任务事件。
|
||||
// 对应 orbit_result_events；Record.ID 与来源 Event.ID 相同，关联由事务校验。
|
||||
type ResultEvent struct {
|
||||
// Record 包含所属命名空间、来源事件标识、固定版本1及UTC时间。
|
||||
core.Record
|
||||
// Sequence 命名空间内提交有序的结果序号，首个为1。
|
||||
Sequence uint64 `db:"sequence"`
|
||||
// QueueID 来源任务队列，订阅必须具备该队列的消费权限。
|
||||
QueueID core.ID `db:"queue_id"`
|
||||
// Snapshot 版本化结果快照JSON，保留数值精度，不包含任务输入或租约秘密。
|
||||
Snapshot core.JSON `db:"snapshot"`
|
||||
}
|
||||
|
||||
func (ResultEvent) TableName() string { return "orbit_result_events" }
|
||||
@@ -13,10 +13,20 @@ type Subscription struct {
|
||||
Name string `db:"name"`
|
||||
// CredentialID 关联的命名空间凭据标识。
|
||||
CredentialID core.ID `db:"credential_id"`
|
||||
// Filter 订阅过滤规则的 JSON 文档。
|
||||
// Filter 冻结的结果事件类型数组JSON；创建后不能改写，以免静默漏消费。
|
||||
Filter core.JSON `db:"filter"`
|
||||
// Enabled 是否启用；关闭后拒绝相应的新接入或新操作。
|
||||
// Enabled 是否启用新交付；暂停不撤销已发出的结果，也不拒绝有效确认。
|
||||
Enabled bool `db:"enabled"`
|
||||
// QueueID 创建时冻结的队列；历史占位记录为空且不允许消费。
|
||||
QueueID *core.ID `db:"queue_id"`
|
||||
// CursorSequence 已扫描并物化交付的结果序号，不表示消费者已确认。
|
||||
CursorSequence uint64 `db:"cursor_sequence"`
|
||||
// PendingCount 未确认交付数，包含次数耗尽记录；每次有效确认只减一次。
|
||||
PendingCount uint32 `db:"pending_count"`
|
||||
// MaxPending 未确认交付上限，1至1000；达到后保留来源日志并暂停物化。
|
||||
MaxPending uint32 `db:"max_pending"`
|
||||
// MaxAttempts 新交付的初始发送次数上限，1至100，不等同于任务执行次数。
|
||||
MaxAttempts uint32 `db:"max_attempts"`
|
||||
}
|
||||
|
||||
func (Subscription) TableName() string { return "orbit_subscriptions" }
|
||||
|
||||
Reference in New Issue
Block a user