feat: add reliable event consumer SDK and process recovery tests

This commit is contained in:
2026-09-17 20:22:41 +08:00
parent 8485a662b2
commit b17d9b1977
25 changed files with 2170 additions and 28 deletions

View File

@@ -1,5 +1,6 @@
// Package orbit 是 Orbit 的独立 Go SDK,提供健康、任务、队列与 Worker 底层协议。
// Package orbit 是 Orbit 的独立 Go SDK,提供健康、任务、队列、计划及结果订阅。
// WorkerConnection 是底层接口;NewWorker/Run 自动管理处理器、续租、恢复与排空。
// EventConnection 是结果交付协议;EventConsumer 自动管理心跳、重连与确认,业务端负责事务去重。
// 流程 SDK 等后续能力按阶段继续开发。
// SDK 不导入服务端模型、迁移或数据库依赖。
package orbit

152
sdk/go/event_codec.go Normal file
View File

@@ -0,0 +1,152 @@
package orbit
import (
"bytes"
"encoding/base64"
"encoding/json"
"reflect"
"strconv"
"strings"
"time"
"github.com/coder/websocket"
)
type eventFrame struct {
Version string `json:"protocol_version"`
Type string `json:"type"`
MessageID string `json:"message_id"`
ReplyTo string `json:"reply_to,omitempty"`
SessionID string `json:"session_id,omitempty"`
Body json.RawMessage `json:"body,omitempty"`
Error *EventError `json:"error,omitempty"`
}
// 仅协议结构严格匹配字段;业务RawMessage保持原数字、键和值。
func eventFields(raw []byte, typ reflect.Type) error {
if typ == reflect.TypeOf(json.RawMessage{}) {
return nil
}
if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) {
return ErrEventProtocol
}
if typ.Kind() == reflect.Pointer {
return eventFields(raw, typ.Elem())
}
if typ == reflect.TypeOf(time.Time{}) {
return nil
}
if typ.Kind() != reflect.Struct {
return nil
}
var fields map[string]json.RawMessage
if json.Unmarshal(raw, &fields) != nil || fields == nil {
return ErrEventProtocol
}
for i := 0; i < typ.NumField(); i++ {
f := typ.Field(i)
tags := strings.Split(f.Tag.Get("json"), ",")
if tags[0] == "" || tags[0] == "-" {
continue
}
v, found := fields[tags[0]]
optional, quoted := false, false
for _, tag := range tags[1:] {
optional = optional || tag == "omitempty" || tag == "omitzero"
quoted = quoted || tag == "string"
}
if !found {
if optional {
continue
}
return ErrEventProtocol
}
delete(fields, tags[0])
if f.Type.Kind() == reflect.Uint64 && quoted {
var s string
if json.Unmarshal(v, &s) != nil {
return ErrEventProtocol
}
n, err := strconv.ParseUint(s, 10, 64)
if err != nil || n == 0 || strconv.FormatUint(n, 10) != s {
return ErrEventProtocol
}
}
if err := eventFields(v, f.Type); err != nil {
return err
}
}
if len(fields) > 0 {
return ErrEventProtocol
}
return nil
}
func decodeEventValue(raw []byte, out any) error {
if validateWorkerJSON(raw) != nil || eventFields(raw, reflect.TypeOf(out).Elem()) != nil || json.Unmarshal(raw, out) != nil {
return ErrEventProtocol
}
return nil
}
func parseEventFrame(kind websocket.MessageType, raw []byte) (eventFrame, error) {
var f eventFrame
if kind != websocket.MessageText || len(raw) > 512<<10 || decodeEventValue(raw, &f) != nil || f.Version != "1" || !validID(f.MessageID) {
return f, ErrEventProtocol
}
var fields map[string]json.RawMessage
_ = json.Unmarshal(raw, &fields)
if fields["reply_to"] != nil && !validID(f.ReplyTo) || fields["session_id"] != nil && !validID(f.SessionID) {
return f, ErrEventProtocol
}
switch f.Type {
case "error":
if f.Error == nil || fields["body"] != nil || len(f.Error.Message) == 0 || len(f.Error.Message) > 1024 {
return f, ErrEventProtocol
}
switch f.Error.Code {
case "INVALID_ARGUMENT", "UNAUTHENTICATED", "PERMISSION_DENIED", "NOT_FOUND", "STATE_CONFLICT", "DELIVERY_LEASE_LOST", "EVENT_HISTORY_EXPIRED", "UNAVAILABLE":
default:
return f, ErrEventProtocol
}
case "events.welcome", "ack", "event.deliver":
if fields["error"] != nil || len(f.Body) == 0 || bytes.Equal(bytes.TrimSpace(f.Body), []byte("null")) || !validID(f.SessionID) {
return f, ErrEventProtocol
}
if f.Type == "event.deliver" {
if fields["reply_to"] != nil {
return f, ErrEventProtocol
}
} else if !validID(f.ReplyTo) {
return f, ErrEventProtocol
}
default:
return f, ErrEventProtocol
}
return f, nil
}
func validEventProof(p EventProof) bool {
b, err := base64.RawURLEncoding.Strict().DecodeString(p.LeaseSecret)
return validID(p.SubscriptionID) && validID(p.EventID) && validID(p.DeliveryID) && p.Epoch > 0 && len(p.LeaseSecret) == 43 && err == nil && len(b) == 32
}
func validEventGrant(g EventGrant, sub string) bool {
d, e, s := g.Delivery, g.Event, g.Event.Snapshot
if !validEventProof(g.Proof()) || d.SubscriptionID != sub || d.EventID != e.ID || d.Sequence != e.Sequence || d.Sequence == 0 || d.Revision == 0 || d.State != DeliveryLeased || d.CreatedAt.IsZero() || d.AvailableAt.IsZero() || d.LeaseUntil == nil || d.LeaseUntil.IsZero() || d.AcknowledgedAt != nil || d.Attempts == 0 || d.Attempts > d.AttemptBudget || d.AttemptBudget > 1000000 {
return false
}
if e.AggregateKind != "task" || !validID(e.AggregateID) || e.AggregateID != s.TaskID || e.AggregateRevision != s.Revision || s.Revision == 0 || e.OccurredAt.IsZero() || s.SchemaVersion != 1 || !validID(s.QueueID) || s.CurrentAttemptID != nil && !validID(*s.CurrentAttemptID) {
return false
}
if len(bytes.TrimSpace(e.Details)) == 0 || bytes.TrimSpace(e.Details)[0] != '{' {
return false
}
switch e.Type {
case "task.succeeded":
return s.State == TaskSucceeded
case "task.failed":
return s.State == TaskFailed
case "task.cancelled":
return s.State == TaskCancelled
case "task.needs_review":
return s.State == TaskNeedsReview
}
return false
}

285
sdk/go/event_connection.go Normal file
View File

@@ -0,0 +1,285 @@
package orbit
import (
"context"
"errors"
"fmt"
"net/http"
"sync"
"time"
"github.com/coder/websocket"
)
const eventSubprotocol = "orbit.events.v1"
const eventMaxPending = 32
// EventConnection 管理单次连接、请求关联和有界收发。可并发调用Ack/Ping。
// 本类型不自动心跳/重连/调用处理器;高层消费请使用EventConsumer。
type EventConnection struct {
conn *websocket.Conn
ctx context.Context
cancel context.CancelFunc
welcome EventWelcome
deliveries chan EventGrant
done chan struct{}
mu sync.Mutex
err error
pending map[string]chan eventFrame
}
// DialEvents 的ctx仅控制握手;成功后必须Close,取消握手ctx不会关闭已建立的连接。
func (c *Client) DialEvents(ctx context.Context, in EventRegistration) (*EventConnection, error) {
if c.token == "" || !validID(in.SubscriptionID) || in.Capacity < 1 || in.Capacity > 32 {
return nil, errors.New("authenticated client, valid subscription and capacity 1..32 are required")
}
bounded, stop := context.WithTimeout(ctx, 8*time.Second)
defer stop()
headers := http.Header{}
headers.Set("Authorization", "Bearer "+c.token)
conn, response, err := websocket.Dial(bounded, c.baseURL+"/v1/events/connect", &websocket.DialOptions{HTTPClient: c.http, HTTPHeader: headers, Subprotocols: []string{eventSubprotocol}, CompressionMode: websocket.CompressionDisabled})
if err != nil {
if response != nil && response.StatusCode != http.StatusSwitchingProtocols {
return nil, &HTTPError{StatusCode: response.StatusCode}
}
return nil, eventIOError(err)
}
ok := false
defer func() {
if !ok {
_ = conn.CloseNow()
}
}()
if conn.Subprotocol() != eventSubprotocol {
return nil, ErrEventProtocol
}
conn.SetReadLimit(512 << 10)
id, err := NewEventMessageID()
if err != nil {
return nil, err
}
body, _ := marshalWire(in)
raw, _ := marshalWire(eventFrame{Version: "1", Type: "events.hello", MessageID: id, Body: body})
if err = conn.Write(bounded, websocket.MessageText, raw); err != nil {
return nil, eventIOError(err)
}
kind, raw, err := conn.Read(bounded)
if err != nil {
return nil, eventIOError(err)
}
f, err := parseEventFrame(kind, raw)
if err != nil || f.ReplyTo != id {
return nil, ErrEventProtocol
}
if f.Type == "error" {
return nil, f.Error
}
var welcome EventWelcome
if f.Type != "events.welcome" || decodeEventValue(f.Body, &welcome) != nil || welcome.SessionID != f.SessionID || welcome.SubscriptionID != in.SubscriptionID || welcome.Capacity != in.Capacity || welcome.ServerTime.IsZero() || welcome.LeaseDurationMS != 30000 || welcome.HeartbeatIntervalMS != 15000 {
return nil, ErrEventProtocol
}
lifetime, cancel := context.WithCancel(context.Background())
e := &EventConnection{conn: conn, ctx: lifetime, cancel: cancel, welcome: welcome, deliveries: make(chan EventGrant, in.Capacity), done: make(chan struct{}), pending: make(map[string]chan eventFrame)}
ok = true
go e.readLoop()
return e, nil
}
func (e *EventConnection) Welcome() EventWelcome { return e.welcome }
func (e *EventConnection) Done() <-chan struct{} { return e.done }
func (e *EventConnection) Err() error { e.mu.Lock(); defer e.mu.Unlock(); return e.err }
func (e *EventConnection) Close() error { e.fail(ErrEventClosed); <-e.done; return nil }
func (e *EventConnection) fail(err error) {
e.mu.Lock()
if e.err != nil {
e.mu.Unlock()
return
}
e.err = err
e.cancel()
e.mu.Unlock()
_ = e.conn.CloseNow()
}
func (e *EventConnection) readLoop() {
defer close(e.done)
for {
ctx, stop := context.WithTimeout(e.ctx, 60*time.Second)
kind, raw, err := e.conn.Read(ctx)
stop()
if err != nil {
e.fail(eventIOError(err))
return
}
f, err := parseEventFrame(kind, raw)
if err != nil || f.SessionID != e.welcome.SessionID {
e.fail(ErrEventProtocol)
return
}
if f.Type == "event.deliver" {
var grant EventGrant
if decodeEventValue(f.Body, &grant) != nil || !validEventGrant(grant, e.welcome.SubscriptionID) {
e.fail(ErrEventProtocol)
return
}
select {
case e.deliveries <- grant:
default:
e.fail(ErrEventBackpressure)
return
}
continue
}
if f.Type != "ack" && f.Type != "error" {
e.fail(ErrEventProtocol)
return
}
if f.Type == "error" && f.ReplyTo == "" {
e.fail(f.Error)
return
}
e.mu.Lock()
waiter := e.pending[f.ReplyTo]
e.mu.Unlock()
if waiter == nil {
e.fail(ErrEventProtocol)
return
}
select {
case waiter <- f:
default:
e.fail(ErrEventProtocol)
return
}
if f.Error != nil && (f.Error.Code == "UNAUTHENTICATED" || f.Error.Code == "PERMISSION_DENIED" || f.Error.Code == "NOT_FOUND" || f.Error.Code == "UNAVAILABLE") {
e.fail(f.Error)
return
}
}
}
// Next 等待下一条交付。ctx到期只中断本次等待;未确认交付按服务端预算补发。
func (e *EventConnection) Next(ctx context.Context) (EventGrant, error) {
if err := ctx.Err(); err != nil {
return EventGrant{}, err
}
select {
case <-ctx.Done():
return EventGrant{}, ctx.Err()
case <-e.ctx.Done():
return EventGrant{}, e.Err()
case g := <-e.deliveries:
if err := e.Err(); err != nil {
return EventGrant{}, err
}
return g, nil
}
}
func (e *EventConnection) call(ctx context.Context, kind, id string, in, out any) error {
if !validID(id) {
return errors.New("invalid event message id")
}
if err := ctx.Err(); err != nil {
return err
}
bounded, stop := context.WithTimeout(ctx, 10*time.Second)
defer stop()
waiter := make(chan eventFrame, 1)
e.mu.Lock()
if e.err != nil {
err := e.err
e.mu.Unlock()
return err
}
if len(e.pending) >= eventMaxPending {
e.mu.Unlock()
return ErrEventBusy
}
if e.pending[id] != nil {
e.mu.Unlock()
return errors.New("event message id is already in flight")
}
e.pending[id] = waiter
e.mu.Unlock()
defer func() { e.mu.Lock(); delete(e.pending, id); e.mu.Unlock() }()
body, err := marshalWire(in)
if err != nil {
return errors.New("invalid event request")
}
raw, err := marshalWire(eventFrame{Version: "1", Type: kind, MessageID: id, SessionID: e.welcome.SessionID, Body: body})
if err != nil || len(raw) > 512<<10 {
return errors.New("event request exceeds frame limit")
}
if err = e.conn.Write(bounded, websocket.MessageText, raw); err != nil {
err = eventIOError(err)
e.fail(err)
return err
}
var f eventFrame
select {
case f = <-waiter:
case <-e.ctx.Done():
select {
case f = <-waiter:
default:
return e.Err()
}
case <-bounded.Done():
e.fail(bounded.Err())
return bounded.Err()
}
if f.Error != nil {
return f.Error
}
if decodeEventValue(f.Body, out) != nil {
e.fail(ErrEventProtocol)
return ErrEventProtocol
}
return nil
}
// Ack 仅在业务事务已提交或持久化去重命中后调用。重连可用原Proof重报,MessageID可新建。
// 超时/断线不证明ACK未提交;DELIVERY_LEASE_LOST表示证明已失效,不应撤销已经提交的业务。
func (e *EventConnection) Ack(ctx context.Context, id string, p EventProof) (EventAckResult, error) {
var out EventAckResult
if !validEventProof(p) || p.SubscriptionID != e.welcome.SubscriptionID {
return out, errors.New("invalid event proof or subscription")
}
in := struct {
DeliveryID string `json:"delivery_id"`
Epoch uint64 `json:"epoch,string"`
LeaseSecret string `json:"lease_secret"`
}{p.DeliveryID, p.Epoch, p.LeaseSecret}
err := e.call(ctx, "event.ack", id, in, &out)
if err == nil && (out.DeliveryID != p.DeliveryID || out.EventID != p.EventID || out.Epoch != p.Epoch || out.AcknowledgedAt.IsZero()) {
err = ErrEventProtocol
e.fail(err)
}
return out, err
}
// Ping 重新核验凭据及订阅,并维持连接;不会续期交付租约。
func (e *EventConnection) Ping(ctx context.Context, id string) (EventPong, error) {
var out EventPong
err := e.call(ctx, "events.ping", id, struct{}{}, &out)
if err == nil && out.ServerTime.IsZero() {
err = ErrEventProtocol
e.fail(err)
}
return out, err
}
func eventIOError(err error) error {
if errors.Is(err, context.Canceled) {
return context.Canceled
}
if errors.Is(err, context.DeadlineExceeded) {
return context.DeadlineExceeded
}
switch websocket.CloseStatus(err) {
case websocket.StatusProtocolError, websocket.StatusUnsupportedData, websocket.StatusInvalidFramePayloadData, websocket.StatusPolicyViolation, websocket.StatusMessageTooBig:
return ErrEventProtocol
}
return ErrEventDisconnected
}
func (e *EventConnection) String() string {
return fmt.Sprintf("EventConnection{SubscriptionID:%s SessionID:%s secrets:redacted}", e.welcome.SubscriptionID, e.welcome.SessionID)
}
func (e *EventConnection) GoString() string { return e.String() }

View File

@@ -0,0 +1,283 @@
package orbit
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/coder/websocket"
)
const eventTestSub = "55555555555555555555555555555555"
const eventTestID = "66666666666666666666666666666666"
const eventTestDelivery = "77777777777777777777777777777777"
func eventTestGrant() EventGrant {
now := time.Now().UTC()
until := now.Add(30 * time.Second)
return EventGrant{Delivery: EventDelivery{ID: eventTestDelivery, SubscriptionID: eventTestSub, EventID: eventTestID, Sequence: 9007199254740993, Revision: 2, State: DeliveryLeased, CreatedAt: now, AvailableAt: now, LeaseEpoch: 1, LeaseUntil: &until, Attempts: 1, AttemptBudget: 20}, Event: ResultEvent{ID: eventTestID, Sequence: 9007199254740993, Type: "task.succeeded", AggregateKind: "task", AggregateID: workerTestTask, AggregateRevision: 9007199254740993, OccurredAt: now, Details: json.RawMessage(`{}`), Snapshot: TaskResultSnapshot{SchemaVersion: 1, TaskID: workerTestTask, QueueID: workerTestQueue, State: TaskSucceeded, Revision: 9007199254740993, Result: json.RawMessage(`{"n":9007199254740993,"text":"<&>"}`)}}, LeaseSecret: strings.Repeat("A", 43)}
}
func eventTestRead(ctx context.Context, c *websocket.Conn) (eventFrame, error) {
kind, raw, err := c.Read(ctx)
var f eventFrame
if err != nil {
return f, err
}
if kind != websocket.MessageText || json.Unmarshal(raw, &f) != nil {
return f, ErrEventProtocol
}
return f, nil
}
func eventTestWrite(ctx context.Context, c *websocket.Conn, f eventFrame) error {
f.Version = "1"
f.SessionID = workerTestSession
f.MessageID, _ = NewEventMessageID()
raw, err := marshalWire(f)
if err != nil {
return err
}
return c.Write(ctx, websocket.MessageText, raw)
}
func eventBody(in any) json.RawMessage { b, _ := marshalWire(in); return b }
func eventMock(t *testing.T, serve func(context.Context, *websocket.Conn, EventRegistration, int)) *Client {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
var count atomic.Int64
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/events/connect" || r.URL.RawQuery != "" || r.Header.Get("Authorization") != "Bearer event-test-token" {
t.Error("unexpected event handshake")
}
c, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{eventSubprotocol}})
if err != nil {
return
}
defer c.CloseNow()
f, err := eventTestRead(ctx, c)
var in EventRegistration
if err != nil || f.Type != "events.hello" || f.SessionID != "" || !validID(f.MessageID) || json.Unmarshal(f.Body, &in) != nil {
t.Error("invalid event hello")
return
}
welcome := EventWelcome{SessionID: workerTestSession, SubscriptionID: in.SubscriptionID, Capacity: in.Capacity, LeaseDurationMS: 30000, HeartbeatIntervalMS: 15000, ServerTime: time.Now().UTC(), Enabled: true}
if eventTestWrite(ctx, c, eventFrame{Type: "events.welcome", ReplyTo: f.MessageID, Body: eventBody(welcome)}) != nil {
return
}
serve(ctx, c, in, int(count.Add(1)))
}))
t.Cleanup(func() { cancel(); server.Close() })
client, err := NewClient(server.URL, Options{Token: "event-test-token", HTTPClient: server.Client()})
if err != nil {
t.Fatal(err)
}
return client
}
func eventTestDial(t *testing.T, client *Client, capacity uint32) *EventConnection {
t.Helper()
ctx, stop := context.WithTimeout(context.Background(), 3*time.Second)
defer stop()
c, err := client.DialEvents(ctx, EventRegistration{SubscriptionID: eventTestSub, Capacity: capacity})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { c.Close() })
return c
}
func eventAckFor(g EventGrant) EventAckResult {
return EventAckResult{DeliveryID: g.Delivery.ID, EventID: g.Event.ID, Epoch: g.Delivery.LeaseEpoch, AcknowledgedAt: time.Now().UTC()}
}
func TestEventConnectionMultiplexAndPrecision(t *testing.T) {
g := eventTestGrant()
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
if eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(g)}) != nil {
return
}
first, err := eventTestRead(ctx, c)
if err != nil {
return
}
second, err := eventTestRead(ctx, c)
if err != nil {
return
}
for _, f := range []eventFrame{second, first} {
var body any = EventPong{ServerTime: time.Now().UTC(), Enabled: true}
if f.Type == "event.ack" {
body = eventAckFor(g)
}
if eventTestWrite(ctx, c, eventFrame{Type: "ack", ReplyTo: f.MessageID, Body: eventBody(body)}) != nil {
return
}
}
_, _ = eventTestRead(ctx, c)
})
c := eventTestDial(t, client, 1)
ctx, stop := context.WithTimeout(context.Background(), 3*time.Second)
defer stop()
got, err := c.Next(ctx)
if err != nil || got.Event.Sequence != 9007199254740993 || string(got.Event.Snapshot.Result) != string(g.Event.Snapshot.Result) {
t.Fatal("event snapshot changed", err)
}
errs := make(chan error, 2)
go func() { _, err := c.Ping(ctx, workerTestID(t)); errs <- err }()
go func() { _, err := c.Ack(ctx, workerTestID(t), got.Proof()); errs <- err }()
for range 2 {
if err := <-errs; err != nil {
t.Fatal(err)
}
}
if c.Err() != nil {
t.Fatal(c.Err())
}
for _, value := range []any{g, g.Proof(), c, &EventError{Code: "UNAVAILABLE", Message: "secret-text"}} {
for _, format := range []string{"%v", "%+v", "%#v"} {
printed := fmt.Sprintf(format, value)
if strings.Contains(printed, g.LeaseSecret) || strings.Contains(printed, "secret-text") || strings.Contains(printed, "9007199254740993") {
t.Fatal("event formatter exposed secrets or business data")
}
}
}
}
func TestEventConnectionWaitCancellationAndAckUncertainty(t *testing.T) {
received := make(chan struct{}, 1)
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
f, err := eventTestRead(ctx, c)
if err != nil {
return
}
if f.Type != "event.ack" {
t.Error("unexpected control")
}
received <- struct{}{}
_, _ = eventTestRead(ctx, c)
})
c := eventTestDial(t, client, 1)
short, stop := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer stop()
if _, err := c.Next(short); !errors.Is(err, context.DeadlineExceeded) || c.Err() != nil {
t.Fatal("Next timeout closed connection", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { _, err := c.Ack(ctx, workerTestID(t), eventTestGrant().Proof()); done <- err }()
<-received
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
select {
case <-c.Done():
case <-time.After(time.Second):
t.Fatal("uncertain ACK did not close connection")
}
}
func TestEventConnectionBoundedControlsAndDeliveries(t *testing.T) {
t.Run("controls", func(t *testing.T) {
received := make(chan struct{}, 40)
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
for {
if _, err := eventTestRead(ctx, c); err != nil {
return
}
received <- struct{}{}
}
})
c := eventTestDial(t, client, 1)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan error, 32)
for range 32 {
go func() { _, err := c.Ping(ctx, workerTestID(t)); done <- err }()
<-received
}
if _, err := c.Ping(ctx, workerTestID(t)); !errors.Is(err, ErrEventBusy) {
t.Fatal("unbounded pending controls", err)
}
cancel()
for range 32 {
<-done
}
})
t.Run("deliveries", func(t *testing.T) {
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
for range 2 {
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(eventTestGrant())})
}
_, _ = eventTestRead(ctx, c)
})
c := eventTestDial(t, client, 1)
select {
case <-c.Done():
case <-time.After(time.Second):
t.Fatal("unbounded deliveries")
}
if !errors.Is(c.Err(), ErrEventBackpressure) {
t.Fatal(c.Err())
}
})
}
func TestEventConnectionRejectsUntrustedProtocol(t *testing.T) {
g := eventTestGrant()
valid := eventFrame{Version: "1", Type: "event.deliver", MessageID: workerTestQueue, SessionID: workerTestSession, Body: eventBody(g)}
raw, _ := marshalWire(valid)
cases := map[string][]byte{
"duplicate": []byte(strings.Replace(string(raw), `"protocol_version":"1"`, `"protocol_version":"1","protocol_version":"1"`, 1)),
"alias": []byte(strings.Replace(string(raw), `"lease_secret"`, `"Lease_Secret"`, 1)),
"noncanonical": []byte(strings.Replace(string(raw), `"sequence":"9007199254740993"`, `"sequence":"09007199254740993"`, 1)),
"null_bool": []byte(strings.Replace(string(raw), `"effect_uncertain":false`, `"effect_uncertain":null`, 1)),
"null_error": []byte(strings.Replace(string(raw), `"body":`, `"error":null,"body":`, 1)),
"mismatched_event": []byte(strings.Replace(string(raw), `"event_id":"`+eventTestID+`"`, `"event_id":"`+workerTestTask+`"`, 1)),
"missing_created": []byte(strings.Replace(string(raw), `"created_at":`+string(eventBody(g.Delivery.CreatedAt))+`,`, "", 1)),
"wrong_state": []byte(strings.Replace(string(raw), `"state":"SUCCEEDED"`, `"state":"FAILED"`, 1)),
"unknown_body_field": []byte(strings.Replace(string(raw), `"schema_version":1`, `"extra":true,"schema_version":1`, 1)),
}
for name, bad := range cases {
t.Run(name, func(t *testing.T) {
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
_ = c.Write(ctx, websocket.MessageText, bad)
_, _ = eventTestRead(ctx, c)
})
c := eventTestDial(t, client, 1)
ctx, stop := context.WithTimeout(context.Background(), time.Second)
defer stop()
if _, err := c.Next(ctx); !errors.Is(err, ErrEventProtocol) {
t.Fatal("invalid response accepted", err)
}
})
}
}
func TestEventConnectionRejectsMismatchedAck(t *testing.T) {
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
f, err := eventTestRead(ctx, c)
if err != nil {
return
}
out := eventAckFor(eventTestGrant())
out.Epoch++
_ = eventTestWrite(ctx, c, eventFrame{Type: "ack", ReplyTo: f.MessageID, Body: eventBody(out)})
_, _ = eventTestRead(ctx, c)
})
c := eventTestDial(t, client, 1)
if _, err := c.Ack(context.Background(), workerTestID(t), eventTestGrant().Proof()); !errors.Is(err, ErrEventProtocol) {
t.Fatal("wrong receipt accepted", err)
}
}
func TestEventConnectionPolicyClosureIsPermanent(t *testing.T) {
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
_ = c.Close(websocket.StatusPolicyViolation, "private peer reason")
})
c := eventTestDial(t, client, 1)
ctx, stop := context.WithTimeout(context.Background(), time.Second)
defer stop()
if _, err := c.Next(ctx); !errors.Is(err, ErrEventProtocol) || eventRetryable(err) {
t.Fatal("policy violation classified as retryable", err)
}
}

368
sdk/go/event_consumer.go Normal file
View File

@@ -0,0 +1,368 @@
package orbit
import (
"context"
"errors"
"fmt"
"math/rand/v2"
"sync/atomic"
"time"
)
var ErrEventConsumerAlreadyRun = errors.New("event consumer may only Run once")
var ErrEventShutdownTimeout = errors.New("event consumer shutdown grace period expired")
// EventConsumerOptions 定义自动消费器。Handle必须以事件ID做持久化去重,
// 在去重记录和业务效果同一事务提交后才返回nil;提交结果未知时返回error。
// error或panic不ACK,平台在租约到期后按原预算补发;SDK不在内存中立即重跑。
type EventConsumerOptions struct {
SubscriptionID string
Capacity uint32 // Capacity 最大本地在途事件/处理器,默认4,范围1至32。
Handle func(context.Context, ResultEvent) error // Handle不接收租约秘密;必须响应context取消。
ShutdownTimeout time.Duration // ShutdownTimeout停止时取消处理器,等待返回及ACK,默认30秒,最多5分钟。
ReconnectMinDelay time.Duration // ReconnectMinDelay默认200毫秒,至少10毫秒。
ReconnectMaxDelay time.Duration // ReconnectMaxDelay默认5秒,最多30秒;指数退避并加入抖动。
}
// EventConsumerEvent 只用于本地观察,缓冲满可丢弃;不是可靠结果或审计。
type EventConsumerEvent struct {
Type string // Type为connected、reconnecting、handler_started、handler_failed、acknowledged、proof_lost、draining。
EventID string
Code string // Code不含业务异常、数据或秘密。
}
type EventConsumerStats struct {
Running int64
Started uint64
HandlerFailures uint64
Acknowledged uint64
Reconnects uint64
DroppedEvents uint64
}
type EventConsumerRunError struct {
Cause error
Unconfirmed int // Unconfirmed本进程已知且未确认的事件数,不是订阅全局积压。
Running int64 // Running仍未退出的处理器;Go不能强杀忽略context的函数。
}
func (e *EventConsumerRunError) Error() string {
return fmt.Sprintf("event consumer stopped with %d unconfirmed events and %d running handlers: %v", e.Unconfirmed, e.Running, e.Cause)
}
func (e *EventConsumerRunError) Unwrap() error { return e.Cause }
// EventConsumer 自动维护心跳、重连、处理器并发和ACK核对。只能Run一次。
// 服务端交付租约不是业务锁;跨进程重复及并行消费由业务Inbox唯一键和事务处理。
type EventConsumer struct {
client *Client
options EventConsumerOptions
used atomic.Bool
events chan EventConsumerEvent
running atomic.Int64
started, failures, acked, reconnects, dropped atomic.Uint64
}
func NewEventConsumer(client *Client, in EventConsumerOptions) (*EventConsumer, error) {
if client == nil || client.token == "" || !validID(in.SubscriptionID) || in.Handle == nil {
return nil, errors.New("authenticated client, subscription and durable event handler are required")
}
if in.Capacity == 0 {
in.Capacity = 4
}
if in.ShutdownTimeout == 0 {
in.ShutdownTimeout = 30 * time.Second
}
if in.ReconnectMinDelay == 0 {
in.ReconnectMinDelay = 200 * time.Millisecond
}
if in.ReconnectMaxDelay == 0 {
in.ReconnectMaxDelay = 5 * time.Second
}
if in.Capacity > 32 || in.ShutdownTimeout < time.Millisecond || in.ShutdownTimeout > 5*time.Minute || in.ReconnectMinDelay < 10*time.Millisecond || in.ReconnectMaxDelay < in.ReconnectMinDelay || in.ReconnectMaxDelay > 30*time.Second {
return nil, errors.New("invalid event consumer capacity or durations")
}
return &EventConsumer{client: client, options: in, events: make(chan EventConsumerEvent, 128)}, nil
}
func (c *EventConsumer) Events() <-chan EventConsumerEvent { return c.events }
func (c *EventConsumer) Stats() EventConsumerStats {
return EventConsumerStats{Running: c.running.Load(), Started: c.started.Load(), HandlerFailures: c.failures.Load(), Acknowledged: c.acked.Load(), Reconnects: c.reconnects.Load(), DroppedEvents: c.dropped.Load()}
}
func (c *EventConsumer) emit(typ, id, code string) {
select {
case c.events <- EventConsumerEvent{Type: typ, EventID: id, Code: code}:
default:
c.dropped.Add(1)
}
}
func (c *EventConsumer) String() string {
return fmt.Sprintf("EventConsumer{SubscriptionID:%s Capacity:%d secrets:redacted}", c.options.SubscriptionID, c.options.Capacity)
}
func (c *EventConsumer) GoString() string { return c.String() }
type consumerJob struct {
proof EventProof
complete bool
}
type consumerResult struct {
id string
failed bool
}
// Run在ctx取消后停止启动新处理器,取消在途处理器,在收尾时限内确认已提交结果。
// 收到认证/协议等永久错误时也取消并等待在途处理器;未确认结果仍保留在服务端。
// 重连先核对已完成处理器的原证明;证明被替代时等待新投递,由业务去重命中后确认。
func (c *EventConsumer) Run(parent context.Context) error {
if !c.used.CompareAndSwap(false, true) {
return ErrEventConsumerAlreadyRun
}
defer close(c.events)
if parent.Err() != nil {
return nil
}
ctx, cancel := context.WithCancel(context.WithoutCancel(parent))
defer cancel()
handlerCtx, stopHandlers := context.WithCancel(ctx)
defer stopHandlers()
abort := make(chan struct{})
watchDone := make(chan struct{})
go func() {
defer close(watchDone)
select {
case <-ctx.Done():
return
case <-parent.Done():
case <-abort:
}
stopHandlers()
timer := time.NewTimer(c.options.ShutdownTimeout)
defer timer.Stop()
select {
case <-ctx.Done():
case <-timer.C:
cancel()
}
}()
defer func() { cancel(); <-watchDone }()
var conn *EventConnection
var heartbeatDone <-chan struct{}
closeConnection := func() {
if conn != nil {
_ = conn.Close()
<-heartbeatDone
conn = nil
}
}
defer closeConnection()
jobs := make(map[string]*consumerJob)
results := make(chan consumerResult, c.options.Capacity)
draining := false
var cause error
beginDrain := func(err error) {
if !draining {
draining = true
stopHandlers()
close(abort)
c.emit("draining", "", "")
}
if cause == nil {
cause = err
}
}
finish := func(err error) error {
if err == nil {
return nil
}
return &EventConsumerRunError{Cause: err, Unconfirmed: len(jobs), Running: c.running.Load()}
}
backoff := c.options.ReconnectMinDelay
var dialAt time.Time
everConnected := false
disconnect := func(err error) {
closeConnection()
if !eventRetryable(err) {
beginDrain(err)
return
}
floor := backoff / 2
dialAt = time.Now().Add(floor + time.Duration(rand.Int64N(int64(backoff-floor)+1)))
backoff = min(backoff*2, c.options.ReconnectMaxDelay)
c.emit("reconnecting", "", eventErrorCode(err))
}
for {
if parent.Err() != nil {
beginDrain(nil)
}
if ctx.Err() != nil {
return finish(ErrEventShutdownTimeout)
}
if draining && (len(jobs) == 0 || cause != nil && c.running.Load() == 0) {
return finish(cause)
}
if conn == nil && cause == nil && !time.Now().Before(dialAt) {
if everConnected {
c.reconnects.Add(1)
}
var err error
conn, err = c.client.DialEvents(ctx, EventRegistration{SubscriptionID: c.options.SubscriptionID, Capacity: c.options.Capacity})
if err != nil {
disconnect(err)
continue
}
everConnected = true
heartbeatDone = startEventHeartbeat(ctx, conn)
c.emit("connected", "", "")
continue
}
if conn != nil {
if err := conn.Err(); err != nil {
disconnect(err)
continue
}
// 一个ACK在途;读取与自动心跳独立,不需要接收方不断调用Next维持连接。
acknowledged := false
for id, j := range jobs {
if !j.complete {
continue
}
messageID, err := NewEventMessageID()
if err == nil {
_, err = conn.Ack(ctx, messageID, j.proof)
}
if err == nil {
delete(jobs, id)
c.acked.Add(1)
c.emit("acknowledged", id, "")
backoff = c.options.ReconnectMinDelay
} else {
var remote *EventError
if errors.As(err, &remote) && remote.Code == "DELIVERY_LEASE_LOST" {
delete(jobs, id)
c.emit("proof_lost", id, remote.Code)
} else {
disconnect(err)
}
}
acknowledged = true
break
}
if acknowledged {
continue
}
}
var deliveries <-chan EventGrant
var disconnected <-chan struct{}
if conn != nil {
disconnected = conn.Done()
if !draining && len(jobs) < int(c.options.Capacity) {
deliveries = conn.deliveries
}
}
var parentDone <-chan struct{}
if !draining {
parentDone = parent.Done()
}
// 定时器只负责重连;活跃连接没有每轮轮询。
var wake <-chan time.Time
var timer *time.Timer
if conn == nil && cause == nil {
timer = time.NewTimer(max(time.Millisecond, time.Until(dialAt)))
wake = timer.C
}
select {
case <-ctx.Done():
case <-parentDone:
beginDrain(nil)
case <-wake:
case <-disconnected:
disconnect(conn.Err())
case result := <-results:
j := jobs[result.id]
if result.failed {
delete(jobs, result.id)
c.failures.Add(1)
c.emit("handler_failed", result.id, "HANDLER_FAILED")
} else {
j.complete = true
}
case grant := <-deliveries:
if parent.Err() != nil || conn.Err() != nil {
break
}
id := grant.Event.ID
if j := jobs[id]; j != nil {
if j.proof.DeliveryID != grant.Delivery.ID {
disconnect(ErrEventProtocol)
break
}
if grant.Delivery.LeaseEpoch > j.proof.Epoch {
j.proof = grant.Proof()
}
break
}
jobs[id] = &consumerJob{proof: grant.Proof()}
c.running.Add(1)
c.started.Add(1)
c.emit("handler_started", id, "")
go func(event ResultEvent) {
failed := true
defer func() {
_ = recover()
c.running.Add(-1)
select {
case results <- consumerResult{id: event.ID, failed: failed}:
case <-ctx.Done():
}
}()
failed = c.options.Handle(handlerCtx, event) != nil
}(grant.Event)
}
if timer != nil {
timer.Stop()
}
}
}
func startEventHeartbeat(ctx context.Context, conn *EventConnection) <-chan struct{} {
done := make(chan struct{})
go func() {
defer close(done)
ticker := time.NewTicker(time.Duration(conn.Welcome().HeartbeatIntervalMS) * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-conn.Done():
return
case <-ticker.C:
id, err := NewEventMessageID()
if err == nil {
_, err = conn.Ping(ctx, id)
}
if err != nil {
conn.fail(err)
return
}
}
}
}()
return done
}
func eventRetryable(err error) bool {
if errors.Is(err, ErrEventDisconnected) || errors.Is(err, ErrEventBackpressure) || errors.Is(err, context.DeadlineExceeded) {
return true
}
var remote *EventError
if errors.As(err, &remote) {
return remote.Code == "UNAVAILABLE"
}
var httpErr *HTTPError
if errors.As(err, &httpErr) {
return httpErr.StatusCode == 408 || httpErr.StatusCode == 429 || httpErr.StatusCode >= 500 && httpErr.StatusCode <= 599
}
return false
}
func eventErrorCode(err error) string {
var remote *EventError
if errors.As(err, &remote) {
return remote.Code
}
return "TRANSPORT_INTERRUPTED"
}

View File

@@ -0,0 +1,342 @@
package orbit
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"runtime"
"sync/atomic"
"testing"
"time"
"github.com/coder/websocket"
)
func runEventConsumer(t *testing.T, client *Client, handle func(context.Context, ResultEvent) error) (*EventConsumer, context.CancelFunc, <-chan error) {
t.Helper()
consumer, err := NewEventConsumer(client, EventConsumerOptions{SubscriptionID: eventTestSub, Capacity: 1, Handle: handle, ReconnectMinDelay: 10 * time.Millisecond, ReconnectMaxDelay: 50 * time.Millisecond, ShutdownTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- consumer.Run(ctx) }()
t.Cleanup(cancel)
return consumer, cancel, done
}
func awaitConsumerEvent(t *testing.T, c *EventConsumer, typ string) EventConsumerEvent {
t.Helper()
timer := time.NewTimer(3 * time.Second)
defer timer.Stop()
for {
select {
case event, ok := <-c.Events():
if !ok {
t.Fatal("consumer stopped before " + typ)
}
if event.Type == typ {
return event
}
case <-timer.C:
t.Fatal("consumer did not emit " + typ)
}
}
}
func awaitConsumerStop(t *testing.T, done <-chan error) error {
t.Helper()
select {
case err := <-done:
return err
case <-time.After(3 * time.Second):
t.Fatal("consumer did not stop")
return nil
}
}
func TestEventConsumerReconcilesLostAckWithoutRerunningHandler(t *testing.T) {
g := eventTestGrant()
var calls atomic.Int64
var proofs [2]EventProof
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, n int) {
if n > 2 {
t.Error("unexpected third connection")
return
}
if n == 1 {
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(g)})
}
f, err := eventTestRead(ctx, c)
if err != nil {
return
}
if f.Type != "event.ack" || json.Unmarshal(f.Body, &proofs[n-1]) != nil {
t.Error("missing ACK proof")
return
}
if n == 1 {
return
} // ACK已在服务端提交,回应在网络中丢失。
if proofs[0].DeliveryID != proofs[1].DeliveryID || proofs[0].Epoch != proofs[1].Epoch || proofs[0].LeaseSecret != proofs[1].LeaseSecret {
t.Error("reconnect changed acknowledgement identity")
}
out := eventAckFor(g)
out.Deduplicated = true
_ = eventTestWrite(ctx, c, eventFrame{Type: "ack", ReplyTo: f.MessageID, Body: eventBody(out)})
_, _ = eventTestRead(ctx, c)
})
consumer, cancel, done := runEventConsumer(t, client, func(context.Context, ResultEvent) error { calls.Add(1); return nil })
awaitConsumerEvent(t, consumer, "acknowledged")
cancel()
if err := awaitConsumerStop(t, done); err != nil {
t.Fatal(err)
}
if calls.Load() != 1 || consumer.Stats().Acknowledged != 1 || consumer.Stats().Reconnects != 1 {
t.Fatal("lost receipt re-executed committed handler", consumer.Stats())
}
if !errors.Is(consumer.Run(context.Background()), ErrEventConsumerAlreadyRun) {
t.Fatal("consumer ran twice")
}
}
func TestEventConsumerSupersededProofRequiresDurableDeduplication(t *testing.T) {
g := eventTestGrant()
var calls, effects atomic.Int64
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(g)})
f, err := eventTestRead(ctx, c)
if err != nil {
return
}
_ = eventTestWrite(ctx, c, eventFrame{Type: "error", ReplyTo: f.MessageID, Error: &EventError{Code: "DELIVERY_LEASE_LOST", Message: "proof superseded"}})
next := g
next.Delivery.LeaseEpoch = 2
next.Delivery.Attempts = 2
next.Delivery.Revision = 4
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(next)})
f, err = eventTestRead(ctx, c)
if err != nil {
return
}
_ = eventTestWrite(ctx, c, eventFrame{Type: "ack", ReplyTo: f.MessageID, Body: eventBody(eventAckFor(next))})
_, _ = eventTestRead(ctx, c)
})
consumer, cancel, done := runEventConsumer(t, client, func(context.Context, ResultEvent) error { calls.Add(1); effects.CompareAndSwap(0, 1); return nil })
awaitConsumerEvent(t, consumer, "acknowledged")
cancel()
if err := awaitConsumerStop(t, done); err != nil {
t.Fatal(err)
}
if calls.Load() != 2 || effects.Load() != 1 {
t.Fatal("replacement proof bypassed handler deduplication")
}
}
func TestEventConsumerHandlerFailureDoesNotAcknowledge(t *testing.T) {
for _, panicHandler := range []bool{false, true} {
name := "error"
if panicHandler {
name = "panic"
}
t.Run(name, func(t *testing.T) {
g := eventTestGrant()
retry := make(chan struct{})
var calls atomic.Int64
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(g)})
select {
case <-retry:
case <-ctx.Done():
return
}
next := g
next.Delivery.LeaseEpoch = 2
next.Delivery.Attempts = 2
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(next)})
f, err := eventTestRead(ctx, c)
if err != nil {
return
}
var proof EventProof
if f.Type != "event.ack" || json.Unmarshal(f.Body, &proof) != nil || proof.Epoch != 2 {
t.Error("failed handler acknowledged original delivery")
}
_ = eventTestWrite(ctx, c, eventFrame{Type: "ack", ReplyTo: f.MessageID, Body: eventBody(eventAckFor(next))})
_, _ = eventTestRead(ctx, c)
})
consumer, cancel, done := runEventConsumer(t, client, func(context.Context, ResultEvent) error {
if calls.Add(1) == 1 {
if panicHandler {
panic("private handler data")
}
return errors.New("private handler data")
}
return nil
})
awaitConsumerEvent(t, consumer, "handler_failed")
if calls.Load() != 1 {
t.Fatal("handler retried locally")
}
close(retry)
awaitConsumerEvent(t, consumer, "acknowledged")
cancel()
if err := awaitConsumerStop(t, done); err != nil {
t.Fatal(err)
}
if consumer.Stats().HandlerFailures != 1 || consumer.Stats().Acknowledged != 1 {
t.Fatal(consumer.Stats())
}
})
}
}
func TestEventConsumerShutdownCancelsAndConfirmsCommittedHandler(t *testing.T) {
g := eventTestGrant()
started := make(chan struct{})
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(g)})
f, err := eventTestRead(ctx, c)
if err != nil {
return
}
_ = eventTestWrite(ctx, c, eventFrame{Type: "ack", ReplyTo: f.MessageID, Body: eventBody(eventAckFor(g))})
_, _ = eventTestRead(ctx, c)
})
consumer, cancel, done := runEventConsumer(t, client, func(ctx context.Context, _ ResultEvent) error { close(started); <-ctx.Done(); return nil })
<-started
cancel()
if err := awaitConsumerStop(t, done); err != nil {
t.Fatal(err)
}
if consumer.Stats().Acknowledged != 1 || consumer.Stats().Running != 0 {
t.Fatal("committed result not drained", consumer.Stats())
}
}
func TestEventConsumerShutdownBoundsIgnoringHandler(t *testing.T) {
started, release := make(chan struct{}), make(chan struct{})
defer close(release)
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(eventTestGrant())})
_, _ = eventTestRead(ctx, c)
})
consumer, err := NewEventConsumer(client, EventConsumerOptions{SubscriptionID: eventTestSub, Capacity: 1, ShutdownTimeout: 30 * time.Millisecond, Handle: func(context.Context, ResultEvent) error { close(started); <-release; return nil }})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan error, 1)
go func() { done <- consumer.Run(ctx) }()
<-started
cancel()
err = awaitConsumerStop(t, done)
var run *EventConsumerRunError
if !errors.Is(err, ErrEventShutdownTimeout) || !errors.As(err, &run) || run.Unconfirmed != 1 || run.Running != 1 {
t.Fatal("uncooperative handler not reported", err)
}
}
func TestEventConsumerPermanentFailureDoesNotReconnect(t *testing.T) {
var calls atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); w.WriteHeader(http.StatusUnauthorized) }))
defer server.Close()
client, _ := NewClient(server.URL, Options{Token: "token"})
_, _, done := runEventConsumer(t, client, func(context.Context, ResultEvent) error { t.Error("unauthorized consumer invoked handler"); return nil })
err := awaitConsumerStop(t, done)
var status *HTTPError
if !errors.As(err, &status) || status.StatusCode != 401 || calls.Load() != 1 {
t.Fatal("permanent handshake error retried", err)
}
}
func TestEventConsumerHandlerGoexitReleasesSlotWithoutAck(t *testing.T) {
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(eventTestGrant())})
if f, err := eventTestRead(ctx, c); err == nil && f.Type == "event.ack" {
t.Error("Goexit handler was acknowledged")
}
})
consumer, cancel, done := runEventConsumer(t, client, func(context.Context, ResultEvent) error { runtime.Goexit(); return nil })
awaitConsumerEvent(t, consumer, "handler_failed")
cancel()
if err := awaitConsumerStop(t, done); err != nil || consumer.Stats().Running != 0 {
t.Fatal("Goexit leaked handler capacity", err)
}
}
func TestEventConsumerConcurrentCapacity(t *testing.T) {
start := make(chan string, 3)
release := make(chan struct{})
var active, peak atomic.Int64
grants := []EventGrant{eventTestGrant(), eventTestGrant(), eventTestGrant()}
for i := range grants {
grants[i].Event.ID = workerTestID(t)
grants[i].Delivery.ID = workerTestID(t)
grants[i].Delivery.EventID = grants[i].Event.ID
}
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, in EventRegistration, _ int) {
if in.Capacity != 2 {
t.Error("capacity changed")
}
for _, g := range grants[:2] {
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(g)})
}
for i := 0; i < 3; i++ {
f, err := eventTestRead(ctx, c)
if err != nil {
return
}
var p EventProof
if f.Type != "event.ack" || json.Unmarshal(f.Body, &p) != nil {
t.Error("missing ACK")
return
}
var grant EventGrant
for _, g := range grants {
if g.Delivery.ID == p.DeliveryID {
grant = g
}
}
_ = eventTestWrite(ctx, c, eventFrame{Type: "ack", ReplyTo: f.MessageID, Body: eventBody(eventAckFor(grant))})
if i == 0 {
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(grants[2])})
}
}
_, _ = eventTestRead(ctx, c)
})
consumer, err := NewEventConsumer(client, EventConsumerOptions{SubscriptionID: eventTestSub, Capacity: 2, Handle: func(ctx context.Context, event ResultEvent) error {
n := active.Add(1)
defer active.Add(-1)
for {
old := peak.Load()
if n <= old || peak.CompareAndSwap(old, n) {
break
}
}
start <- event.ID
select {
case <-release:
return nil
case <-ctx.Done():
return ctx.Err()
}
}})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
done := make(chan error, 1)
go func() { done <- consumer.Run(ctx) }()
for range 2 {
select {
case <-start:
case <-ctx.Done():
t.Fatal("capacity not utilized")
}
}
close(release)
for range 3 {
awaitConsumerEvent(t, consumer, "acknowledged")
}
cancel()
if err := awaitConsumerStop(t, done); err != nil || peak.Load() != 2 || consumer.Stats().Started != 3 {
t.Fatal("capacity or drain invariant", err, consumer.Stats())
}
}

View File

@@ -0,0 +1,190 @@
package orbit
import (
"context"
"encoding/json"
"errors"
"sync/atomic"
"testing"
"time"
)
func integrationEventSubscription(t *testing.T, ctx context.Context, api *Client, maxAttempts uint32) Subscription {
t.Helper()
queue, err := api.CreateQueue(ctx, workerTestID(t), CreateQueueRequest{Name: "event-sdk-" + workerTestID(t), MaxConcurrency: 1, RatePerSecond: 10})
if err != nil {
t.Fatal(err)
}
sub, err := api.CreateSubscription(ctx, workerTestID(t), CreateSubscriptionRequest{Name: "sdk-" + workerTestID(t), QueueID: queue.Queue.ID, Types: []string{"task.cancelled"}, MaxPending: 10, MaxAttempts: maxAttempts})
if err != nil {
t.Fatal(err)
}
return sub.Subscription
}
func integrationCancelledEvent(t *testing.T, ctx context.Context, api *Client, sub Subscription) Task {
t.Helper()
task, err := api.SubmitTask(ctx, workerTestID(t), SubmitTaskRequest{QueueID: sub.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)})
if err != nil {
t.Fatal(err)
}
cancelled, err := api.CancelTask(ctx, task.Task.ID, workerTestID(t), CancelTaskRequest{ExpectedRevision: task.Task.Revision, Reason: "subscription SDK regression"})
if err != nil {
t.Fatal(err)
}
return cancelled.Task
}
func eventDialIntegration(t *testing.T, ctx context.Context, api *Client, sub Subscription, capacity uint32) *EventConnection {
t.Helper()
c, err := api.DialEvents(ctx, EventRegistration{SubscriptionID: sub.ID, Capacity: capacity})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { c.Close() })
return c
}
func TestIntegrationEventConnectionSDK(t *testing.T) {
api, _, _ := integrationWorkerClient(t)
ctx, stop := context.WithTimeout(context.Background(), 15*time.Second)
defer stop()
sub := integrationEventSubscription(t, ctx, api, 3)
task := integrationCancelledEvent(t, ctx, api, sub)
a := eventDialIntegration(t, ctx, api, sub, 1)
grant, err := a.Next(ctx)
if err != nil || grant.Event.Snapshot.TaskID != task.ID || grant.Event.Snapshot.Revision != task.Revision || grant.Event.Type != "task.cancelled" {
t.Fatal("SDK snapshot mismatch", err)
}
paused, err := api.ConfigureSubscription(ctx, sub.ID, workerTestID(t), ConfigureSubscriptionRequest{ExpectedRevision: sub.Revision, Reason: "pause with active delivery", Enabled: false})
if err != nil {
t.Fatal(err)
}
pong, err := a.Ping(ctx, workerTestID(t))
if err != nil || pong.Enabled {
t.Fatal("paused subscription heartbeat", err)
}
first, err := a.Ack(ctx, workerTestID(t), grant.Proof())
if err != nil {
t.Fatal(err)
}
a.Close()
b := eventDialIntegration(t, ctx, api, sub, 1)
duplicate, err := b.Ack(ctx, workerTestID(t), grant.Proof())
if err != nil || !duplicate.Deduplicated || !duplicate.AcknowledgedAt.Equal(first.AcknowledgedAt) {
t.Fatal("ACK proof could not cross connections", err)
}
_ = integrationCancelledEvent(t, ctx, api, sub)
short, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
_, err = b.Next(short)
cancel()
if !errors.Is(err, context.DeadlineExceeded) || b.Err() != nil {
t.Fatal("paused subscription delivered or Next closed connection", err)
}
if _, err = api.ConfigureSubscription(ctx, sub.ID, workerTestID(t), ConfigureSubscriptionRequest{ExpectedRevision: paused.Subscription.Revision, Reason: "resume", Enabled: true}); err != nil {
t.Fatal(err)
}
second, err := b.Next(ctx)
if err != nil || second.Event.ID == grant.Event.ID {
t.Fatal("resume failed", err)
}
if _, err = b.Ack(ctx, workerTestID(t), second.Proof()); err != nil {
t.Fatal(err)
}
current, err := api.GetSubscription(ctx, sub.ID)
if err != nil || current.PendingCount != 0 {
t.Fatal("SDK acknowledgement accounting", err)
}
}
func TestIntegrationEventConsumerHeartbeatAndLostAck(t *testing.T) {
api, _, token := integrationWorkerClient(t)
ctx, stop := context.WithTimeout(context.Background(), 25*time.Second)
defer stop()
sub := integrationEventSubscription(t, ctx, api, 3)
_ = integrationCancelledEvent(t, ctx, api, sub)
gateway := newProtocolFaultGateway(t, api.baseURL, "/v1/events/connect", eventSubprotocol)
gateway.drop.Store("event.ack")
client, err := NewClient(gateway.server.URL, Options{Token: token, HTTPClient: gateway.server.Client()})
if err != nil {
t.Fatal(err)
}
var calls atomic.Int64
consumer, err := NewEventConsumer(client, EventConsumerOptions{SubscriptionID: sub.ID, Capacity: 1, ShutdownTimeout: time.Second, Handle: func(work context.Context, event ResultEvent) error {
calls.Add(1)
// 慢业务期间真实等待15秒自动心跳;确认计数来自上游服务端ACK,不是客户端写入。
ticker := time.NewTicker(20 * time.Millisecond)
defer ticker.Stop()
for gateway.confirmedEventPings.Load() == 0 {
select {
case <-work.Done():
return work.Err()
case <-ticker.C:
}
}
return nil
}})
if err != nil {
t.Fatal(err)
}
run, cancel := context.WithCancel(ctx)
defer cancel()
done := make(chan error, 1)
go func() { done <- consumer.Run(run) }()
for consumer.Stats().Acknowledged == 0 {
select {
case <-ctx.Done():
t.Fatal("consumer did not reconcile committed ACK")
case err := <-done:
t.Fatal("consumer stopped", err)
case <-time.After(20 * time.Millisecond):
}
}
cancel()
if err := awaitConsumerStop(t, done); err != nil {
t.Fatal(err)
}
if calls.Load() != 1 || consumer.Stats().Reconnects < 1 || gateway.confirmedEventPings.Load() < 1 {
t.Fatal("heartbeat or reconciliation missing", consumer.Stats())
}
page, err := api.ListEventDeliveries(ctx, sub.ID, DeliveryPageRequest{State: DeliveryAcked})
if err != nil || len(page.Items) != 1 || page.Items[0].Attempts != 1 {
t.Fatal("lost ACK created another delivery attempt", err)
}
t.Log("real 15-second heartbeat and committed ACK loss recovered; handler and delivery attempt each remained one")
}
func TestIntegrationEventConsumerFailureRedeliversNaturally(t *testing.T) {
api, _, _ := integrationWorkerClient(t)
ctx, stop := context.WithTimeout(context.Background(), 45*time.Second)
defer stop()
sub := integrationEventSubscription(t, ctx, api, 2)
_ = integrationCancelledEvent(t, ctx, api, sub)
var calls atomic.Int64
consumer, err := NewEventConsumer(api, EventConsumerOptions{SubscriptionID: sub.ID, Capacity: 1, ShutdownTimeout: time.Second, Handle: func(context.Context, ResultEvent) error {
if calls.Add(1) == 1 {
return errors.New("receiver transaction rolled back")
}
return nil
}})
if err != nil {
t.Fatal(err)
}
run, cancel := context.WithCancel(ctx)
defer cancel()
done := make(chan error, 1)
go func() { done <- consumer.Run(run) }()
for consumer.Stats().Acknowledged == 0 {
select {
case <-ctx.Done():
t.Fatal("delivery did not retry after natural lease expiry")
case err := <-done:
t.Fatal("consumer stopped", err)
case <-time.After(50 * time.Millisecond):
}
}
cancel()
if err := awaitConsumerStop(t, done); err != nil {
t.Fatal(err)
}
page, err := api.ListEventDeliveries(ctx, sub.ID, DeliveryPageRequest{State: DeliveryAcked})
if err != nil || len(page.Items) != 1 || page.Items[0].Attempts != 2 || calls.Load() != 2 || consumer.Stats().HandlerFailures != 1 {
t.Fatal("handler error or retry budget mishandled", err)
}
t.Log("handler error caused no ACK; same delivery reappeared after its real 30-second lease and backoff, then acknowledged at attempt two")
}

View File

@@ -0,0 +1,103 @@
package orbit
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
)
func TestIntegrationEventProcessLeaseRecovery(t *testing.T) {
a := startOrbitProcess(t)
_, _, token := integrationWorkerClient(t)
api := processClient(t, a, token)
ctx, stop := context.WithTimeout(context.Background(), 45*time.Second)
defer stop()
sub := integrationEventSubscription(t, ctx, api, 3)
_ = integrationCancelledEvent(t, ctx, api, sub)
firstConn := eventDialIntegration(t, ctx, api, sub, 1)
first, err := firstConn.Next(ctx)
if err != nil {
t.Fatal(err)
}
a.kill(t)
select {
case <-firstConn.Done():
case <-time.After(3 * time.Second):
t.Fatal("dead server connection remained live")
}
b := startOrbitProcess(t)
other := processClient(t, b, token)
secondConn := eventDialIntegration(t, ctx, other, sub, 1)
second, err := secondConn.Next(ctx)
if err != nil || first.Event.ID != second.Event.ID || first.Delivery.ID != second.Delivery.ID || second.Delivery.LeaseEpoch != 2 || second.Delivery.Attempts != 2 {
t.Fatal("node restart did not recover original delivery", err)
}
_, err = secondConn.Ack(ctx, workerTestID(t), first.Proof())
var remote *EventError
if !errors.As(err, &remote) || remote.Code != "DELIVERY_LEASE_LOST" {
t.Fatal("dead node proof acknowledged newer lease", err)
}
if _, err = secondConn.Ack(ctx, workerTestID(t), second.Proof()); err != nil {
t.Fatal(err)
}
current, err := other.GetSubscription(ctx, sub.ID)
if err != nil || current.PendingCount != 0 {
t.Fatal("recovered acknowledgement not persisted", err)
}
t.Log("SIGKILL confirmed by Wait; new process recovered original delivery after natural lease expiry, rejected old proof and committed new ACK")
}
func TestIntegrationEventProcessAckCommitResponseLoss(t *testing.T) {
a := startOrbitProcess(t)
_, _, token := integrationWorkerClient(t)
api := processClient(t, a, token)
ctx, stop := context.WithTimeout(context.Background(), 20*time.Second)
defer stop()
sub := integrationEventSubscription(t, ctx, api, 3)
_ = integrationCancelledEvent(t, ctx, api, sub)
gateway := newProtocolFaultGateway(t, a.url, "/v1/events/connect", eventSubprotocol)
gateway.drop.Store("event.ack")
gateway.dropOffline.Store(true)
client, err := NewClient(gateway.server.URL, Options{Token: token, HTTPClient: gateway.server.Client()})
if err != nil {
t.Fatal(err)
}
var calls atomic.Int64
consumer, err := NewEventConsumer(client, EventConsumerOptions{SubscriptionID: sub.ID, Capacity: 1, ShutdownTimeout: time.Second, Handle: func(context.Context, ResultEvent) error { calls.Add(1); return nil }})
if err != nil {
t.Fatal(err)
}
run, cancel := context.WithCancel(ctx)
defer cancel()
done := make(chan error, 1)
go func() { done <- consumer.Run(run) }()
awaitProcessDrop(t, ctx, gateway, "event.ack")
before, err := api.ListEventDeliveries(ctx, sub.ID, DeliveryPageRequest{State: DeliveryAcked})
if err != nil || len(before.Items) != 1 || before.Items[0].AcknowledgedAt == nil {
t.Fatal("fault occurred before durable ACK", err)
}
a.kill(t)
b := startOrbitProcess(t)
other := processClient(t, b, token)
gateway.target.Store(b.url)
gateway.offline.Store(false)
for consumer.Stats().Acknowledged == 0 {
select {
case <-ctx.Done():
t.Fatal("consumer failed to reconcile ACK on new process")
case err := <-done:
t.Fatal("consumer stopped", err)
case <-time.After(20 * time.Millisecond):
}
}
cancel()
if err := awaitConsumerStop(t, done); err != nil {
t.Fatal(err)
}
after, err := other.ListEventDeliveries(ctx, sub.ID, DeliveryPageRequest{State: DeliveryAcked})
if err != nil || len(after.Items) != 1 || after.Items[0].Attempts != 1 || after.Items[0].AcknowledgedAt == nil || !after.Items[0].AcknowledgedAt.Equal(*before.Items[0].AcknowledgedAt) || calls.Load() != 1 {
t.Fatal("ACK receipt changed or committed handler reran", err)
}
t.Log("ACK commit verified before response loss and SIGKILL; replacement process returned original receipt without another handler or delivery attempt")
}

87
sdk/go/event_types.go Normal file
View File

@@ -0,0 +1,87 @@
package orbit
import (
"errors"
"fmt"
"time"
)
var (
ErrEventClosed = errors.New("event connection closed")
ErrEventDisconnected = errors.New("event connection interrupted; acknowledgement may be unconfirmed")
ErrEventProtocol = errors.New("invalid event protocol response")
ErrEventBusy = errors.New("event control request limit reached")
ErrEventBackpressure = errors.New("event delivery buffer exhausted")
)
// EventError 是服务端公开错误。Error/String/GoString只展示分类,不输出远端正文。
type EventError struct {
Code string `json:"code"`
Message string `json:"message"`
}
func (e *EventError) Error() string { return "orbit event " + e.Code }
func (e *EventError) String() string { return e.Error() }
func (e *EventError) GoString() string { return e.Error() }
// EventRegistration 绑定一个已有订阅。容量固定为1至32,重连可以重新选择。
type EventRegistration struct {
SubscriptionID string `json:"subscription_id"` // SubscriptionID 已创建且属于当前凭据的订阅。
Capacity uint32 `json:"capacity"`
}
// EventWelcome 是当前连接的握手快照;Enabled变化通过Ping查询。
type EventWelcome struct {
SessionID string `json:"session_id"`
SubscriptionID string `json:"subscription_id"`
Capacity uint32 `json:"capacity"`
LeaseDurationMS int64 `json:"lease_duration_ms"`
HeartbeatIntervalMS int64 `json:"heartbeat_interval_ms"`
ServerTime time.Time `json:"server_time"`
Enabled bool `json:"enabled"`
}
type EventPong struct {
ServerTime time.Time `json:"server_time"`
Enabled bool `json:"enabled"`
}
// EventGrant 包含冻结结果及当前交付证明。不得将JSON写到普通日志。
// 租约只控制平台补发,不授权接收方跳过去重;不同消费者仍可能并行收到同一事件。
type EventGrant struct {
Delivery EventDelivery `json:"delivery"`
Event ResultEvent `json:"event"`
LeaseSecret string `json:"lease_secret"`
}
// EventProof 保留原身份及秘密,在同订阅的新连接上仍可核对ACK。
// SubscriptionID和EventID用于SDK本地关联校验;只把后三个字段发送到服务端。
type EventProof struct {
SubscriptionID string `json:"subscription_id"`
EventID string `json:"event_id"`
DeliveryID string `json:"delivery_id"`
Epoch uint64 `json:"epoch,string"`
LeaseSecret string `json:"lease_secret"`
}
func (g EventGrant) Proof() EventProof {
return EventProof{SubscriptionID: g.Delivery.SubscriptionID, EventID: g.Event.ID, DeliveryID: g.Delivery.ID, Epoch: g.Delivery.LeaseEpoch, LeaseSecret: g.LeaseSecret}
}
type EventAckResult struct {
DeliveryID string `json:"delivery_id"`
EventID string `json:"event_id"`
Epoch uint64 `json:"epoch,string"`
AcknowledgedAt time.Time `json:"acknowledged_at"`
Deduplicated bool `json:"deduplicated"`
}
// NewEventMessageID 生成控制消息关联ID;ACK的持久化身份是交付ID、代次和证明。
func NewEventMessageID() (string, error) { return NewWorkerMessageID() }
func (g EventGrant) String() string {
return fmt.Sprintf("EventGrant{EventID:%s DeliveryID:%s data:redacted}", g.Event.ID, g.Delivery.ID)
}
func (g EventGrant) GoString() string { return g.String() }
func (p EventProof) String() string {
return fmt.Sprintf("EventProof{DeliveryID:%s Epoch:%d secrets:redacted}", p.DeliveryID, p.Epoch)
}
func (p EventProof) GoString() string { return p.String() }

View File

@@ -0,0 +1,112 @@
// event-protocol 验证冻结结果、连接切换与ACK重报;不执行业务写入。
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"time"
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
)
func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run() error {
token, queue := os.Getenv("ORBIT_TOKEN"), os.Getenv("ORBIT_QUEUE_ID")
if path := os.Getenv("ORBIT_CREDENTIALS_FILE"); path != "" {
raw, err := os.ReadFile(path)
if err != nil {
return err
}
var credential struct {
Token string `json:"token"`
QueueID string `json:"queue_id"`
}
if json.Unmarshal(raw, &credential) != nil {
return errors.New("invalid credential file")
}
if token == "" {
token = credential.Token
}
if queue == "" {
queue = credential.QueueID
}
}
if token == "" || queue == "" {
return errors.New("event-enabled token and isolated example queue are required")
}
api, err := orbit.NewClient(os.Getenv("ORBIT_URL"), orbit.Options{Token: token})
if err != nil {
return err
}
ctx, stop := context.WithTimeout(context.Background(), 15*time.Second)
defer stop()
id, err := orbit.NewEventMessageID()
if err != nil {
return err
}
created, err := api.CreateSubscription(ctx, "sub-"+id, orbit.CreateSubscriptionRequest{Name: "protocol-" + id, QueueID: queue, Types: []string{"task.cancelled"}, MaxPending: 10, MaxAttempts: 3})
if err != nil {
return err
}
sub := created.Subscription
task, err := api.SubmitTask(ctx, "task-"+id, orbit.SubmitTaskRequest{QueueID: queue, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)})
if err != nil {
return err
}
if _, err = api.CancelTask(ctx, task.Task.ID, "cancel-"+id, orbit.CancelTaskRequest{ExpectedRevision: task.Task.Revision, Reason: "event protocol example"}); err != nil {
return err
}
first, err := api.DialEvents(ctx, orbit.EventRegistration{SubscriptionID: sub.ID, Capacity: 1})
if err != nil {
return err
}
defer first.Close()
grant, err := first.Next(ctx)
if err != nil {
return err
}
if grant.Event.Snapshot.TaskID != task.Task.ID || grant.Event.Snapshot.State != orbit.TaskCancelled {
return errors.New("unexpected example queue event")
}
// 本例只读取并验证结果,没有业务数据库写入;业务处理必须先提交Inbox事务再ACK。
ackID, err := orbit.NewEventMessageID()
if err != nil {
return err
}
confirmed, err := first.Ack(ctx, ackID, grant.Proof())
if err != nil {
return err
}
first.Close()
second, err := api.DialEvents(ctx, orbit.EventRegistration{SubscriptionID: sub.ID, Capacity: 1})
if err != nil {
return err
}
defer second.Close()
replayID, err := orbit.NewEventMessageID()
if err != nil {
return err
}
replayed, err := second.Ack(ctx, replayID, grant.Proof())
if err != nil {
return err
}
if !replayed.Deduplicated || !replayed.AcknowledgedAt.Equal(confirmed.AcknowledgedAt) {
return errors.New("acknowledgement receipt changed across connections")
}
paused, err := api.ConfigureSubscription(ctx, sub.ID, "pause-"+id, orbit.ConfigureSubscriptionRequest{ExpectedRevision: sub.Revision, Enabled: false, Reason: "protocol example completed"})
if err != nil {
return err
}
fmt.Printf("event=%s task=%s state=%s\n", grant.Event.ID, task.Task.ID, grant.Event.Snapshot.State)
fmt.Printf("ack_replayed=%t same_receipt=true pending=%d subscription_enabled=%t\n", replayed.Deduplicated, paused.Subscription.PendingCount, paused.Subscription.Enabled)
return nil
}

View File

@@ -48,18 +48,23 @@ func integrationWorkerClient(t *testing.T) (*Client, string, string) {
// workerFaultGateway 在实际服务端确认后丢弃指定 ACK;SDK 到网关使用真实 TLS。
// 只丢响应,不伪造任务状态,测试凭据不进入 URL 或输出。
type workerFaultGateway struct {
drop atomic.Value
target atomic.Value
dropOffline atomic.Bool
dropCancel atomic.Bool
offline atomic.Bool
mu sync.Mutex
active map[*websocket.Conn]struct{}
dropped chan string
server *httptest.Server
confirmedEventPings atomic.Uint64
drop atomic.Value
target atomic.Value
dropOffline atomic.Bool
dropCancel atomic.Bool
offline atomic.Bool
mu sync.Mutex
active map[*websocket.Conn]struct{}
dropped chan string
server *httptest.Server
}
func newWorkerFaultGateway(t *testing.T, upstream string) *workerFaultGateway {
return newProtocolFaultGateway(t, upstream, "/v1/worker/connect", workerSubprotocol)
}
func newProtocolFaultGateway(t *testing.T, upstream, path, protocol string) *workerFaultGateway {
t.Helper()
g := &workerFaultGateway{dropped: make(chan string, 4), active: make(map[*websocket.Conn]struct{})}
g.drop.Store("")
@@ -77,14 +82,14 @@ func newWorkerFaultGateway(t *testing.T, upstream string) *workerFaultGateway {
defer stop()
header := http.Header{}
header.Set("Authorization", r.Header.Get("Authorization"))
up, _, err := websocket.Dial(bridge, g.target.Load().(string)+"/v1/worker/connect", &websocket.DialOptions{HTTPHeader: header, Subprotocols: []string{workerSubprotocol}})
up, _, err := websocket.Dial(bridge, g.target.Load().(string)+path, &websocket.DialOptions{HTTPHeader: header, Subprotocols: []string{protocol}})
if err != nil {
http.Error(w, "upstream unavailable", 503)
return
}
defer up.CloseNow()
up.SetReadLimit(512 << 10)
down, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{workerSubprotocol}})
down, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{protocol}})
if err != nil {
return
}
@@ -138,6 +143,9 @@ func newWorkerFaultGateway(t *testing.T, upstream string) *workerFaultGateway {
operation := operations[f.ReplyTo]
delete(operations, f.ReplyTo)
mu.Unlock()
if f.Type == "ack" && operation == "events.ping" {
g.confirmedEventPings.Add(1)
}
if f.Type == "task.cancel" && g.dropCancel.CompareAndSwap(true, false) {
g.dropped <- "task.cancel"
return