feat: add reliable event consumer SDK and process recovery tests
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
// Package orbit 是 Orbit 的独立 Go SDK,提供健康、任务、队列与 Worker 底层协议。
|
||||
// Package orbit 是 Orbit 的独立 Go SDK,提供健康、任务、队列、计划及结果订阅。
|
||||
// WorkerConnection 是底层接口;NewWorker/Run 自动管理处理器、续租、恢复与排空。
|
||||
// EventConnection 是结果交付协议;EventConsumer 自动管理心跳、重连与确认,业务端负责事务去重。
|
||||
// 流程 SDK 等后续能力按阶段继续开发。
|
||||
// SDK 不导入服务端模型、迁移或数据库依赖。
|
||||
package orbit
|
||||
|
||||
152
sdk/go/event_codec.go
Normal file
152
sdk/go/event_codec.go
Normal file
@@ -0,0 +1,152 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
type eventFrame struct {
|
||||
Version string `json:"protocol_version"`
|
||||
Type string `json:"type"`
|
||||
MessageID string `json:"message_id"`
|
||||
ReplyTo string `json:"reply_to,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Body json.RawMessage `json:"body,omitempty"`
|
||||
Error *EventError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// 仅协议结构严格匹配字段;业务RawMessage保持原数字、键和值。
|
||||
func eventFields(raw []byte, typ reflect.Type) error {
|
||||
if typ == reflect.TypeOf(json.RawMessage{}) {
|
||||
return nil
|
||||
}
|
||||
if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) {
|
||||
return ErrEventProtocol
|
||||
}
|
||||
if typ.Kind() == reflect.Pointer {
|
||||
return eventFields(raw, typ.Elem())
|
||||
}
|
||||
if typ == reflect.TypeOf(time.Time{}) {
|
||||
return nil
|
||||
}
|
||||
if typ.Kind() != reflect.Struct {
|
||||
return nil
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(raw, &fields) != nil || fields == nil {
|
||||
return ErrEventProtocol
|
||||
}
|
||||
for i := 0; i < typ.NumField(); i++ {
|
||||
f := typ.Field(i)
|
||||
tags := strings.Split(f.Tag.Get("json"), ",")
|
||||
if tags[0] == "" || tags[0] == "-" {
|
||||
continue
|
||||
}
|
||||
v, found := fields[tags[0]]
|
||||
optional, quoted := false, false
|
||||
for _, tag := range tags[1:] {
|
||||
optional = optional || tag == "omitempty" || tag == "omitzero"
|
||||
quoted = quoted || tag == "string"
|
||||
}
|
||||
if !found {
|
||||
if optional {
|
||||
continue
|
||||
}
|
||||
return ErrEventProtocol
|
||||
}
|
||||
delete(fields, tags[0])
|
||||
if f.Type.Kind() == reflect.Uint64 && quoted {
|
||||
var s string
|
||||
if json.Unmarshal(v, &s) != nil {
|
||||
return ErrEventProtocol
|
||||
}
|
||||
n, err := strconv.ParseUint(s, 10, 64)
|
||||
if err != nil || n == 0 || strconv.FormatUint(n, 10) != s {
|
||||
return ErrEventProtocol
|
||||
}
|
||||
}
|
||||
if err := eventFields(v, f.Type); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(fields) > 0 {
|
||||
return ErrEventProtocol
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func decodeEventValue(raw []byte, out any) error {
|
||||
if validateWorkerJSON(raw) != nil || eventFields(raw, reflect.TypeOf(out).Elem()) != nil || json.Unmarshal(raw, out) != nil {
|
||||
return ErrEventProtocol
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func parseEventFrame(kind websocket.MessageType, raw []byte) (eventFrame, error) {
|
||||
var f eventFrame
|
||||
if kind != websocket.MessageText || len(raw) > 512<<10 || decodeEventValue(raw, &f) != nil || f.Version != "1" || !validID(f.MessageID) {
|
||||
return f, ErrEventProtocol
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
_ = json.Unmarshal(raw, &fields)
|
||||
if fields["reply_to"] != nil && !validID(f.ReplyTo) || fields["session_id"] != nil && !validID(f.SessionID) {
|
||||
return f, ErrEventProtocol
|
||||
}
|
||||
switch f.Type {
|
||||
case "error":
|
||||
if f.Error == nil || fields["body"] != nil || len(f.Error.Message) == 0 || len(f.Error.Message) > 1024 {
|
||||
return f, ErrEventProtocol
|
||||
}
|
||||
switch f.Error.Code {
|
||||
case "INVALID_ARGUMENT", "UNAUTHENTICATED", "PERMISSION_DENIED", "NOT_FOUND", "STATE_CONFLICT", "DELIVERY_LEASE_LOST", "EVENT_HISTORY_EXPIRED", "UNAVAILABLE":
|
||||
default:
|
||||
return f, ErrEventProtocol
|
||||
}
|
||||
case "events.welcome", "ack", "event.deliver":
|
||||
if fields["error"] != nil || len(f.Body) == 0 || bytes.Equal(bytes.TrimSpace(f.Body), []byte("null")) || !validID(f.SessionID) {
|
||||
return f, ErrEventProtocol
|
||||
}
|
||||
if f.Type == "event.deliver" {
|
||||
if fields["reply_to"] != nil {
|
||||
return f, ErrEventProtocol
|
||||
}
|
||||
} else if !validID(f.ReplyTo) {
|
||||
return f, ErrEventProtocol
|
||||
}
|
||||
default:
|
||||
return f, ErrEventProtocol
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
func validEventProof(p EventProof) bool {
|
||||
b, err := base64.RawURLEncoding.Strict().DecodeString(p.LeaseSecret)
|
||||
return validID(p.SubscriptionID) && validID(p.EventID) && validID(p.DeliveryID) && p.Epoch > 0 && len(p.LeaseSecret) == 43 && err == nil && len(b) == 32
|
||||
}
|
||||
func validEventGrant(g EventGrant, sub string) bool {
|
||||
d, e, s := g.Delivery, g.Event, g.Event.Snapshot
|
||||
if !validEventProof(g.Proof()) || d.SubscriptionID != sub || d.EventID != e.ID || d.Sequence != e.Sequence || d.Sequence == 0 || d.Revision == 0 || d.State != DeliveryLeased || d.CreatedAt.IsZero() || d.AvailableAt.IsZero() || d.LeaseUntil == nil || d.LeaseUntil.IsZero() || d.AcknowledgedAt != nil || d.Attempts == 0 || d.Attempts > d.AttemptBudget || d.AttemptBudget > 1000000 {
|
||||
return false
|
||||
}
|
||||
if e.AggregateKind != "task" || !validID(e.AggregateID) || e.AggregateID != s.TaskID || e.AggregateRevision != s.Revision || s.Revision == 0 || e.OccurredAt.IsZero() || s.SchemaVersion != 1 || !validID(s.QueueID) || s.CurrentAttemptID != nil && !validID(*s.CurrentAttemptID) {
|
||||
return false
|
||||
}
|
||||
if len(bytes.TrimSpace(e.Details)) == 0 || bytes.TrimSpace(e.Details)[0] != '{' {
|
||||
return false
|
||||
}
|
||||
switch e.Type {
|
||||
case "task.succeeded":
|
||||
return s.State == TaskSucceeded
|
||||
case "task.failed":
|
||||
return s.State == TaskFailed
|
||||
case "task.cancelled":
|
||||
return s.State == TaskCancelled
|
||||
case "task.needs_review":
|
||||
return s.State == TaskNeedsReview
|
||||
}
|
||||
return false
|
||||
}
|
||||
285
sdk/go/event_connection.go
Normal file
285
sdk/go/event_connection.go
Normal file
@@ -0,0 +1,285 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
const eventSubprotocol = "orbit.events.v1"
|
||||
const eventMaxPending = 32
|
||||
|
||||
// EventConnection 管理单次连接、请求关联和有界收发。可并发调用Ack/Ping。
|
||||
// 本类型不自动心跳/重连/调用处理器;高层消费请使用EventConsumer。
|
||||
type EventConnection struct {
|
||||
conn *websocket.Conn
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
welcome EventWelcome
|
||||
deliveries chan EventGrant
|
||||
done chan struct{}
|
||||
mu sync.Mutex
|
||||
err error
|
||||
pending map[string]chan eventFrame
|
||||
}
|
||||
|
||||
// DialEvents 的ctx仅控制握手;成功后必须Close,取消握手ctx不会关闭已建立的连接。
|
||||
func (c *Client) DialEvents(ctx context.Context, in EventRegistration) (*EventConnection, error) {
|
||||
if c.token == "" || !validID(in.SubscriptionID) || in.Capacity < 1 || in.Capacity > 32 {
|
||||
return nil, errors.New("authenticated client, valid subscription and capacity 1..32 are required")
|
||||
}
|
||||
bounded, stop := context.WithTimeout(ctx, 8*time.Second)
|
||||
defer stop()
|
||||
headers := http.Header{}
|
||||
headers.Set("Authorization", "Bearer "+c.token)
|
||||
conn, response, err := websocket.Dial(bounded, c.baseURL+"/v1/events/connect", &websocket.DialOptions{HTTPClient: c.http, HTTPHeader: headers, Subprotocols: []string{eventSubprotocol}, CompressionMode: websocket.CompressionDisabled})
|
||||
if err != nil {
|
||||
if response != nil && response.StatusCode != http.StatusSwitchingProtocols {
|
||||
return nil, &HTTPError{StatusCode: response.StatusCode}
|
||||
}
|
||||
return nil, eventIOError(err)
|
||||
}
|
||||
ok := false
|
||||
defer func() {
|
||||
if !ok {
|
||||
_ = conn.CloseNow()
|
||||
}
|
||||
}()
|
||||
if conn.Subprotocol() != eventSubprotocol {
|
||||
return nil, ErrEventProtocol
|
||||
}
|
||||
conn.SetReadLimit(512 << 10)
|
||||
id, err := NewEventMessageID()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, _ := marshalWire(in)
|
||||
raw, _ := marshalWire(eventFrame{Version: "1", Type: "events.hello", MessageID: id, Body: body})
|
||||
if err = conn.Write(bounded, websocket.MessageText, raw); err != nil {
|
||||
return nil, eventIOError(err)
|
||||
}
|
||||
kind, raw, err := conn.Read(bounded)
|
||||
if err != nil {
|
||||
return nil, eventIOError(err)
|
||||
}
|
||||
f, err := parseEventFrame(kind, raw)
|
||||
if err != nil || f.ReplyTo != id {
|
||||
return nil, ErrEventProtocol
|
||||
}
|
||||
if f.Type == "error" {
|
||||
return nil, f.Error
|
||||
}
|
||||
var welcome EventWelcome
|
||||
if f.Type != "events.welcome" || decodeEventValue(f.Body, &welcome) != nil || welcome.SessionID != f.SessionID || welcome.SubscriptionID != in.SubscriptionID || welcome.Capacity != in.Capacity || welcome.ServerTime.IsZero() || welcome.LeaseDurationMS != 30000 || welcome.HeartbeatIntervalMS != 15000 {
|
||||
return nil, ErrEventProtocol
|
||||
}
|
||||
lifetime, cancel := context.WithCancel(context.Background())
|
||||
e := &EventConnection{conn: conn, ctx: lifetime, cancel: cancel, welcome: welcome, deliveries: make(chan EventGrant, in.Capacity), done: make(chan struct{}), pending: make(map[string]chan eventFrame)}
|
||||
ok = true
|
||||
go e.readLoop()
|
||||
return e, nil
|
||||
}
|
||||
func (e *EventConnection) Welcome() EventWelcome { return e.welcome }
|
||||
func (e *EventConnection) Done() <-chan struct{} { return e.done }
|
||||
func (e *EventConnection) Err() error { e.mu.Lock(); defer e.mu.Unlock(); return e.err }
|
||||
func (e *EventConnection) Close() error { e.fail(ErrEventClosed); <-e.done; return nil }
|
||||
func (e *EventConnection) fail(err error) {
|
||||
e.mu.Lock()
|
||||
if e.err != nil {
|
||||
e.mu.Unlock()
|
||||
return
|
||||
}
|
||||
e.err = err
|
||||
e.cancel()
|
||||
e.mu.Unlock()
|
||||
_ = e.conn.CloseNow()
|
||||
}
|
||||
func (e *EventConnection) readLoop() {
|
||||
defer close(e.done)
|
||||
for {
|
||||
ctx, stop := context.WithTimeout(e.ctx, 60*time.Second)
|
||||
kind, raw, err := e.conn.Read(ctx)
|
||||
stop()
|
||||
if err != nil {
|
||||
e.fail(eventIOError(err))
|
||||
return
|
||||
}
|
||||
f, err := parseEventFrame(kind, raw)
|
||||
if err != nil || f.SessionID != e.welcome.SessionID {
|
||||
e.fail(ErrEventProtocol)
|
||||
return
|
||||
}
|
||||
if f.Type == "event.deliver" {
|
||||
var grant EventGrant
|
||||
if decodeEventValue(f.Body, &grant) != nil || !validEventGrant(grant, e.welcome.SubscriptionID) {
|
||||
e.fail(ErrEventProtocol)
|
||||
return
|
||||
}
|
||||
select {
|
||||
case e.deliveries <- grant:
|
||||
default:
|
||||
e.fail(ErrEventBackpressure)
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if f.Type != "ack" && f.Type != "error" {
|
||||
e.fail(ErrEventProtocol)
|
||||
return
|
||||
}
|
||||
if f.Type == "error" && f.ReplyTo == "" {
|
||||
e.fail(f.Error)
|
||||
return
|
||||
}
|
||||
e.mu.Lock()
|
||||
waiter := e.pending[f.ReplyTo]
|
||||
e.mu.Unlock()
|
||||
if waiter == nil {
|
||||
e.fail(ErrEventProtocol)
|
||||
return
|
||||
}
|
||||
select {
|
||||
case waiter <- f:
|
||||
default:
|
||||
e.fail(ErrEventProtocol)
|
||||
return
|
||||
}
|
||||
if f.Error != nil && (f.Error.Code == "UNAUTHENTICATED" || f.Error.Code == "PERMISSION_DENIED" || f.Error.Code == "NOT_FOUND" || f.Error.Code == "UNAVAILABLE") {
|
||||
e.fail(f.Error)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Next 等待下一条交付。ctx到期只中断本次等待;未确认交付按服务端预算补发。
|
||||
func (e *EventConnection) Next(ctx context.Context) (EventGrant, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return EventGrant{}, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return EventGrant{}, ctx.Err()
|
||||
case <-e.ctx.Done():
|
||||
return EventGrant{}, e.Err()
|
||||
case g := <-e.deliveries:
|
||||
if err := e.Err(); err != nil {
|
||||
return EventGrant{}, err
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
}
|
||||
func (e *EventConnection) call(ctx context.Context, kind, id string, in, out any) error {
|
||||
if !validID(id) {
|
||||
return errors.New("invalid event message id")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
bounded, stop := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer stop()
|
||||
waiter := make(chan eventFrame, 1)
|
||||
e.mu.Lock()
|
||||
if e.err != nil {
|
||||
err := e.err
|
||||
e.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
if len(e.pending) >= eventMaxPending {
|
||||
e.mu.Unlock()
|
||||
return ErrEventBusy
|
||||
}
|
||||
if e.pending[id] != nil {
|
||||
e.mu.Unlock()
|
||||
return errors.New("event message id is already in flight")
|
||||
}
|
||||
e.pending[id] = waiter
|
||||
e.mu.Unlock()
|
||||
defer func() { e.mu.Lock(); delete(e.pending, id); e.mu.Unlock() }()
|
||||
body, err := marshalWire(in)
|
||||
if err != nil {
|
||||
return errors.New("invalid event request")
|
||||
}
|
||||
raw, err := marshalWire(eventFrame{Version: "1", Type: kind, MessageID: id, SessionID: e.welcome.SessionID, Body: body})
|
||||
if err != nil || len(raw) > 512<<10 {
|
||||
return errors.New("event request exceeds frame limit")
|
||||
}
|
||||
if err = e.conn.Write(bounded, websocket.MessageText, raw); err != nil {
|
||||
err = eventIOError(err)
|
||||
e.fail(err)
|
||||
return err
|
||||
}
|
||||
var f eventFrame
|
||||
select {
|
||||
case f = <-waiter:
|
||||
case <-e.ctx.Done():
|
||||
select {
|
||||
case f = <-waiter:
|
||||
default:
|
||||
return e.Err()
|
||||
}
|
||||
case <-bounded.Done():
|
||||
e.fail(bounded.Err())
|
||||
return bounded.Err()
|
||||
}
|
||||
if f.Error != nil {
|
||||
return f.Error
|
||||
}
|
||||
if decodeEventValue(f.Body, out) != nil {
|
||||
e.fail(ErrEventProtocol)
|
||||
return ErrEventProtocol
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ack 仅在业务事务已提交或持久化去重命中后调用。重连可用原Proof重报,MessageID可新建。
|
||||
// 超时/断线不证明ACK未提交;DELIVERY_LEASE_LOST表示证明已失效,不应撤销已经提交的业务。
|
||||
func (e *EventConnection) Ack(ctx context.Context, id string, p EventProof) (EventAckResult, error) {
|
||||
var out EventAckResult
|
||||
if !validEventProof(p) || p.SubscriptionID != e.welcome.SubscriptionID {
|
||||
return out, errors.New("invalid event proof or subscription")
|
||||
}
|
||||
in := struct {
|
||||
DeliveryID string `json:"delivery_id"`
|
||||
Epoch uint64 `json:"epoch,string"`
|
||||
LeaseSecret string `json:"lease_secret"`
|
||||
}{p.DeliveryID, p.Epoch, p.LeaseSecret}
|
||||
err := e.call(ctx, "event.ack", id, in, &out)
|
||||
if err == nil && (out.DeliveryID != p.DeliveryID || out.EventID != p.EventID || out.Epoch != p.Epoch || out.AcknowledgedAt.IsZero()) {
|
||||
err = ErrEventProtocol
|
||||
e.fail(err)
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
|
||||
// Ping 重新核验凭据及订阅,并维持连接;不会续期交付租约。
|
||||
func (e *EventConnection) Ping(ctx context.Context, id string) (EventPong, error) {
|
||||
var out EventPong
|
||||
err := e.call(ctx, "events.ping", id, struct{}{}, &out)
|
||||
if err == nil && out.ServerTime.IsZero() {
|
||||
err = ErrEventProtocol
|
||||
e.fail(err)
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
func eventIOError(err error) error {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return context.Canceled
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return context.DeadlineExceeded
|
||||
}
|
||||
switch websocket.CloseStatus(err) {
|
||||
case websocket.StatusProtocolError, websocket.StatusUnsupportedData, websocket.StatusInvalidFramePayloadData, websocket.StatusPolicyViolation, websocket.StatusMessageTooBig:
|
||||
return ErrEventProtocol
|
||||
}
|
||||
return ErrEventDisconnected
|
||||
}
|
||||
func (e *EventConnection) String() string {
|
||||
return fmt.Sprintf("EventConnection{SubscriptionID:%s SessionID:%s secrets:redacted}", e.welcome.SubscriptionID, e.welcome.SessionID)
|
||||
}
|
||||
func (e *EventConnection) GoString() string { return e.String() }
|
||||
283
sdk/go/event_connection_test.go
Normal file
283
sdk/go/event_connection_test.go
Normal file
@@ -0,0 +1,283 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
const eventTestSub = "55555555555555555555555555555555"
|
||||
const eventTestID = "66666666666666666666666666666666"
|
||||
const eventTestDelivery = "77777777777777777777777777777777"
|
||||
|
||||
func eventTestGrant() EventGrant {
|
||||
now := time.Now().UTC()
|
||||
until := now.Add(30 * time.Second)
|
||||
return EventGrant{Delivery: EventDelivery{ID: eventTestDelivery, SubscriptionID: eventTestSub, EventID: eventTestID, Sequence: 9007199254740993, Revision: 2, State: DeliveryLeased, CreatedAt: now, AvailableAt: now, LeaseEpoch: 1, LeaseUntil: &until, Attempts: 1, AttemptBudget: 20}, Event: ResultEvent{ID: eventTestID, Sequence: 9007199254740993, Type: "task.succeeded", AggregateKind: "task", AggregateID: workerTestTask, AggregateRevision: 9007199254740993, OccurredAt: now, Details: json.RawMessage(`{}`), Snapshot: TaskResultSnapshot{SchemaVersion: 1, TaskID: workerTestTask, QueueID: workerTestQueue, State: TaskSucceeded, Revision: 9007199254740993, Result: json.RawMessage(`{"n":9007199254740993,"text":"<&>"}`)}}, LeaseSecret: strings.Repeat("A", 43)}
|
||||
}
|
||||
func eventTestRead(ctx context.Context, c *websocket.Conn) (eventFrame, error) {
|
||||
kind, raw, err := c.Read(ctx)
|
||||
var f eventFrame
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
if kind != websocket.MessageText || json.Unmarshal(raw, &f) != nil {
|
||||
return f, ErrEventProtocol
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
func eventTestWrite(ctx context.Context, c *websocket.Conn, f eventFrame) error {
|
||||
f.Version = "1"
|
||||
f.SessionID = workerTestSession
|
||||
f.MessageID, _ = NewEventMessageID()
|
||||
raw, err := marshalWire(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.Write(ctx, websocket.MessageText, raw)
|
||||
}
|
||||
func eventBody(in any) json.RawMessage { b, _ := marshalWire(in); return b }
|
||||
func eventMock(t *testing.T, serve func(context.Context, *websocket.Conn, EventRegistration, int)) *Client {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
var count atomic.Int64
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/events/connect" || r.URL.RawQuery != "" || r.Header.Get("Authorization") != "Bearer event-test-token" {
|
||||
t.Error("unexpected event handshake")
|
||||
}
|
||||
c, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{eventSubprotocol}})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.CloseNow()
|
||||
f, err := eventTestRead(ctx, c)
|
||||
var in EventRegistration
|
||||
if err != nil || f.Type != "events.hello" || f.SessionID != "" || !validID(f.MessageID) || json.Unmarshal(f.Body, &in) != nil {
|
||||
t.Error("invalid event hello")
|
||||
return
|
||||
}
|
||||
welcome := EventWelcome{SessionID: workerTestSession, SubscriptionID: in.SubscriptionID, Capacity: in.Capacity, LeaseDurationMS: 30000, HeartbeatIntervalMS: 15000, ServerTime: time.Now().UTC(), Enabled: true}
|
||||
if eventTestWrite(ctx, c, eventFrame{Type: "events.welcome", ReplyTo: f.MessageID, Body: eventBody(welcome)}) != nil {
|
||||
return
|
||||
}
|
||||
serve(ctx, c, in, int(count.Add(1)))
|
||||
}))
|
||||
t.Cleanup(func() { cancel(); server.Close() })
|
||||
client, err := NewClient(server.URL, Options{Token: "event-test-token", HTTPClient: server.Client()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
func eventTestDial(t *testing.T, client *Client, capacity uint32) *EventConnection {
|
||||
t.Helper()
|
||||
ctx, stop := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer stop()
|
||||
c, err := client.DialEvents(ctx, EventRegistration{SubscriptionID: eventTestSub, Capacity: capacity})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { c.Close() })
|
||||
return c
|
||||
}
|
||||
func eventAckFor(g EventGrant) EventAckResult {
|
||||
return EventAckResult{DeliveryID: g.Delivery.ID, EventID: g.Event.ID, Epoch: g.Delivery.LeaseEpoch, AcknowledgedAt: time.Now().UTC()}
|
||||
}
|
||||
|
||||
func TestEventConnectionMultiplexAndPrecision(t *testing.T) {
|
||||
g := eventTestGrant()
|
||||
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
|
||||
if eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(g)}) != nil {
|
||||
return
|
||||
}
|
||||
first, err := eventTestRead(ctx, c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
second, err := eventTestRead(ctx, c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, f := range []eventFrame{second, first} {
|
||||
var body any = EventPong{ServerTime: time.Now().UTC(), Enabled: true}
|
||||
if f.Type == "event.ack" {
|
||||
body = eventAckFor(g)
|
||||
}
|
||||
if eventTestWrite(ctx, c, eventFrame{Type: "ack", ReplyTo: f.MessageID, Body: eventBody(body)}) != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
_, _ = eventTestRead(ctx, c)
|
||||
})
|
||||
c := eventTestDial(t, client, 1)
|
||||
ctx, stop := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer stop()
|
||||
got, err := c.Next(ctx)
|
||||
if err != nil || got.Event.Sequence != 9007199254740993 || string(got.Event.Snapshot.Result) != string(g.Event.Snapshot.Result) {
|
||||
t.Fatal("event snapshot changed", err)
|
||||
}
|
||||
errs := make(chan error, 2)
|
||||
go func() { _, err := c.Ping(ctx, workerTestID(t)); errs <- err }()
|
||||
go func() { _, err := c.Ack(ctx, workerTestID(t), got.Proof()); errs <- err }()
|
||||
for range 2 {
|
||||
if err := <-errs; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if c.Err() != nil {
|
||||
t.Fatal(c.Err())
|
||||
}
|
||||
for _, value := range []any{g, g.Proof(), c, &EventError{Code: "UNAVAILABLE", Message: "secret-text"}} {
|
||||
for _, format := range []string{"%v", "%+v", "%#v"} {
|
||||
printed := fmt.Sprintf(format, value)
|
||||
if strings.Contains(printed, g.LeaseSecret) || strings.Contains(printed, "secret-text") || strings.Contains(printed, "9007199254740993") {
|
||||
t.Fatal("event formatter exposed secrets or business data")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestEventConnectionWaitCancellationAndAckUncertainty(t *testing.T) {
|
||||
received := make(chan struct{}, 1)
|
||||
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
|
||||
f, err := eventTestRead(ctx, c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if f.Type != "event.ack" {
|
||||
t.Error("unexpected control")
|
||||
}
|
||||
received <- struct{}{}
|
||||
_, _ = eventTestRead(ctx, c)
|
||||
})
|
||||
c := eventTestDial(t, client, 1)
|
||||
short, stop := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer stop()
|
||||
if _, err := c.Next(short); !errors.Is(err, context.DeadlineExceeded) || c.Err() != nil {
|
||||
t.Fatal("Next timeout closed connection", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := c.Ack(ctx, workerTestID(t), eventTestGrant().Proof()); done <- err }()
|
||||
<-received
|
||||
cancel()
|
||||
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-c.Done():
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("uncertain ACK did not close connection")
|
||||
}
|
||||
}
|
||||
func TestEventConnectionBoundedControlsAndDeliveries(t *testing.T) {
|
||||
t.Run("controls", func(t *testing.T) {
|
||||
received := make(chan struct{}, 40)
|
||||
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
|
||||
for {
|
||||
if _, err := eventTestRead(ctx, c); err != nil {
|
||||
return
|
||||
}
|
||||
received <- struct{}{}
|
||||
}
|
||||
})
|
||||
c := eventTestDial(t, client, 1)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan error, 32)
|
||||
for range 32 {
|
||||
go func() { _, err := c.Ping(ctx, workerTestID(t)); done <- err }()
|
||||
<-received
|
||||
}
|
||||
if _, err := c.Ping(ctx, workerTestID(t)); !errors.Is(err, ErrEventBusy) {
|
||||
t.Fatal("unbounded pending controls", err)
|
||||
}
|
||||
cancel()
|
||||
for range 32 {
|
||||
<-done
|
||||
}
|
||||
})
|
||||
t.Run("deliveries", func(t *testing.T) {
|
||||
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
|
||||
for range 2 {
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(eventTestGrant())})
|
||||
}
|
||||
_, _ = eventTestRead(ctx, c)
|
||||
})
|
||||
c := eventTestDial(t, client, 1)
|
||||
select {
|
||||
case <-c.Done():
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("unbounded deliveries")
|
||||
}
|
||||
if !errors.Is(c.Err(), ErrEventBackpressure) {
|
||||
t.Fatal(c.Err())
|
||||
}
|
||||
})
|
||||
}
|
||||
func TestEventConnectionRejectsUntrustedProtocol(t *testing.T) {
|
||||
g := eventTestGrant()
|
||||
valid := eventFrame{Version: "1", Type: "event.deliver", MessageID: workerTestQueue, SessionID: workerTestSession, Body: eventBody(g)}
|
||||
raw, _ := marshalWire(valid)
|
||||
cases := map[string][]byte{
|
||||
"duplicate": []byte(strings.Replace(string(raw), `"protocol_version":"1"`, `"protocol_version":"1","protocol_version":"1"`, 1)),
|
||||
"alias": []byte(strings.Replace(string(raw), `"lease_secret"`, `"Lease_Secret"`, 1)),
|
||||
"noncanonical": []byte(strings.Replace(string(raw), `"sequence":"9007199254740993"`, `"sequence":"09007199254740993"`, 1)),
|
||||
"null_bool": []byte(strings.Replace(string(raw), `"effect_uncertain":false`, `"effect_uncertain":null`, 1)),
|
||||
"null_error": []byte(strings.Replace(string(raw), `"body":`, `"error":null,"body":`, 1)),
|
||||
"mismatched_event": []byte(strings.Replace(string(raw), `"event_id":"`+eventTestID+`"`, `"event_id":"`+workerTestTask+`"`, 1)),
|
||||
"missing_created": []byte(strings.Replace(string(raw), `"created_at":`+string(eventBody(g.Delivery.CreatedAt))+`,`, "", 1)),
|
||||
"wrong_state": []byte(strings.Replace(string(raw), `"state":"SUCCEEDED"`, `"state":"FAILED"`, 1)),
|
||||
"unknown_body_field": []byte(strings.Replace(string(raw), `"schema_version":1`, `"extra":true,"schema_version":1`, 1)),
|
||||
}
|
||||
for name, bad := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
|
||||
_ = c.Write(ctx, websocket.MessageText, bad)
|
||||
_, _ = eventTestRead(ctx, c)
|
||||
})
|
||||
c := eventTestDial(t, client, 1)
|
||||
ctx, stop := context.WithTimeout(context.Background(), time.Second)
|
||||
defer stop()
|
||||
if _, err := c.Next(ctx); !errors.Is(err, ErrEventProtocol) {
|
||||
t.Fatal("invalid response accepted", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestEventConnectionRejectsMismatchedAck(t *testing.T) {
|
||||
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
|
||||
f, err := eventTestRead(ctx, c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
out := eventAckFor(eventTestGrant())
|
||||
out.Epoch++
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "ack", ReplyTo: f.MessageID, Body: eventBody(out)})
|
||||
_, _ = eventTestRead(ctx, c)
|
||||
})
|
||||
c := eventTestDial(t, client, 1)
|
||||
if _, err := c.Ack(context.Background(), workerTestID(t), eventTestGrant().Proof()); !errors.Is(err, ErrEventProtocol) {
|
||||
t.Fatal("wrong receipt accepted", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventConnectionPolicyClosureIsPermanent(t *testing.T) {
|
||||
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
|
||||
_ = c.Close(websocket.StatusPolicyViolation, "private peer reason")
|
||||
})
|
||||
c := eventTestDial(t, client, 1)
|
||||
ctx, stop := context.WithTimeout(context.Background(), time.Second)
|
||||
defer stop()
|
||||
if _, err := c.Next(ctx); !errors.Is(err, ErrEventProtocol) || eventRetryable(err) {
|
||||
t.Fatal("policy violation classified as retryable", err)
|
||||
}
|
||||
}
|
||||
368
sdk/go/event_consumer.go
Normal file
368
sdk/go/event_consumer.go
Normal file
@@ -0,0 +1,368 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrEventConsumerAlreadyRun = errors.New("event consumer may only Run once")
|
||||
var ErrEventShutdownTimeout = errors.New("event consumer shutdown grace period expired")
|
||||
|
||||
// EventConsumerOptions 定义自动消费器。Handle必须以事件ID做持久化去重,
|
||||
// 在去重记录和业务效果同一事务提交后才返回nil;提交结果未知时返回error。
|
||||
// error或panic不ACK,平台在租约到期后按原预算补发;SDK不在内存中立即重跑。
|
||||
type EventConsumerOptions struct {
|
||||
SubscriptionID string
|
||||
Capacity uint32 // Capacity 最大本地在途事件/处理器,默认4,范围1至32。
|
||||
Handle func(context.Context, ResultEvent) error // Handle不接收租约秘密;必须响应context取消。
|
||||
ShutdownTimeout time.Duration // ShutdownTimeout停止时取消处理器,等待返回及ACK,默认30秒,最多5分钟。
|
||||
ReconnectMinDelay time.Duration // ReconnectMinDelay默认200毫秒,至少10毫秒。
|
||||
ReconnectMaxDelay time.Duration // ReconnectMaxDelay默认5秒,最多30秒;指数退避并加入抖动。
|
||||
}
|
||||
|
||||
// EventConsumerEvent 只用于本地观察,缓冲满可丢弃;不是可靠结果或审计。
|
||||
type EventConsumerEvent struct {
|
||||
Type string // Type为connected、reconnecting、handler_started、handler_failed、acknowledged、proof_lost、draining。
|
||||
EventID string
|
||||
Code string // Code不含业务异常、数据或秘密。
|
||||
}
|
||||
type EventConsumerStats struct {
|
||||
Running int64
|
||||
Started uint64
|
||||
HandlerFailures uint64
|
||||
Acknowledged uint64
|
||||
Reconnects uint64
|
||||
DroppedEvents uint64
|
||||
}
|
||||
type EventConsumerRunError struct {
|
||||
Cause error
|
||||
Unconfirmed int // Unconfirmed本进程已知且未确认的事件数,不是订阅全局积压。
|
||||
Running int64 // Running仍未退出的处理器;Go不能强杀忽略context的函数。
|
||||
}
|
||||
|
||||
func (e *EventConsumerRunError) Error() string {
|
||||
return fmt.Sprintf("event consumer stopped with %d unconfirmed events and %d running handlers: %v", e.Unconfirmed, e.Running, e.Cause)
|
||||
}
|
||||
func (e *EventConsumerRunError) Unwrap() error { return e.Cause }
|
||||
|
||||
// EventConsumer 自动维护心跳、重连、处理器并发和ACK核对。只能Run一次。
|
||||
// 服务端交付租约不是业务锁;跨进程重复及并行消费由业务Inbox唯一键和事务处理。
|
||||
type EventConsumer struct {
|
||||
client *Client
|
||||
options EventConsumerOptions
|
||||
used atomic.Bool
|
||||
events chan EventConsumerEvent
|
||||
running atomic.Int64
|
||||
started, failures, acked, reconnects, dropped atomic.Uint64
|
||||
}
|
||||
|
||||
func NewEventConsumer(client *Client, in EventConsumerOptions) (*EventConsumer, error) {
|
||||
if client == nil || client.token == "" || !validID(in.SubscriptionID) || in.Handle == nil {
|
||||
return nil, errors.New("authenticated client, subscription and durable event handler are required")
|
||||
}
|
||||
if in.Capacity == 0 {
|
||||
in.Capacity = 4
|
||||
}
|
||||
if in.ShutdownTimeout == 0 {
|
||||
in.ShutdownTimeout = 30 * time.Second
|
||||
}
|
||||
if in.ReconnectMinDelay == 0 {
|
||||
in.ReconnectMinDelay = 200 * time.Millisecond
|
||||
}
|
||||
if in.ReconnectMaxDelay == 0 {
|
||||
in.ReconnectMaxDelay = 5 * time.Second
|
||||
}
|
||||
if in.Capacity > 32 || in.ShutdownTimeout < time.Millisecond || in.ShutdownTimeout > 5*time.Minute || in.ReconnectMinDelay < 10*time.Millisecond || in.ReconnectMaxDelay < in.ReconnectMinDelay || in.ReconnectMaxDelay > 30*time.Second {
|
||||
return nil, errors.New("invalid event consumer capacity or durations")
|
||||
}
|
||||
return &EventConsumer{client: client, options: in, events: make(chan EventConsumerEvent, 128)}, nil
|
||||
}
|
||||
func (c *EventConsumer) Events() <-chan EventConsumerEvent { return c.events }
|
||||
func (c *EventConsumer) Stats() EventConsumerStats {
|
||||
return EventConsumerStats{Running: c.running.Load(), Started: c.started.Load(), HandlerFailures: c.failures.Load(), Acknowledged: c.acked.Load(), Reconnects: c.reconnects.Load(), DroppedEvents: c.dropped.Load()}
|
||||
}
|
||||
func (c *EventConsumer) emit(typ, id, code string) {
|
||||
select {
|
||||
case c.events <- EventConsumerEvent{Type: typ, EventID: id, Code: code}:
|
||||
default:
|
||||
c.dropped.Add(1)
|
||||
}
|
||||
}
|
||||
func (c *EventConsumer) String() string {
|
||||
return fmt.Sprintf("EventConsumer{SubscriptionID:%s Capacity:%d secrets:redacted}", c.options.SubscriptionID, c.options.Capacity)
|
||||
}
|
||||
func (c *EventConsumer) GoString() string { return c.String() }
|
||||
|
||||
type consumerJob struct {
|
||||
proof EventProof
|
||||
complete bool
|
||||
}
|
||||
type consumerResult struct {
|
||||
id string
|
||||
failed bool
|
||||
}
|
||||
|
||||
// Run在ctx取消后停止启动新处理器,取消在途处理器,在收尾时限内确认已提交结果。
|
||||
// 收到认证/协议等永久错误时也取消并等待在途处理器;未确认结果仍保留在服务端。
|
||||
// 重连先核对已完成处理器的原证明;证明被替代时等待新投递,由业务去重命中后确认。
|
||||
func (c *EventConsumer) Run(parent context.Context) error {
|
||||
if !c.used.CompareAndSwap(false, true) {
|
||||
return ErrEventConsumerAlreadyRun
|
||||
}
|
||||
defer close(c.events)
|
||||
if parent.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.WithoutCancel(parent))
|
||||
defer cancel()
|
||||
handlerCtx, stopHandlers := context.WithCancel(ctx)
|
||||
defer stopHandlers()
|
||||
abort := make(chan struct{})
|
||||
watchDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(watchDone)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-parent.Done():
|
||||
case <-abort:
|
||||
}
|
||||
stopHandlers()
|
||||
timer := time.NewTimer(c.options.ShutdownTimeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-timer.C:
|
||||
cancel()
|
||||
}
|
||||
}()
|
||||
defer func() { cancel(); <-watchDone }()
|
||||
var conn *EventConnection
|
||||
var heartbeatDone <-chan struct{}
|
||||
closeConnection := func() {
|
||||
if conn != nil {
|
||||
_ = conn.Close()
|
||||
<-heartbeatDone
|
||||
conn = nil
|
||||
}
|
||||
}
|
||||
defer closeConnection()
|
||||
jobs := make(map[string]*consumerJob)
|
||||
results := make(chan consumerResult, c.options.Capacity)
|
||||
draining := false
|
||||
var cause error
|
||||
beginDrain := func(err error) {
|
||||
if !draining {
|
||||
draining = true
|
||||
stopHandlers()
|
||||
close(abort)
|
||||
c.emit("draining", "", "")
|
||||
}
|
||||
if cause == nil {
|
||||
cause = err
|
||||
}
|
||||
}
|
||||
finish := func(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return &EventConsumerRunError{Cause: err, Unconfirmed: len(jobs), Running: c.running.Load()}
|
||||
}
|
||||
backoff := c.options.ReconnectMinDelay
|
||||
var dialAt time.Time
|
||||
everConnected := false
|
||||
disconnect := func(err error) {
|
||||
closeConnection()
|
||||
if !eventRetryable(err) {
|
||||
beginDrain(err)
|
||||
return
|
||||
}
|
||||
floor := backoff / 2
|
||||
dialAt = time.Now().Add(floor + time.Duration(rand.Int64N(int64(backoff-floor)+1)))
|
||||
backoff = min(backoff*2, c.options.ReconnectMaxDelay)
|
||||
c.emit("reconnecting", "", eventErrorCode(err))
|
||||
}
|
||||
for {
|
||||
if parent.Err() != nil {
|
||||
beginDrain(nil)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return finish(ErrEventShutdownTimeout)
|
||||
}
|
||||
if draining && (len(jobs) == 0 || cause != nil && c.running.Load() == 0) {
|
||||
return finish(cause)
|
||||
}
|
||||
if conn == nil && cause == nil && !time.Now().Before(dialAt) {
|
||||
if everConnected {
|
||||
c.reconnects.Add(1)
|
||||
}
|
||||
var err error
|
||||
conn, err = c.client.DialEvents(ctx, EventRegistration{SubscriptionID: c.options.SubscriptionID, Capacity: c.options.Capacity})
|
||||
if err != nil {
|
||||
disconnect(err)
|
||||
continue
|
||||
}
|
||||
everConnected = true
|
||||
heartbeatDone = startEventHeartbeat(ctx, conn)
|
||||
c.emit("connected", "", "")
|
||||
continue
|
||||
}
|
||||
if conn != nil {
|
||||
if err := conn.Err(); err != nil {
|
||||
disconnect(err)
|
||||
continue
|
||||
}
|
||||
// 一个ACK在途;读取与自动心跳独立,不需要接收方不断调用Next维持连接。
|
||||
acknowledged := false
|
||||
for id, j := range jobs {
|
||||
if !j.complete {
|
||||
continue
|
||||
}
|
||||
messageID, err := NewEventMessageID()
|
||||
if err == nil {
|
||||
_, err = conn.Ack(ctx, messageID, j.proof)
|
||||
}
|
||||
if err == nil {
|
||||
delete(jobs, id)
|
||||
c.acked.Add(1)
|
||||
c.emit("acknowledged", id, "")
|
||||
backoff = c.options.ReconnectMinDelay
|
||||
} else {
|
||||
var remote *EventError
|
||||
if errors.As(err, &remote) && remote.Code == "DELIVERY_LEASE_LOST" {
|
||||
delete(jobs, id)
|
||||
c.emit("proof_lost", id, remote.Code)
|
||||
} else {
|
||||
disconnect(err)
|
||||
}
|
||||
}
|
||||
acknowledged = true
|
||||
break
|
||||
}
|
||||
if acknowledged {
|
||||
continue
|
||||
}
|
||||
}
|
||||
var deliveries <-chan EventGrant
|
||||
var disconnected <-chan struct{}
|
||||
if conn != nil {
|
||||
disconnected = conn.Done()
|
||||
if !draining && len(jobs) < int(c.options.Capacity) {
|
||||
deliveries = conn.deliveries
|
||||
}
|
||||
}
|
||||
var parentDone <-chan struct{}
|
||||
if !draining {
|
||||
parentDone = parent.Done()
|
||||
}
|
||||
// 定时器只负责重连;活跃连接没有每轮轮询。
|
||||
var wake <-chan time.Time
|
||||
var timer *time.Timer
|
||||
if conn == nil && cause == nil {
|
||||
timer = time.NewTimer(max(time.Millisecond, time.Until(dialAt)))
|
||||
wake = timer.C
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-parentDone:
|
||||
beginDrain(nil)
|
||||
case <-wake:
|
||||
case <-disconnected:
|
||||
disconnect(conn.Err())
|
||||
case result := <-results:
|
||||
j := jobs[result.id]
|
||||
if result.failed {
|
||||
delete(jobs, result.id)
|
||||
c.failures.Add(1)
|
||||
c.emit("handler_failed", result.id, "HANDLER_FAILED")
|
||||
} else {
|
||||
j.complete = true
|
||||
}
|
||||
case grant := <-deliveries:
|
||||
if parent.Err() != nil || conn.Err() != nil {
|
||||
break
|
||||
}
|
||||
id := grant.Event.ID
|
||||
if j := jobs[id]; j != nil {
|
||||
if j.proof.DeliveryID != grant.Delivery.ID {
|
||||
disconnect(ErrEventProtocol)
|
||||
break
|
||||
}
|
||||
if grant.Delivery.LeaseEpoch > j.proof.Epoch {
|
||||
j.proof = grant.Proof()
|
||||
}
|
||||
break
|
||||
}
|
||||
jobs[id] = &consumerJob{proof: grant.Proof()}
|
||||
c.running.Add(1)
|
||||
c.started.Add(1)
|
||||
c.emit("handler_started", id, "")
|
||||
go func(event ResultEvent) {
|
||||
failed := true
|
||||
defer func() {
|
||||
_ = recover()
|
||||
c.running.Add(-1)
|
||||
select {
|
||||
case results <- consumerResult{id: event.ID, failed: failed}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
failed = c.options.Handle(handlerCtx, event) != nil
|
||||
}(grant.Event)
|
||||
}
|
||||
if timer != nil {
|
||||
timer.Stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
func startEventHeartbeat(ctx context.Context, conn *EventConnection) <-chan struct{} {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
ticker := time.NewTicker(time.Duration(conn.Welcome().HeartbeatIntervalMS) * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-conn.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
id, err := NewEventMessageID()
|
||||
if err == nil {
|
||||
_, err = conn.Ping(ctx, id)
|
||||
}
|
||||
if err != nil {
|
||||
conn.fail(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return done
|
||||
}
|
||||
func eventRetryable(err error) bool {
|
||||
if errors.Is(err, ErrEventDisconnected) || errors.Is(err, ErrEventBackpressure) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return true
|
||||
}
|
||||
var remote *EventError
|
||||
if errors.As(err, &remote) {
|
||||
return remote.Code == "UNAVAILABLE"
|
||||
}
|
||||
var httpErr *HTTPError
|
||||
if errors.As(err, &httpErr) {
|
||||
return httpErr.StatusCode == 408 || httpErr.StatusCode == 429 || httpErr.StatusCode >= 500 && httpErr.StatusCode <= 599
|
||||
}
|
||||
return false
|
||||
}
|
||||
func eventErrorCode(err error) string {
|
||||
var remote *EventError
|
||||
if errors.As(err, &remote) {
|
||||
return remote.Code
|
||||
}
|
||||
return "TRANSPORT_INTERRUPTED"
|
||||
}
|
||||
342
sdk/go/event_consumer_test.go
Normal file
342
sdk/go/event_consumer_test.go
Normal file
@@ -0,0 +1,342 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"runtime"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
func runEventConsumer(t *testing.T, client *Client, handle func(context.Context, ResultEvent) error) (*EventConsumer, context.CancelFunc, <-chan error) {
|
||||
t.Helper()
|
||||
consumer, err := NewEventConsumer(client, EventConsumerOptions{SubscriptionID: eventTestSub, Capacity: 1, Handle: handle, ReconnectMinDelay: 10 * time.Millisecond, ReconnectMaxDelay: 50 * time.Millisecond, ShutdownTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- consumer.Run(ctx) }()
|
||||
t.Cleanup(cancel)
|
||||
return consumer, cancel, done
|
||||
}
|
||||
func awaitConsumerEvent(t *testing.T, c *EventConsumer, typ string) EventConsumerEvent {
|
||||
t.Helper()
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-c.Events():
|
||||
if !ok {
|
||||
t.Fatal("consumer stopped before " + typ)
|
||||
}
|
||||
if event.Type == typ {
|
||||
return event
|
||||
}
|
||||
case <-timer.C:
|
||||
t.Fatal("consumer did not emit " + typ)
|
||||
}
|
||||
}
|
||||
}
|
||||
func awaitConsumerStop(t *testing.T, done <-chan error) error {
|
||||
t.Helper()
|
||||
select {
|
||||
case err := <-done:
|
||||
return err
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("consumer did not stop")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
func TestEventConsumerReconcilesLostAckWithoutRerunningHandler(t *testing.T) {
|
||||
g := eventTestGrant()
|
||||
var calls atomic.Int64
|
||||
var proofs [2]EventProof
|
||||
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, n int) {
|
||||
if n > 2 {
|
||||
t.Error("unexpected third connection")
|
||||
return
|
||||
}
|
||||
if n == 1 {
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(g)})
|
||||
}
|
||||
f, err := eventTestRead(ctx, c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if f.Type != "event.ack" || json.Unmarshal(f.Body, &proofs[n-1]) != nil {
|
||||
t.Error("missing ACK proof")
|
||||
return
|
||||
}
|
||||
if n == 1 {
|
||||
return
|
||||
} // ACK已在服务端提交,回应在网络中丢失。
|
||||
if proofs[0].DeliveryID != proofs[1].DeliveryID || proofs[0].Epoch != proofs[1].Epoch || proofs[0].LeaseSecret != proofs[1].LeaseSecret {
|
||||
t.Error("reconnect changed acknowledgement identity")
|
||||
}
|
||||
out := eventAckFor(g)
|
||||
out.Deduplicated = true
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "ack", ReplyTo: f.MessageID, Body: eventBody(out)})
|
||||
_, _ = eventTestRead(ctx, c)
|
||||
})
|
||||
consumer, cancel, done := runEventConsumer(t, client, func(context.Context, ResultEvent) error { calls.Add(1); return nil })
|
||||
awaitConsumerEvent(t, consumer, "acknowledged")
|
||||
cancel()
|
||||
if err := awaitConsumerStop(t, done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls.Load() != 1 || consumer.Stats().Acknowledged != 1 || consumer.Stats().Reconnects != 1 {
|
||||
t.Fatal("lost receipt re-executed committed handler", consumer.Stats())
|
||||
}
|
||||
if !errors.Is(consumer.Run(context.Background()), ErrEventConsumerAlreadyRun) {
|
||||
t.Fatal("consumer ran twice")
|
||||
}
|
||||
}
|
||||
func TestEventConsumerSupersededProofRequiresDurableDeduplication(t *testing.T) {
|
||||
g := eventTestGrant()
|
||||
var calls, effects atomic.Int64
|
||||
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(g)})
|
||||
f, err := eventTestRead(ctx, c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "error", ReplyTo: f.MessageID, Error: &EventError{Code: "DELIVERY_LEASE_LOST", Message: "proof superseded"}})
|
||||
next := g
|
||||
next.Delivery.LeaseEpoch = 2
|
||||
next.Delivery.Attempts = 2
|
||||
next.Delivery.Revision = 4
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(next)})
|
||||
f, err = eventTestRead(ctx, c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "ack", ReplyTo: f.MessageID, Body: eventBody(eventAckFor(next))})
|
||||
_, _ = eventTestRead(ctx, c)
|
||||
})
|
||||
consumer, cancel, done := runEventConsumer(t, client, func(context.Context, ResultEvent) error { calls.Add(1); effects.CompareAndSwap(0, 1); return nil })
|
||||
awaitConsumerEvent(t, consumer, "acknowledged")
|
||||
cancel()
|
||||
if err := awaitConsumerStop(t, done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls.Load() != 2 || effects.Load() != 1 {
|
||||
t.Fatal("replacement proof bypassed handler deduplication")
|
||||
}
|
||||
}
|
||||
func TestEventConsumerHandlerFailureDoesNotAcknowledge(t *testing.T) {
|
||||
for _, panicHandler := range []bool{false, true} {
|
||||
name := "error"
|
||||
if panicHandler {
|
||||
name = "panic"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
g := eventTestGrant()
|
||||
retry := make(chan struct{})
|
||||
var calls atomic.Int64
|
||||
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(g)})
|
||||
select {
|
||||
case <-retry:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
next := g
|
||||
next.Delivery.LeaseEpoch = 2
|
||||
next.Delivery.Attempts = 2
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(next)})
|
||||
f, err := eventTestRead(ctx, c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var proof EventProof
|
||||
if f.Type != "event.ack" || json.Unmarshal(f.Body, &proof) != nil || proof.Epoch != 2 {
|
||||
t.Error("failed handler acknowledged original delivery")
|
||||
}
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "ack", ReplyTo: f.MessageID, Body: eventBody(eventAckFor(next))})
|
||||
_, _ = eventTestRead(ctx, c)
|
||||
})
|
||||
consumer, cancel, done := runEventConsumer(t, client, func(context.Context, ResultEvent) error {
|
||||
if calls.Add(1) == 1 {
|
||||
if panicHandler {
|
||||
panic("private handler data")
|
||||
}
|
||||
return errors.New("private handler data")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
awaitConsumerEvent(t, consumer, "handler_failed")
|
||||
if calls.Load() != 1 {
|
||||
t.Fatal("handler retried locally")
|
||||
}
|
||||
close(retry)
|
||||
awaitConsumerEvent(t, consumer, "acknowledged")
|
||||
cancel()
|
||||
if err := awaitConsumerStop(t, done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if consumer.Stats().HandlerFailures != 1 || consumer.Stats().Acknowledged != 1 {
|
||||
t.Fatal(consumer.Stats())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestEventConsumerShutdownCancelsAndConfirmsCommittedHandler(t *testing.T) {
|
||||
g := eventTestGrant()
|
||||
started := make(chan struct{})
|
||||
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(g)})
|
||||
f, err := eventTestRead(ctx, c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "ack", ReplyTo: f.MessageID, Body: eventBody(eventAckFor(g))})
|
||||
_, _ = eventTestRead(ctx, c)
|
||||
})
|
||||
consumer, cancel, done := runEventConsumer(t, client, func(ctx context.Context, _ ResultEvent) error { close(started); <-ctx.Done(); return nil })
|
||||
<-started
|
||||
cancel()
|
||||
if err := awaitConsumerStop(t, done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if consumer.Stats().Acknowledged != 1 || consumer.Stats().Running != 0 {
|
||||
t.Fatal("committed result not drained", consumer.Stats())
|
||||
}
|
||||
}
|
||||
func TestEventConsumerShutdownBoundsIgnoringHandler(t *testing.T) {
|
||||
started, release := make(chan struct{}), make(chan struct{})
|
||||
defer close(release)
|
||||
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(eventTestGrant())})
|
||||
_, _ = eventTestRead(ctx, c)
|
||||
})
|
||||
consumer, err := NewEventConsumer(client, EventConsumerOptions{SubscriptionID: eventTestSub, Capacity: 1, ShutdownTimeout: 30 * time.Millisecond, Handle: func(context.Context, ResultEvent) error { close(started); <-release; return nil }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- consumer.Run(ctx) }()
|
||||
<-started
|
||||
cancel()
|
||||
err = awaitConsumerStop(t, done)
|
||||
var run *EventConsumerRunError
|
||||
if !errors.Is(err, ErrEventShutdownTimeout) || !errors.As(err, &run) || run.Unconfirmed != 1 || run.Running != 1 {
|
||||
t.Fatal("uncooperative handler not reported", err)
|
||||
}
|
||||
}
|
||||
func TestEventConsumerPermanentFailureDoesNotReconnect(t *testing.T) {
|
||||
var calls atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); w.WriteHeader(http.StatusUnauthorized) }))
|
||||
defer server.Close()
|
||||
client, _ := NewClient(server.URL, Options{Token: "token"})
|
||||
_, _, done := runEventConsumer(t, client, func(context.Context, ResultEvent) error { t.Error("unauthorized consumer invoked handler"); return nil })
|
||||
err := awaitConsumerStop(t, done)
|
||||
var status *HTTPError
|
||||
if !errors.As(err, &status) || status.StatusCode != 401 || calls.Load() != 1 {
|
||||
t.Fatal("permanent handshake error retried", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventConsumerHandlerGoexitReleasesSlotWithoutAck(t *testing.T) {
|
||||
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, _ EventRegistration, _ int) {
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(eventTestGrant())})
|
||||
if f, err := eventTestRead(ctx, c); err == nil && f.Type == "event.ack" {
|
||||
t.Error("Goexit handler was acknowledged")
|
||||
}
|
||||
})
|
||||
consumer, cancel, done := runEventConsumer(t, client, func(context.Context, ResultEvent) error { runtime.Goexit(); return nil })
|
||||
awaitConsumerEvent(t, consumer, "handler_failed")
|
||||
cancel()
|
||||
if err := awaitConsumerStop(t, done); err != nil || consumer.Stats().Running != 0 {
|
||||
t.Fatal("Goexit leaked handler capacity", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventConsumerConcurrentCapacity(t *testing.T) {
|
||||
start := make(chan string, 3)
|
||||
release := make(chan struct{})
|
||||
var active, peak atomic.Int64
|
||||
grants := []EventGrant{eventTestGrant(), eventTestGrant(), eventTestGrant()}
|
||||
for i := range grants {
|
||||
grants[i].Event.ID = workerTestID(t)
|
||||
grants[i].Delivery.ID = workerTestID(t)
|
||||
grants[i].Delivery.EventID = grants[i].Event.ID
|
||||
}
|
||||
client := eventMock(t, func(ctx context.Context, c *websocket.Conn, in EventRegistration, _ int) {
|
||||
if in.Capacity != 2 {
|
||||
t.Error("capacity changed")
|
||||
}
|
||||
for _, g := range grants[:2] {
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(g)})
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
f, err := eventTestRead(ctx, c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var p EventProof
|
||||
if f.Type != "event.ack" || json.Unmarshal(f.Body, &p) != nil {
|
||||
t.Error("missing ACK")
|
||||
return
|
||||
}
|
||||
var grant EventGrant
|
||||
for _, g := range grants {
|
||||
if g.Delivery.ID == p.DeliveryID {
|
||||
grant = g
|
||||
}
|
||||
}
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "ack", ReplyTo: f.MessageID, Body: eventBody(eventAckFor(grant))})
|
||||
if i == 0 {
|
||||
_ = eventTestWrite(ctx, c, eventFrame{Type: "event.deliver", Body: eventBody(grants[2])})
|
||||
}
|
||||
}
|
||||
_, _ = eventTestRead(ctx, c)
|
||||
})
|
||||
consumer, err := NewEventConsumer(client, EventConsumerOptions{SubscriptionID: eventTestSub, Capacity: 2, Handle: func(ctx context.Context, event ResultEvent) error {
|
||||
n := active.Add(1)
|
||||
defer active.Add(-1)
|
||||
for {
|
||||
old := peak.Load()
|
||||
if n <= old || peak.CompareAndSwap(old, n) {
|
||||
break
|
||||
}
|
||||
}
|
||||
start <- event.ID
|
||||
select {
|
||||
case <-release:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- consumer.Run(ctx) }()
|
||||
for range 2 {
|
||||
select {
|
||||
case <-start:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("capacity not utilized")
|
||||
}
|
||||
}
|
||||
close(release)
|
||||
for range 3 {
|
||||
awaitConsumerEvent(t, consumer, "acknowledged")
|
||||
}
|
||||
cancel()
|
||||
if err := awaitConsumerStop(t, done); err != nil || peak.Load() != 2 || consumer.Stats().Started != 3 {
|
||||
t.Fatal("capacity or drain invariant", err, consumer.Stats())
|
||||
}
|
||||
}
|
||||
190
sdk/go/event_integration_test.go
Normal file
190
sdk/go/event_integration_test.go
Normal file
@@ -0,0 +1,190 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func integrationEventSubscription(t *testing.T, ctx context.Context, api *Client, maxAttempts uint32) Subscription {
|
||||
t.Helper()
|
||||
queue, err := api.CreateQueue(ctx, workerTestID(t), CreateQueueRequest{Name: "event-sdk-" + workerTestID(t), MaxConcurrency: 1, RatePerSecond: 10})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sub, err := api.CreateSubscription(ctx, workerTestID(t), CreateSubscriptionRequest{Name: "sdk-" + workerTestID(t), QueueID: queue.Queue.ID, Types: []string{"task.cancelled"}, MaxPending: 10, MaxAttempts: maxAttempts})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return sub.Subscription
|
||||
}
|
||||
func integrationCancelledEvent(t *testing.T, ctx context.Context, api *Client, sub Subscription) Task {
|
||||
t.Helper()
|
||||
task, err := api.SubmitTask(ctx, workerTestID(t), SubmitTaskRequest{QueueID: sub.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancelled, err := api.CancelTask(ctx, task.Task.ID, workerTestID(t), CancelTaskRequest{ExpectedRevision: task.Task.Revision, Reason: "subscription SDK regression"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return cancelled.Task
|
||||
}
|
||||
func eventDialIntegration(t *testing.T, ctx context.Context, api *Client, sub Subscription, capacity uint32) *EventConnection {
|
||||
t.Helper()
|
||||
c, err := api.DialEvents(ctx, EventRegistration{SubscriptionID: sub.ID, Capacity: capacity})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { c.Close() })
|
||||
return c
|
||||
}
|
||||
func TestIntegrationEventConnectionSDK(t *testing.T) {
|
||||
api, _, _ := integrationWorkerClient(t)
|
||||
ctx, stop := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer stop()
|
||||
sub := integrationEventSubscription(t, ctx, api, 3)
|
||||
task := integrationCancelledEvent(t, ctx, api, sub)
|
||||
a := eventDialIntegration(t, ctx, api, sub, 1)
|
||||
grant, err := a.Next(ctx)
|
||||
if err != nil || grant.Event.Snapshot.TaskID != task.ID || grant.Event.Snapshot.Revision != task.Revision || grant.Event.Type != "task.cancelled" {
|
||||
t.Fatal("SDK snapshot mismatch", err)
|
||||
}
|
||||
paused, err := api.ConfigureSubscription(ctx, sub.ID, workerTestID(t), ConfigureSubscriptionRequest{ExpectedRevision: sub.Revision, Reason: "pause with active delivery", Enabled: false})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pong, err := a.Ping(ctx, workerTestID(t))
|
||||
if err != nil || pong.Enabled {
|
||||
t.Fatal("paused subscription heartbeat", err)
|
||||
}
|
||||
first, err := a.Ack(ctx, workerTestID(t), grant.Proof())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.Close()
|
||||
b := eventDialIntegration(t, ctx, api, sub, 1)
|
||||
duplicate, err := b.Ack(ctx, workerTestID(t), grant.Proof())
|
||||
if err != nil || !duplicate.Deduplicated || !duplicate.AcknowledgedAt.Equal(first.AcknowledgedAt) {
|
||||
t.Fatal("ACK proof could not cross connections", err)
|
||||
}
|
||||
_ = integrationCancelledEvent(t, ctx, api, sub)
|
||||
short, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
|
||||
_, err = b.Next(short)
|
||||
cancel()
|
||||
if !errors.Is(err, context.DeadlineExceeded) || b.Err() != nil {
|
||||
t.Fatal("paused subscription delivered or Next closed connection", err)
|
||||
}
|
||||
if _, err = api.ConfigureSubscription(ctx, sub.ID, workerTestID(t), ConfigureSubscriptionRequest{ExpectedRevision: paused.Subscription.Revision, Reason: "resume", Enabled: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := b.Next(ctx)
|
||||
if err != nil || second.Event.ID == grant.Event.ID {
|
||||
t.Fatal("resume failed", err)
|
||||
}
|
||||
if _, err = b.Ack(ctx, workerTestID(t), second.Proof()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, err := api.GetSubscription(ctx, sub.ID)
|
||||
if err != nil || current.PendingCount != 0 {
|
||||
t.Fatal("SDK acknowledgement accounting", err)
|
||||
}
|
||||
}
|
||||
func TestIntegrationEventConsumerHeartbeatAndLostAck(t *testing.T) {
|
||||
api, _, token := integrationWorkerClient(t)
|
||||
ctx, stop := context.WithTimeout(context.Background(), 25*time.Second)
|
||||
defer stop()
|
||||
sub := integrationEventSubscription(t, ctx, api, 3)
|
||||
_ = integrationCancelledEvent(t, ctx, api, sub)
|
||||
gateway := newProtocolFaultGateway(t, api.baseURL, "/v1/events/connect", eventSubprotocol)
|
||||
gateway.drop.Store("event.ack")
|
||||
client, err := NewClient(gateway.server.URL, Options{Token: token, HTTPClient: gateway.server.Client()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var calls atomic.Int64
|
||||
consumer, err := NewEventConsumer(client, EventConsumerOptions{SubscriptionID: sub.ID, Capacity: 1, ShutdownTimeout: time.Second, Handle: func(work context.Context, event ResultEvent) error {
|
||||
calls.Add(1)
|
||||
// 慢业务期间真实等待15秒自动心跳;确认计数来自上游服务端ACK,不是客户端写入。
|
||||
ticker := time.NewTicker(20 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for gateway.confirmedEventPings.Load() == 0 {
|
||||
select {
|
||||
case <-work.Done():
|
||||
return work.Err()
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- consumer.Run(run) }()
|
||||
for consumer.Stats().Acknowledged == 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatal("consumer did not reconcile committed ACK")
|
||||
case err := <-done:
|
||||
t.Fatal("consumer stopped", err)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
if err := awaitConsumerStop(t, done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls.Load() != 1 || consumer.Stats().Reconnects < 1 || gateway.confirmedEventPings.Load() < 1 {
|
||||
t.Fatal("heartbeat or reconciliation missing", consumer.Stats())
|
||||
}
|
||||
page, err := api.ListEventDeliveries(ctx, sub.ID, DeliveryPageRequest{State: DeliveryAcked})
|
||||
if err != nil || len(page.Items) != 1 || page.Items[0].Attempts != 1 {
|
||||
t.Fatal("lost ACK created another delivery attempt", err)
|
||||
}
|
||||
t.Log("real 15-second heartbeat and committed ACK loss recovered; handler and delivery attempt each remained one")
|
||||
}
|
||||
func TestIntegrationEventConsumerFailureRedeliversNaturally(t *testing.T) {
|
||||
api, _, _ := integrationWorkerClient(t)
|
||||
ctx, stop := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer stop()
|
||||
sub := integrationEventSubscription(t, ctx, api, 2)
|
||||
_ = integrationCancelledEvent(t, ctx, api, sub)
|
||||
var calls atomic.Int64
|
||||
consumer, err := NewEventConsumer(api, EventConsumerOptions{SubscriptionID: sub.ID, Capacity: 1, ShutdownTimeout: time.Second, Handle: func(context.Context, ResultEvent) error {
|
||||
if calls.Add(1) == 1 {
|
||||
return errors.New("receiver transaction rolled back")
|
||||
}
|
||||
return nil
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- consumer.Run(run) }()
|
||||
for consumer.Stats().Acknowledged == 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatal("delivery did not retry after natural lease expiry")
|
||||
case err := <-done:
|
||||
t.Fatal("consumer stopped", err)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
if err := awaitConsumerStop(t, done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
page, err := api.ListEventDeliveries(ctx, sub.ID, DeliveryPageRequest{State: DeliveryAcked})
|
||||
if err != nil || len(page.Items) != 1 || page.Items[0].Attempts != 2 || calls.Load() != 2 || consumer.Stats().HandlerFailures != 1 {
|
||||
t.Fatal("handler error or retry budget mishandled", err)
|
||||
}
|
||||
t.Log("handler error caused no ACK; same delivery reappeared after its real 30-second lease and backoff, then acknowledged at attempt two")
|
||||
}
|
||||
103
sdk/go/event_process_integration_test.go
Normal file
103
sdk/go/event_process_integration_test.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIntegrationEventProcessLeaseRecovery(t *testing.T) {
|
||||
a := startOrbitProcess(t)
|
||||
_, _, token := integrationWorkerClient(t)
|
||||
api := processClient(t, a, token)
|
||||
ctx, stop := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer stop()
|
||||
sub := integrationEventSubscription(t, ctx, api, 3)
|
||||
_ = integrationCancelledEvent(t, ctx, api, sub)
|
||||
firstConn := eventDialIntegration(t, ctx, api, sub, 1)
|
||||
first, err := firstConn.Next(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.kill(t)
|
||||
select {
|
||||
case <-firstConn.Done():
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("dead server connection remained live")
|
||||
}
|
||||
b := startOrbitProcess(t)
|
||||
other := processClient(t, b, token)
|
||||
secondConn := eventDialIntegration(t, ctx, other, sub, 1)
|
||||
second, err := secondConn.Next(ctx)
|
||||
if err != nil || first.Event.ID != second.Event.ID || first.Delivery.ID != second.Delivery.ID || second.Delivery.LeaseEpoch != 2 || second.Delivery.Attempts != 2 {
|
||||
t.Fatal("node restart did not recover original delivery", err)
|
||||
}
|
||||
_, err = secondConn.Ack(ctx, workerTestID(t), first.Proof())
|
||||
var remote *EventError
|
||||
if !errors.As(err, &remote) || remote.Code != "DELIVERY_LEASE_LOST" {
|
||||
t.Fatal("dead node proof acknowledged newer lease", err)
|
||||
}
|
||||
if _, err = secondConn.Ack(ctx, workerTestID(t), second.Proof()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, err := other.GetSubscription(ctx, sub.ID)
|
||||
if err != nil || current.PendingCount != 0 {
|
||||
t.Fatal("recovered acknowledgement not persisted", err)
|
||||
}
|
||||
t.Log("SIGKILL confirmed by Wait; new process recovered original delivery after natural lease expiry, rejected old proof and committed new ACK")
|
||||
}
|
||||
func TestIntegrationEventProcessAckCommitResponseLoss(t *testing.T) {
|
||||
a := startOrbitProcess(t)
|
||||
_, _, token := integrationWorkerClient(t)
|
||||
api := processClient(t, a, token)
|
||||
ctx, stop := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer stop()
|
||||
sub := integrationEventSubscription(t, ctx, api, 3)
|
||||
_ = integrationCancelledEvent(t, ctx, api, sub)
|
||||
gateway := newProtocolFaultGateway(t, a.url, "/v1/events/connect", eventSubprotocol)
|
||||
gateway.drop.Store("event.ack")
|
||||
gateway.dropOffline.Store(true)
|
||||
client, err := NewClient(gateway.server.URL, Options{Token: token, HTTPClient: gateway.server.Client()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var calls atomic.Int64
|
||||
consumer, err := NewEventConsumer(client, EventConsumerOptions{SubscriptionID: sub.ID, Capacity: 1, ShutdownTimeout: time.Second, Handle: func(context.Context, ResultEvent) error { calls.Add(1); return nil }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- consumer.Run(run) }()
|
||||
awaitProcessDrop(t, ctx, gateway, "event.ack")
|
||||
before, err := api.ListEventDeliveries(ctx, sub.ID, DeliveryPageRequest{State: DeliveryAcked})
|
||||
if err != nil || len(before.Items) != 1 || before.Items[0].AcknowledgedAt == nil {
|
||||
t.Fatal("fault occurred before durable ACK", err)
|
||||
}
|
||||
a.kill(t)
|
||||
b := startOrbitProcess(t)
|
||||
other := processClient(t, b, token)
|
||||
gateway.target.Store(b.url)
|
||||
gateway.offline.Store(false)
|
||||
for consumer.Stats().Acknowledged == 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatal("consumer failed to reconcile ACK on new process")
|
||||
case err := <-done:
|
||||
t.Fatal("consumer stopped", err)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
if err := awaitConsumerStop(t, done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, err := other.ListEventDeliveries(ctx, sub.ID, DeliveryPageRequest{State: DeliveryAcked})
|
||||
if err != nil || len(after.Items) != 1 || after.Items[0].Attempts != 1 || after.Items[0].AcknowledgedAt == nil || !after.Items[0].AcknowledgedAt.Equal(*before.Items[0].AcknowledgedAt) || calls.Load() != 1 {
|
||||
t.Fatal("ACK receipt changed or committed handler reran", err)
|
||||
}
|
||||
t.Log("ACK commit verified before response loss and SIGKILL; replacement process returned original receipt without another handler or delivery attempt")
|
||||
}
|
||||
87
sdk/go/event_types.go
Normal file
87
sdk/go/event_types.go
Normal file
@@ -0,0 +1,87 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrEventClosed = errors.New("event connection closed")
|
||||
ErrEventDisconnected = errors.New("event connection interrupted; acknowledgement may be unconfirmed")
|
||||
ErrEventProtocol = errors.New("invalid event protocol response")
|
||||
ErrEventBusy = errors.New("event control request limit reached")
|
||||
ErrEventBackpressure = errors.New("event delivery buffer exhausted")
|
||||
)
|
||||
|
||||
// EventError 是服务端公开错误。Error/String/GoString只展示分类,不输出远端正文。
|
||||
type EventError struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
func (e *EventError) Error() string { return "orbit event " + e.Code }
|
||||
func (e *EventError) String() string { return e.Error() }
|
||||
func (e *EventError) GoString() string { return e.Error() }
|
||||
|
||||
// EventRegistration 绑定一个已有订阅。容量固定为1至32,重连可以重新选择。
|
||||
type EventRegistration struct {
|
||||
SubscriptionID string `json:"subscription_id"` // SubscriptionID 已创建且属于当前凭据的订阅。
|
||||
Capacity uint32 `json:"capacity"`
|
||||
}
|
||||
|
||||
// EventWelcome 是当前连接的握手快照;Enabled变化通过Ping查询。
|
||||
type EventWelcome struct {
|
||||
SessionID string `json:"session_id"`
|
||||
SubscriptionID string `json:"subscription_id"`
|
||||
Capacity uint32 `json:"capacity"`
|
||||
LeaseDurationMS int64 `json:"lease_duration_ms"`
|
||||
HeartbeatIntervalMS int64 `json:"heartbeat_interval_ms"`
|
||||
ServerTime time.Time `json:"server_time"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
type EventPong struct {
|
||||
ServerTime time.Time `json:"server_time"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
// EventGrant 包含冻结结果及当前交付证明。不得将JSON写到普通日志。
|
||||
// 租约只控制平台补发,不授权接收方跳过去重;不同消费者仍可能并行收到同一事件。
|
||||
type EventGrant struct {
|
||||
Delivery EventDelivery `json:"delivery"`
|
||||
Event ResultEvent `json:"event"`
|
||||
LeaseSecret string `json:"lease_secret"`
|
||||
}
|
||||
|
||||
// EventProof 保留原身份及秘密,在同订阅的新连接上仍可核对ACK。
|
||||
// SubscriptionID和EventID用于SDK本地关联校验;只把后三个字段发送到服务端。
|
||||
type EventProof struct {
|
||||
SubscriptionID string `json:"subscription_id"`
|
||||
EventID string `json:"event_id"`
|
||||
DeliveryID string `json:"delivery_id"`
|
||||
Epoch uint64 `json:"epoch,string"`
|
||||
LeaseSecret string `json:"lease_secret"`
|
||||
}
|
||||
|
||||
func (g EventGrant) Proof() EventProof {
|
||||
return EventProof{SubscriptionID: g.Delivery.SubscriptionID, EventID: g.Event.ID, DeliveryID: g.Delivery.ID, Epoch: g.Delivery.LeaseEpoch, LeaseSecret: g.LeaseSecret}
|
||||
}
|
||||
|
||||
type EventAckResult struct {
|
||||
DeliveryID string `json:"delivery_id"`
|
||||
EventID string `json:"event_id"`
|
||||
Epoch uint64 `json:"epoch,string"`
|
||||
AcknowledgedAt time.Time `json:"acknowledged_at"`
|
||||
Deduplicated bool `json:"deduplicated"`
|
||||
}
|
||||
|
||||
// NewEventMessageID 生成控制消息关联ID;ACK的持久化身份是交付ID、代次和证明。
|
||||
func NewEventMessageID() (string, error) { return NewWorkerMessageID() }
|
||||
func (g EventGrant) String() string {
|
||||
return fmt.Sprintf("EventGrant{EventID:%s DeliveryID:%s data:redacted}", g.Event.ID, g.Delivery.ID)
|
||||
}
|
||||
func (g EventGrant) GoString() string { return g.String() }
|
||||
func (p EventProof) String() string {
|
||||
return fmt.Sprintf("EventProof{DeliveryID:%s Epoch:%d secrets:redacted}", p.DeliveryID, p.Epoch)
|
||||
}
|
||||
func (p EventProof) GoString() string { return p.String() }
|
||||
112
sdk/go/examples/event-protocol/main.go
Normal file
112
sdk/go/examples/event-protocol/main.go
Normal file
@@ -0,0 +1,112 @@
|
||||
// event-protocol 验证冻结结果、连接切换与ACK重报;不执行业务写入。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
func run() error {
|
||||
token, queue := os.Getenv("ORBIT_TOKEN"), os.Getenv("ORBIT_QUEUE_ID")
|
||||
if path := os.Getenv("ORBIT_CREDENTIALS_FILE"); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var credential struct {
|
||||
Token string `json:"token"`
|
||||
QueueID string `json:"queue_id"`
|
||||
}
|
||||
if json.Unmarshal(raw, &credential) != nil {
|
||||
return errors.New("invalid credential file")
|
||||
}
|
||||
if token == "" {
|
||||
token = credential.Token
|
||||
}
|
||||
if queue == "" {
|
||||
queue = credential.QueueID
|
||||
}
|
||||
}
|
||||
if token == "" || queue == "" {
|
||||
return errors.New("event-enabled token and isolated example queue are required")
|
||||
}
|
||||
api, err := orbit.NewClient(os.Getenv("ORBIT_URL"), orbit.Options{Token: token})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, stop := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer stop()
|
||||
id, err := orbit.NewEventMessageID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
created, err := api.CreateSubscription(ctx, "sub-"+id, orbit.CreateSubscriptionRequest{Name: "protocol-" + id, QueueID: queue, Types: []string{"task.cancelled"}, MaxPending: 10, MaxAttempts: 3})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sub := created.Subscription
|
||||
task, err := api.SubmitTask(ctx, "task-"+id, orbit.SubmitTaskRequest{QueueID: queue, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = api.CancelTask(ctx, task.Task.ID, "cancel-"+id, orbit.CancelTaskRequest{ExpectedRevision: task.Task.Revision, Reason: "event protocol example"}); err != nil {
|
||||
return err
|
||||
}
|
||||
first, err := api.DialEvents(ctx, orbit.EventRegistration{SubscriptionID: sub.ID, Capacity: 1})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer first.Close()
|
||||
grant, err := first.Next(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if grant.Event.Snapshot.TaskID != task.Task.ID || grant.Event.Snapshot.State != orbit.TaskCancelled {
|
||||
return errors.New("unexpected example queue event")
|
||||
}
|
||||
// 本例只读取并验证结果,没有业务数据库写入;业务处理必须先提交Inbox事务再ACK。
|
||||
ackID, err := orbit.NewEventMessageID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
confirmed, err := first.Ack(ctx, ackID, grant.Proof())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
first.Close()
|
||||
second, err := api.DialEvents(ctx, orbit.EventRegistration{SubscriptionID: sub.ID, Capacity: 1})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer second.Close()
|
||||
replayID, err := orbit.NewEventMessageID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
replayed, err := second.Ack(ctx, replayID, grant.Proof())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !replayed.Deduplicated || !replayed.AcknowledgedAt.Equal(confirmed.AcknowledgedAt) {
|
||||
return errors.New("acknowledgement receipt changed across connections")
|
||||
}
|
||||
paused, err := api.ConfigureSubscription(ctx, sub.ID, "pause-"+id, orbit.ConfigureSubscriptionRequest{ExpectedRevision: sub.Revision, Enabled: false, Reason: "protocol example completed"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("event=%s task=%s state=%s\n", grant.Event.ID, task.Task.ID, grant.Event.Snapshot.State)
|
||||
fmt.Printf("ack_replayed=%t same_receipt=true pending=%d subscription_enabled=%t\n", replayed.Deduplicated, paused.Subscription.PendingCount, paused.Subscription.Enabled)
|
||||
return nil
|
||||
}
|
||||
@@ -48,18 +48,23 @@ func integrationWorkerClient(t *testing.T) (*Client, string, string) {
|
||||
// workerFaultGateway 在实际服务端确认后丢弃指定 ACK;SDK 到网关使用真实 TLS。
|
||||
// 只丢响应,不伪造任务状态,测试凭据不进入 URL 或输出。
|
||||
type workerFaultGateway struct {
|
||||
drop atomic.Value
|
||||
target atomic.Value
|
||||
dropOffline atomic.Bool
|
||||
dropCancel atomic.Bool
|
||||
offline atomic.Bool
|
||||
mu sync.Mutex
|
||||
active map[*websocket.Conn]struct{}
|
||||
dropped chan string
|
||||
server *httptest.Server
|
||||
confirmedEventPings atomic.Uint64
|
||||
drop atomic.Value
|
||||
target atomic.Value
|
||||
dropOffline atomic.Bool
|
||||
dropCancel atomic.Bool
|
||||
offline atomic.Bool
|
||||
mu sync.Mutex
|
||||
active map[*websocket.Conn]struct{}
|
||||
dropped chan string
|
||||
server *httptest.Server
|
||||
}
|
||||
|
||||
func newWorkerFaultGateway(t *testing.T, upstream string) *workerFaultGateway {
|
||||
return newProtocolFaultGateway(t, upstream, "/v1/worker/connect", workerSubprotocol)
|
||||
}
|
||||
|
||||
func newProtocolFaultGateway(t *testing.T, upstream, path, protocol string) *workerFaultGateway {
|
||||
t.Helper()
|
||||
g := &workerFaultGateway{dropped: make(chan string, 4), active: make(map[*websocket.Conn]struct{})}
|
||||
g.drop.Store("")
|
||||
@@ -77,14 +82,14 @@ func newWorkerFaultGateway(t *testing.T, upstream string) *workerFaultGateway {
|
||||
defer stop()
|
||||
header := http.Header{}
|
||||
header.Set("Authorization", r.Header.Get("Authorization"))
|
||||
up, _, err := websocket.Dial(bridge, g.target.Load().(string)+"/v1/worker/connect", &websocket.DialOptions{HTTPHeader: header, Subprotocols: []string{workerSubprotocol}})
|
||||
up, _, err := websocket.Dial(bridge, g.target.Load().(string)+path, &websocket.DialOptions{HTTPHeader: header, Subprotocols: []string{protocol}})
|
||||
if err != nil {
|
||||
http.Error(w, "upstream unavailable", 503)
|
||||
return
|
||||
}
|
||||
defer up.CloseNow()
|
||||
up.SetReadLimit(512 << 10)
|
||||
down, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{workerSubprotocol}})
|
||||
down, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{protocol}})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -138,6 +143,9 @@ func newWorkerFaultGateway(t *testing.T, upstream string) *workerFaultGateway {
|
||||
operation := operations[f.ReplyTo]
|
||||
delete(operations, f.ReplyTo)
|
||||
mu.Unlock()
|
||||
if f.Type == "ack" && operation == "events.ping" {
|
||||
g.confirmedEventPings.Add(1)
|
||||
}
|
||||
if f.Type == "task.cancel" && g.dropCancel.CompareAndSwap(true, false) {
|
||||
g.dropped <- "task.cancel"
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user