feat: expose result subscription HTTP and WebSocket APIs
This commit is contained in:
356
internal/transport/httpserver/event_hub.go
Normal file
356
internal/transport/httpserver/event_hub.go
Normal file
@@ -0,0 +1,356 @@
|
||||
package httpserver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/dispatch"
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/jsonwire"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/events"
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
// EventHub 仅负责已持久化结果的传输；断线不删除租约，消费者按事件ID持久化去重。
|
||||
// 节点最多128条连接，领取池全局4个、每应用1个在途，队列加在途共128个。
|
||||
type EventHub struct {
|
||||
service *events.Service
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
mu sync.Mutex
|
||||
closing bool
|
||||
slots chan struct{}
|
||||
wg sync.WaitGroup
|
||||
dispatch *dispatch.Pool
|
||||
poll time.Duration
|
||||
}
|
||||
|
||||
func NewEventHub(parent context.Context, service *events.Service) *EventHub {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
h := &EventHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, 128), poll: time.Second}
|
||||
h.dispatch = dispatch.New(ctx, 4, 128)
|
||||
h.wg.Add(1)
|
||||
go func() { defer h.wg.Done(); <-h.dispatch.Done() }()
|
||||
return h
|
||||
}
|
||||
func RegisterEventConsumers(server *http.Server, hub *EventHub) {
|
||||
server.Handler.(*http.ServeMux).Handle("GET /v1/events/connect", hub)
|
||||
}
|
||||
func (h *EventHub) Shutdown(ctx context.Context) error {
|
||||
h.mu.Lock()
|
||||
h.closing = true
|
||||
h.cancel()
|
||||
h.mu.Unlock()
|
||||
done := make(chan struct{})
|
||||
go func() { h.wg.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
func (h *EventHub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
requestID, err := core.NewID()
|
||||
if err != nil {
|
||||
writeError(w, "", fault.Unavailable)
|
||||
return
|
||||
}
|
||||
w.Header().Set("X-Request-ID", string(requestID))
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
if err = noEventQuery(r); err != nil {
|
||||
writeError(w, string(requestID), err)
|
||||
return
|
||||
}
|
||||
values := r.Header.Values("Authorization")
|
||||
if len(values) != 1 {
|
||||
writeError(w, string(requestID), fault.Unauthorized)
|
||||
return
|
||||
}
|
||||
scheme, value, ok := strings.Cut(values[0], " ")
|
||||
if !ok || !strings.EqualFold(scheme, "Bearer") {
|
||||
writeError(w, string(requestID), fault.Unauthorized)
|
||||
return
|
||||
}
|
||||
token, err := auth.Parse(value)
|
||||
if err != nil {
|
||||
writeError(w, string(requestID), err)
|
||||
return
|
||||
}
|
||||
supported := false
|
||||
for _, header := range r.Header.Values("Sec-WebSocket-Protocol") {
|
||||
for _, p := range strings.Split(header, ",") {
|
||||
supported = supported || strings.TrimSpace(p) == EventSubprotocol
|
||||
}
|
||||
}
|
||||
if !supported {
|
||||
writeJSON(w, http.StatusUpgradeRequired, map[string]any{"request_id": requestID, "error": fault.New("PROTOCOL_REQUIRED", "orbit.events.v1 subprotocol is required")})
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
if h.closing || h.ctx.Err() != nil {
|
||||
h.mu.Unlock()
|
||||
writeError(w, string(requestID), fault.Unavailable)
|
||||
return
|
||||
}
|
||||
select {
|
||||
case h.slots <- struct{}{}:
|
||||
h.wg.Add(1)
|
||||
default:
|
||||
h.mu.Unlock()
|
||||
w.Header().Set("Retry-After", "1")
|
||||
writeJSON(w, 429, map[string]any{"request_id": requestID, "error": fault.New("CAPACITY_EXCEEDED", "node event connection capacity exceeded")})
|
||||
return
|
||||
}
|
||||
h.mu.Unlock()
|
||||
defer func() { <-h.slots; h.wg.Done() }()
|
||||
authorize, stop := context.WithTimeout(r.Context(), 5*time.Second)
|
||||
err = h.service.Authorize(authorize, token)
|
||||
stop()
|
||||
if err != nil {
|
||||
writeError(w, string(requestID), err)
|
||||
return
|
||||
}
|
||||
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{EventSubprotocol}, CompressionMode: websocket.CompressionDisabled})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
conn.SetReadLimit(eventMaxFrame)
|
||||
ctx, cancel := context.WithCancel(h.ctx)
|
||||
defer cancel()
|
||||
helloCtx, helloStop := context.WithTimeout(ctx, 5*time.Second)
|
||||
kind, raw, err := conn.Read(helloCtx)
|
||||
helloStop()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
frame, decodeErr := decodeEventRequest(raw, true)
|
||||
var input eventHello
|
||||
if kind != websocket.MessageText || decodeErr != nil || decodeEventJSON(frame.Body, &input) != nil || input.SubscriptionID.Validate() != nil || input.Capacity < 1 || input.Capacity > 32 {
|
||||
_ = conn.Close(websocket.StatusPolicyViolation, "invalid event hello")
|
||||
return
|
||||
}
|
||||
call, callStop := context.WithTimeout(ctx, 8*time.Second)
|
||||
binding, err := h.service.Bind(call, token, input.SubscriptionID)
|
||||
callStop()
|
||||
if err != nil {
|
||||
_ = writeEventFrame(ctx, conn, eventFrame{Version: "1", Type: "error", MessageID: requestID, ReplyTo: frame.MessageID, Error: eventFault(err)})
|
||||
return
|
||||
}
|
||||
session, err := core.NewID()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
welcome := eventWelcome{SessionID: session, SubscriptionID: input.SubscriptionID, Capacity: input.Capacity, LeaseDurationMS: events.LeaseDuration.Milliseconds(), HeartbeatIntervalMS: 15000, ServerTime: binding.ServerTime, Enabled: binding.Enabled}
|
||||
body, err := jsonwire.Marshal(welcome)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err = writeEventFrame(ctx, conn, eventFrame{Version: "1", Type: "events.welcome", MessageID: requestID, ReplyTo: frame.MessageID, SessionID: session, Body: body}); err != nil {
|
||||
return
|
||||
}
|
||||
h.consume(ctx, cancel, conn, token, binding, session, input.Capacity)
|
||||
}
|
||||
|
||||
type eventClaimResult struct {
|
||||
grant *events.Grant
|
||||
err error
|
||||
}
|
||||
|
||||
func (h *EventHub) claim(ctx context.Context, t auth.Token, sub, connection core.ID, capacity uint32) (out eventClaimResult) {
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
out = eventClaimResult{err: fault.Unavailable}
|
||||
}
|
||||
}()
|
||||
call, stop := context.WithTimeout(ctx, 8*time.Second)
|
||||
defer stop()
|
||||
out.grant, out.err = h.service.Claim(call, t, sub, events.ClaimInput{ConnectionID: connection, Capacity: capacity})
|
||||
return out
|
||||
}
|
||||
func (h *EventHub) consume(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn, t auth.Token, b events.ConsumerBinding, session core.ID, capacity uint32) {
|
||||
incoming := make(chan eventFrame, 4)
|
||||
outgoing := make(chan outboundFrame, 8)
|
||||
var loops sync.WaitGroup
|
||||
loops.Add(2)
|
||||
go func() {
|
||||
defer loops.Done()
|
||||
defer cancel()
|
||||
for {
|
||||
readCtx, stop := context.WithTimeout(ctx, 60*time.Second)
|
||||
kind, raw, err := conn.Read(readCtx)
|
||||
stop()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
frame, err := decodeEventRequest(raw, false)
|
||||
if kind != websocket.MessageText || err != nil || frame.SessionID != session {
|
||||
_ = conn.Close(websocket.StatusPolicyViolation, "invalid event frame")
|
||||
return
|
||||
}
|
||||
select {
|
||||
case incoming <- frame:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer loops.Done()
|
||||
defer cancel()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case frame := <-outgoing:
|
||||
writeCtx, stop := context.WithTimeout(ctx, 5*time.Second)
|
||||
err := conn.Write(writeCtx, websocket.MessageText, frame.data)
|
||||
stop()
|
||||
if frame.done != nil {
|
||||
frame.done <- err
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
defer func() { cancel(); _ = conn.CloseNow(); loops.Wait() }()
|
||||
send := func(frame eventFrame, wait bool) bool {
|
||||
frame.Version = "1"
|
||||
frame.SessionID = session
|
||||
if frame.MessageID == "" {
|
||||
var err error
|
||||
frame.MessageID, err = core.NewID()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
raw, err := jsonwire.Marshal(frame)
|
||||
if err != nil || len(raw) > eventMaxFrame {
|
||||
return false
|
||||
}
|
||||
out := outboundFrame{data: raw}
|
||||
if wait {
|
||||
out.done = make(chan error, 1)
|
||||
}
|
||||
select {
|
||||
case outgoing <- out:
|
||||
default:
|
||||
return false
|
||||
}
|
||||
if !wait {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case err := <-out.done:
|
||||
return err == nil
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
results := make(chan eventClaimResult, 1)
|
||||
pending, burst := false, 0
|
||||
request := func() {
|
||||
if pending || burst == 0 || len(outgoing) >= 4 {
|
||||
return
|
||||
}
|
||||
pending = h.dispatch.Submit(dispatch.Job{Context: ctx, Application: string(b.ApplicationID), Queue: string(t.NamespaceID) + "/" + string(b.SubscriptionID), Run: func(job context.Context) { results <- h.claim(job, t, b.SubscriptionID, session, capacity) }})
|
||||
if pending {
|
||||
burst--
|
||||
}
|
||||
}
|
||||
ticker := time.NewTicker(h.poll)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case result := <-results:
|
||||
pending = false
|
||||
if result.err != nil {
|
||||
send(eventFrame{Type: "error", Error: eventFault(result.err)}, true)
|
||||
return
|
||||
}
|
||||
if result.grant == nil {
|
||||
burst = 0
|
||||
continue
|
||||
}
|
||||
raw, err := jsonwire.Marshal(result.grant)
|
||||
if err != nil || !send(eventFrame{Type: "event.deliver", Body: raw}, false) {
|
||||
return
|
||||
}
|
||||
request()
|
||||
case frame := <-incoming:
|
||||
call, stop := context.WithTimeout(ctx, 8*time.Second)
|
||||
var body any
|
||||
var err error
|
||||
switch frame.Type {
|
||||
case "event.ack":
|
||||
var in events.AckInput
|
||||
if err = decodeEventJSON(frame.Body, &in); err == nil {
|
||||
body, err = h.service.Ack(call, t, b.SubscriptionID, in)
|
||||
}
|
||||
case "events.ping":
|
||||
var in struct{}
|
||||
if err = decodeEventJSON(frame.Body, &in); err == nil {
|
||||
var current events.ConsumerBinding
|
||||
current, err = h.service.Bind(call, t, b.SubscriptionID)
|
||||
if err == nil {
|
||||
body = eventPong{ServerTime: current.ServerTime, Enabled: current.Enabled}
|
||||
}
|
||||
}
|
||||
default:
|
||||
err = fault.Invalid("unsupported event message type")
|
||||
}
|
||||
stop()
|
||||
reply := eventFrame{Type: "ack", ReplyTo: frame.MessageID}
|
||||
if err != nil {
|
||||
reply.Type = "error"
|
||||
reply.Error = eventFault(err)
|
||||
} else {
|
||||
var marshalErr error
|
||||
reply.Body, marshalErr = jsonwire.Marshal(body)
|
||||
if marshalErr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
fatal := err != nil && (fault.Code(err) == "UNAUTHENTICATED" || fault.Code(err) == "PERMISSION_DENIED" || fault.Code(err) == "NOT_FOUND" || fault.Code(err) == "UNAVAILABLE")
|
||||
if !send(reply, fatal) || fatal {
|
||||
return
|
||||
}
|
||||
case <-ticker.C:
|
||||
if !pending {
|
||||
burst = 4
|
||||
request()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
func eventFault(err error) *fault.Error {
|
||||
var known *fault.Error
|
||||
if errors.As(err, &known) {
|
||||
return known
|
||||
}
|
||||
return fault.Unavailable
|
||||
}
|
||||
func writeEventFrame(ctx context.Context, conn *websocket.Conn, frame eventFrame) error {
|
||||
raw, err := jsonwire.Marshal(frame)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(raw) > eventMaxFrame {
|
||||
return fault.Invalid("event frame too large")
|
||||
}
|
||||
bounded, stop := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer stop()
|
||||
return conn.Write(bounded, websocket.MessageText, raw)
|
||||
}
|
||||
Reference in New Issue
Block a user