feat: add durable Worker execution and WSS server
This commit is contained in:
401
internal/transport/httpserver/worker.go
Normal file
401
internal/transport/httpserver/worker.go
Normal file
@@ -0,0 +1,401 @@
|
||||
package httpserver
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/jsonvalue"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
const WorkerSubprotocol = "pulse.worker.v1"
|
||||
|
||||
// WorkerHub 管理节点内连接生命周期；任务归属和容量仍由 MySQL 决定。
|
||||
type WorkerHub struct {
|
||||
service *worker.Service
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
slots chan struct{}
|
||||
mu sync.Mutex
|
||||
closing bool
|
||||
wg sync.WaitGroup
|
||||
poll time.Duration
|
||||
}
|
||||
|
||||
func NewWorkerHub(parent context.Context, service *worker.Service) *WorkerHub {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
return &WorkerHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, 128), poll: time.Second}
|
||||
}
|
||||
func RegisterWorkers(server *http.Server, hub *WorkerHub) {
|
||||
server.Handler.(*http.ServeMux).Handle("GET /v1/worker/connect", hub)
|
||||
}
|
||||
|
||||
// Shutdown 停止派发并关闭本节点连接；不清除持久化租约，由重連核对或到期恢复处理。
|
||||
func (h *WorkerHub) Shutdown(ctx context.Context) error {
|
||||
h.mu.Lock()
|
||||
h.closing = true
|
||||
h.cancel()
|
||||
h.mu.Unlock()
|
||||
done := make(chan struct{})
|
||||
go func() { h.wg.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
type workerFrame struct {
|
||||
Version string `json:"protocol_version"`
|
||||
Type string `json:"type"`
|
||||
MessageID core.ID `json:"message_id"`
|
||||
ReplyTo core.ID `json:"reply_to,omitempty"`
|
||||
SessionID core.ID `json:"session_id,omitempty"`
|
||||
Body json.RawMessage `json:"body,omitempty"`
|
||||
Error *fault.Error `json:"error,omitempty"`
|
||||
}
|
||||
type outboundFrame struct {
|
||||
data []byte
|
||||
done chan error
|
||||
}
|
||||
|
||||
func (h *WorkerHub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
requestID, _ := core.NewID()
|
||||
w.Header().Set("X-Request-ID", string(requestID))
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
if r.URL.RawQuery != "" {
|
||||
writeError(w, string(requestID), fault.Invalid("Worker credentials and options must not be placed in URL"))
|
||||
return
|
||||
}
|
||||
values := r.Header.Values("Authorization")
|
||||
if len(values) != 1 {
|
||||
writeError(w, string(requestID), fault.Unauthorized)
|
||||
return
|
||||
}
|
||||
scheme, value, ok := strings.Cut(values[0], " ")
|
||||
if !ok || !strings.EqualFold(scheme, "Bearer") {
|
||||
writeError(w, string(requestID), fault.Unauthorized)
|
||||
return
|
||||
}
|
||||
token, err := auth.Parse(value)
|
||||
if err != nil {
|
||||
writeError(w, string(requestID), err)
|
||||
return
|
||||
}
|
||||
supported := false
|
||||
for _, header := range r.Header.Values("Sec-WebSocket-Protocol") {
|
||||
for _, protocol := range strings.Split(header, ",") {
|
||||
supported = supported || strings.TrimSpace(protocol) == WorkerSubprotocol
|
||||
}
|
||||
}
|
||||
if !supported {
|
||||
writeJSON(w, http.StatusUpgradeRequired, map[string]any{"request_id": requestID, "error": fault.New("PROTOCOL_REQUIRED", "pulse.worker.v1 subprotocol is required")})
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
if h.closing || h.ctx.Err() != nil {
|
||||
h.mu.Unlock()
|
||||
writeError(w, string(requestID), fault.Unavailable)
|
||||
return
|
||||
}
|
||||
select {
|
||||
case h.slots <- struct{}{}:
|
||||
h.wg.Add(1)
|
||||
default:
|
||||
h.mu.Unlock()
|
||||
writeJSON(w, 429, map[string]any{"request_id": requestID, "error": fault.New("CAPACITY_EXCEEDED", "node connection capacity exceeded")})
|
||||
return
|
||||
}
|
||||
h.mu.Unlock()
|
||||
defer func() { <-h.slots; h.wg.Done() }()
|
||||
authorize, cancel := context.WithTimeout(r.Context(), 5*time.Second)
|
||||
err = h.service.Authorize(authorize, token)
|
||||
cancel()
|
||||
if err != nil {
|
||||
writeError(w, string(requestID), err)
|
||||
return
|
||||
}
|
||||
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{WorkerSubprotocol}, CompressionMode: websocket.CompressionDisabled})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
conn.SetReadLimit(512 << 10)
|
||||
ctx, cancel := context.WithCancel(h.ctx)
|
||||
defer cancel()
|
||||
helloCtx, helloCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
kind, raw, err := conn.Read(helloCtx)
|
||||
helloCancel()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var hello workerFrame
|
||||
var input worker.ConnectInput
|
||||
if kind != websocket.MessageText || decodeWorkerJSON(raw, &hello) != nil || hello.Version != "1" || hello.Type != "worker.hello" || hello.MessageID.Validate() != nil || hello.SessionID != "" || hello.ReplyTo != "" || hello.Error != nil || decodeWorkerJSON(hello.Body, &input) != nil {
|
||||
_ = conn.Close(websocket.StatusPolicyViolation, "invalid worker hello")
|
||||
return
|
||||
}
|
||||
call, cancelCall := context.WithTimeout(ctx, 8*time.Second)
|
||||
binding, welcome, err := h.service.Connect(call, token, input)
|
||||
cancelCall()
|
||||
if err != nil {
|
||||
_ = writeWorkerFrame(ctx, conn, workerFrame{Version: "1", Type: "error", MessageID: requestID, ReplyTo: hello.MessageID, Error: publicWorkerError(err)})
|
||||
return
|
||||
}
|
||||
welcomeRaw, err := json.Marshal(welcome)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err := writeWorkerFrame(ctx, conn, workerFrame{Version: "1", Type: "worker.welcome", MessageID: requestID, ReplyTo: hello.MessageID, SessionID: binding.SessionID, Body: welcomeRaw}); err != nil {
|
||||
return
|
||||
}
|
||||
// 有界读写队列与 SQL 控制循环分开，处理器绝不运行在这里。
|
||||
incoming := make(chan workerFrame, 4)
|
||||
outgoing := make(chan outboundFrame, 8)
|
||||
var loops sync.WaitGroup
|
||||
loops.Add(2)
|
||||
go func() {
|
||||
defer loops.Done()
|
||||
defer cancel()
|
||||
for {
|
||||
readCtx, stop := context.WithTimeout(ctx, 60*time.Second)
|
||||
kind, raw, err := conn.Read(readCtx)
|
||||
stop()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var frame workerFrame
|
||||
if kind != websocket.MessageText || decodeWorkerJSON(raw, &frame) != nil || frame.Version != "1" || frame.MessageID.Validate() != nil || frame.SessionID != binding.SessionID || frame.ReplyTo != "" || frame.Error != nil {
|
||||
_ = conn.Close(websocket.StatusPolicyViolation, "invalid worker frame")
|
||||
return
|
||||
}
|
||||
select {
|
||||
case incoming <- frame:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer loops.Done()
|
||||
defer cancel()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case frame := <-outgoing:
|
||||
writeCtx, stop := context.WithTimeout(ctx, 5*time.Second)
|
||||
err := conn.Write(writeCtx, websocket.MessageText, frame.data)
|
||||
stop()
|
||||
if frame.done != nil {
|
||||
frame.done <- err
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
defer func() { cancel(); _ = conn.CloseNow(); loops.Wait() }()
|
||||
send := func(frame workerFrame, wait bool) bool {
|
||||
frame.Version = "1"
|
||||
frame.SessionID = binding.SessionID
|
||||
if frame.MessageID == "" {
|
||||
var err error
|
||||
frame.MessageID, err = core.NewID()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
raw, err := json.Marshal(frame)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
out := outboundFrame{data: raw}
|
||||
if wait {
|
||||
out.done = make(chan error, 1)
|
||||
}
|
||||
select {
|
||||
case outgoing <- out:
|
||||
default:
|
||||
return false
|
||||
}
|
||||
if !wait {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case err := <-out.done:
|
||||
return err == nil
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
capacity := input.Capacity
|
||||
draining := welcome.Draining
|
||||
eligible := false
|
||||
leases := map[core.ID]time.Time{}
|
||||
ticker := time.NewTicker(h.poll)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case frame := <-incoming:
|
||||
call, stop := context.WithTimeout(ctx, 8*time.Second)
|
||||
var body any
|
||||
var commandErr error
|
||||
switch frame.Type {
|
||||
case "worker.capacity":
|
||||
var in worker.CapacityInput
|
||||
if commandErr = decodeWorkerJSON(frame.Body, &in); commandErr == nil {
|
||||
var out worker.Welcome
|
||||
out, commandErr = h.service.Capacity(call, token, binding, in)
|
||||
body = out
|
||||
if commandErr == nil {
|
||||
capacity = in.Capacity
|
||||
draining = out.Draining
|
||||
eligible = true
|
||||
}
|
||||
}
|
||||
case "worker.resume":
|
||||
var ref worker.AttemptRef
|
||||
if commandErr = decodeWorkerJSON(frame.Body, &ref); commandErr == nil {
|
||||
var out worker.LeaseStatus
|
||||
out, commandErr = h.service.Resume(call, token, binding, ref)
|
||||
body = out
|
||||
if commandErr == nil {
|
||||
if out.Active {
|
||||
leases[ref.AttemptID] = out.LeaseUntil
|
||||
} else {
|
||||
delete(leases, ref.AttemptID)
|
||||
}
|
||||
}
|
||||
}
|
||||
case "task.start", "task.renew":
|
||||
var ref worker.AttemptRef
|
||||
if commandErr = decodeWorkerJSON(frame.Body, &ref); commandErr == nil {
|
||||
var out worker.CommandResult
|
||||
if frame.Type == "task.start" {
|
||||
out, commandErr = h.service.Start(call, token, binding, frame.MessageID, ref)
|
||||
} else {
|
||||
out, commandErr = h.service.Renew(call, token, binding, frame.MessageID, ref)
|
||||
}
|
||||
body = out
|
||||
if commandErr == nil {
|
||||
if out.Lease.Active {
|
||||
leases[ref.AttemptID] = out.Lease.LeaseUntil
|
||||
} else {
|
||||
delete(leases, ref.AttemptID)
|
||||
}
|
||||
}
|
||||
}
|
||||
case "task.finish":
|
||||
var in worker.FinishInput
|
||||
if commandErr = decodeWorkerJSON(frame.Body, &in); commandErr == nil {
|
||||
var out worker.CommandResult
|
||||
out, commandErr = h.service.Finish(call, token, binding, frame.MessageID, in)
|
||||
body = out
|
||||
if commandErr == nil && !out.Lease.Active {
|
||||
delete(leases, in.Ref.AttemptID)
|
||||
}
|
||||
}
|
||||
default:
|
||||
commandErr = fault.Invalid("unsupported worker message type")
|
||||
}
|
||||
stop()
|
||||
reply := workerFrame{Type: "ack", ReplyTo: frame.MessageID}
|
||||
if commandErr != nil {
|
||||
reply.Type = "error"
|
||||
reply.Error = publicWorkerError(commandErr)
|
||||
} else {
|
||||
reply.Body, commandErr = json.Marshal(body)
|
||||
if commandErr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
fatal := commandErr != nil && (fault.Code(commandErr) == "STALE_SESSION" || fault.Code(commandErr) == "UNAUTHENTICATED" || fault.Code(commandErr) == "PERMISSION_DENIED")
|
||||
if !send(reply, fatal) || fatal {
|
||||
return
|
||||
}
|
||||
case <-ticker.C:
|
||||
// 本地记录仅减少空轮询；SQL 仍原子验证总容量、队列额度和有效会话。
|
||||
now := time.Now()
|
||||
for attempt, until := range leases {
|
||||
if !until.After(now) {
|
||||
delete(leases, attempt)
|
||||
}
|
||||
}
|
||||
if !eligible || draining || capacity == 0 || len(leases) >= int(capacity) || len(outgoing) >= 4 {
|
||||
continue
|
||||
}
|
||||
for burst := 0; burst < 4 && len(leases) < int(capacity) && len(outgoing) < 4; burst++ {
|
||||
call, stop := context.WithTimeout(ctx, 8*time.Second)
|
||||
assignment, err := h.service.Claim(call, token, binding)
|
||||
stop()
|
||||
if err != nil {
|
||||
if fault.Code(err) == "STALE_SESSION" || fault.Code(err) == "UNAUTHENTICATED" || fault.Code(err) == "PERMISSION_DENIED" {
|
||||
send(workerFrame{Type: "error", Error: publicWorkerError(err)}, true)
|
||||
return
|
||||
}
|
||||
break
|
||||
}
|
||||
if assignment == nil {
|
||||
break
|
||||
}
|
||||
raw, err := json.Marshal(assignment)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
leases[assignment.AttemptID] = assignment.LeaseUntil
|
||||
if !send(workerFrame{Type: "task.assign", Body: raw}, false) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
func publicWorkerError(err error) *fault.Error {
|
||||
var known *fault.Error
|
||||
if errors.As(err, &known) {
|
||||
return known
|
||||
}
|
||||
return fault.Unavailable
|
||||
}
|
||||
func writeWorkerFrame(ctx context.Context, conn *websocket.Conn, frame workerFrame) error {
|
||||
raw, err := json.Marshal(frame)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bounded, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
return conn.Write(bounded, websocket.MessageText, raw)
|
||||
}
|
||||
func decodeWorkerJSON(raw []byte, out any) error {
|
||||
if len(raw) == 0 || len(raw) > 512<<10 {
|
||||
return fault.Invalid("invalid worker message size")
|
||||
}
|
||||
if _, err := jsonvalue.Canonical(raw); err != nil {
|
||||
return fault.Invalid("invalid worker JSON")
|
||||
}
|
||||
if err := exactFields(raw, reflect.TypeOf(out).Elem()); err != nil {
|
||||
return fault.Invalid("unknown or invalid worker field")
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(out); err != nil {
|
||||
return fault.Invalid("invalid worker field value")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user