403 lines
12 KiB
Go
403 lines
12 KiB
Go
package httpserver
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"net/http"
|
||
"reflect"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||
"git.yuntuo.hk/k2safe/orbit/internal/jsonvalue"
|
||
"git.yuntuo.hk/k2safe/orbit/internal/jsonwire"
|
||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||
"github.com/coder/websocket"
|
||
)
|
||
|
||
const WorkerSubprotocol = "pulse.worker.v1"
|
||
|
||
// WorkerHub 管理节点内连接生命周期；任务归属和容量仍由 MySQL 决定。
|
||
type WorkerHub struct {
|
||
service *worker.Service
|
||
ctx context.Context
|
||
cancel context.CancelFunc
|
||
slots chan struct{}
|
||
mu sync.Mutex
|
||
closing bool
|
||
wg sync.WaitGroup
|
||
poll time.Duration
|
||
}
|
||
|
||
func NewWorkerHub(parent context.Context, service *worker.Service) *WorkerHub {
|
||
ctx, cancel := context.WithCancel(parent)
|
||
return &WorkerHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, 128), poll: time.Second}
|
||
}
|
||
func RegisterWorkers(server *http.Server, hub *WorkerHub) {
|
||
server.Handler.(*http.ServeMux).Handle("GET /v1/worker/connect", hub)
|
||
}
|
||
|
||
// Shutdown 停止派发并关闭本节点连接；不清除持久化租约，由重連核对或到期恢复处理。
|
||
func (h *WorkerHub) Shutdown(ctx context.Context) error {
|
||
h.mu.Lock()
|
||
h.closing = true
|
||
h.cancel()
|
||
h.mu.Unlock()
|
||
done := make(chan struct{})
|
||
go func() { h.wg.Wait(); close(done) }()
|
||
select {
|
||
case <-done:
|
||
return nil
|
||
case <-ctx.Done():
|
||
return ctx.Err()
|
||
}
|
||
}
|
||
|
||
type workerFrame struct {
|
||
Version string `json:"protocol_version"`
|
||
Type string `json:"type"`
|
||
MessageID core.ID `json:"message_id"`
|
||
ReplyTo core.ID `json:"reply_to,omitempty"`
|
||
SessionID core.ID `json:"session_id,omitempty"`
|
||
Body json.RawMessage `json:"body,omitempty"`
|
||
Error *fault.Error `json:"error,omitempty"`
|
||
}
|
||
type outboundFrame struct {
|
||
data []byte
|
||
done chan error
|
||
}
|
||
|
||
func (h *WorkerHub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||
requestID, _ := core.NewID()
|
||
w.Header().Set("X-Request-ID", string(requestID))
|
||
w.Header().Set("Cache-Control", "no-store")
|
||
if r.URL.RawQuery != "" {
|
||
writeError(w, string(requestID), fault.Invalid("Worker credentials and options must not be placed in URL"))
|
||
return
|
||
}
|
||
values := r.Header.Values("Authorization")
|
||
if len(values) != 1 {
|
||
writeError(w, string(requestID), fault.Unauthorized)
|
||
return
|
||
}
|
||
scheme, value, ok := strings.Cut(values[0], " ")
|
||
if !ok || !strings.EqualFold(scheme, "Bearer") {
|
||
writeError(w, string(requestID), fault.Unauthorized)
|
||
return
|
||
}
|
||
token, err := auth.Parse(value)
|
||
if err != nil {
|
||
writeError(w, string(requestID), err)
|
||
return
|
||
}
|
||
supported := false
|
||
for _, header := range r.Header.Values("Sec-WebSocket-Protocol") {
|
||
for _, protocol := range strings.Split(header, ",") {
|
||
supported = supported || strings.TrimSpace(protocol) == WorkerSubprotocol
|
||
}
|
||
}
|
||
if !supported {
|
||
writeJSON(w, http.StatusUpgradeRequired, map[string]any{"request_id": requestID, "error": fault.New("PROTOCOL_REQUIRED", "pulse.worker.v1 subprotocol is required")})
|
||
return
|
||
}
|
||
h.mu.Lock()
|
||
if h.closing || h.ctx.Err() != nil {
|
||
h.mu.Unlock()
|
||
writeError(w, string(requestID), fault.Unavailable)
|
||
return
|
||
}
|
||
select {
|
||
case h.slots <- struct{}{}:
|
||
h.wg.Add(1)
|
||
default:
|
||
h.mu.Unlock()
|
||
writeJSON(w, 429, map[string]any{"request_id": requestID, "error": fault.New("CAPACITY_EXCEEDED", "node connection capacity exceeded")})
|
||
return
|
||
}
|
||
h.mu.Unlock()
|
||
defer func() { <-h.slots; h.wg.Done() }()
|
||
authorize, cancel := context.WithTimeout(r.Context(), 5*time.Second)
|
||
err = h.service.Authorize(authorize, token)
|
||
cancel()
|
||
if err != nil {
|
||
writeError(w, string(requestID), err)
|
||
return
|
||
}
|
||
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{WorkerSubprotocol}, CompressionMode: websocket.CompressionDisabled})
|
||
if err != nil {
|
||
return
|
||
}
|
||
defer conn.CloseNow()
|
||
conn.SetReadLimit(512 << 10)
|
||
ctx, cancel := context.WithCancel(h.ctx)
|
||
defer cancel()
|
||
helloCtx, helloCancel := context.WithTimeout(ctx, 5*time.Second)
|
||
kind, raw, err := conn.Read(helloCtx)
|
||
helloCancel()
|
||
if err != nil {
|
||
return
|
||
}
|
||
var hello workerFrame
|
||
var input worker.ConnectInput
|
||
if kind != websocket.MessageText || decodeWorkerJSON(raw, &hello) != nil || hello.Version != "1" || hello.Type != "worker.hello" || hello.MessageID.Validate() != nil || hello.SessionID != "" || hello.ReplyTo != "" || hello.Error != nil || decodeWorkerJSON(hello.Body, &input) != nil {
|
||
_ = conn.Close(websocket.StatusPolicyViolation, "invalid worker hello")
|
||
return
|
||
}
|
||
call, cancelCall := context.WithTimeout(ctx, 8*time.Second)
|
||
binding, welcome, err := h.service.Connect(call, token, input)
|
||
cancelCall()
|
||
if err != nil {
|
||
_ = writeWorkerFrame(ctx, conn, workerFrame{Version: "1", Type: "error", MessageID: requestID, ReplyTo: hello.MessageID, Error: publicWorkerError(err)})
|
||
return
|
||
}
|
||
welcomeRaw, err := jsonwire.Marshal(welcome)
|
||
if err != nil {
|
||
return
|
||
}
|
||
if err := writeWorkerFrame(ctx, conn, workerFrame{Version: "1", Type: "worker.welcome", MessageID: requestID, ReplyTo: hello.MessageID, SessionID: binding.SessionID, Body: welcomeRaw}); err != nil {
|
||
return
|
||
}
|
||
// 有界读写队列与 SQL 控制循环分开，处理器绝不运行在这里。
|
||
incoming := make(chan workerFrame, 4)
|
||
outgoing := make(chan outboundFrame, 8)
|
||
var loops sync.WaitGroup
|
||
loops.Add(2)
|
||
go func() {
|
||
defer loops.Done()
|
||
defer cancel()
|
||
for {
|
||
readCtx, stop := context.WithTimeout(ctx, 60*time.Second)
|
||
kind, raw, err := conn.Read(readCtx)
|
||
stop()
|
||
if err != nil {
|
||
return
|
||
}
|
||
var frame workerFrame
|
||
if kind != websocket.MessageText || decodeWorkerJSON(raw, &frame) != nil || frame.Version != "1" || frame.MessageID.Validate() != nil || frame.SessionID != binding.SessionID || frame.ReplyTo != "" || frame.Error != nil {
|
||
_ = conn.Close(websocket.StatusPolicyViolation, "invalid worker frame")
|
||
return
|
||
}
|
||
select {
|
||
case incoming <- frame:
|
||
case <-ctx.Done():
|
||
return
|
||
}
|
||
}
|
||
}()
|
||
go func() {
|
||
defer loops.Done()
|
||
defer cancel()
|
||
for {
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case frame := <-outgoing:
|
||
writeCtx, stop := context.WithTimeout(ctx, 5*time.Second)
|
||
err := conn.Write(writeCtx, websocket.MessageText, frame.data)
|
||
stop()
|
||
if frame.done != nil {
|
||
frame.done <- err
|
||
}
|
||
if err != nil {
|
||
return
|
||
}
|
||
}
|
||
}
|
||
}()
|
||
defer func() { cancel(); _ = conn.CloseNow(); loops.Wait() }()
|
||
send := func(frame workerFrame, wait bool) bool {
|
||
frame.Version = "1"
|
||
frame.SessionID = binding.SessionID
|
||
if frame.MessageID == "" {
|
||
var err error
|
||
frame.MessageID, err = core.NewID()
|
||
if err != nil {
|
||
return false
|
||
}
|
||
}
|
||
raw, err := jsonwire.Marshal(frame)
|
||
if err != nil {
|
||
return false
|
||
}
|
||
out := outboundFrame{data: raw}
|
||
if wait {
|
||
out.done = make(chan error, 1)
|
||
}
|
||
select {
|
||
case outgoing <- out:
|
||
default:
|
||
return false
|
||
}
|
||
if !wait {
|
||
return true
|
||
}
|
||
select {
|
||
case err := <-out.done:
|
||
return err == nil
|
||
case <-ctx.Done():
|
||
return false
|
||
}
|
||
}
|
||
capacity := input.Capacity
|
||
draining := welcome.Draining
|
||
eligible := false
|
||
leases := map[core.ID]time.Time{}
|
||
ticker := time.NewTicker(h.poll)
|
||
defer ticker.Stop()
|
||
for {
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case frame := <-incoming:
|
||
call, stop := context.WithTimeout(ctx, 8*time.Second)
|
||
var body any
|
||
var commandErr error
|
||
switch frame.Type {
|
||
case "worker.capacity":
|
||
var in worker.CapacityInput
|
||
if commandErr = decodeWorkerJSON(frame.Body, &in); commandErr == nil {
|
||
var out worker.Welcome
|
||
out, commandErr = h.service.Capacity(call, token, binding, in)
|
||
body = out
|
||
if commandErr == nil {
|
||
capacity = in.Capacity
|
||
draining = out.Draining
|
||
eligible = true
|
||
}
|
||
}
|
||
case "worker.resume":
|
||
var ref worker.AttemptRef
|
||
if commandErr = decodeWorkerJSON(frame.Body, &ref); commandErr == nil {
|
||
var out worker.LeaseStatus
|
||
out, commandErr = h.service.Resume(call, token, binding, ref)
|
||
body = out
|
||
if commandErr == nil {
|
||
if out.Active {
|
||
leases[ref.AttemptID] = out.LeaseUntil
|
||
} else {
|
||
delete(leases, ref.AttemptID)
|
||
}
|
||
}
|
||
}
|
||
case "task.start", "task.renew":
|
||
var ref worker.AttemptRef
|
||
if commandErr = decodeWorkerJSON(frame.Body, &ref); commandErr == nil {
|
||
var out worker.CommandResult
|
||
if frame.Type == "task.start" {
|
||
out, commandErr = h.service.Start(call, token, binding, frame.MessageID, ref)
|
||
} else {
|
||
out, commandErr = h.service.Renew(call, token, binding, frame.MessageID, ref)
|
||
}
|
||
body = out
|
||
if commandErr == nil {
|
||
if out.Lease.Active {
|
||
leases[ref.AttemptID] = out.Lease.LeaseUntil
|
||
} else {
|
||
delete(leases, ref.AttemptID)
|
||
}
|
||
}
|
||
}
|
||
case "task.finish":
|
||
var in worker.FinishInput
|
||
if commandErr = decodeWorkerJSON(frame.Body, &in); commandErr == nil {
|
||
var out worker.CommandResult
|
||
out, commandErr = h.service.Finish(call, token, binding, frame.MessageID, in)
|
||
body = out
|
||
if commandErr == nil && !out.Lease.Active {
|
||
delete(leases, in.Ref.AttemptID)
|
||
}
|
||
}
|
||
default:
|
||
commandErr = fault.Invalid("unsupported worker message type")
|
||
}
|
||
stop()
|
||
reply := workerFrame{Type: "ack", ReplyTo: frame.MessageID}
|
||
if commandErr != nil {
|
||
reply.Type = "error"
|
||
reply.Error = publicWorkerError(commandErr)
|
||
} else {
|
||
reply.Body, commandErr = jsonwire.Marshal(body)
|
||
if commandErr != nil {
|
||
return
|
||
}
|
||
}
|
||
fatal := commandErr != nil && (fault.Code(commandErr) == "STALE_SESSION" || fault.Code(commandErr) == "UNAUTHENTICATED" || fault.Code(commandErr) == "PERMISSION_DENIED")
|
||
if !send(reply, fatal) || fatal {
|
||
return
|
||
}
|
||
case <-ticker.C:
|
||
// 本地记录仅减少空轮询；SQL 仍原子验证总容量、队列额度和有效会话。
|
||
now := time.Now()
|
||
for attempt, until := range leases {
|
||
if !until.After(now) {
|
||
delete(leases, attempt)
|
||
}
|
||
}
|
||
if !eligible || draining || capacity == 0 || len(leases) >= int(capacity) || len(outgoing) >= 4 {
|
||
continue
|
||
}
|
||
for burst := 0; burst < 4 && len(leases) < int(capacity) && len(outgoing) < 4; burst++ {
|
||
call, stop := context.WithTimeout(ctx, 8*time.Second)
|
||
assignment, err := h.service.Claim(call, token, binding)
|
||
stop()
|
||
if err != nil {
|
||
if fault.Code(err) == "STALE_SESSION" || fault.Code(err) == "UNAUTHENTICATED" || fault.Code(err) == "PERMISSION_DENIED" {
|
||
send(workerFrame{Type: "error", Error: publicWorkerError(err)}, true)
|
||
return
|
||
}
|
||
break
|
||
}
|
||
if assignment == nil {
|
||
break
|
||
}
|
||
raw, err := jsonwire.Marshal(assignment)
|
||
if err != nil {
|
||
return
|
||
}
|
||
leases[assignment.AttemptID] = assignment.LeaseUntil
|
||
if !send(workerFrame{Type: "task.assign", Body: raw}, false) {
|
||
return
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
func publicWorkerError(err error) *fault.Error {
|
||
var known *fault.Error
|
||
if errors.As(err, &known) {
|
||
return known
|
||
}
|
||
return fault.Unavailable
|
||
}
|
||
func writeWorkerFrame(ctx context.Context, conn *websocket.Conn, frame workerFrame) error {
|
||
raw, err := jsonwire.Marshal(frame)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
bounded, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||
defer cancel()
|
||
return conn.Write(bounded, websocket.MessageText, raw)
|
||
}
|
||
func decodeWorkerJSON(raw []byte, out any) error {
|
||
if len(raw) == 0 || len(raw) > 512<<10 {
|
||
return fault.Invalid("invalid worker message size")
|
||
}
|
||
if _, err := jsonvalue.Canonical(raw); err != nil {
|
||
return fault.Invalid("invalid worker JSON")
|
||
}
|
||
if err := exactFields(raw, reflect.TypeOf(out).Elem()); err != nil {
|
||
return fault.Invalid("unknown or invalid worker field")
|
||
}
|
||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||
decoder.DisallowUnknownFields()
|
||
if err := decoder.Decode(out); err != nil {
|
||
return fault.Invalid("invalid worker field value")
|
||
}
|
||
return nil
|
||
}
|