feat: add bounded task priority and fair application dispatch

internal/transport/httpserver/worker.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 	"time"
 
+	"git.yuntuo.hk/k2safe/orbit/internal/dispatch"
 	"git.yuntuo.hk/k2safe/orbit/internal/fault"
 	"git.yuntuo.hk/k2safe/orbit/internal/jsonvalue"
 	"git.yuntuo.hk/k2safe/orbit/internal/jsonwire"
@@ -25,19 +26,25 @@ const WorkerSubprotocol = "pulse.worker.v1"
 
 // WorkerHub 管理节点内连接生命周期；任务归属和容量仍由 MySQL 决定。
 type WorkerHub struct {
-	service *worker.Service
-	ctx     context.Context
-	cancel  context.CancelFunc
-	slots   chan struct{}
-	mu      sync.Mutex
-	closing bool
-	wg      sync.WaitGroup
-	poll    time.Duration
+	service    *worker.Service
+	ctx        context.Context
+	cancel     context.CancelFunc
+	slots      chan struct{}
+	mu         sync.Mutex
+	closing    bool
+	wg         sync.WaitGroup
+	poll       time.Duration
+	dispatch   *dispatch.Pool
+	emptyPolls map[[32]byte]time.Time
 }
 
 func NewWorkerHub(parent context.Context, service *worker.Service) *WorkerHub {
 	ctx, cancel := context.WithCancel(parent)
-	return &WorkerHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, 128), poll: time.Second}
+	h := &WorkerHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, 128), poll: time.Second, emptyPolls: make(map[[32]byte]time.Time)}
+	h.dispatch = dispatch.New(ctx, 4, 128)
+	h.wg.Add(1)
+	go func() { defer h.wg.Done(); <-h.dispatch.Done() }()
+	return h
 }
 func RegisterWorkers(server *http.Server, hub *WorkerHub) {
 	server.Handler.(*http.ServeMux).Handle("GET /v1/worker/connect", hub)
@@ -249,12 +256,53 @@ func (h *WorkerHub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	eligible := false
 	leases := map[core.ID]time.Time{}
 	cancelSent := map[core.ID]bool{}
+	results := make(chan claimResult, 1)
+	pending, burstLeft := false, 0
+	pollKey := workerPollKey(token.NamespaceID, input)
+	canRequest := func() bool {
+		return eligible && !draining && capacity > 0 && len(leases) < int(capacity) && len(outgoing) < 4
+	}
+	requestClaim := func() {
+		if pending || burstLeft == 0 || !canRequest() {
+			return
+		}
+		pending = h.dispatch.Submit(dispatch.Job{Context: ctx, Application: string(binding.ApplicationID), Queue: string(token.NamespaceID) + "/" + string(input.QueueID), Run: func(jobCtx context.Context) {
+			results <- h.claimFair(jobCtx, token, binding, pollKey)
+		}})
+		if pending {
+			burstLeft--
+		}
+	}
 	ticker := time.NewTicker(h.poll)
 	defer ticker.Stop()
 	for {
 		select {
 		case <-ctx.Done():
 			return
+		case result := <-results:
+			pending = false
+			if result.err != nil {
+				if fault.Code(result.err) == "STALE_SESSION" || fault.Code(result.err) == "UNAUTHENTICATED" || fault.Code(result.err) == "PERMISSION_DENIED" {
+					send(workerFrame{Type: "error", Error: publicWorkerError(result.err)}, true)
+					return
+				}
+				burstLeft = 0
+				continue
+			}
+			if result.assignment == nil {
+				burstLeft = 0
+				continue
+			}
+			assignment := result.assignment
+			raw, err := jsonwire.Marshal(assignment)
+			if err != nil {
+				return
+			}
+			leases[assignment.AttemptID] = assignment.LeaseUntil
+			if !send(workerFrame{Type: "task.assign", Body: raw}, false) {
+				return
+			}
+			requestClaim()
 		case frame := <-incoming:
 			call, stop := context.WithTimeout(ctx, 8*time.Second)
 			var body any
@@ -370,31 +418,9 @@ func (h *WorkerHub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 					delete(leases, attempt)
 				}
 			}
-			if !eligible || draining || capacity == 0 || len(leases) >= int(capacity) || len(outgoing) >= 4 {
-				continue
-			}
-			for burst := 0; burst < 4 && len(leases) < int(capacity) && len(outgoing) < 4; burst++ {
-				call, stop := context.WithTimeout(ctx, 8*time.Second)
-				assignment, err := h.service.Claim(call, token, binding)
-				stop()
-				if err != nil {
-					if fault.Code(err) == "STALE_SESSION" || fault.Code(err) == "UNAUTHENTICATED" || fault.Code(err) == "PERMISSION_DENIED" {
-						send(workerFrame{Type: "error", Error: publicWorkerError(err)}, true)
-						return
-					}
-					break
-				}
-				if assignment == nil {
-					break
-				}
-				raw, err := jsonwire.Marshal(assignment)
-				if err != nil {
-					return
-				}
-				leases[assignment.AttemptID] = assignment.LeaseUntil
-				if !send(workerFrame{Type: "task.assign", Body: raw}, false) {
-					return
-				}
+			if !pending {
+				burstLeft = 4
+				requestClaim()
 			}
 		}
 	}

internal/transport/httpserver/worker_contract_test.go

@@ -194,6 +194,7 @@ func TestWorkerPublishedContract(t *testing.T) {
 	assignment := worker.Assignment{Task: task, AttemptID: id, Epoch: 1, LeaseSecret: secret, LeaseUntil: lease.LeaseUntil, ServerTime: now}
 	keyed := assignment
 	keyed.Task.ConcurrencyKey = "object/共享资源"
+	keyed.Task.Priority = 9
 	keyed.ConcurrencyToken = ^uint64(0) - 1
 	for _, tc := range []struct {
 		kind string
@@ -215,6 +216,8 @@ func TestWorkerPublishedContract(t *testing.T) {
 	keyedBody, _ := jsonwire.Marshal(keyed)
 	keyedFrame, _ := jsonwire.Marshal(workerFrame{Version: "1", Type: "task.assign", MessageID: id, SessionID: id, Body: keyedBody})
 	for _, raw := range []string{
+		strings.Replace(string(keyedFrame), `"priority":9`, `"priority":10`, 1),
+		strings.Replace(string(keyedFrame), `"priority":9`, `"priority":-1`, 1),
 		strings.Replace(string(keyedFrame), `"concurrency_token":"18446744073709551614",`, "", 1),
 		strings.Replace(string(keyedFrame), `"concurrency_key":"object/共享资源",`, "", 1),
 		strings.Replace(string(keyedFrame), `"concurrency_token":"18446744073709551614"`, `"concurrency_token":0`, 1),

internal/transport/httpserver/worker_dispatch.go

@@ -0,0 +1,60 @@
+package httpserver
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"time"
+
+	"git.yuntuo.hk/k2safe/orbit/internal/fault"
+	core "git.yuntuo.hk/k2safe/orbit/model"
+	"git.yuntuo.hk/k2safe/orbit/platform/auth"
+	"git.yuntuo.hk/k2safe/orbit/pulse/worker"
+)
+
+type claimResult struct {
+	assignment *worker.Assignment
+	err        error
+}
+
+func workerPollKey(ns core.ID, in worker.ConnectInput) [32]byte {
+	// 正式注册已校验；再次规范化仅取得稳定能力集合，不包含任何秘密。
+	in, _ = worker.NormalizeConnect(in)
+	raw, _ := json.Marshal(struct {
+		Namespace, Queue core.ID
+		Capabilities     []worker.Capability
+		Keys             bool
+	}{ns, in.QueueID, in.Capabilities, in.ConcurrencyKeys})
+	return sha256.Sum256(raw)
+}
+func (h *WorkerHub) claimFair(ctx context.Context, t auth.Token, b worker.Binding, key [32]byte) (out claimResult) {
+	// HTTP handler 原本具有 panic 隔离；异步领取同样不能因一次异常杀死其他连接。
+	defer func() {
+		if recover() != nil {
+			out = claimResult{err: fault.Unavailable}
+		}
+	}()
+	h.mu.Lock()
+	until := h.emptyPolls[key]
+	h.mu.Unlock()
+	if until.After(time.Now()) {
+		return out
+	}
+	call, stop := context.WithTimeout(ctx, 8*time.Second)
+	defer stop()
+	out.assignment, out.err = h.service.Claim(call, t, b)
+	if out.assignment == nil && out.err == nil {
+		h.mu.Lock()
+		now := time.Now()
+		for k, v := range h.emptyPolls {
+			if !v.After(now) {
+				delete(h.emptyPolls, k)
+			}
+		}
+		if len(h.emptyPolls) < 128 {
+			h.emptyPolls[key] = now.Add(h.poll)
+		}
+		h.mu.Unlock()
+	}
+	return out
+}

internal/transport/httpserver/worker_dispatch_test.go

@@ -0,0 +1,75 @@
+package httpserver
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	core "git.yuntuo.hk/k2safe/orbit/model"
+	"git.yuntuo.hk/k2safe/orbit/platform/auth"
+	"git.yuntuo.hk/k2safe/orbit/pulse/worker"
+)
+
+type emptyClaimStore struct {
+	worker.Store
+	calls     int
+	panicNext bool
+}
+
+func (s *emptyClaimStore) Claim(context.Context, auth.Token, worker.Binding) (*worker.Assignment, error) {
+	s.calls++
+	if s.panicNext {
+		s.panicNext = false
+		panic("private database details")
+	}
+	return nil, nil
+}
+func TestEmptyPollCoalescesExactCapabilitiesAndBoundsCache(t *testing.T) {
+	store := &emptyClaimStore{}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	h := NewWorkerHub(ctx, worker.New(store))
+	defer h.Shutdown(context.Background())
+	in := worker.ConnectInput{InstanceID: core.ID(strings.Repeat("1", 32)), InstanceSecret: strings.Repeat("A", 43), QueueID: core.ID(strings.Repeat("2", 32)), Capacity: 1, Capabilities: []worker.Capability{{Handler: "a", HandlerVersion: "v1", PayloadVersion: "v1"}, {Handler: "b", HandlerVersion: "v1", PayloadVersion: "v1"}}}
+	ns := core.ID(strings.Repeat("3", 32))
+	key := workerPollKey(ns, in)
+	for range 20 {
+		if out := h.claimFair(ctx, auth.Token{}, worker.Binding{}, key); out.err != nil || out.assignment != nil {
+			t.Fatal("unexpected poll result")
+		}
+	}
+	if store.calls != 1 {
+		t.Fatal("identical idle connections scanned separately")
+	}
+	in.Capabilities[0], in.Capabilities[1] = in.Capabilities[1], in.Capabilities[0]
+	if workerPollKey(ns, in) != key {
+		t.Fatal("capability order changed coalescing identity")
+	}
+	in.ConcurrencyKeys = true
+	keyed := workerPollKey(ns, in)
+	if keyed == key {
+		t.Fatal("legacy and keyed workers shared an eligibility cache")
+	}
+	h.claimFair(ctx, auth.Token{}, worker.Binding{}, keyed)
+	if store.calls != 2 {
+		t.Fatal("different capabilities hidden by cache")
+	}
+	h.emptyPolls[key] = time.Now().Add(-time.Second)
+	h.claimFair(ctx, auth.Token{}, worker.Binding{}, key)
+	if store.calls != 3 {
+		t.Fatal("expired cache prevented rediscovery")
+	}
+	for i := 0; i < 256; i++ {
+		k := [32]byte{byte(i), 1}
+		h.claimFair(ctx, auth.Token{}, worker.Binding{}, k)
+	}
+	if len(h.emptyPolls) > 128 {
+		t.Fatal("unbounded empty poll cache")
+	}
+	store.panicNext = true
+	out := h.claimFair(ctx, auth.Token{}, worker.Binding{}, [32]byte{1, 2, 3})
+	if out.err == nil || strings.Contains(out.err.Error(), "private") {
+		t.Fatal("claim panic escaped isolation")
+	}
+}