k2safe/orbit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: tune websocket hub backpressure

Browse Source
This commit is contained in:
k2safe
2026-09-20 20:53:50 +08:00
parent 850376624f
commit 478be069e6
16 changed files with 218 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
internal/config/config.go
View File

@@ -10,31 +10,39 @@ import (
)
type Config struct {
HTTPAddr string
MySQLDSN string
MaxOpen int
MaxIdle int
MetricsAddr string
WorkflowTickInterval time.Duration // WorkflowTickInterval Weave 持久化推进扫描间隔，范围100ms–1m。
ScheduleTickInterval time.Duration // ScheduleTickInterval 计划扫描间隔，范围100ms–1m。
RecoveryTickInterval time.Duration // RecoveryTickInterval 执行租约恢复扫描间隔，范围100ms–1m。
RuntimeBatchLimit int // RuntimeBatchLimit 每个后台批次的候选上限，范围1–100。
ShutdownTimeout time.Duration // ShutdownTimeout 优雅排空和关闭窗口，范围1s–5m。
HTTPAddr string
MySQLDSN string
MaxOpen int
MaxIdle int
MetricsAddr string
WorkflowTickInterval time.Duration // WorkflowTickInterval Weave 持久化推进扫描间隔，范围100ms–1m。
ScheduleTickInterval time.Duration // ScheduleTickInterval 计划扫描间隔，范围100ms–1m。
RecoveryTickInterval time.Duration // RecoveryTickInterval 执行租约恢复扫描间隔，范围100ms–1m。
RuntimeBatchLimit int // RuntimeBatchLimit 每个后台批次的候选上限，范围1–100。
ShutdownTimeout time.Duration // ShutdownTimeout 优雅排空和关闭窗口，范围1s–5m。
WorkerConnectionLimit int // WorkerConnectionLimit 单节点 Worker WebSocket 连接上限，范围1–4096。
EventConnectionLimit int // EventConnectionLimit 单节点 Event WebSocket 连接上限，范围1–4096。
DispatchWorkers int // DispatchWorkers 单个 Hub 并发执行数据库领取调用的协程数，范围1–256。
DispatchCapacity int // DispatchCapacity 单个 Hub 领取调用的有界内存排队槽位，范围1–65536。
}
func Load() (Config, error) { return FromEnv(os.Getenv) }
func FromEnv(get func(string) string) (Config, error) {
c := Config{
HTTPAddr: get("ORBIT_HTTP_ADDR"),
MySQLDSN: get("ORBIT_MYSQL_DSN"),
MaxOpen: 20,
MaxIdle: 5,
MetricsAddr: get("ORBIT_METRICS_ADDR"),
WorkflowTickInterval: time.Second,
ScheduleTickInterval: time.Second,
RecoveryTickInterval: 2 * time.Second,
RuntimeBatchLimit: 100,
ShutdownTimeout: 10 * time.Second,
HTTPAddr: get("ORBIT_HTTP_ADDR"),
MySQLDSN: get("ORBIT_MYSQL_DSN"),
MaxOpen: 20,
MaxIdle: 5,
MetricsAddr: get("ORBIT_METRICS_ADDR"),
WorkflowTickInterval: time.Second,
ScheduleTickInterval: time.Second,
RecoveryTickInterval: 2 * time.Second,
RuntimeBatchLimit: 100,
ShutdownTimeout: 10 * time.Second,
WorkerConnectionLimit: 128,
EventConnectionLimit: 128,
DispatchWorkers: 4,
DispatchCapacity: 128,
}
if c.HTTPAddr == "" {
c.HTTPAddr = "127.0.0.1:8080"
@@ -86,6 +94,25 @@ func FromEnv(get func(string) string) (Config, error) {
}
c.ShutdownTimeout = d
}
for _, field := range []struct {
name string
dst *int
min int
max int
}{
{"ORBIT_WORKER_CONNECTION_LIMIT", &c.WorkerConnectionLimit, 1, 4096},
{"ORBIT_EVENT_CONNECTION_LIMIT", &c.EventConnectionLimit, 1, 4096},
{"ORBIT_DISPATCH_WORKERS", &c.DispatchWorkers, 1, 256},
{"ORBIT_DISPATCH_CAPACITY", &c.DispatchCapacity, 1, 65536},
} {
if raw := get(field.name); raw != "" {
n, err := strconv.Atoi(raw)
if err != nil || n < field.min || n > field.max {
return Config{}, fmt.Errorf("%s must be an integer from %d to %d", field.name, field.min, field.max)
}
*field.dst = n
}
}
if c.MetricsAddr != "" {
_, port, err := net.SplitHostPort(c.MetricsAddr)
n, parseErr := strconv.Atoi(port)

22
internal/config/config_test.go
View File

@@ -9,7 +9,7 @@ func TestRuntimeDefaultsAndBoundedOverrides(t *testing.T) {
base := map[string]string{"ORBIT_MYSQL_DSN": "private-dsn"}
get := func(k string) string { return base[k] }
c, err := FromEnv(get)
if err != nil || c.WorkflowTickInterval != time.Second || c.ScheduleTickInterval != time.Second || c.RecoveryTickInterval != 2*time.Second || c.RuntimeBatchLimit != 100 || c.ShutdownTimeout != 10*time.Second {
if err != nil || c.WorkflowTickInterval != time.Second || c.ScheduleTickInterval != time.Second || c.RecoveryTickInterval != 2*time.Second || c.RuntimeBatchLimit != 100 || c.ShutdownTimeout != 10*time.Second || c.WorkerConnectionLimit != 128 || c.EventConnectionLimit != 128 || c.DispatchWorkers != 4 || c.DispatchCapacity != 128 {
t.Fatalf("unexpected runtime defaults: %+v err=%v", c, err)
}
base["ORBIT_WORKFLOW_TICK_INTERVAL"] = "250ms"
@@ -17,16 +17,24 @@ func TestRuntimeDefaultsAndBoundedOverrides(t *testing.T) {
base["ORBIT_RECOVERY_TICK_INTERVAL"] = "1m"
base["ORBIT_RUNTIME_BATCH_LIMIT"] = "37"
base["ORBIT_SHUTDOWN_TIMEOUT"] = "45s"
base["ORBIT_WORKER_CONNECTION_LIMIT"] = "512"
base["ORBIT_EVENT_CONNECTION_LIMIT"] = "256"
base["ORBIT_DISPATCH_WORKERS"] = "16"
base["ORBIT_DISPATCH_CAPACITY"] = "2048"
c, err = FromEnv(get)
if err != nil || c.WorkflowTickInterval != 250*time.Millisecond || c.ScheduleTickInterval != 1500*time.Millisecond || c.RecoveryTickInterval != time.Minute || c.RuntimeBatchLimit != 37 || c.ShutdownTimeout != 45*time.Second {
if err != nil || c.WorkflowTickInterval != 250*time.Millisecond || c.ScheduleTickInterval != 1500*time.Millisecond || c.RecoveryTickInterval != time.Minute || c.RuntimeBatchLimit != 37 || c.ShutdownTimeout != 45*time.Second || c.WorkerConnectionLimit != 512 || c.EventConnectionLimit != 256 || c.DispatchWorkers != 16 || c.DispatchCapacity != 2048 {
t.Fatalf("valid runtime override rejected: %+v err=%v", c, err)
}
for key, value := range map[string]string{
"ORBIT_WORKFLOW_TICK_INTERVAL": "99ms",
"ORBIT_SCHEDULE_TICK_INTERVAL": "2m",
"ORBIT_RECOVERY_TICK_INTERVAL": "not-a-duration",
"ORBIT_RUNTIME_BATCH_LIMIT": "101",
"ORBIT_SHUTDOWN_TIMEOUT": "500ms",
"ORBIT_WORKFLOW_TICK_INTERVAL": "99ms",
"ORBIT_SCHEDULE_TICK_INTERVAL": "2m",
"ORBIT_RECOVERY_TICK_INTERVAL": "not-a-duration",
"ORBIT_RUNTIME_BATCH_LIMIT": "101",
"ORBIT_SHUTDOWN_TIMEOUT": "500ms",
"ORBIT_WORKER_CONNECTION_LIMIT": "0",
"ORBIT_EVENT_CONNECTION_LIMIT": "4097",
"ORBIT_DISPATCH_WORKERS": "257",
"ORBIT_DISPATCH_CAPACITY": "65537",
} {
base[key] = value
if _, err := FromEnv(get); err == nil {

4
internal/observability/catalog.go
View File

@@ -66,5 +66,9 @@ func catalog() map[string]descriptor {
c["orbit_runtime_tick_interval_seconds"] = descriptor{gauge, "Configured background controller tick interval in seconds", map[string][]string{"component": components}}
c["orbit_runtime_batch_limit"] = descriptor{gauge, "Configured maximum candidates per background batch", nil}
c["orbit_shutdown_timeout_seconds"] = descriptor{gauge, "Configured graceful drain and shutdown window in seconds", nil}
c["orbit_worker_connection_limit"] = descriptor{gauge, "Configured maximum Worker WebSocket connections on this node", nil}
c["orbit_event_connection_limit"] = descriptor{gauge, "Configured maximum Event WebSocket connections on this node", nil}
c["orbit_dispatch_workers"] = descriptor{gauge, "Configured database claim workers per WebSocket hub", nil}
c["orbit_dispatch_capacity"] = descriptor{gauge, "Configured bounded in-memory claim queue per WebSocket hub", nil}
return c
}

4
internal/observability/metrics_test.go
View File

@@ -96,7 +96,7 @@ func TestInvalidNumbersAndMonotoneCounters(t *testing.T) {
}
func TestRuntimeObservationDoesNotExposeErrors(t *testing.T) {
r := NewRegistry()
r.RecordRuntimeConfig(RuntimeConfig{WorkflowTickInterval: time.Second, ScheduleTickInterval: 1500 * time.Millisecond, RecoveryTickInterval: 2 * time.Second, BatchLimit: 37, ShutdownTimeout: 45 * time.Second})
r.RecordRuntimeConfig(RuntimeConfig{WorkflowTickInterval: time.Second, ScheduleTickInterval: 1500 * time.Millisecond, RecoveryTickInterval: 2 * time.Second, BatchLimit: 37, ShutdownTimeout: 45 * time.Second, WorkerConnectionLimit: 512, EventConnectionLimit: 256, DispatchWorkers: 16, DispatchCapacity: 2048})
r.RecordDraining(true)
r.RecordBatch("workflow", time.Now().Add(-time.Millisecond), nil)
r.RecordBatch("workflow", time.Now(), errors.New("private-dsn"))
@@ -108,7 +108,7 @@ func TestRuntimeObservationDoesNotExposeErrors(t *testing.T) {
t.Fatal("missing runtime evidence", want)
}
}
for _, want := range []string{`orbit_runtime_tick_interval_seconds{component="workflow"} 1`, `orbit_runtime_tick_interval_seconds{component="schedule"} 1.5`, `orbit_runtime_tick_interval_seconds{component="execution_recovery"} 2`, "orbit_runtime_batch_limit 37\n", "orbit_shutdown_timeout_seconds 45\n", "orbit_process_draining 1\n"} {
for _, want := range []string{`orbit_runtime_tick_interval_seconds{component="workflow"} 1`, `orbit_runtime_tick_interval_seconds{component="schedule"} 1.5`, `orbit_runtime_tick_interval_seconds{component="execution_recovery"} 2`, "orbit_runtime_batch_limit 37\n", "orbit_shutdown_timeout_seconds 45\n", "orbit_worker_connection_limit 512\n", "orbit_event_connection_limit 256\n", "orbit_dispatch_workers 16\n", "orbit_dispatch_capacity 2048\n", "orbit_process_draining 1\n"} {
if !strings.Contains(out, want) {
t.Fatal("missing runtime configuration", want)
}

20
internal/observability/runtime.go
View File

@@ -13,16 +13,20 @@ import (
// RuntimeConfig 是节点启动后实际生效的后台控制参数。
// 它只用于固定基数观测，不参与业务授权或持久化状态判断。
type RuntimeConfig struct {
WorkflowTickInterval time.Duration
ScheduleTickInterval time.Duration
RecoveryTickInterval time.Duration
BatchLimit int
ShutdownTimeout time.Duration
WorkflowTickInterval time.Duration
ScheduleTickInterval time.Duration
RecoveryTickInterval time.Duration
BatchLimit int
ShutdownTimeout time.Duration
WorkerConnectionLimit int // Worker WebSocket 单节点连接上限。
EventConnectionLimit int // Event WebSocket 单节点连接上限。
DispatchWorkers int // 每个 Hub 同时执行数据库领取调用的协程上限。
DispatchCapacity int // 每个 Hub 领取调用的有界内存排队槽位。
}
// RecordRuntimeConfig records the effective bounded controller settings.
func (r *Registry) RecordRuntimeConfig(config RuntimeConfig) {
if r == nil || config.WorkflowTickInterval <= 0 || config.ScheduleTickInterval <= 0 || config.RecoveryTickInterval <= 0 || config.BatchLimit <= 0 || config.ShutdownTimeout <= 0 {
if r == nil || config.WorkflowTickInterval <= 0 || config.ScheduleTickInterval <= 0 || config.RecoveryTickInterval <= 0 || config.BatchLimit <= 0 || config.ShutdownTimeout <= 0 || config.WorkerConnectionLimit <= 0 || config.EventConnectionLimit <= 0 || config.DispatchWorkers <= 0 || config.DispatchCapacity <= 0 {
return
}
r.SetGauge("orbit_runtime_tick_interval_seconds", config.WorkflowTickInterval.Seconds(), Labels{"component": "workflow"})
@@ -30,6 +34,10 @@ func (r *Registry) RecordRuntimeConfig(config RuntimeConfig) {
r.SetGauge("orbit_runtime_tick_interval_seconds", config.RecoveryTickInterval.Seconds(), Labels{"component": "execution_recovery"})
r.SetGauge("orbit_runtime_batch_limit", float64(config.BatchLimit), nil)
r.SetGauge("orbit_shutdown_timeout_seconds", config.ShutdownTimeout.Seconds(), nil)
r.SetGauge("orbit_worker_connection_limit", float64(config.WorkerConnectionLimit), nil)
r.SetGauge("orbit_event_connection_limit", float64(config.EventConnectionLimit), nil)
r.SetGauge("orbit_dispatch_workers", float64(config.DispatchWorkers), nil)
r.SetGauge("orbit_dispatch_capacity", float64(config.DispatchCapacity), nil)
}
// RecordDraining records whether this process has entered graceful shutdown.

7
internal/transport/httpserver/event_hub.go
View File

@@ -31,10 +31,11 @@ type EventHub struct {
poll time.Duration
}
func NewEventHub(parent context.Context, service *events.Service) *EventHub {
func NewEventHub(parent context.Context, service *events.Service, options ...HubOptions) *EventHub {
ctx, cancel := context.WithCancel(parent)
h := &EventHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, 128), poll: time.Second}
h.dispatch = dispatch.New(ctx, 4, 128)
limits := resolveHubOptions(options)
h := &EventHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, limits.ConnectionLimit), poll: time.Second}
h.dispatch = dispatch.New(ctx, limits.DispatchWorkers, limits.DispatchCapacity)
h.wg.Add(1)
go func() { defer h.wg.Done(); <-h.dispatch.Done() }()
return h

48
internal/transport/httpserver/hub_options.go Normal file
View File

@@ -0,0 +1,48 @@
package httpserver
// HubOptions 是节点内 Worker/Event WebSocket Hub 的有界运行参数。
// 这些参数只控制本节点内存和领取并发，不改变 MySQL 中的授权、租约或配额事实。
type HubOptions struct {
// ConnectionLimit 是该 Hub 允许的同时连接数；超过后返回 429。
ConnectionLimit int
// DispatchWorkers 是本 Hub 同时执行数据库领取调用的 Go 协程上限。
DispatchWorkers int
// DispatchCapacity 是领取调用的有界内存排队槽位；满额时连接等待下一轮扫描。
DispatchCapacity int
}
const (
defaultHubConnectionLimit = 128
defaultHubDispatchWorkers = 4
defaultHubDispatchCapacity = 128
maxHubConnectionLimit = 4096
maxHubDispatchWorkers = 256
maxHubDispatchCapacity = 65536
)
func resolveHubOptions(options []HubOptions) HubOptions {
out := HubOptions{
ConnectionLimit: defaultHubConnectionLimit,
DispatchWorkers: defaultHubDispatchWorkers,
DispatchCapacity: defaultHubDispatchCapacity,
}
if len(options) > 1 {
panic("at most one HubOptions value is allowed")
}
if len(options) == 1 {
in := options[0]
if in.ConnectionLimit != 0 {
out.ConnectionLimit = in.ConnectionLimit
}
if in.DispatchWorkers != 0 {
out.DispatchWorkers = in.DispatchWorkers
}
if in.DispatchCapacity != 0 {
out.DispatchCapacity = in.DispatchCapacity
}
}
if out.ConnectionLimit < 1 || out.ConnectionLimit > maxHubConnectionLimit || out.DispatchWorkers < 1 || out.DispatchWorkers > maxHubDispatchWorkers || out.DispatchCapacity < 1 || out.DispatchCapacity > maxHubDispatchCapacity {
panic("HubOptions values exceed bounded range")
}
return out
}

42
internal/transport/httpserver/hub_options_test.go Normal file
View File

@@ -0,0 +1,42 @@
package httpserver
import (
"context"
"testing"
"git.yuntuo.hk/k2safe/orbit/platform/events"
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
)
func TestHubOptionsAreBoundedAndApplied(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
workerOptions := resolveHubOptions([]HubOptions{{ConnectionLimit: 7, DispatchWorkers: 3, DispatchCapacity: 11}})
if workerOptions.ConnectionLimit != 7 || workerOptions.DispatchWorkers != 3 || workerOptions.DispatchCapacity != 11 {
t.Fatalf("worker hub options not resolved: %+v", workerOptions)
}
workerHub := NewWorkerHub(ctx, worker.New(nil), workerOptions)
if cap(workerHub.slots) != 7 {
t.Fatalf("worker connection limit not applied: %d", cap(workerHub.slots))
}
eventOptions := resolveHubOptions([]HubOptions{{ConnectionLimit: 5, DispatchWorkers: 2, DispatchCapacity: 9}})
eventHub := NewEventHub(ctx, events.New(nil), eventOptions)
if cap(eventHub.slots) != 5 {
t.Fatalf("event connection limit not applied: %d", cap(eventHub.slots))
}
workerHub.Shutdown(context.Background())
eventHub.Shutdown(context.Background())
}
func TestHubOptionsRejectInvalidValues(t *testing.T) {
for _, options := range []HubOptions{{ConnectionLimit: -1}, {ConnectionLimit: maxHubConnectionLimit + 1}, {DispatchWorkers: -1}, {DispatchWorkers: maxHubDispatchWorkers + 1}, {DispatchCapacity: -1}, {DispatchCapacity: maxHubDispatchCapacity + 1}} {
t.Run("invalid", func(t *testing.T) {
defer func() {
if recover() == nil {
t.Fatal("invalid hub options did not panic")
}
}()
resolveHubOptions([]HubOptions{options})
})
}
}

7
internal/transport/httpserver/worker.go
View File

@@ -38,10 +38,11 @@ type WorkerHub struct {
emptyPolls map[[32]byte]time.Time
}
func NewWorkerHub(parent context.Context, service *worker.Service) *WorkerHub {
func NewWorkerHub(parent context.Context, service *worker.Service, options ...HubOptions) *WorkerHub {
ctx, cancel := context.WithCancel(parent)
h := &WorkerHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, 128), poll: time.Second, emptyPolls: make(map[[32]byte]time.Time)}
h.dispatch = dispatch.New(ctx, 4, 128)
limits := resolveHubOptions(options)
h := &WorkerHub{service: service, ctx: ctx, cancel: cancel, slots: make(chan struct{}, limits.ConnectionLimit), poll: time.Second, emptyPolls: make(map[[32]byte]time.Time)}
h.dispatch = dispatch.New(ctx, limits.DispatchWorkers, limits.DispatchCapacity)
h.wg.Add(1)
go func() { defer h.wg.Done(); <-h.dispatch.Done() }()
return h