feat: tune websocket hub backpressure

This commit is contained in:
2026-09-20 20:53:50 +08:00
parent 850376624f
commit 478be069e6
16 changed files with 218 additions and 52 deletions

View File

@@ -66,5 +66,9 @@ func catalog() map[string]descriptor {
c["orbit_runtime_tick_interval_seconds"] = descriptor{gauge, "Configured background controller tick interval in seconds", map[string][]string{"component": components}}
c["orbit_runtime_batch_limit"] = descriptor{gauge, "Configured maximum candidates per background batch", nil}
c["orbit_shutdown_timeout_seconds"] = descriptor{gauge, "Configured graceful drain and shutdown window in seconds", nil}
c["orbit_worker_connection_limit"] = descriptor{gauge, "Configured maximum Worker WebSocket connections on this node", nil}
c["orbit_event_connection_limit"] = descriptor{gauge, "Configured maximum Event WebSocket connections on this node", nil}
c["orbit_dispatch_workers"] = descriptor{gauge, "Configured database claim workers per WebSocket hub", nil}
c["orbit_dispatch_capacity"] = descriptor{gauge, "Configured bounded in-memory claim queue per WebSocket hub", nil}
return c
}

View File

@@ -96,7 +96,7 @@ func TestInvalidNumbersAndMonotoneCounters(t *testing.T) {
}
func TestRuntimeObservationDoesNotExposeErrors(t *testing.T) {
r := NewRegistry()
r.RecordRuntimeConfig(RuntimeConfig{WorkflowTickInterval: time.Second, ScheduleTickInterval: 1500 * time.Millisecond, RecoveryTickInterval: 2 * time.Second, BatchLimit: 37, ShutdownTimeout: 45 * time.Second})
r.RecordRuntimeConfig(RuntimeConfig{WorkflowTickInterval: time.Second, ScheduleTickInterval: 1500 * time.Millisecond, RecoveryTickInterval: 2 * time.Second, BatchLimit: 37, ShutdownTimeout: 45 * time.Second, WorkerConnectionLimit: 512, EventConnectionLimit: 256, DispatchWorkers: 16, DispatchCapacity: 2048})
r.RecordDraining(true)
r.RecordBatch("workflow", time.Now().Add(-time.Millisecond), nil)
r.RecordBatch("workflow", time.Now(), errors.New("private-dsn"))
@@ -108,7 +108,7 @@ func TestRuntimeObservationDoesNotExposeErrors(t *testing.T) {
t.Fatal("missing runtime evidence", want)
}
}
for _, want := range []string{`orbit_runtime_tick_interval_seconds{component="workflow"} 1`, `orbit_runtime_tick_interval_seconds{component="schedule"} 1.5`, `orbit_runtime_tick_interval_seconds{component="execution_recovery"} 2`, "orbit_runtime_batch_limit 37\n", "orbit_shutdown_timeout_seconds 45\n", "orbit_process_draining 1\n"} {
for _, want := range []string{`orbit_runtime_tick_interval_seconds{component="workflow"} 1`, `orbit_runtime_tick_interval_seconds{component="schedule"} 1.5`, `orbit_runtime_tick_interval_seconds{component="execution_recovery"} 2`, "orbit_runtime_batch_limit 37\n", "orbit_shutdown_timeout_seconds 45\n", "orbit_worker_connection_limit 512\n", "orbit_event_connection_limit 256\n", "orbit_dispatch_workers 16\n", "orbit_dispatch_capacity 2048\n", "orbit_process_draining 1\n"} {
if !strings.Contains(out, want) {
t.Fatal("missing runtime configuration", want)
}

View File

@@ -13,16 +13,20 @@ import (
// RuntimeConfig 是节点启动后实际生效的后台控制参数。
// 它只用于固定基数观测,不参与业务授权或持久化状态判断。
type RuntimeConfig struct {
WorkflowTickInterval time.Duration
ScheduleTickInterval time.Duration
RecoveryTickInterval time.Duration
BatchLimit int
ShutdownTimeout time.Duration
WorkflowTickInterval time.Duration
ScheduleTickInterval time.Duration
RecoveryTickInterval time.Duration
BatchLimit int
ShutdownTimeout time.Duration
WorkerConnectionLimit int // Worker WebSocket 单节点连接上限。
EventConnectionLimit int // Event WebSocket 单节点连接上限。
DispatchWorkers int // 每个 Hub 同时执行数据库领取调用的协程上限。
DispatchCapacity int // 每个 Hub 领取调用的有界内存排队槽位。
}
// RecordRuntimeConfig records the effective bounded controller settings.
func (r *Registry) RecordRuntimeConfig(config RuntimeConfig) {
if r == nil || config.WorkflowTickInterval <= 0 || config.ScheduleTickInterval <= 0 || config.RecoveryTickInterval <= 0 || config.BatchLimit <= 0 || config.ShutdownTimeout <= 0 {
if r == nil || config.WorkflowTickInterval <= 0 || config.ScheduleTickInterval <= 0 || config.RecoveryTickInterval <= 0 || config.BatchLimit <= 0 || config.ShutdownTimeout <= 0 || config.WorkerConnectionLimit <= 0 || config.EventConnectionLimit <= 0 || config.DispatchWorkers <= 0 || config.DispatchCapacity <= 0 {
return
}
r.SetGauge("orbit_runtime_tick_interval_seconds", config.WorkflowTickInterval.Seconds(), Labels{"component": "workflow"})
@@ -30,6 +34,10 @@ func (r *Registry) RecordRuntimeConfig(config RuntimeConfig) {
r.SetGauge("orbit_runtime_tick_interval_seconds", config.RecoveryTickInterval.Seconds(), Labels{"component": "execution_recovery"})
r.SetGauge("orbit_runtime_batch_limit", float64(config.BatchLimit), nil)
r.SetGauge("orbit_shutdown_timeout_seconds", config.ShutdownTimeout.Seconds(), nil)
r.SetGauge("orbit_worker_connection_limit", float64(config.WorkerConnectionLimit), nil)
r.SetGauge("orbit_event_connection_limit", float64(config.EventConnectionLimit), nil)
r.SetGauge("orbit_dispatch_workers", float64(config.DispatchWorkers), nil)
r.SetGauge("orbit_dispatch_capacity", float64(config.DispatchCapacity), nil)
}
// RecordDraining records whether this process has entered graceful shutdown.