feat: configure graceful shutdown window

This commit is contained in:
2026-09-20 20:07:32 +08:00
parent c3e9610258
commit a557ba5af0
11 changed files with 32 additions and 13 deletions

View File

@@ -19,6 +19,7 @@ type Config struct {
ScheduleTickInterval time.Duration // ScheduleTickInterval 计划扫描间隔,范围100ms–1m。
RecoveryTickInterval time.Duration // RecoveryTickInterval 执行租约恢复扫描间隔,范围100ms–1m。
RuntimeBatchLimit int // RuntimeBatchLimit 每个后台批次的候选上限,范围1–100。
ShutdownTimeout time.Duration // ShutdownTimeout 优雅排空和关闭窗口,范围1s–5m。
}
func Load() (Config, error) { return FromEnv(os.Getenv) }
@@ -33,6 +34,7 @@ func FromEnv(get func(string) string) (Config, error) {
ScheduleTickInterval: time.Second,
RecoveryTickInterval: 2 * time.Second,
RuntimeBatchLimit: 100,
ShutdownTimeout: 10 * time.Second,
}
if c.HTTPAddr == "" {
c.HTTPAddr = "127.0.0.1:8080"
@@ -77,6 +79,13 @@ func FromEnv(get func(string) string) (Config, error) {
}
c.RuntimeBatchLimit = n
}
if raw := get("ORBIT_SHUTDOWN_TIMEOUT"); raw != "" {
d, err := time.ParseDuration(raw)
if err != nil || d < time.Second || d > 5*time.Minute {
return Config{}, errors.New("ORBIT_SHUTDOWN_TIMEOUT must be a duration from 1s to 5m")
}
c.ShutdownTimeout = d
}
if c.MetricsAddr != "" {
_, port, err := net.SplitHostPort(c.MetricsAddr)
n, parseErr := strconv.Atoi(port)

View File

@@ -9,15 +9,16 @@ func TestRuntimeDefaultsAndBoundedOverrides(t *testing.T) {
base := map[string]string{"ORBIT_MYSQL_DSN": "private-dsn"}
get := func(k string) string { return base[k] }
c, err := FromEnv(get)
if err != nil || c.WorkflowTickInterval != time.Second || c.ScheduleTickInterval != time.Second || c.RecoveryTickInterval != 2*time.Second || c.RuntimeBatchLimit != 100 {
if err != nil || c.WorkflowTickInterval != time.Second || c.ScheduleTickInterval != time.Second || c.RecoveryTickInterval != 2*time.Second || c.RuntimeBatchLimit != 100 || c.ShutdownTimeout != 10*time.Second {
t.Fatalf("unexpected runtime defaults: %+v err=%v", c, err)
}
base["ORBIT_WORKFLOW_TICK_INTERVAL"] = "250ms"
base["ORBIT_SCHEDULE_TICK_INTERVAL"] = "1500ms"
base["ORBIT_RECOVERY_TICK_INTERVAL"] = "1m"
base["ORBIT_RUNTIME_BATCH_LIMIT"] = "37"
base["ORBIT_SHUTDOWN_TIMEOUT"] = "45s"
c, err = FromEnv(get)
if err != nil || c.WorkflowTickInterval != 250*time.Millisecond || c.ScheduleTickInterval != 1500*time.Millisecond || c.RecoveryTickInterval != time.Minute || c.RuntimeBatchLimit != 37 {
if err != nil || c.WorkflowTickInterval != 250*time.Millisecond || c.ScheduleTickInterval != 1500*time.Millisecond || c.RecoveryTickInterval != time.Minute || c.RuntimeBatchLimit != 37 || c.ShutdownTimeout != 45*time.Second {
t.Fatalf("valid runtime override rejected: %+v err=%v", c, err)
}
for key, value := range map[string]string{
@@ -25,6 +26,7 @@ func TestRuntimeDefaultsAndBoundedOverrides(t *testing.T) {
"ORBIT_SCHEDULE_TICK_INTERVAL": "2m",
"ORBIT_RECOVERY_TICK_INTERVAL": "not-a-duration",
"ORBIT_RUNTIME_BATCH_LIMIT": "101",
"ORBIT_SHUTDOWN_TIMEOUT": "500ms",
} {
base[key] = value
if _, err := FromEnv(get); err == nil {

View File

@@ -64,5 +64,6 @@ func catalog() map[string]descriptor {
c["orbit_business_snapshot_truncated"] = descriptor{gauge, "Whether a bounded business-state sample hit its per-state cap", nil}
c["orbit_runtime_tick_interval_seconds"] = descriptor{gauge, "Configured background controller tick interval in seconds", map[string][]string{"component": components}}
c["orbit_runtime_batch_limit"] = descriptor{gauge, "Configured maximum candidates per background batch", nil}
c["orbit_shutdown_timeout_seconds"] = descriptor{gauge, "Configured graceful drain and shutdown window in seconds", nil}
return c
}

View File

@@ -96,7 +96,7 @@ func TestInvalidNumbersAndMonotoneCounters(t *testing.T) {
}
func TestRuntimeObservationDoesNotExposeErrors(t *testing.T) {
r := NewRegistry()
r.RecordRuntimeConfig(RuntimeConfig{WorkflowTickInterval: time.Second, ScheduleTickInterval: 1500 * time.Millisecond, RecoveryTickInterval: 2 * time.Second, BatchLimit: 37})
r.RecordRuntimeConfig(RuntimeConfig{WorkflowTickInterval: time.Second, ScheduleTickInterval: 1500 * time.Millisecond, RecoveryTickInterval: 2 * time.Second, BatchLimit: 37, ShutdownTimeout: 45 * time.Second})
r.RecordBatch("workflow", time.Now().Add(-time.Millisecond), nil)
r.RecordBatch("workflow", time.Now(), errors.New("private-dsn"))
r.RecordBatch("schedule", time.Now(), context.Canceled)
@@ -107,7 +107,7 @@ func TestRuntimeObservationDoesNotExposeErrors(t *testing.T) {
t.Fatal("missing runtime evidence", want)
}
}
for _, want := range []string{`orbit_runtime_tick_interval_seconds{component="workflow"} 1`, `orbit_runtime_tick_interval_seconds{component="schedule"} 1.5`, `orbit_runtime_tick_interval_seconds{component="execution_recovery"} 2`, "orbit_runtime_batch_limit 37\n"} {
for _, want := range []string{`orbit_runtime_tick_interval_seconds{component="workflow"} 1`, `orbit_runtime_tick_interval_seconds{component="schedule"} 1.5`, `orbit_runtime_tick_interval_seconds{component="execution_recovery"} 2`, "orbit_runtime_batch_limit 37\n", "orbit_shutdown_timeout_seconds 45\n"} {
if !strings.Contains(out, want) {
t.Fatal("missing runtime configuration", want)
}

View File

@@ -17,17 +17,19 @@ type RuntimeConfig struct {
ScheduleTickInterval time.Duration
RecoveryTickInterval time.Duration
BatchLimit int
ShutdownTimeout time.Duration
}
// RecordRuntimeConfig records the effective bounded controller settings.
func (r *Registry) RecordRuntimeConfig(config RuntimeConfig) {
if r == nil || config.WorkflowTickInterval <= 0 || config.ScheduleTickInterval <= 0 || config.RecoveryTickInterval <= 0 || config.BatchLimit <= 0 {
if r == nil || config.WorkflowTickInterval <= 0 || config.ScheduleTickInterval <= 0 || config.RecoveryTickInterval <= 0 || config.BatchLimit <= 0 || config.ShutdownTimeout <= 0 {
return
}
r.SetGauge("orbit_runtime_tick_interval_seconds", config.WorkflowTickInterval.Seconds(), Labels{"component": "workflow"})
r.SetGauge("orbit_runtime_tick_interval_seconds", config.ScheduleTickInterval.Seconds(), Labels{"component": "schedule"})
r.SetGauge("orbit_runtime_tick_interval_seconds", config.RecoveryTickInterval.Seconds(), Labels{"component": "execution_recovery"})
r.SetGauge("orbit_runtime_batch_limit", float64(config.BatchLimit), nil)
r.SetGauge("orbit_shutdown_timeout_seconds", config.ShutdownTimeout.Seconds(), nil)
}
// initializeProcess is called once by the registry constructor.