feat: expose graceful drain state
This commit is contained in:
@@ -259,6 +259,9 @@ func run() error {
|
||||
return err
|
||||
case <-root.Done():
|
||||
readiness.BeginDrain()
|
||||
if metrics != nil {
|
||||
metrics.RecordDraining(true)
|
||||
}
|
||||
shutdown, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout)
|
||||
defer cancel()
|
||||
if err := server.Shutdown(shutdown); err != nil {
|
||||
|
||||
@@ -69,6 +69,7 @@ curl -fsS http://127.0.0.1:9090/metrics
|
||||
| `orbit_business_snapshot_last_success_timestamp_seconds` | gauge | 最近一次完整成功快照的Unix时间;失败不会推进 |
|
||||
| `orbit_business_snapshot_truncated` | gauge | 任一固定状态达到10000上限时为1,否则为0 |
|
||||
| `orbit_process_start_time_seconds` / `orbit_process_uptime_seconds` | gauge | 本次指标注册器初始化Unix时间/单调时钟经过的秒数;初始化发生在数据库启动校验之后 |
|
||||
| `orbit_process_draining` | gauge | 节点是否正在优雅排空;0为服务中,1为排空中 |
|
||||
| `orbit_go_goroutines` | gauge | Go协程数,按5秒采样 |
|
||||
| `orbit_build_info{go_version,version,revision}` | gauge | 恒为1,导出当前Go版本、发布版本和源码提交;构建时间及完整协议身份通过 `/version` 获取 |
|
||||
| `orbit_observability_rejected_samples_total` | counter | 内部不合法指标更新次数,正常运行应为0 |
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
| 就绪探针排空门控 | `go test -race ./internal/transport/httpserver ./cmd/orbit`;`TestReadinessEntersDrainBeforeShutdown` | PASS;SIGTERM 关闭路径先将 `/health/ready` 置为 503,再进入限定优雅关闭窗口;响应不泄露内部错误 |
|
||||
| 服务端单元与竞态、vet、构建 | Docker `golang:1.27.1`,仓库声明版本;`GOWORK=off go test -race ./... && go vet ./... && go build ./...` | PASS |
|
||||
| 后台运行参数边界 | `internal/config.TestRuntimeDefaultsAndBoundedOverrides`;默认值与五个 `ORBIT_*` 覆盖项 | PASS;循环间隔限定100ms–1m,批量限定1–100,排空超时限定1s–5m,服务启动日志记录生效值 |
|
||||
| 生效运行参数指标 | `internal/observability.TestRuntimeObservationDoesNotExposeErrors`;固定指标目录 | PASS;workflow/schedule/execution_recovery 间隔、批量上限和排空超时以固定基数指标导出,不进入业务或租户标签 |
|
||||
| 生效运行参数指标 | `internal/observability.TestRuntimeObservationDoesNotExposeErrors`;固定指标目录 | PASS;workflow/schedule/execution_recovery 间隔、批量上限和排空超时以固定基数指标导出;排空状态同步为固定 gauge,不进入业务或租户标签 |
|
||||
| SDK 与独立示例竞态、vet、构建 | Docker `golang:1.27.1`;SDK 全 module、全部示例、`orbit-load`、`orbit-exec-load`、`orbit-schedule-load`、`orbit-workflow-load`,以及 result-inbox/task-outbox | PASS;健康/版本 SDK 契约包含 `Client.Version(ctx)`,返回协议身份且不暴露敏感响应 |
|
||||
| 容器发布物构建 | `make image`;宿主平台 linux/arm64;显式 `docker build --platform linux/amd64`;带 `VERSION`、`REVISION`、`BUILD_TIME` 构建参数的版本化镜像;镜像内 `GOWORK=off` | PASS;native 与 amd64 镜像均构建成功,包含 `orbit`、`orbit-migrate`、`orbit-admin`,运行用户为非 root;版本化 amd64 二进制和 OCI 标签均包含注入的构建身份,`GET /version` 契约单测通过;无 DSN 时服务按预期拒绝启动 |
|
||||
| 当前协议门禁 | `make protocol-check` | PASS,当前 v1 契约同步 |
|
||||
|
||||
@@ -29,6 +29,7 @@ func catalog() map[string]descriptor {
|
||||
"orbit_runtime_batch_last_success_timestamp_seconds": {gauge, "Unix time of last successful background batch, zero before first success", map[string][]string{"component": components}},
|
||||
"orbit_process_start_time_seconds": {gauge, "Unix time when the Orbit metrics registry was started", nil},
|
||||
"orbit_process_uptime_seconds": {gauge, "Monotonic elapsed time since metrics registry startup", nil},
|
||||
"orbit_process_draining": {gauge, "Whether this process is draining for graceful shutdown", nil},
|
||||
"orbit_go_goroutines": {gauge, "Current Go goroutine count", nil},
|
||||
"orbit_build_info": {gauge, "Orbit build and protocol information", map[string][]string{"go_version": {build.GoVersion}, "version": {build.Version}, "revision": {build.Revision}}},
|
||||
}
|
||||
|
||||
@@ -97,6 +97,7 @@ func TestInvalidNumbersAndMonotoneCounters(t *testing.T) {
|
||||
func TestRuntimeObservationDoesNotExposeErrors(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
r.RecordRuntimeConfig(RuntimeConfig{WorkflowTickInterval: time.Second, ScheduleTickInterval: 1500 * time.Millisecond, RecoveryTickInterval: 2 * time.Second, BatchLimit: 37, ShutdownTimeout: 45 * time.Second})
|
||||
r.RecordDraining(true)
|
||||
r.RecordBatch("workflow", time.Now().Add(-time.Millisecond), nil)
|
||||
r.RecordBatch("workflow", time.Now(), errors.New("private-dsn"))
|
||||
r.RecordBatch("schedule", time.Now(), context.Canceled)
|
||||
@@ -107,7 +108,7 @@ func TestRuntimeObservationDoesNotExposeErrors(t *testing.T) {
|
||||
t.Fatal("missing runtime evidence", want)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{`orbit_runtime_tick_interval_seconds{component="workflow"} 1`, `orbit_runtime_tick_interval_seconds{component="schedule"} 1.5`, `orbit_runtime_tick_interval_seconds{component="execution_recovery"} 2`, "orbit_runtime_batch_limit 37\n", "orbit_shutdown_timeout_seconds 45\n"} {
|
||||
for _, want := range []string{`orbit_runtime_tick_interval_seconds{component="workflow"} 1`, `orbit_runtime_tick_interval_seconds{component="schedule"} 1.5`, `orbit_runtime_tick_interval_seconds{component="execution_recovery"} 2`, "orbit_runtime_batch_limit 37\n", "orbit_shutdown_timeout_seconds 45\n", "orbit_process_draining 1\n"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatal("missing runtime configuration", want)
|
||||
}
|
||||
|
||||
@@ -32,12 +32,26 @@ func (r *Registry) RecordRuntimeConfig(config RuntimeConfig) {
|
||||
r.SetGauge("orbit_shutdown_timeout_seconds", config.ShutdownTimeout.Seconds(), nil)
|
||||
}
|
||||
|
||||
// RecordDraining records whether this process has entered graceful shutdown.
|
||||
// The value is a fixed, node-local gauge: 0 means serving and 1 means draining.
|
||||
func (r *Registry) RecordDraining(draining bool) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
value := 0.0
|
||||
if draining {
|
||||
value = 1
|
||||
}
|
||||
r.SetGauge("orbit_process_draining", value, nil)
|
||||
}
|
||||
|
||||
// initializeProcess is called once by the registry constructor.
|
||||
func (r *Registry) initializeProcess() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.SetGauge("orbit_process_start_time_seconds", float64(r.started.Unix()), nil)
|
||||
r.RecordDraining(false)
|
||||
build := buildinfo.Current()
|
||||
r.SetGauge("orbit_build_info", 1, Labels{"go_version": build.GoVersion, "version": build.Version, "revision": build.Revision})
|
||||
r.RecordProcess()
|
||||
|
||||
Reference in New Issue
Block a user