feat: add generic phased background shutdown to managed services

2026-09-25 00:35:59 +08:00
parent adcb6f60b6
commit 16deead70e
37 changed files with 1330 additions and 68 deletions
--- a/lifecycle/README.md
+++ b/lifecycle/README.md
@@ -1,5 +1,67 @@
 # lifecycle

+## 后台组件分阶段关闭
+
+从 `v0.15.0-preview` 起，生成项目使用以下顺序：HTTP 停止接收并排空 → 全部 OnStop → 全部 OnDrain → 逆序 OnClose → 关闭日志。初始化、创建 App 或监听失败也经过后台收尾。Runtime 不导入 Orbit、数据库、ORM 或 Redis。
+
+在项目维护的 svc.Init 中登记即可，main/runtimeGen 由生成器更新。例如使用可选的内存 task.Runner：
+
+```go
+jobs, err := task.New(task.Config{Workers: 4, QueueSize: 128})
+if err != nil { return err }
+c.OnStop("jobs", jobs.StopAccepting)
+c.OnDrain("jobs", jobs.Wait)
+// 把 jobs 保存到项目的 svc 字段，供 Logic 使用。
+// 已接受任务使用 c.WorkContext()，保留到 HTTP 和后台正常排空结束。
+```
+
+数据库、Redis 或第三方客户端仍用已有的 `c.OnClose(client.Close)`；框架创建的 DB 已登记，不要再次登记。这里的 jobs 不再额外登记 Close：StopAccepting 停止定时调度/提交，Wait 等待 worker 与回调全部退出，已完成其生命周期。
+
+任何自定义组件都可以登记 `func(context.Context) error`，不必实现或依赖框架任务接口：
+
+- OnStop 只停止接收新工作，不能关闭正在执行的工作所需的资源。每个 OnStop 必须有同名 OnDrain；缺失时拒绝释放共享资源。
+- OnDrain 返回 nil 表示已经收尾，包含最终写入、确认和监控协程退出。返回错误不作为已完成证明；运行中的业务错误可以先通过 RequestShutdown(err) 上报，再等完成后由 Drain 返回 nil。
+- 同一阶段的回调并发执行，不按登记顺序执行；框架等待整个阶段结束后才进入下一阶段。有组件之间的先后依赖时，组合为一个 Stop/Drain 登记项，由该项明确组织内部顺序。Stop 不能等待 Drain 才会完成的工作。
+- OnClose 保留原来的同步逆序执行；普通项目无后台组件时无需登记新钩子。生命周期钩子不能再次调用 Runtime.Close/Finish，否则会等待自身；可以调用 RequestShutdown。
+- 所有依赖初始化成功后立即登记，再启动使用它们的工作。构造器交付前失败的内部资源仍由构造器释放。
+
+### 两种 context
+
+`c.Context()` / `c.Lifecycle.Context()` 是早期退出通知，会随外部信号或后台故障取消。`c.WorkContext()` 保留初始化 context 的值但不继承其取消/截止时间，允许已接受工作在 HTTP 收尾期间继续使用依赖。
+
+停止接收工作由 OnStop 负责，不能仅等待 WorkContext 取消。正常情况下 WorkContext 在后台排空完成后取消；阶段失败或超时会请求取消工作。框架不假设取消已经意味着 goroutine 结束，仍以 Drain 的完成为准。每个业务任务的执行时限继续由项目/任务组件设置。
+
+### 时限与失败
+
+```yaml
+server:
+  shutdownTimeout: 10s # HTTP 排空时限，保持原语义
+  lifecycle:
+    stopTimeout: 5s
+    drainTimeout: 10s
+    cancelTimeout: 5s
+```
+
+这些是各阶段的整体预算，不是每个回调分别等待这么久。零使用默认值，负值在启动前拒绝。可用 `GO_OMNI_LIFECYCLE_STOP_TIMEOUT`、`GO_OMNI_LIFECYCLE_DRAIN_TIMEOUT`、`GO_OMNI_LIFECYCLE_CANCEL_TIMEOUT` 覆盖；生成配置还支持完整的 `GO_OMNI_SERVER_LIFECYCLE_*` 路径，别名优先。配置应在 config 中设置，不放在创建 App 时才执行的 AppOptions 中。
+
+阶段超时会取消阶段 context 和 WorkContext，再给尚未返回的钩子一个 cancelTimeout 收尾窗口。若所有钩子都返回 nil，可以释放依赖，但仍返回超时错误，进程非零退出。task.Wait 收到取消会取消其任务并返回错误；这不证明全部任务已退出，因此仍按以下保守规则处理。
+
+Stop 超时且仍未返回的组件，不会再并发调用它同名的 Drain；其他组件仍可继续收尾。此时共享资源始终保留，避免未退出的 Stop 和 Drain 同时操作同一个组件。
+
+钩子返回错误、panic、缺少匹配 Drain，或窗口结束后仍有钩子未返回时，Runtime 返回可由 `errors.Is(err, lifecycle.ErrResourcesRetained)` 判断的错误，**跳过全部 OnClose**；生成入口也保留借用的日志实例并非零退出，由进程退出释放剩余 OS 资源。不会悄悄重试 Close，也不会让晚到的后台结果触发自动关闭。错误链保留初始化/监听/后台故障及具体 HookError 的阶段、名称和原始原因。
+
+手写入口必须处理这个错误并终止该服务，不能继续运行半关闭的 Runtime；嵌入式宿主需自行安排隔离/终止策略。Go 不能强杀不响应取消的 goroutine，有限等待与保证它已经结束不能同时无条件成立。OnClose 仍要求自身能返回；本版阶段时限不会强制中断资源 Close。HTTP 超时沿用现有传输层规则，业务处理器仍须响应请求取消。
+
+手写项目使用 `lifecycle.NewRuntimeWithConfig(ctx, cfg)`；完成 HTTP 排空后调用 `runtime.Finish(runErr)`。库本身不执行 os.Exit。
+
+### 升级
+
+v0.14.x 受管理项目将 module 和 CLI 一起升级到 v0.15.0-preview，再运行 `go-omni api generate`、`go-omni api generate --check` 和业务测试。生成器更新 main/runtimeGen，保留 svc、Logic 和配置。已有配置不添加新字段也会使用默认时限，需要调整时再编辑 yaml。
+
+要获得后台分阶段保证，项目需把原先 `OnClose(worker.Close)` 的后台组件改成对应 Stop/Drain，并使用适合的工作 context；框架不会猜测旧 Close 回调负责数据库还是 worker。仅在 OnClose 中登记的旧任务继续遵循旧的逆序清理规则。
+
+## 退出通知
+
 Controller 是服务退出通知入口，不管理具体数据库、租约或后台任务。生成项目已经将它接入 svc 与 main，业务在 svc 调用 `c.RequestShutdown(err)` 即可，见 [项目指南](../PROJECT.md#依赖主动申请退出)。

 手写服务也可组合现有 App：
--- a/lifecycle/phases.go
+++ b/lifecycle/phases.go
@@ -0,0 +1,184 @@
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+	"unicode"
+)
+
+// ErrResourcesRetained means background quiescence could not be established.
+// OnClose callbacks were deliberately skipped. Close is not retried; an owning
+// process should exit unsuccessfully instead of continuing with a stopped Runtime.
+var ErrResourcesRetained = errors.New("lifecycle: background shutdown incomplete; shared resources retained")
+
+type ShutdownConfig struct {
+	StopTimeout   time.Duration `json:"stopTimeout"`
+	DrainTimeout  time.Duration `json:"drainTimeout"`
+	CancelTimeout time.Duration `json:"cancelTimeout"`
+}
+
+func DefaultShutdownConfig() ShutdownConfig {
+	return ShutdownConfig{StopTimeout: 5 * time.Second, DrainTimeout: 10 * time.Second, CancelTimeout: 5 * time.Second}
+}
+func (c ShutdownConfig) Validate() error {
+	if c.StopTimeout < 0 || c.DrainTimeout < 0 || c.CancelTimeout < 0 {
+		return errors.New("lifecycle: shutdown timeouts must be nonnegative")
+	}
+	return nil
+}
+func (c ShutdownConfig) defaults() ShutdownConfig {
+	d := DefaultShutdownConfig()
+	if c.StopTimeout == 0 {
+		c.StopTimeout = d.StopTimeout
+	}
+	if c.DrainTimeout == 0 {
+		c.DrainTimeout = d.DrainTimeout
+	}
+	if c.CancelTimeout == 0 {
+		c.CancelTimeout = d.CancelTimeout
+	}
+	return c
+}
+
+// HookError identifies the failed phase/component and preserves the original error.
+type HookError struct {
+	Phase, Name string
+	Err         error
+}
+
+func (e *HookError) Error() string { return fmt.Sprintf("lifecycle %s %q: %v", e.Phase, e.Name, e.Err) }
+func (e *HookError) Unwrap() error { return e.Err }
+
+// phaseError keeps diagnostics separate from proof that every hook succeeded.
+// A deadline is still reported when all callbacks finish during cancellation grace.
+type phaseError struct {
+	err      error
+	unsafe   bool
+	unjoined map[string]bool
+}
+
+func (e *phaseError) Error() string { return e.err.Error() }
+func (e *phaseError) Unwrap() error { return e.err }
+func unsafePhase(err error) bool    { var p *phaseError; return errors.As(err, &p) && p.unsafe }
+
+type shutdownHook struct {
+	name string
+	fn   func(context.Context) error
+}
+
+// OnStop registers a prompt, idempotent stop-admission callback. It must not
+// close resources used by accepted work. All stop hooks precede all drain hooks.
+// Hooks within a phase run concurrently: combine dependent components in one hook.
+// Every OnStop requires a matching OnDrain with the same name.
+func (r *Runtime) OnStop(name string, fn func(context.Context) error) { r.addHook(&r.stops, name, fn) }
+
+// OnDrain registers proof of quiescence. Return nil only after the component can
+// no longer use shared resources, including final acknowledgements/cleanup.
+// Deadline cancellation asks the component to stop; returning ctx.Err alone does
+// not prove completion. A callback may finish during the cancellation grace.
+func (r *Runtime) OnDrain(name string, fn func(context.Context) error) {
+	r.addHook(&r.drains, name, fn)
+}
+
+func (r *Runtime) addHook(list *[]shutdownHook, name string, fn func(context.Context) error) {
+	if fn == nil || name == "" || len(name) > 128 || strings.TrimSpace(name) != name || strings.IndexFunc(name, unicode.IsControl) >= 0 {
+		panic("lifecycle: valid hook name and function required")
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.closed {
+		panic("lifecycle: register hook after Close")
+	}
+	for _, h := range *list {
+		if h.name == name {
+			panic("lifecycle: duplicate phase hook name")
+		}
+	}
+	*list = append(*list, shutdownHook{name, fn})
+}
+
+func callHook(ctx context.Context, h shutdownHook) (err error) {
+	defer func() {
+		if p := recover(); p != nil {
+			if cause, ok := p.(error); ok {
+				err = fmt.Errorf("hook panic: %w", cause)
+			} else {
+				err = fmt.Errorf("hook panic: %v", p)
+			}
+		}
+	}()
+	return h.fn(ctx)
+}
+
+func (r *Runtime) runPhase(phase string, hooks []shutdownHook, timeout time.Duration) error {
+	if len(hooks) == 0 {
+		return nil
+	}
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(r.work), timeout)
+	defer cancel()
+	type result struct {
+		index int
+		err   error
+	}
+	results := make(chan result, len(hooks))
+	pending := make([]bool, len(hooks))
+	for i, h := range hooks {
+		pending[i] = true
+		go func(i int, h shutdownHook) { results <- result{i, callHook(ctx, h)} }(i, h)
+	}
+	remaining := len(hooks)
+	var errs error
+	unsafe := false
+	accept := func(v result) {
+		pending[v.index] = false
+		remaining--
+		if v.err != nil {
+			unsafe = true
+			errs = errors.Join(errs, &HookError{phase, hooks[v.index].name, v.err})
+			r.cancelWork()
+		}
+	}
+	for remaining > 0 {
+		select {
+		case v := <-results:
+			accept(v)
+		case <-ctx.Done():
+			r.cancelWork()
+			errs = errors.Join(errs, &HookError{phase, "deadline", ctx.Err()})
+			grace := time.NewTimer(r.shutdown.CancelTimeout)
+			defer grace.Stop()
+			for remaining > 0 {
+				select {
+				case v := <-results:
+					accept(v)
+				case <-grace.C:
+					// Collect callbacks already returned at the deadline before marking pending ones.
+					for {
+						select {
+						case v := <-results:
+							accept(v)
+						default:
+							goto collected
+						}
+					}
+				collected:
+					unjoined := make(map[string]bool)
+					for i, active := range pending {
+						if active {
+							unjoined[hooks[i].name] = true
+							errs = errors.Join(errs, &HookError{phase, hooks[i].name, context.DeadlineExceeded})
+						}
+					}
+					return &phaseError{err: errs, unsafe: unsafe || remaining > 0, unjoined: unjoined}
+				}
+			}
+		}
+	}
+	if errs != nil {
+		return &phaseError{err: errs, unsafe: unsafe}
+	}
+	return nil
+}
--- a/lifecycle/phases_test.go
+++ b/lifecycle/phases_test.go
@@ -0,0 +1,234 @@
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func testRuntime(t *testing.T) *Runtime {
+	t.Helper()
+	r, err := NewRuntimeWithConfig(context.Background(), ShutdownConfig{StopTimeout: time.Second, DrainTimeout: time.Second, CancelTimeout: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return r
+}
+
+func TestPhasesBarrierAndWorkLifetime(t *testing.T) {
+	parent, cancel := context.WithCancel(context.WithValue(context.Background(), struct{}{}, "value"))
+	defer cancel()
+	r := NewRuntime(parent)
+	var stops, drains atomic.Int32
+	var mu sync.Mutex
+	var closed []int
+	started := make(chan struct{}, 2)
+	release := make(chan struct{})
+	// Deliberately interleave registration. Phases take precedence over order.
+	r.OnClose(func() error { mu.Lock(); defer mu.Unlock(); closed = append(closed, 1); return nil })
+	for _, name := range []string{"producer", "consumer"} {
+		r.OnStop(name, func(ctx context.Context) error {
+			if r.WorkContext().Err() != nil || ctx.Err() != nil {
+				return errors.New("work canceled before drain")
+			}
+			stops.Add(1)
+			started <- struct{}{}
+			<-release
+			return nil
+		})
+		r.OnDrain(name, func(ctx context.Context) error {
+			if stops.Load() != 2 || r.WorkContext().Err() != nil || ctx.Err() != nil {
+				return errors.New("stage barrier failed")
+			}
+			drains.Add(1)
+			return nil
+		})
+	}
+	r.OnClose(func() error {
+		if drains.Load() != 2 {
+			return errors.New("dependency closed early")
+		}
+		mu.Lock()
+		defer mu.Unlock()
+		closed = append(closed, 2)
+		return nil
+	})
+	cancel()
+	if r.WorkContext().Err() != nil || r.WorkContext().Value(struct{}{}) != "value" {
+		t.Fatal("work context inherited cancellation or lost values")
+	}
+	done := make(chan error, 1)
+	go func() { done <- r.Finish(nil) }()
+	for i := 0; i < 2; i++ {
+		select {
+		case <-started:
+		case <-time.After(time.Second):
+			t.Fatal("stop callbacks did not run concurrently")
+		}
+	}
+	if drains.Load() != 0 {
+		t.Fatal("drain ran before stop barrier")
+	}
+	close(release)
+	if err := <-done; err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(closed, []int{2, 1}) || r.WorkContext().Err() == nil {
+		t.Fatal(closed)
+	}
+}
+
+func TestPhasesFailuresRetainDependenciesAndCauses(t *testing.T) {
+	for _, phase := range []string{"stop", "drain"} {
+		for _, panics := range []bool{false, true} {
+			t.Run(phase+map[bool]string{false: "Error", true: "Panic"}[panics], func(t *testing.T) {
+				r := testRuntime(t)
+				cause := errors.New("component cause")
+				fatal := errors.New("fatal monitor")
+				cleanup := atomic.Int32{}
+				other := atomic.Int32{}
+				r.OnClose(func() error { cleanup.Add(1); return nil })
+				hook := func(context.Context) error {
+					r.RequestShutdown(fatal)
+					if panics {
+						panic(cause)
+					}
+					return cause
+				}
+				if phase == "stop" {
+					r.OnStop("broken", hook)
+				} else {
+					r.OnDrain("broken", hook)
+				}
+				r.OnDrain("other", func(context.Context) error { other.Add(1); return nil })
+				err := r.Finish(nil)
+				var detail *HookError
+				if !errors.Is(err, ErrResourcesRetained) || !errors.Is(err, cause) || !errors.Is(err, fatal) || !errors.As(err, &detail) || detail.Phase != phase {
+					t.Fatal(err)
+				}
+				if cleanup.Load() != 0 || other.Load() != 1 || r.WorkContext().Err() == nil {
+					t.Fatal("unsafe cleanup or omitted drain")
+				}
+				if !errors.Is(r.Close(), ErrResourcesRetained) || other.Load() != 1 {
+					t.Fatal("close retried hooks")
+				}
+			})
+		}
+	}
+}
+
+func TestPhasesDeadlineCancelsWorkAndJoinsBeforeClose(t *testing.T) {
+	r, err := NewRuntimeWithConfig(context.Background(), ShutdownConfig{DrainTimeout: 20 * time.Millisecond, CancelTimeout: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var joined atomic.Bool
+	var closed atomic.Bool
+	r.OnDrain("worker", func(ctx context.Context) error {
+		<-ctx.Done()
+		<-r.WorkContext().Done()
+		// Cleanup can still use shared resources after forced cancellation.
+		if closed.Load() {
+			return errors.New("dependency closed during cancellation")
+		}
+		joined.Store(true)
+		return nil
+	})
+	r.OnClose(func() error {
+		if !joined.Load() {
+			t.Error("worker not joined")
+		}
+		closed.Store(true)
+		return nil
+	})
+	err = r.Close()
+	if !errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrResourcesRetained) || !closed.Load() {
+		t.Fatal(err)
+	}
+}
+
+func TestPhasesUncooperativeHookIsBoundedAndNeverClosesDependencies(t *testing.T) {
+	for _, phase := range []string{"stop", "drain"} {
+		t.Run(phase, func(t *testing.T) {
+			r, err := NewRuntimeWithConfig(context.Background(), ShutdownConfig{StopTimeout: 20 * time.Millisecond, DrainTimeout: 20 * time.Millisecond, CancelTimeout: 20 * time.Millisecond})
+			if err != nil {
+				t.Fatal(err)
+			}
+			release := make(chan struct{})
+			finished := make(chan struct{})
+			var closed atomic.Int32
+			hook := func(context.Context) error { defer close(finished); <-release; return nil }
+			if phase == "stop" {
+				r.OnStop("stuck", hook)
+				r.OnDrain("stuck", func(context.Context) error { t.Error("Drain overlapped unfinished Stop"); return nil })
+			} else {
+				r.OnDrain("stuck", hook)
+			}
+			r.OnClose(func() error { closed.Add(1); return nil })
+			began := time.Now()
+			err = r.Finish(errors.New("listen failed"))
+			close(release)
+			<-finished
+			if time.Since(began) > time.Second || !errors.Is(err, ErrResourcesRetained) || !errors.Is(err, context.DeadlineExceeded) || closed.Load() != 0 {
+				t.Fatal(err)
+			}
+			_ = r.Close()
+			if closed.Load() != 0 {
+				t.Fatal("late cleanup raced work")
+			}
+		})
+	}
+}
+
+func TestPhasesConcurrentFinishAndValidation(t *testing.T) {
+	if _, err := NewRuntimeWithConfig(nil, ShutdownConfig{DrainTimeout: -1}); err == nil {
+		t.Fatal("negative timeout accepted")
+	}
+	r := testRuntime(t)
+	var called atomic.Int32
+	r.OnStop("worker", func(context.Context) error { called.Add(1); return nil })
+	r.OnDrain("worker", func(context.Context) error { called.Add(1); return nil })
+	r.OnClose(func() error { called.Add(1); return nil })
+	var wg sync.WaitGroup
+	for i := 0; i < 20; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := r.Finish(nil); err != nil {
+				t.Error(err)
+			}
+		}()
+	}
+	wg.Wait()
+	if called.Load() != 3 {
+		t.Fatal(called.Load())
+	}
+	for _, fn := range []func(){func() { r.OnStop("late", func(context.Context) error { return nil }) }, func() { NewRuntime(nil).OnDrain("bad\nname", func(context.Context) error { return nil }) }, func() {
+		x := NewRuntime(nil)
+		x.OnStop("duplicate", func(context.Context) error { return nil })
+		x.OnStop("duplicate", func(context.Context) error { return nil })
+	}} {
+		func() {
+			defer func() {
+				if recover() == nil {
+					t.Error("invalid registration accepted")
+				}
+			}()
+			fn()
+		}()
+	}
+}
+
+func TestMissingDrainRetainsResources(t *testing.T) {
+	r := testRuntime(t)
+	r.OnStop("worker", func(context.Context) error { return nil })
+	closed := false
+	r.OnClose(func() error { closed = true; return nil })
+	if err := r.Close(); !errors.Is(err, ErrResourcesRetained) || closed {
+		t.Fatal("missing completion proof accepted", err)
+	}
+}
--- a/lifecycle/runtime.go
+++ b/lifecycle/runtime.go
@@ -11,17 +11,42 @@ import (
 // must finish draining before calling Close. Embed *Runtime in the project's svc.
 // The logger and any other borrowed resources must not be registered here.
 type Runtime struct {
-	Lifecycle *Controller
-	mu        sync.Mutex
-	cleanups  []func() error
-	closed    bool
-	once      sync.Once
-	closeErr  error
+	Lifecycle     *Controller
+	mu            sync.Mutex
+	cleanups      []func() error
+	closed        bool
+	once          sync.Once
+	closeErr      error
+	stops, drains []shutdownHook
+	shutdown      ShutdownConfig
+	work          context.Context
+	cancelWork    context.CancelFunc
 }

-func NewRuntime(ctx context.Context) *Runtime { return &Runtime{Lifecycle: New(ctx)} }
-func (r *Runtime) Context() context.Context   { return r.Lifecycle.Context() }
-func (r *Runtime) RequestShutdown(err error)  { r.Lifecycle.RequestShutdown(err) }
+func NewRuntime(ctx context.Context) *Runtime {
+	r, _ := NewRuntimeWithConfig(ctx, DefaultShutdownConfig())
+	return r
+}
+
+// NewRuntimeWithConfig separates the stop notification from the lifetime of
+// accepted background work. Zero timeout fields use documented defaults.
+func NewRuntimeWithConfig(ctx context.Context, cfg ShutdownConfig) (*Runtime, error) {
+	if err := cfg.Validate(); err != nil {
+		return nil, err
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	work, cancel := context.WithCancel(context.WithoutCancel(ctx))
+	return &Runtime{Lifecycle: New(ctx), shutdown: cfg.defaults(), work: work, cancelWork: cancel}, nil
+}
+func (r *Runtime) Context() context.Context  { return r.Lifecycle.Context() }
+func (r *Runtime) RequestShutdown(err error) { r.Lifecycle.RequestShutdown(err) }
+
+// WorkContext stays alive during HTTP draining and graceful background draining.
+// It is canceled on a shutdown failure/deadline, or after successful draining.
+// Context() remains the early stop notification; do not use it for accepted work.
+func (r *Runtime) WorkContext() context.Context { return r.work }

 // OnClose transfers cleanup ownership. Register immediately after creation,
 // before exposing resources to other goroutines. Registration after Close is a
@@ -38,8 +63,10 @@ func (r *Runtime) OnClose(cleanup func() error) {
 	r.cleanups = append(r.cleanups, cleanup)
 }

-// Close releases all resources in reverse order, including after a failed
-// initialization. All callbacks are attempted even if one fails or panics.
+// Close runs stop and drain barriers before releasing resources in reverse
+// order, including after failed initialization. If quiescence is unproven it
+// skips resource callbacks and returns ErrResourcesRetained. Once release is
+// safe, all OnClose callbacks are attempted even if one fails or panics.
 func (r *Runtime) Close() error {
 	if r == nil {
 		return nil
@@ -48,9 +75,43 @@ func (r *Runtime) Close() error {
 		r.RequestShutdown(nil)
 		r.mu.Lock()
 		r.closed = true
-		cleanups := r.cleanups
+		cleanups, stops, drains := r.cleanups, r.stops, r.drains
 		r.cleanups = nil
+		r.stops, r.drains = nil, nil
 		r.mu.Unlock()
+		stopErr := r.runPhase("stop", stops, r.shutdown.StopTimeout)
+		var stopped *phaseError
+		errors.As(stopErr, &stopped)
+		for _, stop := range stops {
+			found := false
+			for _, drain := range drains {
+				if drain.name == stop.name {
+					found = true
+					break
+				}
+			}
+			if !found {
+				stopErr = &phaseError{err: errors.Join(stopErr, &HookError{"drain", stop.name, errors.New("matching OnDrain required")}), unsafe: true}
+			}
+		}
+		if stopErr != nil {
+			r.cancelWork()
+		}
+		// A timed-out Stop can still be executing. Never overlap its own Drain;
+		// other components can still establish their completion independently.
+		eligible := make([]shutdownHook, 0, len(drains))
+		for _, drain := range drains {
+			if stopped == nil || !stopped.unjoined[drain.name] {
+				eligible = append(eligible, drain)
+			}
+		}
+		drainErr := r.runPhase("drain", eligible, r.shutdown.DrainTimeout)
+		r.cancelWork()
+		r.closeErr = errors.Join(stopErr, drainErr)
+		if unsafePhase(stopErr) || unsafePhase(drainErr) {
+			r.closeErr = errors.Join(r.closeErr, ErrResourcesRetained)
+			return
+		}
 		for i := len(cleanups) - 1; i >= 0; i-- {
 			r.closeErr = errors.Join(r.closeErr, runCleanup(cleanups[i]))
 		}