feat: complete API-driven projects with infrastructure and business acceptance

2026-09-23 17:33:50 +08:00
parent 04a1f7999d
commit 2c74dc0859
137 changed files with 11926 additions and 105 deletions
--- a/logx/README.md
+++ b/logx/README.md
@@ -1,31 +1,62 @@
 # logx

-`logx` 是 GoOmni 的业务层结构化日志入口。业务代码不需要依赖具体网络库：
+业务通过 `logx` 记录结构化日志，独立于 HTTP、队列和具体业务字段。普通 `context.Context` 继续由业务管理；框架不会约定或自动注入 userId、tenantId。

 ```go
-import "git.yuntuo.hk/k2safe/go-omni/logx"
-
-func (s *UserService) Create(ctx context.Context, input CreateInput) error {
-	logx.InfoContext(ctx, "user.create.start", "email", input.Email)
-	// ...
-	return nil
-}
+logx.InfoContext(ctx, "order.created", "orderId", orderID)
+logx.ErrorContext(ctx, "order.failed", "error", err)
+// 可选：附上本次记录位置的当前 goroutine 堆栈。
+logx.ErrorStackContext(ctx, "unexpected.failure", "error", err)
+// 固定字段可使用子日志对象，无需重复传入。
+logger := logx.With("component", "invoice")
+logger.InfoContext(ctx, "invoice.saved", "invoiceId", invoiceID)
 ```

-应用启动时可以替换默认输出：
+`Debug/Info/Warn/Error` 有包级和实例方法，也有带 Context 的方法。`requestId` 自动跟随 HTTP Context；接入 telemetry 后可带 `traceId`。不要在 Service 中改用 `context.Background()`，否则会丢失取消信号和关联字段。
+
+## 文件轮转

 ```go
 closeLog, err := logx.Setup(logx.Config{
-	Service:     "billing-api",
-	Environment: "production",
-	Level:       "info",
-	Format:      "json",
-	File:        "./var/log/billing-api.log",
+    Service: "billing-api", Environment: "production",
+    Level: "info", Format: "json", File: "./var/log/billing-api.log",
+    Rotation: logx.RotationConfig{
+        Enabled: true, MaxSizeMB: 100, MaxBackups: 10,
+        MaxAgeDays: 7, Compress: true,
+    },
+    RedactKeys: []string{"bankAccount", "identityNumber"},
 })
-if err != nil {
-	return err
-}
+if err != nil { return err }
 defer closeLog()
 ```

-HTTP 请求的 `X-Request-ID` 会被放进标准 `context.Context`，之后的 `InfoContext`、`ErrorContext` 会自动带上 `requestId`。后续可以在不改变业务调用方式的前提下增加文件轮转、采样和 OpenTelemetry 导出。
+也可用 `omni.WithLogConfig(...)` 交给应用运行生命周期管理。子日志共用输出、级别和关闭状态。独立 `logx.New` 返回的实例由创建者负责关闭。
+
+轮转显式开启；三个数值为 0 时分别取 100 MiB、10 份、7 天。超过大小后轮转，旧文件按份数/年龄清理，可 gzip 压缩；保留天数不表示每天定时切割。压缩和历史清理由底层异步执行，不承诺 Close 等待压缩完成。轮转使用 [lumberjack v2](https://github.com/natefinch/lumberjack/tree/v2.2.1)，**一个日志文件只能由一个进程拥有**。容器通常直接用 stdout/stderr，让运行平台管理日志文件。
+
+`rotation.enabled` 与自定义 Writer 不能同时使用；只设轮转但未提供 File 会报错。开启轮转时会创建父目录，并在初始化阶段检查文件可打开。自定义 Writer 的并发和生命周期由接入方负责。普通文件输出仍然是追加写入；未开启轮转不自动清理。
+
+## 结构化脱敏
+
+默认匹配 password、passwd、secret、clientSecret、token、accessToken、refreshToken、authorization、cookie、setCookie、apiKey、privateKey、signedUrl。匹配忽略大小写、下划线和连字符；`RedactKeys` 追加业务自行定义的键，不替换默认列表。
+
+覆盖普通属性、With 属性、slog Group/LogValuer、嵌套字符串键 map、切片和导出 struct 字段。敏感值变为 `[REDACTED]`，不修改调用者的数据；循环/过深结构截断。嵌套字节数组隐藏为 `[BINARY]`，非字符串键 map 隐藏为 `[UNSUPPORTED MAP]`，不调用对象自定义 JSON 序列化方法。
+
+**脱敏按字段名处理，不解析自由文本。** 把密码拼进 msg/error/URL 字符串，或使用不在列表中的字段名，仍可能泄露。签名链接、完整请求体和认证头不应写入日志。业务隐私字段自行放进 `RedactKeys`；日志中不会自动添加身份/租户约定。
+
+## HTTP 异常与慢请求
+
+```go
+application := omni.New(
+    omni.WithLogger(true),
+    omni.WithSlowRequestThreshold(500*time.Millisecond),
+)
+```
+
+`WithSlowRequestThreshold` 同时开启访问日志，0 禁用慢请求识别。也可以在 JSON/YAML 中配 `enableLogger: true`、`slowRequestThreshold: "500ms"`，或使用 `GO_OMNI_SLOW_REQUEST_THRESHOLD` 配合 `GO_OMNI_ENABLE_LOGGER=true`。
+
+每个请求记录一条 `http.request`：方法、路径、匹配路由、实际状态码、耗时、客户端 IP 和关联字段。正常请求 INFO，超过阈值 WARN（`slow: true`），5xx ERROR；返回的 error 会记录到结构化字段。计时覆盖 Handler 调用，不包含框架之外的代理排队，也不自动计入 Handler 返回后继续写入的自管流。
+
+Recovery 自动记录单独的 ERROR `http.panic` 和当前堆栈，再走原有统一错误响应；如果响应已提交，不改写已发送状态。客户端不会收到日志堆栈。普通业务 error 默认不抓堆栈，需要时显式使用 `ErrorStackContext`，它记录调用位置，不回溯 error 最初创建位置。
+
+运行时可通过 `logger.SetLevel("debug")` 调整级别；文件轮转和脱敏配置属于静态设置，变更后重建实例/重启。当前不包含日志采样、异步缓冲、日志平台导出或慢 SQL 记录。
--- a/logx/logx.go
+++ b/logx/logx.go
@@ -10,6 +10,7 @@ import (
 	"io"
 	"log/slog"
 	"os"
+	"runtime/debug"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -21,14 +22,16 @@ import (
 // standard stream. File takes precedence over Output and appends to the named
 // file.
 type Config struct {
-	Level       string    `json:"level"`
-	Format      string    `json:"format"`
-	Output      string    `json:"output"`
-	File        string    `json:"file"`
-	Service     string    `json:"service"`
-	Environment string    `json:"environment"`
-	AddSource   bool      `json:"addSource"`
-	Writer      io.Writer `json:"-"`
+	Level       string         `json:"level"`
+	Format      string         `json:"format"`
+	Output      string         `json:"output"`
+	File        string         `json:"file"`
+	Service     string         `json:"service"`
+	Environment string         `json:"environment"`
+	AddSource   bool           `json:"addSource"`
+	Writer      io.Writer      `json:"-"`
+	Rotation    RotationConfig `json:"rotation"`
+	RedactKeys  []string       `json:"redactKeys"`
 }

 // Default returns the default production-friendly logger configuration.
@@ -62,13 +65,17 @@ var (
 func init() {
 	// Keep package-level logging useful without requiring every service to
 	// repeat setup. Applications can replace this logger with Setup.
-	defaultLogger.Store(&Logger{logger: slog.New(slog.NewJSONHandler(os.Stderr, nil))})
+	logger, _ := New(Default())
+	defaultLogger.Store(logger)
 }

 // New creates an independent logger. The caller owns its lifecycle and should
 // call Close when the process is shutting down.
 func New(cfg Config) (*Logger, error) {
 	cfg = normalizeConfig(cfg)
+	if err := cfg.Rotation.validate(cfg); err != nil {
+		return nil, err
+	}
 	level, err := parseLevel(cfg.Level)
 	if err != nil {
 		return nil, err
@@ -81,7 +88,7 @@ func New(cfg Config) (*Logger, error) {

 	dynamicLevel := new(slog.LevelVar)
 	dynamicLevel.Set(level)
-	opts := &slog.HandlerOptions{Level: dynamicLevel, AddSource: cfg.AddSource}
+	opts := &slog.HandlerOptions{Level: dynamicLevel, AddSource: cfg.AddSource, ReplaceAttr: redactor(cfg.RedactKeys)}
 	var handler slog.Handler
 	if cfg.Format == "text" {
 		handler = slog.NewTextHandler(writer, opts)
@@ -208,6 +215,19 @@ func (l *Logger) ErrorContext(ctx context.Context, msg string, args ...any) {
 	l.logger.Log(ctx, slog.LevelError, msg, contextAttrs(ctx, args)...)
 }

+// ErrorStackContext records the current goroutine's stack at the logging call,
+// not the historical location where an error was originally constructed.
+func (l *Logger) ErrorStackContext(ctx context.Context, msg string, args ...any) {
+	attrs := make([]any, 0, len(args)+2)
+	attrs = append(attrs, args...)
+	attrs = append(attrs, "stack", string(debug.Stack()))
+	l.ErrorContext(ctx, msg, attrs...)
+}
+
+func ErrorStackContext(ctx context.Context, msg string, args ...any) {
+	defaultLogger.Load().ErrorStackContext(ctx, msg, args...)
+}
+
 func (l *Logger) Debug(msg string, args ...any) { l.DebugContext(context.Background(), msg, args...) }
 func (l *Logger) Info(msg string, args ...any)  { l.InfoContext(context.Background(), msg, args...) }
 func (l *Logger) Warn(msg string, args ...any)  { l.WarnContext(context.Background(), msg, args...) }
@@ -324,6 +344,9 @@ func openWriter(cfg Config) (io.Writer, func() error, error) {
 		return cfg.Writer, nil, nil
 	}
 	if cfg.File != "" {
+		if cfg.Rotation.Enabled {
+			return rotatingWriter(cfg)
+		}
 		file, err := os.OpenFile(cfg.File, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
 		if err != nil {
 			return nil, nil, err
--- a/logx/production_test.go
+++ b/logx/production_test.go
@@ -0,0 +1,157 @@
+package logx
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+type credentialsValue struct{}
+
+func (credentialsValue) LogValue() slog.Value {
+	return slog.GroupValue(slog.String("token", "do-not-print"), slog.String("label", "keep"))
+}
+func TestStructuredRedaction(t *testing.T) {
+	for _, format := range []string{"json", "text"} {
+		t.Run(format, func(t *testing.T) {
+			var out bytes.Buffer
+			logger, err := New(Config{Writer: &out, Format: format, RedactKeys: []string{"bankAccount"}})
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer logger.Close()
+			type account struct {
+				Name     string `json:"name"`
+				Password string `json:"password"`
+				Account  string `json:"bankAccount"`
+			}
+			nested := map[string]any{"profile": account{"visible", "do-not-print", "do-not-print"}, "Authorization": "do-not-print", "items": []any{map[string]string{"API_KEY": "do-not-print", "label": "visible"}}}
+			logger.With("refreshToken", "do-not-print").WithGroup("metadata").Info("event", "body", nested, slog.Group("password", "value", "do-not-print"), "valuer", credentialsValue{})
+			if strings.Contains(out.String(), "do-not-print") || !strings.Contains(out.String(), "visible") || !strings.Contains(out.String(), redacted) {
+				t.Fatal(out.String())
+			}
+			if format == "json" {
+				var value map[string]any
+				if err := json.Unmarshal(out.Bytes(), &value); err != nil {
+					t.Fatal(err)
+				}
+			}
+		})
+	}
+}
+func TestRedactionDoesNotMutateOrLoop(t *testing.T) {
+	var out bytes.Buffer
+	logger, _ := New(Config{Writer: &out})
+	m := map[string]any{"password": "original"}
+	m["self"] = m
+	logger.Info("event", "map", m)
+	if m["password"] != "original" || strings.Contains(out.String(), "original") || !strings.Contains(out.String(), "TRUNCATED") {
+		t.Fatal(out.String())
+	}
+}
+func TestRotationCompressRetentionAndClose(t *testing.T) {
+	dir := t.TempDir()
+	file := filepath.Join(dir, "logs", "app.log")
+	logger, err := New(Config{File: file, Rotation: RotationConfig{Enabled: true, MaxSizeMB: 1, MaxBackups: 2, MaxAgeDays: 1, Compress: true}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	payload := strings.Repeat("x", 600<<10)
+	for i := 0; i < 5; i++ {
+		logger.Info("rotation", "value", payload)
+		time.Sleep(3 * time.Millisecond)
+	}
+	deadline := time.Now().Add(3 * time.Second)
+	var backups []string
+	for time.Now().Before(deadline) {
+		backups, _ = filepath.Glob(filepath.Join(dir, "logs", "app-*.gz"))
+		plain, _ := filepath.Glob(filepath.Join(dir, "logs", "app-*.log"))
+		if len(backups) == 2 && len(plain) == 0 {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if len(backups) != 2 {
+		t.Fatalf("backups: %v", backups)
+	}
+	for _, path := range backups {
+		f, err := os.Open(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		z, err := gzip.NewReader(f)
+		if err != nil {
+			t.Fatal(err)
+		}
+		data, err := io.ReadAll(z)
+		z.Close()
+		f.Close()
+		if err != nil || !json.Valid(bytes.TrimSpace(data)) {
+			t.Fatal("corrupt backup", err)
+		}
+	}
+	child := logger.With("component", "test")
+	if err = logger.Close(); err != nil {
+		t.Fatal(err)
+	}
+	before, _ := os.Stat(file)
+	child.Info("must not reopen")
+	after, _ := os.Stat(file)
+	if before.Size() != after.Size() {
+		t.Fatal("closed logger wrote")
+	}
+}
+func TestRotationConcurrentWritesAndValidation(t *testing.T) {
+	for _, cfg := range []Config{{Rotation: RotationConfig{Enabled: true}}, {File: "unused", Writer: io.Discard, Rotation: RotationConfig{Enabled: true}}, {Rotation: RotationConfig{MaxBackups: -1}}} {
+		if _, err := New(cfg); err == nil {
+			t.Fatal("invalid rotation accepted")
+		}
+	}
+	file := filepath.Join(t.TempDir(), "app.log")
+	logger, err := New(Config{File: file, Rotation: RotationConfig{Enabled: true}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var wg sync.WaitGroup
+	for i := 0; i < 8; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < 100; j++ {
+				logger.Info("concurrent", "n", j)
+			}
+		}()
+	}
+	wg.Wait()
+	logger.Close()
+	raw, _ := os.ReadFile(file)
+	if len(bytes.Split(bytes.TrimSpace(raw), []byte("\n"))) != 800 {
+		t.Fatal("lost logs")
+	}
+}
+
+func TestBusinessErrorStack(t *testing.T) {
+	var out bytes.Buffer
+	logger, _ := New(Config{Writer: &out, AddSource: true})
+	ctx := WithRequestID(context.Background(), "request")
+	logger.ErrorStackContext(ctx, "failure", "password", "do-not-print")
+	var entry map[string]any
+	if err := json.Unmarshal(out.Bytes(), &entry); err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(entry["stack"].(string), "TestBusinessErrorStack") || entry["requestId"] != "request" || strings.Contains(out.String(), "do-not-print") {
+		t.Fatal(out.String())
+	}
+	if entry["source"].(map[string]any)["file"] == nil {
+		t.Fatal("source schema changed")
+	}
+}
--- a/logx/redaction.go
+++ b/logx/redaction.go
@@ -0,0 +1,133 @@
+package logx
+
+import (
+	"log/slog"
+	"reflect"
+	"strings"
+	"time"
+	"unicode"
+)
+
+const redacted = "[REDACTED]"
+
+// Redaction applies to structured keys, including nested maps and structs.
+// Free text (messages, errors, URLs embedded in strings) is not parsed.
+func redactor(extra []string) func([]string, slog.Attr) slog.Attr {
+	keys := make(map[string]bool)
+	for _, key := range append([]string{"password", "passwd", "secret", "clientSecret", "token", "accessToken", "refreshToken", "authorization", "cookie", "setCookie", "apiKey", "privateKey", "signedUrl"}, extra...) {
+		keys[normalizedKey(key)] = true
+	}
+	return func(groups []string, a slog.Attr) slog.Attr {
+		for _, group := range groups {
+			if keys[normalizedKey(group)] {
+				return slog.String(a.Key, redacted)
+			}
+		}
+		if keys[normalizedKey(a.Key)] {
+			return slog.String(a.Key, redacted)
+		}
+		a.Value = a.Value.Resolve()
+		if a.Value.Kind() == slog.KindAny {
+			a.Value = slog.AnyValue(sanitize(reflect.ValueOf(a.Value.Any()), keys, 0))
+		}
+		return a
+	}
+}
+
+func normalizedKey(key string) string {
+	return strings.Map(func(r rune) rune {
+		if unicode.IsLetter(r) || unicode.IsDigit(r) {
+			return unicode.ToLower(r)
+		}
+		return -1
+	}, key)
+}
+
+func sanitize(v reflect.Value, keys map[string]bool, depth int) any {
+	if !v.IsValid() {
+		return nil
+	}
+	if depth >= 16 {
+		return "[TRUNCATED]"
+	}
+	if (v.Kind() == reflect.Pointer || v.Kind() == reflect.Interface) && v.IsNil() {
+		return nil
+	}
+	if v.CanInterface() {
+		switch x := v.Interface().(type) {
+		case *slog.Source:
+			return sanitize(reflect.ValueOf(map[string]any{"function": x.Function, "file": x.File, "line": x.Line}), keys, depth+1)
+		case time.Time:
+			return x
+		case error:
+			return x.Error()
+		case []slog.Attr:
+			out := make(map[string]any, len(x))
+			for _, a := range x {
+				if keys[normalizedKey(a.Key)] {
+					out[a.Key] = redacted
+				} else {
+					out[a.Key] = sanitize(reflect.ValueOf(a.Value.Resolve().Any()), keys, depth+1)
+				}
+			}
+			return out
+		case slog.LogValuer:
+			value := slog.AnyValue(x).Resolve()
+			return sanitize(reflect.ValueOf(value.Any()), keys, depth+1)
+		}
+	}
+	switch v.Kind() {
+	case reflect.Interface, reflect.Pointer:
+		return sanitize(v.Elem(), keys, depth+1)
+	case reflect.Map:
+		if v.Type().Key().Kind() != reflect.String {
+			return "[UNSUPPORTED MAP]"
+		}
+		out := make(map[string]any, v.Len())
+		it := v.MapRange()
+		for it.Next() {
+			key := it.Key().String()
+			if keys[normalizedKey(key)] {
+				out[key] = redacted
+			} else {
+				out[key] = sanitize(it.Value(), keys, depth+1)
+			}
+		}
+		return out
+	case reflect.Struct:
+		out := make(map[string]any)
+		for i := 0; i < v.NumField(); i++ {
+			field := v.Type().Field(i)
+			if !field.IsExported() {
+				continue
+			}
+			name := strings.Split(field.Tag.Get("json"), ",")[0]
+			if name == "-" {
+				continue
+			}
+			if name == "" {
+				name = field.Name
+			}
+			if keys[normalizedKey(name)] || keys[normalizedKey(field.Name)] {
+				out[name] = redacted
+			} else {
+				out[name] = sanitize(v.Field(i), keys, depth+1)
+			}
+		}
+		return out
+	case reflect.Slice, reflect.Array:
+		if v.Type().Elem().Kind() == reflect.Uint8 {
+			return "[BINARY]"
+		}
+		out := make([]any, v.Len())
+		for i := range out {
+			out[i] = sanitize(v.Index(i), keys, depth+1)
+		}
+		return out
+	default:
+		if v.CanInterface() {
+			return v.Interface()
+		}
+		return nil
+	}
+}
--- a/logx/rotation.go
+++ b/logx/rotation.go
@@ -0,0 +1,83 @@
+package logx
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"gopkg.in/natefinch/lumberjack.v2"
+)
+
+// RotationConfig opts a file logger into size-based rotation. Zero limits use
+// 100 MiB, 10 backups and 7 days. Each file must have a single process owner.
+type RotationConfig struct {
+	Enabled    bool `json:"enabled"`
+	MaxSizeMB  int  `json:"maxSizeMb"`
+	MaxBackups int  `json:"maxBackups"`
+	MaxAgeDays int  `json:"maxAgeDays"`
+	Compress   bool `json:"compress"`
+}
+
+func (r RotationConfig) validate(cfg Config) error {
+	if r.MaxSizeMB < 0 || r.MaxSizeMB > 1024*1024 || r.MaxBackups < 0 || r.MaxBackups > 10000 || r.MaxAgeDays < 0 || r.MaxAgeDays > 36500 {
+		return errors.New("logx: invalid rotation limits")
+	}
+	if r.Enabled && (cfg.File == "" || cfg.Writer != nil) {
+		return errors.New("logx: rotation requires a file and no custom writer")
+	}
+	return nil
+}
+
+func rotatingWriter(cfg Config) (io.Writer, func() error, error) {
+	r := cfg.Rotation
+	if r.MaxSizeMB == 0 {
+		r.MaxSizeMB = 100
+	}
+	if r.MaxBackups == 0 {
+		r.MaxBackups = 10
+	}
+	if r.MaxAgeDays == 0 {
+		r.MaxAgeDays = 7
+	}
+	if err := os.MkdirAll(filepath.Dir(cfg.File), 0700); err != nil {
+		return nil, nil, err
+	}
+	// Fail at setup, rather than silently dropping the first log record.
+	f, err := os.OpenFile(cfg.File, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
+	if err != nil {
+		return nil, nil, err
+	}
+	if err = f.Close(); err != nil {
+		return nil, nil, err
+	}
+	w := &rotationWriter{writer: &lumberjack.Logger{Filename: cfg.File, MaxSize: r.MaxSizeMB, MaxBackups: r.MaxBackups, MaxAge: r.MaxAgeDays, Compress: r.Compress}}
+	return w, w.Close, nil
+}
+
+// Lumberjack can reopen after Close; prevent children of a closed logger from
+// resurrecting its file. The lock also serializes shutdown with pending writes.
+type rotationWriter struct {
+	mu     sync.Mutex
+	writer *lumberjack.Logger
+	closed bool
+}
+
+func (w *rotationWriter) Write(p []byte) (int, error) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.closed {
+		return 0, io.ErrClosedPipe
+	}
+	return w.writer.Write(p)
+}
+func (w *rotationWriter) Close() error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.closed {
+		return nil
+	}
+	w.closed = true
+	return w.writer.Close()
+}