feat: add cache, outbound HTTP, request protection, tasks and module scaffolding

2026-09-23 13:38:25 +08:00
parent 467c890363
commit fe66221a9c
34 changed files with 3100 additions and 1 deletions
--- a/examples/modules/README.md
+++ b/examples/modules/README.md
@@ -0,0 +1,33 @@
+# 基础模块组合示例
+
+在仓库根目录执行：
+
+```sh
+go run ./examples/modules
+```
+
+默认监听 `127.0.0.1:8080`，使用仅限开发的 `Bearer dev-token`。
+配置 `GO_OMNI_TRANSPORT=native` 可直接切换自研网络层，TLS 配置与主框架一致。
+
+```sh
+curl -H 'Authorization: Bearer dev-token' http://127.0.0.1:8080/api/catalog
+curl -H 'Authorization: Bearer dev-token' -H 'Content-Type: application/json' \
+  -H 'Idempotency-Key: report-1' -d '{"name":"report"}' http://127.0.0.1:8080/api/jobs
+curl -H 'Authorization: Bearer dev-token' http://127.0.0.1:8080/api/jobs
+```
+
+catalog 缓存 30 秒；可通过启动环境变量 `GO_OMNI_DEMO_UPSTREAM` 指定可信 JSON 上游，
+演示 HTTP 连接池、请求 ID 传递与合并回源。URL 来自服务配置，不接受请求参数传入。
+
+jobs 展示固定 worker、队列、结构化日志与状态查询；相同 Idempotency-Key 和请求内容重放
+第一次 202 响应，内容不同返回 409。每分钟的统计任务演示间隔调度。退出先排空任务，再关闭
+HTTP 客户端和缓存。这个示例的任务、缓存和幂等记录均保存在进程内。
+
+组合测试通过真实 TLS 连接覆盖 nethttp、native 和实验 HTTP/2，检查缓存回源次数、
+幂等重放、认证不可被重放绕过、任务只提交一次与关闭流程：
+
+```sh
+go test -race ./examples/modules
+```
+
+完整 API、默认上限和适用范围见 [业务基础模块](../../MODULES.md)。
--- a/examples/modules/main.go
+++ b/examples/modules/main.go
@@ -0,0 +1,133 @@
+// modules combines caching, outbound calls, middleware and background jobs.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"log"
+	"os"
+	"os/signal"
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	omni "git.yuntuo.hk/k2safe/go-omni"
+	"git.yuntuo.hk/k2safe/go-omni/auth"
+	"git.yuntuo.hk/k2safe/go-omni/cache"
+	"git.yuntuo.hk/k2safe/go-omni/httpc"
+	"git.yuntuo.hk/k2safe/go-omni/logx"
+	"git.yuntuo.hk/k2safe/go-omni/middleware"
+	"git.yuntuo.hk/k2safe/go-omni/task"
+)
+
+func newApp(settings omni.Config, token, upstream string) (application *omni.App, err error) {
+	application = omni.New(omni.WithConfig(settings))
+	defer func(app *omni.App) {
+		if err != nil {
+			_ = app.Close()
+		}
+	}(application)
+	store, err := cache.NewMemory(cache.MemoryConfig{MaxEntries: 128, MaxBytes: 1 << 20})
+	if err != nil {
+		return nil, err
+	}
+	application.OnShutdown(store.Close)
+	loader, err := cache.NewLoader(store, cache.LoaderConfig{MaxLoads: 8, Timeout: time.Second})
+	if err != nil {
+		return application, err
+	}
+	client, err := httpc.New(httpc.Config{Timeout: time.Second, MaxRetries: 1, MaxConnsPerHost: 8})
+	if err != nil {
+		return application, err
+	}
+	application.OnShutdown(client.Close)
+	jobs, err := task.New(task.Config{Workers: 2, QueueSize: 16, TaskTimeout: 2 * time.Second})
+	if err != nil {
+		return application, err
+	}
+	// Reverse cleanup order: drain jobs before closing their dependencies.
+	application.OnShutdown(jobs.Close)
+	if err = jobs.Every("reportStats", time.Minute, func(ctx context.Context) error {
+		logx.InfoContext(ctx, "jobs.stats", "stats", jobs.Stats())
+		return nil
+	}); err != nil {
+		return application, err
+	}
+	limiter, err := middleware.NewRateLimiter(middleware.RateLimitConfig{Rate: 20, Burst: 40, MaxKeys: 128, Key: func(c *omni.Context) string { p, _ := auth.FromContext(c); return p.UserID }})
+	if err != nil {
+		return application, err
+	}
+	idempotency, err := middleware.NewIdempotency(middleware.IdempotencyConfig{Scope: func(c *omni.Context) (string, error) { p, _ := auth.FromContext(c); return p.UserID, nil }})
+	if err != nil {
+		return application, err
+	}
+	api := application.Group("/api", middleware.Timeout(2*time.Second), auth.RequireAuth(auth.StaticAuthenticator{token: {UserID: "demo-user"}}), limiter.Middleware())
+	api.GET("/catalog", func(c *omni.Context) error {
+		value, err := loader.GetOrLoad(c.Context(), "publicCatalog:v1", 30*time.Second, func(ctx context.Context) ([]byte, error) {
+			data := map[string]any{"name": "demo catalog"}
+			if upstream != "" {
+				if err := client.JSON(ctx, "GET", upstream, nil, &data); err != nil {
+					return nil, err
+				}
+			}
+			return json.Marshal(data)
+		})
+		if err != nil {
+			return err
+		}
+		return c.OK(json.RawMessage(value))
+	})
+	var nextID atomic.Uint64
+	api.POST("/jobs", func(c *omni.Context) error {
+		var input struct {
+			Name string `json:"name" binding:"required,max=100"`
+		}
+		if err := c.BindJSON(&input); err != nil {
+			return err
+		}
+		id := nextID.Add(1)
+		// This job deliberately outlives the HTTP request. Runner still applies
+		// its own deadline and shutdown cancellation; log correlation survives.
+		_, err := jobs.Submit(context.WithoutCancel(c.Context()), "demoJob", func(ctx context.Context) error {
+			logx.InfoContext(ctx, "job.finished", "jobId", id, "name", input.Name)
+			return nil
+		})
+		if errors.Is(err, task.ErrQueueFull) {
+			return omni.NewError(503, "QUEUE_FULL", "job queue is full")
+		}
+		if err != nil {
+			return err
+		}
+		return c.Accepted(map[string]uint64{"jobId": id})
+	}, idempotency.Middleware())
+	api.GET("/jobs", func(c *omni.Context) error { return c.OK(jobs.Stats()) })
+	return application, nil
+}
+
+func run(ctx context.Context) error {
+	settings, err := omni.LoadConfig("")
+	if err != nil {
+		return err
+	}
+	if os.Getenv("GO_OMNI_HTTP_ADDR") == "" {
+		settings.Address = "127.0.0.1:8080"
+	}
+	token := os.Getenv("GO_OMNI_DEV_TOKEN")
+	if token == "" {
+		token = "dev-token"
+	}
+	application, err := newApp(settings, token, os.Getenv("GO_OMNI_DEMO_UPSTREAM"))
+	if err != nil {
+		return err
+	}
+	return application.RunContext(ctx)
+}
+
+func main() {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	if err := run(ctx); err != nil {
+		log.Fatal(err)
+	}
+}
--- a/examples/modules/main_test.go
+++ b/examples/modules/main_test.go
@@ -0,0 +1,126 @@
+package main
+
+import (
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	omni "git.yuntuo.hk/k2safe/go-omni"
+	"git.yuntuo.hk/k2safe/go-omni/internal/testutil"
+)
+
+// Exercise the composed feature chain through real sockets and all runners.
+func TestModulesConfiguredTransports(t *testing.T) {
+	for _, transport := range []string{omni.TransportNetHTTP, omni.TransportNative, omni.TransportHTTP2} {
+		t.Run(transport, func(t *testing.T) {
+			var loads atomic.Int32
+			upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				loads.Add(1)
+				if r.Header.Get("X-Request-ID") == "" {
+					t.Error("missing outbound correlation")
+				}
+				io.WriteString(w, `{"name":"upstream catalog"}`)
+			}))
+			defer upstream.Close()
+			settings, err := omni.LoadConfig("MODULES_TEST_")
+			if err != nil {
+				t.Fatal(err)
+			}
+			cert, key, tlsClient := testutil.CertificateFiles(t)
+			settings.Transport = transport
+			settings.TLS.CertFile = cert
+			settings.TLS.KeyFile = key
+			application, err := newApp(settings, "demo", upstream.URL)
+			if err != nil {
+				t.Fatal(err)
+			}
+			listener, err := net.Listen("tcp", "127.0.0.1:0")
+			if err != nil {
+				t.Fatal(err)
+			}
+			ctx, cancel := context.WithCancel(context.Background())
+			done := make(chan error, 1)
+			go func() { done <- application.ServeContext(ctx, listener) }()
+			defer func() {
+				cancel()
+				select {
+				case err := <-done:
+					if err != nil {
+						t.Error(err)
+					}
+				case <-time.After(5 * time.Second):
+					listener.Close()
+					t.Error("server failed to stop")
+				}
+			}()
+			client := &http.Client{Timeout: 3 * time.Second, Transport: &http.Transport{TLSClientConfig: tlsClient, ForceAttemptHTTP2: true}}
+			defer client.CloseIdleConnections()
+			request := func(method, path, body, key, requestID, token string) (*http.Response, string) {
+				r, err := http.NewRequest(method, "https://"+listener.Addr().String()+path, strings.NewReader(body))
+				if err != nil {
+					t.Fatal(err)
+				}
+				r.Header.Set("Content-Type", "application/json")
+				r.Header.Set("Idempotency-Key", key)
+				r.Header.Set("X-Request-ID", requestID)
+				r.Header.Set("Authorization", "Bearer "+token)
+				response, err := client.Do(r)
+				if err != nil {
+					t.Fatal(err)
+				}
+				defer response.Body.Close()
+				data, err := io.ReadAll(response.Body)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if response.TLS == nil || len(response.TLS.VerifiedChains) == 0 {
+					t.Fatal("unverified TLS")
+				}
+				if transport == omni.TransportHTTP2 && response.ProtoMajor != 2 {
+					t.Fatal(response.Proto)
+				}
+				return response, string(data)
+			}
+			for n := 0; n < 2; n++ {
+				response, body := request("GET", "/api/catalog", "", "", "catalogRequest", "demo")
+				if response.StatusCode != 200 || !strings.Contains(body, "upstream catalog") {
+					t.Fatal(response.StatusCode, body)
+				}
+			}
+			if loads.Load() != 1 {
+				t.Fatal("cache missed", loads.Load())
+			}
+			first, original := request("POST", "/api/jobs", `{"name":"report"}`, "job1", "firstRequest", "demo")
+			second, replayed := request("POST", "/api/jobs", `{"name":"report"}`, "job1", "secondRequest", "demo")
+			if first.StatusCode != 202 || second.StatusCode != 202 || original != replayed || second.Header.Get("Idempotency-Replayed") != "true" || second.Header.Get("X-Request-ID") != "firstRequest" {
+				t.Fatal(first.StatusCode, second.StatusCode, original, replayed)
+			}
+			if response, _ := request("POST", "/api/jobs", `{"name":"other"}`, "job1", "conflictRequest", "demo"); response.StatusCode != 409 {
+				t.Fatal(response.StatusCode)
+			}
+			if response, _ := request("POST", "/api/jobs", `{"name":"report"}`, "job1", "unauthorized", "wrong"); response.StatusCode != 401 {
+				t.Fatal("replay bypassed authentication", response.StatusCode)
+			}
+			deadline := time.Now().Add(time.Second)
+			for {
+				response, body := request("GET", "/api/jobs", "", "", "statsRequest", "demo")
+				if response.StatusCode != 200 {
+					t.Fatal(response.StatusCode)
+				}
+				if strings.Contains(body, `"completed":1`) {
+					break
+				}
+				if time.Now().After(deadline) {
+					t.Fatal("job did not finish once", body)
+				}
+				time.Sleep(10 * time.Millisecond)
+			}
+		})
+	}
+}