From e29dec717f79e606778324ea8073f57a31fd13f2 Mon Sep 17 00:00:00 2001 From: k2safe Date: Thu, 24 Sep 2026 20:54:36 +0800 Subject: [PATCH] feat: add reusable MySQL outbox and HTTP lifecycle example --- .gitignore | 2 + AGENTS.md | 1 + Makefile | 13 +- README.md | 4 + adapters/outbox/mysql/README.md | 10 + .../outbox/mysql/business_fixture_test.go | 60 ++++ .../outbox/mysql}/cli_integration_test.go | 3 +- .../mysql}/commit_loss_integration_test.go | 2 +- adapters/outbox/mysql/go.mod | 13 + adapters/outbox/mysql/go.sum | 8 + .../outbox/mysql}/http_integration_test.go | 4 +- adapters/outbox/mysql/model.go | 23 ++ adapters/outbox/mysql/options.go | 19 ++ .../outbox/mysql}/process_integration_test.go | 2 +- adapters/outbox/mysql/runtime.go | 76 +++++ adapters/outbox/mysql/schema.sql | 32 ++ .../outbox/mysql}/sender.go | 0 adapters/outbox/mysql/store.go | 311 +++++++++++++++++ .../outbox/mysql}/store_integration_test.go | 6 +- .../outbox/mysql}/store_test.go | 0 .../mysql/transaction_integration_test.go | 179 ++++++++++ docs/08-development-plan.md | 9 +- docs/examples/README.md | 4 + docs/examples/http-service.md | 129 +++++++ docs/examples/task-outbox.md | 4 +- docs/reports/outbox-http-integration.md | 44 +++ docs/reports/phase-3-task-outbox.md | 14 +- docs/sdk/README.md | 2 +- docs/sdk/outbox-mysql.md | 123 +++++++ docs/sdk/production-integration.md | 18 +- examples/http-service/README.md | 11 + .../cmd/server/integration_test.go | 251 ++++++++++++++ examples/http-service/cmd/server/lifecycle.go | 99 ++++++ .../http-service/cmd/server/lifecycle_test.go | 90 +++++ examples/http-service/cmd/server/main.go | 234 +++++++++++++ examples/http-service/go.mod | 16 + examples/http-service/go.sum | 8 + examples/http-service/internal/inbox/model.go | 28 ++ .../http-service/internal/inbox/schema.sql | 32 ++ examples/http-service/internal/inbox/store.go | 314 +++++++++++++++++ .../http-service/internal/inbox/store_test.go | 59 ++++ .../http-service/internal/jobs/schema.sql | 9 + examples/http-service/internal/jobs/store.go | 126 +++++++ examples/task-outbox/README.md | 4 +- examples/task-outbox/go.mod | 7 +- examples/task-outbox/internal/outbox/model.go | 27 +- .../task-outbox/internal/outbox/schema.sql | 33 -- examples/task-outbox/internal/outbox/store.go | 316 +++--------------- go.work | 5 +- go.work.sum | 1 + scripts/test-adapters.sh | 52 +++ scripts/test-mysql.sh | 8 +- 52 files changed, 2488 insertions(+), 357 deletions(-) create mode 100644 adapters/outbox/mysql/README.md create mode 100644 adapters/outbox/mysql/business_fixture_test.go rename {examples/task-outbox/internal/outbox => adapters/outbox/mysql}/cli_integration_test.go (96%) rename {examples/task-outbox/internal/outbox => adapters/outbox/mysql}/commit_loss_integration_test.go (99%) create mode 100644 adapters/outbox/mysql/go.mod create mode 100644 adapters/outbox/mysql/go.sum rename {examples/task-outbox/internal/outbox => adapters/outbox/mysql}/http_integration_test.go (97%) create mode 100644 adapters/outbox/mysql/model.go create mode 100644 adapters/outbox/mysql/options.go rename {examples/task-outbox/internal/outbox => adapters/outbox/mysql}/process_integration_test.go (98%) create mode 100644 adapters/outbox/mysql/runtime.go create mode 100644 adapters/outbox/mysql/schema.sql rename {examples/task-outbox/internal/outbox => adapters/outbox/mysql}/sender.go (100%) create mode 100644 adapters/outbox/mysql/store.go rename {examples/task-outbox/internal/outbox => adapters/outbox/mysql}/store_integration_test.go (98%) rename {examples/task-outbox/internal/outbox => adapters/outbox/mysql}/store_test.go (100%) create mode 100644 adapters/outbox/mysql/transaction_integration_test.go create mode 100644 docs/examples/http-service.md create mode 100644 docs/reports/outbox-http-integration.md create mode 100644 docs/sdk/outbox-mysql.md create mode 100644 examples/http-service/README.md create mode 100644 examples/http-service/cmd/server/integration_test.go create mode 100644 examples/http-service/cmd/server/lifecycle.go create mode 100644 examples/http-service/cmd/server/lifecycle_test.go create mode 100644 examples/http-service/cmd/server/main.go create mode 100644 examples/http-service/go.mod create mode 100644 examples/http-service/go.sum create mode 100644 examples/http-service/internal/inbox/model.go create mode 100644 examples/http-service/internal/inbox/schema.sql create mode 100644 examples/http-service/internal/inbox/store.go create mode 100644 examples/http-service/internal/inbox/store_test.go create mode 100644 examples/http-service/internal/jobs/schema.sql create mode 100644 examples/http-service/internal/jobs/store.go create mode 100644 go.work.sum create mode 100755 scripts/test-adapters.sh diff --git a/.gitignore b/.gitignore index f53c56c..04dd2d9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ *.log /.local/ *.credentials.json + +/examples/http-service/server diff --git a/AGENTS.md b/AGENTS.md index 63bebc9..43955aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,6 +26,7 @@ - 任务来源通过独立只读路由查询，tasks:read仅授予当前任务的有限来源引用；计划定义仍需schedules:read。显式核对命名空间、队列、计划、触发与固定版本，悬空/错误/未实现所有者不得默认为DIRECT；不依赖外键，也不向派发路径加入来源查询。 - 业务事务Outbox示例使用独立module和业务数据库，核心SDK不得引入SQL驱动。业务记录/待发命令同事务；原请求、操作ID、目标及提交键冻结，HTTP在事务外，确认按最新租约代次与业务关联同事务。24小时/100次预算内原键核对，禁止换键、重置预算或把ACCEPTED解释成业务成功；REVIEW保留待核查数据。 +- 可复用业务Outbox位于 adapters/outbox/mysql 独立module。EnqueueTx只写调用方事务，不Commit/Rollback/HTTP；重复核对使用共享锁当前读，避免重复INSERT锁升级死锁。受理钩子与最新代次确认同事务，只允许同库短SQL，固定Outbox→业务行锁序。旧CLI与HTTP宿主共用发送核心；宿主关闭顺序为HTTP收尾→发送/执行收尾→结果消费收尾→数据库关闭，未结清记录保留原身份。 - 相对延迟delay_ms只在首次受理事务按数据库时间计算；原时长参与摘要，零值omitempty保持旧摘要。重报先返回原回执，禁止重算执行时间；总时限包含等待。计划模板拒绝非零delay_ms，人工重试不自动继承首次等待。 diff --git a/Makefile b/Makefile index f0abb76..09e7659 100644 --- a/Makefile +++ b/Makefile @@ -1,25 +1,33 @@ GO ?= go -.PHONY: fmt test vet build image integration console-e2e compose-check mysql-replication mysql-compatibility sdk-check protocol-check docs-check stability-check schedule-backlog release-manifest release-local-check +.PHONY: fmt test vet build adapters-check image integration console-e2e compose-check mysql-replication mysql-compatibility sdk-check protocol-check docs-check stability-check schedule-backlog release-manifest release-local-check fmt: $(GO) fmt ./... cd sdk/go && $(GO) fmt ./... cd examples/result-inbox && GOWORK=off $(GO) fmt ./... cd examples/task-outbox && GOWORK=off $(GO) fmt ./... + cd adapters/outbox/mysql && GOWORK=off $(GO) fmt ./... + cd examples/http-service && GOWORK=off $(GO) fmt ./... test: $(GO) test -race ./... cd sdk/go && $(GO) test -race ./... cd examples/result-inbox && GOWORK=off $(GO) test -race ./... cd examples/task-outbox && GOWORK=off $(GO) test -race ./... + cd adapters/outbox/mysql && GOWORK=off $(GO) test -race ./... + cd examples/http-service && GOWORK=off $(GO) test -race ./... vet: $(GO) vet ./... cd sdk/go && $(GO) vet ./... cd examples/result-inbox && GOWORK=off $(GO) vet ./... cd examples/task-outbox && GOWORK=off $(GO) vet ./... + cd adapters/outbox/mysql && GOWORK=off $(GO) vet ./... + cd examples/http-service && GOWORK=off $(GO) vet ./... build: + cd adapters/outbox/mysql && GOWORK=off $(GO) build ./... + cd examples/http-service && GOWORK=off $(GO) build -o ../../bin/orbit-http-example ./cmd/server $(GO) build -o bin/orbit ./cmd/orbit $(GO) build -o bin/orbit-migrate ./cmd/orbit-migrate $(GO) build -o bin/orbit-admin ./cmd/orbit-admin @@ -68,3 +76,6 @@ release-local-check: # Runs the bounded, intentionally incomplete persistent backlog rehearsal in an isolated MySQL. schedule-backlog: ./scripts/test-schedule-backlog.sh + +adapters-check: + ./scripts/test-adapters.sh diff --git a/README.md b/README.md index a5660ec..690041c 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ internal/config/ 配置校验 internal/storage/mysql/ 连接、事务、迁移及集成测试 internal/transport/httpserver/ HTTP 接入 sdk/go/ 独立 Go SDK module 与示例 +adapters/outbox/mysql/ 可直接引用的业务事务 Outbox 独立模块 +examples/http-service/ HTTP + Outbox + Worker + Inbox 完整服务示例 scripts/ 可复现的回归工具 docs/ 设计、接口、SDK、场景与验证报告 ``` @@ -116,3 +118,5 @@ go get git.yuntuo.hk/k2safe/orbit/sdk/go@v0.1.0 [流程实例筛选分页](docs/api/workflow-queries.md)已提供独立SDK和[完整示例](docs/examples/workflow-query.md)，按固定版本全部队列鉴权，数据库保持零外键。 - [周期归档与作业管理](docs/api/archive-scheduler.md) + +业务接入可直接使用[MySQL 事务 Outbox 适配包](docs/sdk/outbox-mysql.md)和[完整 HTTP 生命周期示例](docs/examples/http-service.md)，覆盖已有事务、后台发送、结果入库以及有界关闭。 diff --git a/adapters/outbox/mysql/README.md b/adapters/outbox/mysql/README.md new file mode 100644 index 0000000..a1cc2dd --- /dev/null +++ b/adapters/outbox/mysql/README.md @@ -0,0 +1,10 @@ +# Orbit MySQL 事务 Outbox 适配包 + +独立 Go module：`git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql`，包名 `outbox`。依赖公开 SDK v0.1.0 与 MySQL 驱动，不依赖平台服务端代码；核心 SDK 不反向依赖本包。 + +- [接口、事务边界、错误与恢复](../../../docs/sdk/outbox-mysql.md) +- [完整 HTTP 服务接入](../../../docs/examples/http-service.md) +- [中文模型](model.go)、[无外键表结构](schema.sql) +- [测试报告](../../../docs/reports/outbox-http-integration.md) + +以业务自己的数据库存储待发命令，调用 `EnqueueTx` 加入已有事务，后台运行 `Run` 或 `SendOnce`。本包确认的是平台受理，任务执行完成通过 Worker 和可靠结果订阅衔接。 diff --git a/adapters/outbox/mysql/business_fixture_test.go b/adapters/outbox/mysql/business_fixture_test.go new file mode 100644 index 0000000..94ff97d --- /dev/null +++ b/adapters/outbox/mysql/business_fixture_test.go @@ -0,0 +1,60 @@ +package outbox + +import ( + "context" + "database/sql" + "strings" + "unicode" + "unicode/utf8" + + orbit "git.yuntuo.hk/k2safe/orbit/sdk/go" +) + +func businessStore(db *sql.DB, namespace, queue string) (*Store, error) { + return New(db, namespace, queue, Options{OnAccepted: accepted}) +} +func accepted(ctx context.Context, tx *sql.Tx, r Record, receipt orbit.SubmitTaskResult) error { + var existing *string + if err := tx.QueryRowContext(ctx, `SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, r.NamespaceID, r.OperationID).Scan(&existing); err != nil { + return ErrStorage + } + if existing != nil && *existing != receipt.Task.ID { + return ErrConflict + } + _, err := tx.ExecContext(ctx, `UPDATE sender_jobs SET task_id=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, receipt.Task.ID, r.NamespaceID, r.OperationID) + if err != nil { + return ErrStorage + } + return nil +} +func (s *Store) Enqueue(ctx context.Context, id, source string, in orbit.SubmitTaskRequest) (bool, error) { + if !utf8.ValidString(source) || strings.TrimSpace(source) == "" || utf8.RuneCountInString(source) > 512 || strings.ContainsFunc(source, unicode.IsControl) { + return false, ErrInvalid + } + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) + if err != nil { + return false, ErrStorage + } + defer tx.Rollback() + duplicate, err := s.EnqueueTx(ctx, tx, id, in) + if err != nil { + return false, err + } + if duplicate { + var saved string + if err = tx.QueryRowContext(ctx, `SELECT source_ref FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, s.namespace, id).Scan(&saved); err != nil { + return false, ErrStorage + } + if saved != source { + return false, ErrConflict + } + } else { + if _, err = tx.ExecContext(ctx, `INSERT INTO sender_jobs(namespace_id,operation_id,source_ref,created_at,updated_at) VALUES(?,?,?,UTC_TIMESTAMP(6),UTC_TIMESTAMP(6))`, s.namespace, id, source); err != nil { + return false, ErrStorage + } + } + if tx.Commit() != nil { + return false, ErrCommitUnconfirmed + } + return duplicate, nil +} diff --git a/examples/task-outbox/internal/outbox/cli_integration_test.go b/adapters/outbox/mysql/cli_integration_test.go similarity index 96% rename from examples/task-outbox/internal/outbox/cli_integration_test.go rename to adapters/outbox/mysql/cli_integration_test.go index fea22fc..a0f8a28 100644 --- a/examples/task-outbox/internal/outbox/cli_integration_test.go +++ b/adapters/outbox/mysql/cli_integration_test.go @@ -17,7 +17,8 @@ func TestIntegrationSenderCLI(t *testing.T) { ctx, stop := context.WithTimeout(context.Background(), 30*time.Second) defer stop() binary := filepath.Join(t.TempDir(), "sender") - build := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "../../cmd/sender") + build := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "./cmd/sender") + build.Dir = "../../../examples/task-outbox" if raw, err := build.CombinedOutput(); err != nil { t.Fatalf("build sender: %v %s", err, raw) } diff --git a/examples/task-outbox/internal/outbox/commit_loss_integration_test.go b/adapters/outbox/mysql/commit_loss_integration_test.go similarity index 99% rename from examples/task-outbox/internal/outbox/commit_loss_integration_test.go rename to adapters/outbox/mysql/commit_loss_integration_test.go index 63a2fbc..0f8fc25 100644 --- a/examples/task-outbox/internal/outbox/commit_loss_integration_test.go +++ b/adapters/outbox/mysql/commit_loss_integration_test.go @@ -160,7 +160,7 @@ func TestIntegrationSenderCommitResponseLoss(t *testing.T) { for _, phase := range []string{"enqueue", "claim", "confirm"} { t.Run(phase, func(t *testing.T) { s := testStore(t, proxied) - fresh, _ := New(direct, s.namespace, s.queue) + fresh, _ := businessStore(direct, s.namespace, s.queue) id := testID(t) in := testRequest(s.queue) if phase == "enqueue" { diff --git a/adapters/outbox/mysql/go.mod b/adapters/outbox/mysql/go.mod new file mode 100644 index 0000000..9f6a405 --- /dev/null +++ b/adapters/outbox/mysql/go.mod @@ -0,0 +1,13 @@ +module git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql + +go 1.27.1 + +require ( + git.yuntuo.hk/k2safe/orbit/sdk/go v0.1.0 + github.com/go-sql-driver/mysql v1.10.1 +) + +require ( + filippo.io/edwards25519 v1.2.0 // indirect + github.com/coder/websocket v1.8.15 // indirect +) diff --git a/adapters/outbox/mysql/go.sum b/adapters/outbox/mysql/go.sum new file mode 100644 index 0000000..a74aa43 --- /dev/null +++ b/adapters/outbox/mysql/go.sum @@ -0,0 +1,8 @@ +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +git.yuntuo.hk/k2safe/orbit/sdk/go v0.1.0 h1:OJp+U8dDNKjiXy4ZhO4MMq68Ek8TkjphUAqsjOOg77c= +git.yuntuo.hk/k2safe/orbit/sdk/go v0.1.0/go.mod h1:9KlMW7Gy+EGaDoEwr3wHHFY47mnAymSfYeQpMS7h4ag= +github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= +github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= +github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= +github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= diff --git a/examples/task-outbox/internal/outbox/http_integration_test.go b/adapters/outbox/mysql/http_integration_test.go similarity index 97% rename from examples/task-outbox/internal/outbox/http_integration_test.go rename to adapters/outbox/mysql/http_integration_test.go index c9108ab..2729b54 100644 --- a/examples/task-outbox/internal/outbox/http_integration_test.go +++ b/adapters/outbox/mysql/http_integration_test.go @@ -53,7 +53,7 @@ func realStore(t *testing.T) (*Store, *orbit.Client, string, string) { if err != nil { t.Fatal(err) } - s, err := New(db, queue.Queue.NamespaceID, queue.Queue.ID) + s, err := businessStore(db, queue.Queue.NamespaceID, queue.Queue.ID) if err != nil { t.Fatal(err) } @@ -165,7 +165,7 @@ func TestIntegrationSenderHTTPReceiptLoss(t *testing.T) { t.Fatal("remote duplicate task", len(page.Items), err) } // 相同凭据搭配错误的持久化目标命名空间，必须在提交前停止。 - wrong, _ := New(s.db, testID(t), s.queue) + wrong, _ := businessStore(s.db, testID(t), s.queue) wrongID := enqueue(t, wrong) r, err = wrong.SendOnce(ctx, through) if err != nil || r.OperationID != wrongID || r.State != "REVIEW" || r.LastCode != "TARGET_MISMATCH" || posts.Load() != 2 { diff --git a/adapters/outbox/mysql/model.go b/adapters/outbox/mysql/model.go new file mode 100644 index 0000000..df89d8e --- /dev/null +++ b/adapters/outbox/mysql/model.go @@ -0,0 +1,23 @@ +package outbox + +import "time" + +// Record 是业务数据库中的待发命令；只确认平台受理，不能表示业务执行成功。 +type Record struct { + NamespaceID string // NamespaceID 固定目标命名空间，换凭据不能改变归属。 + QueueID string // QueueID 固定目标队列，发送前通过当前凭据核对归属。 + OperationID string // OperationID 调用方持久化的业务操作ID，同命名空间内唯一。 + IdempotencyKey string // IdempotencyKey 由操作ID派生的固定提交键，重报禁止换键。 + RequestJSON []byte // RequestJSON 冻结的任务提交正文，保留数字精度，不含凭据。 + RequestDigest []byte // RequestDigest 冻结正文的SHA256，发送前重新校验，非平台语义摘要。 + State string // State PENDING待发、SENDING领取中、ACCEPTED已受理、REVIEW需核查。 + SendCount uint32 // SendCount 持久化领取次数，包含领取后未发HTTP即退出的次数，上限100。 + LeaseEpoch uint64 // LeaseEpoch 每次领取递增；旧代次不能更新新代次的本地结果。 + WakeAt *time.Time // WakeAt PENDING的下次领取时间或SENDING的租约期限；终结状态为空，UTC微秒。 + SendDeadlineAt time.Time // SendDeadlineAt 首次入库后24小时停止新发送，短于平台90天幂等窗口。 + LastCode string // LastCode 最近一次稳定分类，不保存HTTP错误正文、凭据或业务秘密。 + TaskID *string // TaskID 仅ACCEPTED记录平台任务ID，其余状态为空。 + ReceiptJSON []byte // ReceiptJSON 仅ACCEPTED保存原受理回执；不是当前执行状态。 + CreatedAt time.Time // CreatedAt 业务事务首次提交命令的UTC时间，微秒精度。 + UpdatedAt time.Time // UpdatedAt 最近状态变化的UTC时间，不能替代租约代次。 +} diff --git a/adapters/outbox/mysql/options.go b/adapters/outbox/mysql/options.go new file mode 100644 index 0000000..b4c8586 --- /dev/null +++ b/adapters/outbox/mysql/options.go @@ -0,0 +1,19 @@ +package outbox + +import ( + "context" + "database/sql" + + orbit "git.yuntuo.hk/k2safe/orbit/sdk/go" +) + +// AcceptedHook 在最新租约确认事务内关联业务数据；可能在事务回滚/提交不明后再次调用。 +// 必须同库、幂等、遵守 context，只做短 SQL；不能 Commit/Rollback，也不能发网络请求。 +// record 是锁定后的发送记录，receipt 是本次已核验的受理回执，不是任务完成结果。 +type AcceptedHook func(context.Context, *sql.Tx, Record, orbit.SubmitTaskResult) error + +// Options 在构造后固定，Store 可以在多个发送 goroutine 中复用。 +type Options struct { + // OnAccepted 可选的业务关联；为空时受理关系仅保存在 Outbox 的 TaskID/ReceiptJSON。 + OnAccepted AcceptedHook +} diff --git a/examples/task-outbox/internal/outbox/process_integration_test.go b/adapters/outbox/mysql/process_integration_test.go similarity index 98% rename from examples/task-outbox/internal/outbox/process_integration_test.go rename to adapters/outbox/mysql/process_integration_test.go index d62fce0..685f3c2 100644 --- a/examples/task-outbox/internal/outbox/process_integration_test.go +++ b/adapters/outbox/mysql/process_integration_test.go @@ -45,7 +45,7 @@ func TestSenderProcessHelper(t *testing.T) { } db, _ := integrationDB(t) _, base, token := realClient(t) - s, err := New(db, os.Getenv("ORBIT_TEST_SENDER_NAMESPACE"), os.Getenv("ORBIT_TEST_SENDER_QUEUE")) + s, err := businessStore(db, os.Getenv("ORBIT_TEST_SENDER_NAMESPACE"), os.Getenv("ORBIT_TEST_SENDER_QUEUE")) if err != nil { t.Fatal(err) } diff --git a/adapters/outbox/mysql/runtime.go b/adapters/outbox/mysql/runtime.go new file mode 100644 index 0000000..8653e60 --- /dev/null +++ b/adapters/outbox/mysql/runtime.go @@ -0,0 +1,76 @@ +package outbox + +import ( + "context" + "errors" + "time" + + orbit "git.yuntuo.hk/k2safe/orbit/sdk/go" +) + +var ErrShutdownTimeout = errors.New("outbox sender shutdown deadline exceeded; reconcile persisted records") + +// RunOptions 控制进程轮询与关闭，不改变每条记录持久化的 24 小时/100 次预算。 +type RunOptions struct { + PollInterval time.Duration // PollInterval 空闲/退避轮询间隔，默认500毫秒，范围10毫秒至10秒。 + ShutdownTimeout time.Duration // ShutdownTimeout 取消后等待当前发送及本地确认，默认20秒，最多5分钟。 +} + +// Run 持续发送；取消后停止领取，限时完成当前交换及确认，调用方必须等待返回。 +// 存储/确认错误返回给宿主处理；HTTP暂时失败持久化退避，REVIEW留在数据库供核查。 +// 可以多进程并行运行；数据库租约协调领取，同一记录的HTTP仍可能重复，始终复用原键。 +func (s *Store) Run(parent context.Context, api *orbit.Client, options RunOptions) error { + if api == nil { + return ErrInvalid + } + if options.PollInterval == 0 { + options.PollInterval = 500 * time.Millisecond + } + if options.ShutdownTimeout == 0 { + options.ShutdownTimeout = 20 * time.Second + } + if options.PollInterval < 10*time.Millisecond || options.PollInterval > 10*time.Second || options.ShutdownTimeout < time.Millisecond || options.ShutdownTimeout > 5*time.Minute { + return ErrInvalid + } + work, cancel := context.WithCancel(context.WithoutCancel(parent)) + done := make(chan struct{}) + expired := make(chan struct{}) + go func() { + defer close(done) + select { + case <-work.Done(): + return + case <-parent.Done(): + } + timer := time.NewTimer(options.ShutdownTimeout) + defer timer.Stop() + select { + case <-work.Done(): + case <-timer.C: + close(expired) + cancel() + } + }() + defer func() { cancel(); <-done }() + for parent.Err() == nil { + _, err := s.SendOnce(work, api) + select { + case <-expired: + return ErrShutdownTimeout + default: + } + if err != nil && !errors.Is(err, ErrIdle) && !errors.Is(err, ErrLeaseLost) { + return err + } + if err == nil { + continue + } + timer := time.NewTimer(options.PollInterval) + select { + case <-parent.Done(): + timer.Stop() + case <-timer.C: + } + } + return nil +} diff --git a/adapters/outbox/mysql/schema.sql b/adapters/outbox/mysql/schema.sql new file mode 100644 index 0000000..a0d11c1 --- /dev/null +++ b/adapters/outbox/mysql/schema.sql @@ -0,0 +1,32 @@ +-- 业务数据库的初始Outbox表，保留原示例表名以兼容，不属于平台迁移，无外键或级联删除。 +CREATE TABLE sender_outbox ( + namespace_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定目标命名空间，重启及换凭据不得改变', + queue_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定目标队列，发送前核对当前授权及命名空间', + operation_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '业务操作标识，同命名空间内唯一，调用方持久化后复用', + idempotency_key VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '冻结平台提交键，由业务操作标识派生，禁止换键补发', + request_json MEDIUMTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '冻结任务提交正文，保留精确数字，不存认证凭据', + request_digest BINARY(32) NOT NULL COMMENT '冻结正文SHA256，防止同身份不同内容，非平台摘要算法', + state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT 'PENDING待发，SENDING领取中，ACCEPTED已受理，REVIEW需核查', + send_count INT UNSIGNED NOT NULL COMMENT '领取次数，包含领取后崩溃的次数，最多100', + lease_epoch BIGINT UNSIGNED NOT NULL COMMENT '领取代次递增，旧发送器不能覆盖新代次结果', + wake_at DATETIME(6) NULL COMMENT '待发下次时间或发送租约期限，终结后为空，UTC微秒', + send_deadline_at DATETIME(6) NOT NULL COMMENT '首次入库24小时后的发送截止，短于平台幂等保留窗口', + last_code VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '最近稳定分类，不保存服务端错误正文或秘密', + task_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL COMMENT '已确认的平台任务标识，只在ACCEPTED非空', + receipt_json MEDIUMTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL COMMENT '原受理回执，仅ACCEPTED非空，不代表当前执行结果', + created_at DATETIME(6) NOT NULL COMMENT '业务事务首次创建命令UTC时间，微秒精度', + updated_at DATETIME(6) NOT NULL COMMENT '本地状态最近变化UTC时间，微秒精度', + PRIMARY KEY (namespace_id,operation_id), + UNIQUE KEY uq_sender_identity(namespace_id,idempotency_key), + KEY ix_sender_due(namespace_id,queue_id,wake_at,operation_id), + CONSTRAINT ck_sender_document CHECK (JSON_VALID(request_json)), + CONSTRAINT ck_sender_receipt CHECK (receipt_json IS NULL OR JSON_VALID(receipt_json)), + CONSTRAINT ck_sender_count CHECK (send_count<=100 AND lease_epoch=send_count), + CONSTRAINT ck_sender_state CHECK ( + (state IN ('PENDING','SENDING') AND wake_at IS NOT NULL AND task_id IS NULL AND receipt_json IS NULL) + OR (state='REVIEW' AND wake_at IS NULL AND task_id IS NULL AND receipt_json IS NULL) + OR (state='ACCEPTED' AND wake_at IS NULL AND task_id IS NOT NULL AND receipt_json IS NOT NULL) + ), + CONSTRAINT ck_sender_lease CHECK (state<>'SENDING' OR lease_epoch>0), + CONSTRAINT ck_sender_deadline CHECK (send_deadline_at>created_at AND send_deadline_at<=TIMESTAMPADD(HOUR,24,created_at)) +) ENGINE=InnoDB COMMENT='业务事务Outbox，持久化原请求与有限重发状态，不承担业务恰好一次'; diff --git a/examples/task-outbox/internal/outbox/sender.go b/adapters/outbox/mysql/sender.go similarity index 100% rename from examples/task-outbox/internal/outbox/sender.go rename to adapters/outbox/mysql/sender.go diff --git a/adapters/outbox/mysql/store.go b/adapters/outbox/mysql/store.go new file mode 100644 index 0000000..a7c1bac --- /dev/null +++ b/adapters/outbox/mysql/store.go @@ -0,0 +1,311 @@ +// Package outbox 将待发任务加入调用方的 MySQL 事务，并通过有限租约发送器恢复提交。 +package outbox + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + _ "embed" + "encoding/json" + "errors" + "regexp" + "strings" + "time" + "unicode/utf8" + + orbit "git.yuntuo.hk/k2safe/orbit/sdk/go" + "github.com/go-sql-driver/mysql" +) + +//go:embed schema.sql +var schema string + +var ( + ErrInvalid = errors.New("outbox scope, operation or request is invalid") + ErrConflict = errors.New("outbox operation already has different content; keep the original identity") + ErrStorage = errors.New("outbox storage failed; reconcile the original operation") + ErrCommitUnconfirmed = errors.New("outbox commit is unconfirmed; reconcile the original operation") + ErrIdle = errors.New("no due outbox record") + ErrNotFound = errors.New("outbox operation not found in this scope") + ErrAcceptance = errors.New("outbox acceptance hook failed; confirmation rolled back") + ErrLeaseLost = errors.New("outbox lease was replaced; do not overwrite its result") + idPattern = regexp.MustCompile(`^[a-f0-9]{32}$`) +) + +const maxSends = 100 +const recordColumns = `namespace_id,queue_id,operation_id,idempotency_key,request_json,request_digest,state,send_count,lease_epoch,wake_at,send_deadline_at,last_code,task_id,receipt_json,created_at,updated_at` + +// Store 固定一个业务数据库及目标范围，不拥有连接池的关闭权。 +type Store struct { + db *sql.DB + namespace, queue string + onAccepted AcceptedHook +} + +// Open 仅连接业务方数据库，不创建表，不输出原始DSN或数据库错误。 +func Open(ctx context.Context, dsn string) (*sql.DB, error) { + cfg, err := mysql.ParseDSN(dsn) + if err != nil || cfg.DBName == "" { + return nil, ErrInvalid + } + cfg.ParseTime = true + cfg.Loc = time.UTC + cfg.MultiStatements = false + cfg.AllowAllFiles = false + cfg.Timeout, cfg.ReadTimeout, cfg.WriteTimeout = 5*time.Second, 8*time.Second, 8*time.Second + if cfg.Params == nil { + cfg.Params = map[string]string{} + } + cfg.Params["time_zone"] = "'+00:00'" + connector, err := mysql.NewConnector(cfg) + if err != nil { + return nil, ErrStorage + } + db := sql.OpenDB(connector) + db.SetMaxOpenConns(8) + db.SetMaxIdleConns(4) + db.SetConnMaxLifetime(5 * time.Minute) + if err = db.PingContext(ctx); err != nil { + db.Close() + return nil, ErrStorage + } + return db, nil +} + +// SchemaSQL 返回首版无外键表结构，供业务迁移工具管理；不会自动执行。 +func SchemaSQL() string { return schema } + +// Init 显式创建适配表，已有表会报错；生产升级由业务迁移工具管理。 +func Init(ctx context.Context, db *sql.DB) error { + for _, statement := range strings.Split(schema, ";") { + if strings.TrimSpace(statement) == "" { + continue + } + if _, err := db.ExecContext(ctx, statement); err != nil { + return ErrStorage + } + } + return nil +} + +// New 构造可并发复用的适配器，不执行建表或网络请求。 +func New(db *sql.DB, namespace, queue string, options Options) (*Store, error) { + if db == nil || !idPattern.MatchString(namespace) || !idPattern.MatchString(queue) { + return nil, ErrInvalid + } + return &Store{db: db, namespace: namespace, queue: queue, onAccepted: options.OnAccepted}, nil +} + +// document 冻结当前v1默认值及UTC表示；Outbox按文档字节去重，不冒充平台语义摘要。 +func (s *Store) document(in orbit.SubmitTaskRequest) ([]byte, error) { + // encoding/json会把非法UTF-8替换成U+FFFD；在冻结摘要前拒绝，不能静默改变业务标识。 + for _, value := range []string{in.QueueID, in.Handler, in.HandlerVersion, in.PayloadVersion, in.BusinessKey, in.ConcurrencyKey} { + if !utf8.ValidString(value) { + return nil, ErrInvalid + } + } + if in.QueueID != s.queue || in.Handler == "" || in.HandlerVersion == "" || in.PayloadVersion == "" || len(in.Payload) > 256<<10 || !utf8.Valid(in.Payload) || !json.Valid(in.Payload) || bytes.Equal(bytes.TrimSpace(in.Payload), []byte("null")) { + return nil, ErrInvalid + } + if in.TaskTimeoutMS == 0 { + in.TaskTimeoutMS = 86400000 + } + if in.RecoverySafety == "" { + in.RecoverySafety = orbit.RecoveryReconcileOnly + } + if in.Retry.MaxAttempts == 0 { + in.Retry.MaxAttempts = 1 + } + if in.Retry.AttemptTimeoutMS == 0 { + in.Retry.AttemptTimeoutMS = 30000 + } + if in.Retry.MaxAttempts > 1 { + if in.Retry.InitialDelayMS == 0 { + in.Retry.InitialDelayMS = 1000 + } + if in.Retry.MaxDelayMS == 0 { + in.Retry.MaxDelayMS = 30000 + } + } + if in.AvailableAt != nil { + at := in.AvailableAt.UTC() + in.AvailableAt = &at + } + if in.StartDeadlineAt != nil { + at := in.StartDeadlineAt.UTC() + in.StartDeadlineAt = &at + } + raw, err := json.Marshal(in) + if err != nil || len(raw) > 512<<10 { + return nil, ErrInvalid + } + return raw, nil +} + +// EnqueueTx 只写入调用方事务，不开启/提交/回滚事务，不进行 HTTP 请求。 +// tx 必须来自 New 使用的同一业务数据库。失败时调用方必须回滚整个事务。 +// duplicate 只表示事务内已有同身份同内容记录，调用方仍必须确认 Commit 成功。 +func (s *Store) EnqueueTx(ctx context.Context, tx *sql.Tx, operationID string, in orbit.SubmitTaskRequest) (duplicate bool, err error) { + if tx == nil || !idPattern.MatchString(operationID) { + return false, ErrInvalid + } + raw, err := s.document(in) + if err != nil { + return false, err + } + digest := sha256.Sum256(raw) + _, err = tx.ExecContext(ctx, `INSERT INTO sender_outbox(namespace_id,queue_id,operation_id,idempotency_key,request_json,request_digest,state,send_count,lease_epoch,wake_at,send_deadline_at,last_code,created_at,updated_at) VALUES(?,?,?,?,?,?,'PENDING',0,0,UTC_TIMESTAMP(6),TIMESTAMPADD(HOUR,24,UTC_TIMESTAMP(6)),'',UTC_TIMESTAMP(6),UTC_TIMESTAMP(6))`, s.namespace, s.queue, operationID, "outbox-task-"+operationID, raw, digest[:]) + if err == nil { + return false, nil + } + var duplicateKey *mysql.MySQLError + if !errors.As(err, &duplicateKey) || duplicateKey.Number != 1062 { + return false, ErrStorage + } + var saved []byte + var queue, key string + // 共享锁当前读：核对 RR 旧快照外的竞争提交，且不把重复 INSERT 的共享锁升级为排它锁。 + err = tx.QueryRowContext(ctx, `SELECT request_digest,queue_id,idempotency_key FROM sender_outbox WHERE namespace_id=? AND operation_id=? FOR SHARE`, s.namespace, operationID).Scan(&saved, &queue, &key) + if err != nil { + return false, ErrStorage + } + if queue != s.queue || key != "outbox-task-"+operationID || !bytes.Equal(saved, digest[:]) { + return false, ErrConflict + } + return true, nil +} + +type scanner interface{ Scan(...any) error } + +func scanRecord(row scanner, r *Record, extra ...any) error { + args := []any{&r.NamespaceID, &r.QueueID, &r.OperationID, &r.IdempotencyKey, &r.RequestJSON, &r.RequestDigest, &r.State, &r.SendCount, &r.LeaseEpoch, &r.WakeAt, &r.SendDeadlineAt, &r.LastCode, &r.TaskID, &r.ReceiptJSON, &r.CreatedAt, &r.UpdatedAt} + return row.Scan(append(args, extra...)...) +} + +// Status 读取固定命名空间和队列内的本地发送记录，不查询平台执行结果。 +func (s *Store) Status(parent context.Context, id string) (Record, error) { + if !idPattern.MatchString(id) { + return Record{}, ErrInvalid + } + ctx, stop := context.WithTimeout(parent, 5*time.Second) + defer stop() + var r Record + err := scanRecord(s.db.QueryRowContext(ctx, `SELECT `+recordColumns+` FROM sender_outbox WHERE namespace_id=? AND queue_id=? AND operation_id=?`, s.namespace, s.queue, id), &r) + if errors.Is(err, sql.ErrNoRows) { + return Record{}, ErrNotFound + } + if err != nil { + return Record{}, ErrStorage + } + return r, nil +} + +func (s *Store) claim(parent context.Context) (Record, error) { + ctx, stop := context.WithTimeout(parent, 5*time.Second) + defer stop() + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) + if err != nil { + return Record{}, ErrStorage + } + defer tx.Rollback() + var r Record + var now time.Time + err = scanRecord(tx.QueryRowContext(ctx, `SELECT `+recordColumns+`,UTC_TIMESTAMP(6) FROM sender_outbox FORCE INDEX(ix_sender_due) WHERE namespace_id=? AND queue_id=? AND wake_at<=UTC_TIMESTAMP(6) ORDER BY wake_at,operation_id LIMIT 1 FOR UPDATE SKIP LOCKED`, s.namespace, s.queue), &r, &now) + if errors.Is(err, sql.ErrNoRows) { + return Record{}, ErrIdle + } + if err != nil { + return Record{}, ErrStorage + } + if r.State != "PENDING" && r.State != "SENDING" { + return Record{}, ErrStorage + } + if !now.Before(r.SendDeadlineAt) || r.SendCount >= maxSends { + r.State = "REVIEW" + r.WakeAt = nil + r.LastCode = "SEND_BUDGET_EXHAUSTED" + if !now.Before(r.SendDeadlineAt) { + r.LastCode = "SEND_DEADLINE" + } + } else { + r.State = "SENDING" + r.SendCount++ + r.LeaseEpoch++ + wake := now.Add(30 * time.Second) + if wake.After(r.SendDeadlineAt) { + wake = r.SendDeadlineAt + } + r.WakeAt = &wake + } + _, err = tx.ExecContext(ctx, `UPDATE sender_outbox SET state=?,send_count=?,lease_epoch=?,wake_at=?,last_code=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, r.State, r.SendCount, r.LeaseEpoch, r.WakeAt, r.LastCode, s.namespace, r.OperationID) + if err != nil { + return Record{}, ErrStorage + } + if tx.Commit() != nil { + return Record{}, ErrCommitUnconfirmed + } + return r, nil +} + +// finish 只接受最近代次；过期但尚未被替代的成功回执可落库，重复HTTP只复用原键。 +func (s *Store) finish(parent context.Context, claim Record, state, code string, delay time.Duration, receipt *orbit.SubmitTaskResult) error { + ctx, stop := context.WithTimeout(parent, 5*time.Second) + defer stop() + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) + if err != nil { + return ErrStorage + } + defer tx.Rollback() + var current Record + var now time.Time + err = scanRecord(tx.QueryRowContext(ctx, `SELECT `+recordColumns+`,UTC_TIMESTAMP(6) FROM sender_outbox WHERE namespace_id=? AND queue_id=? AND operation_id=? FOR UPDATE`, s.namespace, s.queue, claim.OperationID), ¤t, &now) + if err != nil { + return ErrStorage + } + if current.State != "SENDING" || current.LeaseEpoch != claim.LeaseEpoch { + return ErrLeaseLost + } + var wake *time.Time + var task *string + var raw []byte + if state == "ACCEPTED" { + if receipt == nil || !validReceipt(current, *receipt) { + return ErrInvalid + } + raw, err = json.Marshal(receipt) + if err != nil || len(raw) > 1<<20 { + return ErrInvalid + } + task = &receipt.Task.ID + // 持锁顺序：Outbox → 业务行。回调只能做短 SQL，不得提交、回滚或执行网络副作用。 + if s.onAccepted != nil { + if err = s.onAccepted(ctx, tx, current, *receipt); err != nil { + return ErrAcceptance + } + } + } else if state == "PENDING" { + if delay < time.Second || delay > time.Hour { + return ErrInvalid + } + if !now.Before(current.SendDeadlineAt) || current.SendCount >= maxSends { + state = "REVIEW" + code = "SEND_BUDGET_EXHAUSTED" + } else { + next := now.Add(delay) + if next.After(current.SendDeadlineAt) { + next = current.SendDeadlineAt + } + wake = &next + } + } else if state != "REVIEW" { + return ErrInvalid + } + if _, err = tx.ExecContext(ctx, `UPDATE sender_outbox SET state=?,last_code=?,wake_at=?,task_id=?,receipt_json=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, state, code, wake, task, raw, s.namespace, current.OperationID); err != nil { + return ErrStorage + } + if tx.Commit() != nil { + return ErrCommitUnconfirmed + } + return nil +} diff --git a/examples/task-outbox/internal/outbox/store_integration_test.go b/adapters/outbox/mysql/store_integration_test.go similarity index 98% rename from examples/task-outbox/internal/outbox/store_integration_test.go rename to adapters/outbox/mysql/store_integration_test.go index c0aed42..67751c5 100644 --- a/examples/task-outbox/internal/outbox/store_integration_test.go +++ b/adapters/outbox/mysql/store_integration_test.go @@ -40,7 +40,7 @@ func integrationDB(t *testing.T) (*sql.DB, string) { } func testStore(t *testing.T, db *sql.DB) *Store { t.Helper() - s, err := New(db, testID(t), testID(t)) + s, err := businessStore(db, testID(t), testID(t)) if err != nil { t.Fatal(err) } @@ -173,7 +173,7 @@ func TestIntegrationSenderConcurrentEnqueueAndConflict(t *testing.T) { if _, err := s.Enqueue(context.Background(), id, "objects/test", in); !errors.Is(err, ErrConflict) { t.Fatal("request content conflict lost", err) } - other, _ := New(db, s.namespace, testID(t)) + other, _ := businessStore(db, s.namespace, testID(t)) if _, err := other.Status(context.Background(), id); !errors.Is(err, ErrNotFound) { t.Fatal("cross queue record returned", err) } @@ -209,7 +209,7 @@ func TestIntegrationSenderBusinessRollbackAndConfirmationAtomicity(t *testing.T) } receipt := fakeReceipt(t, s) execSQL(t, db, `CREATE TRIGGER sender_update_fail BEFORE UPDATE ON sender_jobs FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected business confirmation failure'`) - if err = s.finish(context.Background(), claim, "ACCEPTED", "ACCEPTED", 0, &receipt); !errors.Is(err, ErrStorage) { + if err = s.finish(context.Background(), claim, "ACCEPTED", "ACCEPTED", 0, &receipt); !errors.Is(err, ErrAcceptance) { t.Fatal(err) } if r := status(t, s, id); r.State != "SENDING" || r.TaskID != nil { diff --git a/examples/task-outbox/internal/outbox/store_test.go b/adapters/outbox/mysql/store_test.go similarity index 100% rename from examples/task-outbox/internal/outbox/store_test.go rename to adapters/outbox/mysql/store_test.go diff --git a/adapters/outbox/mysql/transaction_integration_test.go b/adapters/outbox/mysql/transaction_integration_test.go new file mode 100644 index 0000000..0124579 --- /dev/null +++ b/adapters/outbox/mysql/transaction_integration_test.go @@ -0,0 +1,179 @@ +package outbox + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + orbit "git.yuntuo.hk/k2safe/orbit/sdk/go" +) + +func TestIntegrationCallerOwnedTransaction(t *testing.T) { + db, _ := integrationDB(t) + s, err := New(db, testID(t), testID(t), Options{}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + id := testID(t) + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + dup, err := s.EnqueueTx(ctx, tx, id, testRequest(s.queue)) + if err != nil || dup { + t.Fatal(dup, err) + } + if dup, err = s.EnqueueTx(ctx, tx, id, testRequest(s.queue)); err != nil || !dup { + t.Fatal("same transaction duplicate not recognized", dup, err) + } + // 从另一连接看不到未提交的待发任务；包不得偷偷提交，也不得发起HTTP。 + if _, err = s.Status(ctx, id); !errors.Is(err, ErrNotFound) { + t.Fatal("adapter committed caller transaction", err) + } + if _, err = s.claim(ctx); !errors.Is(err, ErrIdle) { + t.Fatal("uncommitted task was claimable", err) + } + if err = tx.Rollback(); err != nil { + t.Fatal(err) + } + if _, err = s.Status(ctx, id); !errors.Is(err, ErrNotFound) { + t.Fatal("caller rollback lost", err) + } + + // 先在RR事务建立旧快照，再由另一事务提交相同身份，重复核对必须是当前读。 + tx, err = db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead}) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + var n int + if err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM sender_outbox WHERE namespace_id=?`, s.namespace).Scan(&n); err != nil { + t.Fatal(err) + } + other, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer other.Rollback() + if _, err = s.EnqueueTx(ctx, other, id, testRequest(s.queue)); err != nil { + t.Fatal(err) + } + if err = other.Commit(); err != nil { + t.Fatal(err) + } + if dup, err = s.EnqueueTx(ctx, tx, id, testRequest(s.queue)); err != nil || !dup { + t.Fatal("RR current read failed", dup, err) + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + claim, err := s.claim(ctx) + if err != nil { + t.Fatal(err) + } + receipt := fakeReceipt(t, s) + if err = s.finish(ctx, claim, "ACCEPTED", "ACCEPTED", 0, &receipt); err != nil { + t.Fatal("optional hook requires business schema", err) + } + if status(t, s, id).State != "ACCEPTED" { + t.Fatal("missing acceptance") + } +} + +func TestIntegrationSenderRuntimeDrain(t *testing.T) { + for _, timeout := range []bool{false, true} { + t.Run(map[bool]string{false: "finish_inflight", true: "bounded_timeout"}[timeout], func(t *testing.T) { + db, _ := integrationDB(t) + s, err := New(db, testID(t), testID(t), Options{}) + if err != nil { + t.Fatal(err) + } + id := testID(t) + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + if _, err = s.EnqueueTx(context.Background(), tx, id, testRequest(s.queue)); err != nil { + t.Fatal(err) + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + entered := make(chan struct{}) + release := make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + receipt := fakeReceipt(t, s) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == "GET" { + writeTestJSON(w, map[string]any{"request_id": id, "data": orbit.Queue{ID: s.queue, NamespaceID: s.namespace}}) + return + } + close(entered) + select { + case <-release: + case <-r.Context().Done(): + return + } + w.WriteHeader(201) + writeTestJSON(w, map[string]any{"request_id": id, "data": receipt}) + })) + defer server.Close() + defer unblock() + api, err := orbit.NewClient(server.URL, orbit.Options{Token: "test"}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + grace := time.Second + if timeout { + grace = 20 * time.Millisecond + } + done := make(chan error, 1) + go func() { done <- s.Run(ctx, api, RunOptions{ShutdownTimeout: grace}) }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("sender did not enter HTTP") + } + cancel() + if !timeout { + select { + case err := <-done: + t.Fatal("did not drain", err) + case <-time.After(25 * time.Millisecond): + } + unblock() + } + select { + case err = <-done: + case <-time.After(5 * time.Second): + t.Fatal("shutdown hung") + } + if timeout { + if !errors.Is(err, ErrShutdownTimeout) { + t.Fatal(err) + } + if status(t, s, id).State == "ACCEPTED" { + t.Fatal("timeout fabricated acceptance") + } + } else { + if err != nil || status(t, s, id).State != "ACCEPTED" { + t.Fatal(err) + } + } + }) + } +} + +func writeTestJSON(w http.ResponseWriter, v any) { _ = json.NewEncoder(w).Encode(v) } diff --git a/docs/08-development-plan.md b/docs/08-development-plan.md index 508984f..0c1db55 100644 --- a/docs/08-development-plan.md +++ b/docs/08-development-plan.md @@ -1,6 +1,6 @@ # Orbit 持续开发与回归计划 -更新：2026-09-22。此计划对应持续开发目标，由开发过程同步维护。数据库已按用户要求采用无外键设计，迁移 000008 移除历史外键，关联一致性由服务端事务保证。 +更新：2026-09-24。此计划对应持续开发目标，由开发过程同步维护。数据库已按用户要求采用无外键设计，迁移 000008 移除历史外键，关联一致性由服务端事务保证。 ## 1. 最终交付目标 @@ -252,3 +252,10 @@ A04派发已提交而消息尚未到达Worker的精确进程故障已通过[专 - 已验证：真实 MySQL 多节点竞争、落盘后确认丢失、文件损坏、策略变化、预算提高与卷预算、分页；标准测试、容器端到端、页面实际操作与全量集成回归均已通过。 - 不计入本批完成项：源数据删除、跨作业增量去重、对象存储、任意网络文件系统语义及目标生产跨故障域验收。 - [接口/部署/恢复示例](api/archive-scheduler.md)与[验证记录](reports/phase-6-archive-scheduler.md)。 + +### 可复用 Outbox 与 HTTP 服务生命周期（本批已完成） + +- 已交付独立 MySQL 适配模块，支持调用方 EnqueueTx、受理确认钩子和有界 Run；旧 CLI 改用公共包，原有表结构及命令兼容。 +- 已交付完整 HTTP 宿主：业务接口、发送器、Worker、持久 Inbox、健康检查、后台错误传播、有界关闭和数据库最后释放。 +- 已通过：标准竞态测试、vet/build、SDK隔离和文档链接；定向真实MySQL/平台回归与平台全量 integration 均退出0。包括事务回滚/重复/旧快照、确认回滚、提交回执丢失、强杀恢复和SIGTERM收尾。 +- [公共包接口](sdk/outbox-mysql.md)、[完整调用和故障示例](examples/http-service.md)、[验收报告](reports/outbox-http-integration.md)。本批不扩展到其他数据库/ORM、流程命令Outbox或源数据删除。 diff --git a/docs/examples/README.md b/docs/examples/README.md index dfd2231..39b196e 100644 --- a/docs/examples/README.md +++ b/docs/examples/README.md @@ -18,6 +18,10 @@ 独立的流程场景另见[有界轮询](workflow-poll.md)与[证据恢复](workflow-recovery.md)：前者区分请求重试与业务轮次，后者实际注入外部提交后HTTP断线，保留UNKNOWN历史和暂停门控。本机渠道是受控夹具，不能代替持久业务证据。 +## 已可运行：嵌入现有 HTTP 服务 + +[完整 HTTP 工程与命令](http-service.md)：业务接口同事务入队 → 后台发送 → 内嵌 Worker → WebSocket → Inbox，包含健康检查、后台错误传播、SIGTERM收尾和SIGKILL恢复。公共包接口见 [MySQL Outbox](../sdk/outbox-mysql.md)。 + ## 已可运行：业务事务与任务提交Outbox - 独立模块：[task-outbox](../../examples/task-outbox/README.md)，[完整前置、命令、Go方法及故障处理](task-outbox.md)。 diff --git a/docs/examples/http-service.md b/docs/examples/http-service.md new file mode 100644 index 0000000..ba077d8 --- /dev/null +++ b/docs/examples/http-service.md @@ -0,0 +1,129 @@ +# HTTP 服务完整接入示例 + +[源码工程](../../examples/http-service/README.md)将 SDK 的发送器、Worker、结果消费器放进业务 HTTP 进程。Orbit 平台仍然独立部署，业务表放在自己的 MySQL 数据库。 + +## 1. 场景与运行结果 + +客户端用固定 operation_id 提交一个对象引用。HTTP 事务同时写入业务行和待发请求，后台发送给 Orbit。Worker 对引用进行纯计算转换，结果通过可靠 WebSocket 订阅写入业务 Inbox，再由 SDK 确认交付。 + +接口返回 202 表示业务本地事务已提交。查询先看到 PENDING/SENDING，随后 ACCEPTED，最后收到 execution=SUCCEEDED 和转换结果。每个阶段以数据库记录为准，HTTP 请求不等待远端任务执行结束。 + +## 2. 准备配置 + +需要 Go 1.27.1、运行中的 Orbit 和 MySQL 8.4。创建一个独立的业务数据库，例如 orbit_http_demo，不使用 Orbit 平台库。由业务迁移账号初始化表，正常运行账号只授予所需业务表读写权限。 + +在仓库根目录创建示例专用应用/队列凭据，下面的 ORBIT_MYSQL_DSN 是**平台库**： + +```sh +umask 077 +mkdir -p .local +export ORBIT_MYSQL_DSN='平台数据库DSN' +go run ./cmd/orbit-admin bootstrap \ + --application http-demo --namespace default --environment development \ + --queue http-demo --worker-access --event-access \ + --output .local/http-demo.credentials.json +``` + +该命令用于一次性准备专用队列；重启复用原凭据，不重新 bootstrap。凭据具有任务提交、队列读取、Worker 和事件所需权限，生产可按角色拆分身份。 + +```sh +# 在仓库根目录保存凭据绝对路径，再进入示例。 +export ORBIT_CREDENTIALS_FILE="$PWD/.local/http-demo.credentials.json" +export ORBIT_URL='http://127.0.0.1:8080' +export APP_MYSQL_DSN='业务账号:密码@tcp(127.0.0.1:3306)/orbit_http_demo' +export APP_HTTP_ADDR='127.0.0.1:8090' +cd examples/http-service +export GOWORK=off + +go run ./cmd/server init +export ORBIT_SUBSCRIPTION_ID="$(go run ./cmd/server provision)" +go build -o ./server ./cmd/server +./server serve +``` + +`init` 仅首次在空业务库执行，不自动升级或覆盖已有表；生产迁移管理四张表：sender_outbox、sender_jobs、receiver_inbox、receiver_task_views。全部字段有中文注释，没有外键。 + +`provision` 在发送任何业务任务前执行，使用固定键和固定内容创建订阅。将输出的订阅 ID 保存在部署配置中，后续启动只需 `serve`，不依赖 Orbit 在线才能接收本地业务请求。运行凭据必须能消费该订阅的绑定身份，轮换按原凭据规则执行。 + +也可用 ORBIT_TOKEN、ORBIT_NAMESPACE_ID、ORBIT_QUEUE_ID 配置代替文件；环境变量优先。APP_HTTP_ADDR 仅接受回环 IP，示例不提供对公网的账号体系。集成到现有 HTTP 服务时复用已有认证与授权中间件，不要直接取消绑定限制暴露演示路由。 + +## 3. HTTP 接口与调用 + +| 接口 | 返回 | +| --- | --- | +| `GET /health/live` | 进程存活：200 | +| `GET /health/ready` | 正在接收请求且业务数据库可用：200；关闭期间/数据库不可用：503 | +| `POST /jobs` | 保存业务事务，202；字段错误400、原操作内容冲突409、存储/提交不明503 | +| `GET /jobs/{operation_id}` | 业务提交与结果视图200；不存在404、存储错误503 | + +```sh +export OPERATION_ID="$(openssl rand -hex 16)" +curl -sS http://127.0.0.1:8090/jobs \ + -H 'Content-Type: application/json' \ + -d "{\"operation_id\":\"$OPERATION_ID\",\"source_ref\":\"objects/demo.txt\"}" + +curl -sS "http://127.0.0.1:8090/jobs/$OPERATION_ID" +``` + +首次提交： + +```json +{"operation_id":"原操作ID","local_committed":true,"duplicate":false} +``` + +等待结果入库后查询： + +```json +{ + "operation_id":"原操作ID", + "submission":"ACCEPTED", + "code":"ACCEPTED", + "task_id":"平台任务ID", + "execution":"SUCCEEDED", + "result":{"source_ref":"objects/demo.txt","normalized_ref":"OBJECTS/DEMO.TXT"} +} +``` + +用同一个 OPERATION_ID 和相同请求重复 POST，返回 duplicate=true，不重复写业务行或创建平台任务。相同 ID 改 source_ref 返回409。客户端超时/503时保留原 ID、原内容重试，不重新生成随机 ID。source_ref 最多512个Unicode字符，不能空白或包含控制字符；接口正文上限4KiB。 + +## 4. 生命周期 + +[生命周期源码](../../examples/http-service/cmd/server/lifecycle.go)包含以下顺序： + +1. 加载配置、连接业务库、核验表、构造 SDK 组件，再创建 HTTP 监听。任何构造/监听失败都关闭已经持有的资源；不会先启动失去宿主管理的 Worker。 +2. 启动 HTTP、发送器、Worker 和结果消费器；各组件使用独立的宿主 context，进程信号只触发统一关闭流程。 +3. HTTP 监听失败或任何后台组件不可恢复退出，撤销 readiness，进入同一关闭路径，错误返回 main 并以非零状态退出。 +4. 收到 SIGTERM/SIGINT 后撤销 readiness，用**新的有期限 context**调用 HTTP Shutdown，等待已进入的业务请求完成；共享数据库及后台组件此时仍可用。 +5. HTTP 结束后取消发送器和 Worker，等待当前发送确认、在途任务续租及结果上报完成。 +6. 最后取消结果消费器，等待处理器/ACK 有界收尾，随后关闭共享数据库连接池。 + +发送器和 Worker 收尾各20s，结果消费器10s；示例每个关闭阶段最大25s，进程平台的终止宽限应大于三个阶段之和，例如90s。超时返回错误，不声称工作全部完成。忽略 context 的自定义处理器不能被 Go 强杀；实际业务必须遵守取消，并为外部副作用建立幂等/查询核对。 + +消费器取消会取消其处理器；事务未确认时不会 ACK，重启后允许重投。关闭前并不要求全局订阅积压归零，进程退出后未消费结果仍由持久订阅保存。示例 Inbox 按事件 ID 去重，按任务版本防止迟到覆盖，提交回执未知返回错误，不能提前确认交付。 + +readiness 表示能安全接受**本地持久请求**，不要求 Orbit 或 WebSocket 始终在线；Orbit暂时不可用时任务留在 Outbox。若某类业务必须同步依赖远端，应单独增加路由级就绪门槛。监控仍需覆盖 Outbox 积压/最老待发年龄/REVIEW、Worker重连和事件积压，不能只看健康接口。 + +## 5. 故障场景 + +| 场景 | 预期行为 | +| --- | --- | +| Orbit 暂时不可用 | HTTP仍可保存业务事务，Outbox有界退避；受理超出24h/100次进入REVIEW | +| HTTP提交响应丢失 | 客户端按原ID与内容核对，业务行与待发请求不重复 | +| Orbit受理响应丢失 | 发送器保留原提交键重新核对，不能换键 | +| 本地业务/确认COMMIT不明 | 保留原身份，不把失败返回当成数据库回滚证明 | +| 进程SIGKILL | 内存丢失；重启用相同数据库、目标、凭据和订阅ID恢复，已有发送租约自然到期 | +| 正常SIGTERM | 按上述顺序收尾；成功退出码0，异常/超时非零 | +| 结果重复投递 | 同业务事务Inbox去重，提交完成后才ACK | + +不要把示例的纯计算 `RecoveryIdempotent` 直接用于支付、发送通知等外部效果。真实业务要实现自己的幂等或查询核对。任务预算、24h发送期限、结果交付预算保持有限，长时间停机可能需要人工核查。 + +## 6. 验证与接入现有框架 + +```sh +# 仓库根目录：脚本自行创建、销毁专用MySQL，不操作本地开发库。 +make adapters-check +make test vet build docs-check sdk-check +make integration +``` + +[验证记录](../reports/outbox-http-integration.md)区分纯生命周期测试和真实数据库/平台/进程证据。Gin、Echo、go-zero 等服务可复用本例的组件装配和关闭顺序，将业务路由挂到原路由器，统一接入原生命周期；不需要另起一套业务 HTTP 服务，也不在每次请求里创建 Worker。 diff --git a/docs/examples/task-outbox.md b/docs/examples/task-outbox.md index ebb94d1..def770a 100644 --- a/docs/examples/task-outbox.md +++ b/docs/examples/task-outbox.md @@ -4,7 +4,7 @@ ## 1. 数据与状态 -[中文模型](../../examples/task-outbox/internal/outbox/model.go)和[SQL](../../examples/task-outbox/internal/outbox/schema.sql)包含2张业务表、22个字段，全部有中文说明，外键为零，不进入Orbit平台迁移。 +[中文模型](../../examples/task-outbox/internal/outbox/model.go)和[SQL](../../examples/task-outbox/internal/outbox/schema.sql)与[公共 Outbox SQL](../../adapters/outbox/mysql/schema.sql)合计包含2张业务表、22个字段，全部有中文说明，外键为零，不进入Orbit平台迁移。 | 表 | 职责与关键约束 | | --- | --- | @@ -103,7 +103,7 @@ GOWORK=off go run ./cmd/sender status "$SENDER_OPERATION_ID" ## 4. Go接入接口 -以下方法属于示例的内部业务适配包，完整源码见[store.go](../../examples/task-outbox/internal/outbox/store.go)及[sender.go](../../examples/task-outbox/internal/outbox/sender.go)。接入自己的业务模块时保留事务顺序与协议，替换sender_jobs为实际业务表；核心Orbit客户端仍使用独立SDK。 +发送与租约实现已抽至[公共适配包](../sdk/outbox-mysql.md)。以下方法是本示例的业务封装，完整源码见[store.go](../../examples/task-outbox/internal/outbox/store.go)及[sender.go](../../adapters/outbox/mysql/sender.go)。接入自己的业务模块时保留事务顺序与协议，替换sender_jobs为实际业务表；核心Orbit客户端仍使用独立SDK。 ```go func Open(ctx context.Context, dsn string) (*sql.DB, error) diff --git a/docs/reports/outbox-http-integration.md b/docs/reports/outbox-http-integration.md new file mode 100644 index 0000000..dc0a43c --- /dev/null +++ b/docs/reports/outbox-http-integration.md @@ -0,0 +1,44 @@ +# 可复用 Outbox 与 HTTP 生命周期验证 + +日期：2026-09-24。本报告仅覆盖本批接入适配与示例，不替代平台目标生产环境容量、高可用及数据库复制确认验收。 + +## 交付 + +- 独立模块 adapters/outbox/mysql，公开 EnqueueTx、可选受理关联回调、Run 和查询接口。无平台内部依赖，核心 SDK 无 SQL 驱动。 +- 旧 CLI 的发送核心及原有故障测试迁入公共模块；旧业务表、命令和固定提交身份兼容。 +- examples/http-service 提供完整装配、业务路由、数据库事务、Worker、持久 Inbox、后台错误传播及有界关闭。 +- 业务库与平台库分离；SQL沿用无外键表、中文字段注释。平台迁移及SDK协议未修改。 + +## 验收场景 + +1. 业务回滚同时撤销待发记录；未提交的记录不能领取；调用方负责 Commit。 +2. 8 个并发重复、同事务重复、RR 旧快照核对；内容/目标冲突拒绝；大整数保真。 +3. 接受钩子失败整笔确认回滚；无钩子也可确认，不依赖示例业务表；旧租约代次不能覆盖新确认。 +4. 原HTTP回执丢失、三个阶段的真实MySQL COMMIT成功包丢失、平台受理后发送器SIGKILL及自然租约接管。 +5. Run取消时等待在途发送完成确认，超时返回错误并保留持久恢复状态。 +6. HTTP请求未结束时后台依赖保持可用；后台致命错误触发宿主关闭；消费器在发送/执行组件之后结束。 +7. 完整进程：代理令Orbit不可用，HTTP仍提交本地事务；SIGKILL旧宿主并确认退出；新进程恢复原操作，实际Worker执行、Inbox入库、平台ACK；重复HTTP不新建业务行，改内容409，SIGTERM退出0。 + +## 验证状态 + +环境：macOS arm64、Go 1.27.1、Docker隔离MySQL 8.4、独立业务库与Orbit平台进程，回环地址；业务与平台使用同一隔离MySQL实例，不代表跨故障域部署。 + +- `make test`：六个模块的标准竞态测试通过（未配置数据库时Integration会跳过，真实数据库证据另列）。 +- `make vet build sdk-check`：通过；SDK 30个包、禁止依赖0项。适配器在GOWORK=off下使用已发布SDK v0.1.0，依赖图无平台内部包。 +- `make docs-check`：166份Markdown的本地链接检查通过。 +- `make adapters-check`：隔离MySQL/平台/业务模块定向回归退出0；公共Outbox 33.792s，完整HTTP进程场景9.52s，全部通过。恢复测试真实等待旧发送租约自然到期。 +- `make integration`：全量平台回归退出0；包含真实存储/升级、SDK、旧CLI、公共Outbox、HTTP宿主及Worker/Event/Workflow进程恢复。全量链路中Outbox 33.934s，HTTP进程场景9.64s，新增同事务重复与RR核对均通过。 + +复现：`make adapters-check`（定向隔离链路）和 `make integration`（平台全量隔离链路）。不操作现有开发数据库或运行容器。 + +## 本批发现与修正 + +新增发送器关闭测试的初版 HTTP 桩缺失协议必填 request_id，SDK正确拒绝无效响应，使测试未进入提交阶段。补齐符合正式协议的响应后重新执行。随后发现超时路径的测试桩仍持有未释放屏障，导致 httptest.Close 等待；已在清理中显式、幂等释放屏障并重跑。两次未通过运行均不计入通过证据。 + +## 边界 + +- 首版适配器仅支持 MySQL、SubmitTask；不会自动写业务表、删除历史或自动解除 REVIEW。 +- 外部副作用仍需业务幂等/核对；示例 Worker 仅做可重复纯计算。 +- 每条记录沿用24小时/100次总发送预算，重启不延长。ACCEPTED仅为平台受理。 +- HTTP示例只监听回环地址；接入真实服务复用业务自身鉴权、监控和部署生命周期。健康就绪不等于全局任务已经执行完毕。 +- Inbox是示例业务实现，不是本批新增的公共适配包；其他数据库/ORM专用集成不在本批交付内。 diff --git a/docs/reports/phase-3-task-outbox.md b/docs/reports/phase-3-task-outbox.md index 789cfcd..faa9065 100644 --- a/docs/reports/phase-3-task-outbox.md +++ b/docs/reports/phase-3-task-outbox.md @@ -7,13 +7,13 @@ - [独立模块](../../examples/task-outbox/README.md)只依赖独立Orbit SDK与业务侧MySQL驱动，核心SDK没有新增依赖。 - [模型](../../examples/task-outbox/internal/outbox/model.go)与[建表](../../examples/task-outbox/internal/outbox/schema.sql)：2个业务实体、22个Go/SQL中文注释字段，零外键；不改变平台35个实体和289个持久化字段。 - [业务事务](../../examples/task-outbox/internal/outbox/store.go)：业务操作与待发命令原子创建，稳定操作/提交身份、冻结请求、摘要、有限次数、租约代次及原回执；确认与业务任务关联再次同事务。 -- [发送器](../../examples/task-outbox/internal/outbox/sender.go)：事务外HTTP、当前凭据核对目标、原键重报、背压退避、未知结果分类、过期接管与旧代次拒绝。 +- [发送器](../../adapters/outbox/mysql/sender.go)：事务外HTTP、当前凭据核对目标、原键重报、背压退避、未知结果分类、过期接管与旧代次拒绝。 - [CLI](../../examples/task-outbox/cmd/sender/main.go)及[完整命令/方法/恢复手册](../examples/task-outbox.md)：init、enqueue、send、status和demo。 - [Makefile](../../Makefile)和[隔离脚本](../../scripts/test-mysql.sh)纳入独立模块，使用单独业务数据库和专用平台凭据，不触碰现有开发数据库。 ## 2. 定向数据库与故障证据 -[数据库回归](../../examples/task-outbox/internal/outbox/store_integration_test.go)验证： +[数据库回归](../../adapters/outbox/mysql/store_integration_test.go)验证： 1. 8个相同业务操作并发入库只有一条业务记录/Outbox，7次重复核对；修改正文、源引用或目标队列不能复用原操作。 2. 实际业务INSERT失败时Outbox一起回滚；业务关联UPDATE失败时ACCEPTED一起回滚，恢复后可保存原回执。 @@ -21,13 +21,13 @@ 4. 错误目标、损坏文档、权限拒绝、幂等冲突和无效回执停止自动发送；背压遵守Retry-After，次数耗尽和截止产生REVIEW，最后一次未知响应不能绕过上限。 5. 实际Schema查询确认22个中文列注释、外键0，通过实际违规写入验证JSON、状态、次数和截止CHECK生效，领取查询使用ix_sender_due且无filesort；大整数9007199254740993在冻结请求与真实回执中保持精确。 -[COMMIT故障代理](../../examples/task-outbox/internal/outbox/commit_loss_integration_test.go)在业务入库、领取和本地确认三处分别截获真实MySQL COMMIT OK包后丢弃。调用方得到ErrCommitUnconfirmed，新Store分别核对已存在业务记录、已提交租约和已确认关联，不假定COMMIT失败即回滚。 +[COMMIT故障代理](../../adapters/outbox/mysql/commit_loss_integration_test.go)在业务入库、领取和本地确认三处分别截获真实MySQL COMMIT OK包后丢弃。调用方得到ErrCommitUnconfirmed，新Store分别核对已存在业务记录、已提交租约和已确认关联，不假定COMMIT失败即回滚。 -[HTTP回执丢失](../../examples/task-outbox/internal/outbox/http_integration_test.go)先取得真实平台受理结果，再向客户端发送不完整响应并断开；Outbox保持PENDING，原键重报后同一任务ACCEPTED、deduplicated=true、发送次数2，队列只有一项任务。错误持久化命名空间在提交前被拒绝。 +[HTTP回执丢失](../../adapters/outbox/mysql/http_integration_test.go)先取得真实平台受理结果，再向客户端发送不完整响应并断开；Outbox保持PENDING，原键重报后同一任务ACCEPTED、deduplicated=true、发送次数2，队列只有一项任务。错误持久化命名空间在提交前被拒绝。 -[发送进程故障](../../examples/task-outbox/internal/outbox/process_integration_test.go)等待真实平台受理后、本地确认前的屏障，再SIGKILL并确认旧进程已退出。新进程等待数据库30秒租约自然到期，以原键取得原回执；最终只存在一个平台任务和一条本地业务记录。测试没有直接改时间来替代这项进程接管。 +[发送进程故障](../../adapters/outbox/mysql/process_integration_test.go)等待真实平台受理后、本地确认前的屏障，再SIGKILL并确认旧进程已退出。新进程等待数据库30秒租约自然到期，以原键取得原回执；最终只存在一个平台任务和一条本地业务记录。测试没有直接改时间来替代这项进程接管。 -定向脚本第三轮正常退出，状态码0：Outbox集成33.009秒，其中发送进程故障31.19秒；init/demo均正常完成。随后增加的[真实CLI测试](../../examples/task-outbox/internal/outbox/cli_integration_test.go)进入本轮全量回归，验证编译后的enqueue重复、status、send及正常信号停止。 +定向脚本第三轮正常退出，状态码0：Outbox集成33.009秒，其中发送进程故障31.19秒；init/demo均正常完成。随后增加的[真实CLI测试](../../adapters/outbox/mysql/cli_integration_test.go)进入本轮全量回归，验证编译后的enqueue重复、status、send及正常信号停止。 ## 3. 发现与修正 @@ -73,3 +73,5 @@ make integration - REVIEW保留原数据，由接入方带证据和审计的管理流程处置；本示例未实现通用业务审核系统或跨业务ORM适配。 - 核心SDK、平台Schema和执行协议均未改变；新增业务示例不是已发布的兼容性SDK包。 - 流程启动与信号Outbox类型需随Weave API实现；P6继续容量、数据库切换、备份恢复及运维验证。当前正确性与进程恢复证据不等同于生产高可用或百万规模容量结论。 + +2026-09-24 源码组织调整：发送核心及原故障回归迁至独立 adapters/outbox/mysql 模块，业务示例调用公共包；Outbox Record/SQL见[公共模型](../../adapters/outbox/mysql/model.go)与[表结构](../../adapters/outbox/mysql/schema.sql)，sender_jobs 保留在旧业务示例。上述历史验证日期不变，新增验证另见[接入适配报告](outbox-http-integration.md)。 diff --git a/docs/sdk/README.md b/docs/sdk/README.md index e9cc5a8..990b2db 100644 --- a/docs/sdk/README.md +++ b/docs/sdk/README.md @@ -88,7 +88,7 @@ fmt.Println(current.ID, current.State, current.Revision) 受理回执不是业务执行结果。同键重报返回原始快照；当前状态应使用 GetTask。业务侧创建操作与平台提交之间存在双写边界，接入方应在自身事务中保存操作记录/Outbox，再使用稳定身份提交，不能依靠随机新键补发来“修复”未知结果。 -[业务事务Outbox完整场景](../examples/task-outbox.md)已提供独立业务数据库、原子业务写入、持久化发送器、原身份恢复及CLI；核心SDK保持无MySQL/ORM依赖。业务适配包的Enqueue/SendOnce/Status与实际运行命令均在该手册说明。 +[业务事务Outbox完整场景](../examples/task-outbox.md)已提供独立业务数据库、原子业务写入、持久化发送器、原身份恢复及CLI；核心SDK保持无MySQL/ORM依赖。现已抽为[可直接导入的 MySQL 适配包](outbox-mysql.md)，支持 EnqueueTx/SendOnce/Run/Status 和受理确认钩子；[完整 HTTP 服务示例](../examples/http-service.md)演示宿主生命周期与结果入库。 ## 3. 取消与错误处理 diff --git a/docs/sdk/outbox-mysql.md b/docs/sdk/outbox-mysql.md new file mode 100644 index 0000000..d82595f --- /dev/null +++ b/docs/sdk/outbox-mysql.md @@ -0,0 +1,123 @@ +# MySQL 事务 Outbox 适配包 + +## 1. 用途与安装 + +业务写库与提交 Orbit 是两次独立操作。Outbox 将业务变更和待发请求放在同一个业务数据库事务中，再通过后台发送器提交任务。提交响应丢失时复用原操作身份，避免业务记录已保存而任务丢失。 + +```sh +go get git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql@codex/orbit-foundation +``` + +Go 会将分支解析为固定版本写入 go.mod。上线应固定已验证的版本，不在每次部署时追踪分支。本包要求 Go 1.27.1，使用 SDK v0.1.0 和 MySQL 8.4 验证；不向核心 SDK 加入 SQL/ORM 依赖，也不依赖 Orbit 服务端模型。其他数据库和 ORM 专用适配尚未提供。 + +```go +import ( + "context" + "database/sql" + "time" + outbox "git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql" + orbit "git.yuntuo.hk/k2safe/orbit/sdk/go" +) +``` + +完整可编译工程在 [HTTP 服务示例](../../examples/http-service/README.md)；[旧 CLI 示例](../examples/task-outbox.md)现在也调用本包，不维护第二份发送实现。 + +## 2. 数据库与事务约束 + +在**业务数据库**应用 [schema.sql](../../adapters/outbox/mysql/schema.sql)。表名保留 `sender_outbox`，兼容原示例已经初始化的表。中文模型见 [Record](../../adapters/outbox/mysql/model.go)。没有外键，不执行数据删除，不属于 Orbit 平台迁移。 + +- `New` 不建表、不连接网络、不接管连接池关闭；显式 `Init` 只创建新表，遇到已有表报错。 +- 生产由业务迁移工具管理 `SchemaSQL()` 的内容；不要在每次启动时执行 Init。 +- `db` 和传入 `tx` 必须来自同一个业务库、同一个主写端；不支持跨库事务或读副本确认。 +- 自行创建连接池时配置 `parseTime=true`、UTC 会话时区和合理连接/读写超时；`Open(ctx, dsn)` 是可选的已配置连接池入口。 +- 事务锁序统一为 **Outbox → 业务行**。先调用 EnqueueTx，再写业务行；受理回调遵守同样顺序。不要持有业务行锁反向等待 Outbox。 +- 业务操作 ID 必须由调用方在首次请求前生成并保存，格式为 32 位小写十六进制；同命名空间唯一。HTTP 调用方丢响应后使用原 ID 与原内容重试。 + +## 3. 公共方法 + +| 方法 | 职责 | +| --- | --- | +| `Open(ctx, dsn) (*sql.DB, error)` | 可选的业务库连接池构造；不会自动建表 | +| `SchemaSQL() string` | 返回初始表结构供业务迁移管理 | +| `Init(ctx, db) error` | 显式创建新表，仅用于首次初始化 | +| `New(db, namespaceID, queueID, Options) (*Store, error)` | 固定目标及可选受理回调，可并发复用 | +| `EnqueueTx(ctx, tx, operationID, SubmitTaskRequest) (duplicate, error)` | 在调用方事务写入待发任务；不 Commit/Rollback、不发 HTTP | +| `SendOnce(ctx, client) (Record, error)` | 有界领取并处理一条到期记录；无到期项返回 ErrIdle | +| `Run(ctx, client, RunOptions) error` | 持续发送；取消后停止领取，限时收尾当前发送 | +| `Status(ctx, operationID) (Record, error)` | 只读固定命名空间及队列中的本地提交状态 | + +基本构造不要求业务回调： + +```go +sender, err := outbox.New(db, namespaceID, queueID, outbox.Options{}) +if err != nil { return err } +``` + +`Options.OnAccepted` 类型为： + +```go +type AcceptedHook func(context.Context, *sql.Tx, Record, orbit.SubmitTaskResult) error +``` + +回调与 Outbox 的 ACCEPTED 更新在同一个短事务内执行，并核对最新领取代次。可在此将平台 TaskID 关联到业务记录。回调必须幂等、响应 context，只做同库 SQL，不能 Commit/Rollback 或调用外部服务；失败整笔确认回滚，返回 `ErrAcceptance`。回调收到的 Record 仍是被锁定的 SENDING 记录，受理 TaskID 从 receipt 读取。不开启回调时，任务关联保存在 Outbox 自身，业务可按 OperationID 查询。 + +## 4. 加入现有业务事务 + +下面是接入函数片段。完整业务重复核对与错误处理见 [jobs/store.go](../../examples/http-service/internal/jobs/store.go)。 + +```go +func save(ctx context.Context, db *sql.DB, sender *outbox.Store, + operationID string, request orbit.SubmitTaskRequest) error { + tx, err := db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) + if err != nil { return err } + defer tx.Rollback() + + duplicate, err := sender.EnqueueTx(ctx, tx, operationID, request) + if err != nil { return err } // 必须回滚整笔事务 + if !duplicate { + // 在这里使用 tx 写入自己的业务表。 + // 每一个 SQL 错误都必须返回；禁止忽略错误后提交。 + } else { + // 核对原业务记录与本次输入一致，不能再次执行扣款、计数等操作。 + } + if err := tx.Commit(); err != nil { + return outbox.ErrCommitUnconfirmed + } + return nil +} +``` + +EnqueueTx 返回 nil 只代表该事务内写入/核对完成。事务提交前另一发送器看不到该记录；调用方回滚会同时撤销待发任务与业务写入。同一事务重复入队返回 duplicate=true，但仍需由调用方确认 Commit。默认 REPEATABLE READ 下的重复核对使用共享锁当前读，不依赖旧快照，也不升级重复 INSERT 持有的共享锁。 + +COMMIT 错误不能解释成“肯定没保存”。保留原 ID 与内容，重做同一业务事务核对；不得生成新 ID 补发。冻结请求按字节摘要核对，目标、提交键、默认值和数字精度保持不变。 + +## 5. 发送器与生命周期 + +```go +err := sender.Run(workerContext, client, outbox.RunOptions{ + PollInterval: 500 * time.Millisecond, + ShutdownTimeout: 20 * time.Second, +}) +``` + +RunOptions 默认轮询间隔 500ms（10ms–10s），关闭期限默认 20s（1ms–5min）。HTTP 在数据库事务外，单次网络阶段最多 15s、发送请求最多 10s；领取租约 30s，SQL 确认最多 5s。数据库时间控制首次入库后 24h、最多 100 次领取的总预算；进程重启不重置预算。 + +Run context 取消后停止新领取，在关闭期限内等待当前交换与本地确认，宿主必须等待 Run 返回。`ErrShutdownTimeout` 表示有未核对的持久记录，应在重启后沿原身份恢复。HTTP 暂时失败在库内退避；数据库错误或受理回调失败会让 Run 返回错误，由宿主降低就绪状态、关闭或按自身策略恢复。REVIEW 记录停止自动发送，需要监控 Status/数据库并人工核查。 + +允许多个 Run/进程共用同一范围，由 MySQL SKIP LOCKED 和递增代次协调。网络请求可能重复，平台使用原幂等键去重；不承诺业务外部效果恰好一次。 + +## 6. 状态与错误 + +| 状态/错误 | 含义及处理 | +| --- | --- | +| PENDING / SENDING | 未完成本地受理确认；不能据此认定 Orbit 没收到 | +| ACCEPTED | 已确认平台受理，TaskID/ReceiptJSON 可用；不是执行成功 | +| REVIEW | 权限、内容冲突、无效回执或预算耗尽；保留证据人工核对 | +| ErrConflict | 原操作的目标或内容不一致，不能覆盖 | +| ErrCommitUnconfirmed | 领取/确认 COMMIT 回执不明；保留原身份核对 | +| ErrLeaseLost | 领取代次已替换，旧发送器不得覆盖新结果；Run 可继续其他项 | +| ErrAcceptance | 本地受理关联回调失败，确认事务未获成功确认 | +| ErrStorage | 存储失败；当前进度以持久记录为准 | +| ErrIdle / ErrNotFound | 无到期记录 / 当前范围没有该操作 | + +本版本只支持 SubmitTask，不提供流程启动/信号 Outbox 命令，不自动删除历史、重置发送预算或解除 REVIEW。任务完成另用可靠订阅与业务 Inbox，详见 [完整 HTTP 示例](../examples/http-service.md)。 diff --git a/docs/sdk/production-integration.md b/docs/sdk/production-integration.md index 770d533..1841dfb 100644 --- a/docs/sdk/production-integration.md +++ b/docs/sdk/production-integration.md @@ -2,6 +2,8 @@ 这份手册把 SDK、HTTP 查询和 WebSocket 结果通知串成一条可落地的接入链路。它只描述平台协议，不包含任何业务系统名称、行业规则或外部副作用。 +完整可运行的业务宿主见[HTTP 服务示例](../examples/http-service.md)，可复用事务能力见[MySQL Outbox 包](outbox-mysql.md)。 + ## 1. 接入边界 业务服务保存操作身份和请求正文，再调用 SDK 提交。`Idempotency-Key` 必须由业务库或 Outbox 持久化，网络超时、进程重启和响应不明时复用原键；不能生成新键猜测服务端是否已提交。受理回执只表示平台已经保存请求，不能当作任务或流程完成。 @@ -24,19 +26,19 @@ taskID := receipt.Task.ID ## 2. 完成通知与权威查询 -结果通知使用 Event WebSocket。连接时声明 `orbit.events.v1` 子协议和允许的来源，SDK 自动发送心跳、在连接断开后重连，并把上一次未确认的投递证明带回新连接。处理器完成业务事务后再调用 `Ack`；处理器失败或进程在提交前退出时不要确认，平台会在租约到期后重投同一交付。 +结果通知使用 Event WebSocket。连接时声明 `orbit.events.v1` 子协议和允许的来源，SDK 自动发送心跳、在连接断开后重连，并把上一次未确认的投递证明带回新连接。自动消费器的处理器完成业务事务后返回 nil，SDK 再发送 ACK；处理器失败或进程在提交前退出时不要确认，平台会在租约到期后重投同一交付。 ```go -consumer := orbit.NewEventConsumer(api, orbit.EventConsumerOptions{ +consumer, err := orbit.NewEventConsumer(api, orbit.EventConsumerOptions{ SubscriptionID: subscriptionID, - MaxHandlers: 8, -}) -err := consumer.Run(ctx, func(ctx context.Context, delivery orbit.EventDelivery) error { - // 先在业务 Inbox 中按 delivery.EventID 去重，再提交业务事务。 - // 返回 nil 后 SDK 才发送 ACK；返回错误会保留交付等待重投。 - return applyOnce(ctx, delivery) + Capacity: 8, + Handle: func(ctx context.Context, event orbit.ResultEvent) error { + // 先按 event.ID 持久化去重，去重与业务更新在同一事务确认后返回 nil。 + return applyOnce(ctx, event) + }, }) if err != nil { return err } +if err := consumer.Run(ctx); err != nil { return err } ``` 通知只推动业务侧再次读取权威状态。收到 `task.completed`、`workflow.stopped` 或 `workflow.review_required` 后，调用 `GetTask`、`GetWorkflow` 或对应历史接口；事件快照不替代当前状态，也不表示外部效果已经完成。没有 WebSocket 时可以使用有界 HTTP 查询作为兜底，但必须使用退避、截止时间和业务键，不得无限轮询。 diff --git a/examples/http-service/README.md b/examples/http-service/README.md new file mode 100644 index 0000000..22b73a0 --- /dev/null +++ b/examples/http-service/README.md @@ -0,0 +1,11 @@ +# 在现有 HTTP 服务中使用 Orbit + +可运行的 Go 服务：HTTP 业务事务 → 公共 MySQL Outbox → Orbit → 内嵌 Worker → WebSocket 结果 → 业务 Inbox。 + +- [完整准备、运行、API、故障与退出说明](../../docs/examples/http-service.md) +- [业务路由与组件装配](cmd/server/main.go) +- [生命周期顺序](cmd/server/lifecycle.go) +- [业务事务](internal/jobs/store.go) +- [业务 Inbox 示例实现](internal/inbox/store.go)，沿用独立 result-inbox 示例的事件去重与版本约束 + +`init` 创建业务库表；`provision` 用稳定身份创建结果订阅；`serve` 使用配置中的订阅 ID，启动时无需 Orbit 在线。默认只监听回环地址 127.0.0.1:8090。接入已有服务时挂载业务路由，并接入原有鉴权、监控及生命周期。 diff --git a/examples/http-service/cmd/server/integration_test.go b/examples/http-service/cmd/server/integration_test.go new file mode 100644 index 0000000..afe964b --- /dev/null +++ b/examples/http-service/cmd/server/integration_test.go @@ -0,0 +1,251 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "syscall" + "testing" + "time" + + outbox "git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql" + orbit "git.yuntuo.hk/k2safe/orbit/sdk/go" + "github.com/go-sql-driver/mysql" +) + +func TestIntegrationHTTPServiceRestartAndResults(t *testing.T) { + dsn, base, path := os.Getenv("ORBIT_TEST_HTTP_DSN"), os.Getenv("ORBIT_TEST_HTTP_URL"), os.Getenv("ORBIT_TEST_HTTP_CREDENTIALS_FILE") + if dsn == "" || base == "" || path == "" { + t.Skip("requires disposable business database and Orbit process") + } + cfg, err := mysql.ParseDSN(dsn) + if err != nil || cfg.DBName != "orbit_http_test" || !strings.HasPrefix(cfg.Addr, "127.0.0.1:") || !strings.HasPrefix(base, "http://127.0.0.1:") { + t.Fatal("requires isolated loopback environment") + } + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Second) + defer cancel() + db, err := outbox.Open(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer db.Close() + binary := filepath.Join(t.TempDir(), "server") + build := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-race", "-o", binary, ".") + if raw, err := build.CombinedOutput(); err != nil { + t.Fatalf("build failed %v %s", err, raw) + } + env := []string{} + for _, v := range os.Environ() { + if !strings.HasPrefix(v, "ORBIT_") && !strings.HasPrefix(v, "APP_") { + env = append(env, v) + } + } + env = append(env, "APP_MYSQL_DSN="+dsn, "APP_HTTP_ADDR=127.0.0.1:0", "ORBIT_URL="+base, "ORBIT_CREDENTIALS_FILE="+path) + call := func(args ...string) string { + t.Helper() + cmd := exec.CommandContext(ctx, binary, args...) + cmd.Env = env + raw, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("command failed: %v %s", err, raw) + } + return strings.TrimSpace(string(raw)) + } + call("init") + sub := call("provision") + if call("provision") != sub { + t.Fatal("provision changed subscription") + } + env = append(env, "ORBIT_SUBSCRIPTION_ID="+sub) + target, _ := url.Parse(base) + proxy := httputil.NewSingleHostReverseProxy(target) + var offline atomic.Bool + offline.Store(true) + gateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if offline.Load() { + http.Error(w, "offline", 503) + return + } + proxy.ServeHTTP(w, r) + })) + defer gateway.Close() + for i, v := range env { + if strings.HasPrefix(v, "ORBIT_URL=") { + env[i] = "ORBIT_URL=" + gateway.URL + } + } + type running struct { + cmd *exec.Cmd + done chan error + url string + stderr *bytes.Buffer + } + start := func() *running { + t.Helper() + cmd := exec.CommandContext(ctx, binary, "serve") + cmd.Env = env + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + stderr := new(bytes.Buffer) + cmd.Stderr = stderr + if err = cmd.Start(); err != nil { + t.Fatal(err) + } + address := make(chan string, 1) + go func() { + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + if value, ok := strings.CutPrefix(scanner.Text(), "http_listening="); ok { + address <- "http://" + value + } + } + close(address) + }() + done := make(chan error, 1) + go func() { done <- cmd.Wait(); close(done) }() + run := &running{cmd: cmd, done: done, stderr: stderr} + t.Cleanup(func() { cmd.Process.Kill(); <-done }) + select { + case run.url = <-address: + if run.url == "" { + <-done + t.Fatalf("startup failed: %s", stderr.String()) + } + case <-ctx.Done(): + t.Fatal("startup timeout") + } + return run + } + httpClient := &http.Client{Timeout: 5 * time.Second} + get := func(base, path string) map[string]any { + t.Helper() + r, err := httpClient.Get(base + path) + if err != nil { + t.Fatal(err) + } + defer r.Body.Close() + if r.StatusCode != 200 { + t.Fatalf("GET %s: %d", path, r.StatusCode) + } + var v map[string]any + if json.NewDecoder(r.Body).Decode(&v) != nil { + t.Fatal("invalid JSON") + } + return v + } + id, err := orbit.NewEventMessageID() + if err != nil { + t.Fatal(err) + } + payload := []byte(`{"operation_id":"` + id + `","source_ref":"objects/http-demo.txt"}`) + post := func(base string, body []byte, status int) map[string]any { + t.Helper() + r, err := httpClient.Post(base+"/jobs", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer r.Body.Close() + if r.StatusCode != status { + raw, _ := io.ReadAll(r.Body) + t.Fatalf("POST: %d %s", r.StatusCode, raw) + } + var out map[string]any + if status == 202 { + json.NewDecoder(r.Body).Decode(&out) + } + return out + } + wait := func(check func() bool) { + t.Helper() + for !check() { + select { + case <-ctx.Done(): + t.Fatal("condition timeout") + case <-time.After(50 * time.Millisecond): + } + } + } + first := start() + wait(func() bool { + r, err := httpClient.Get(first.url + "/health/ready") + if err != nil { + return false + } + r.Body.Close() + return r.StatusCode == 200 + }) + if v := post(first.url, payload, 202); v["local_committed"] != true || v["duplicate"] != false { + t.Fatal(v) + } + wait(func() bool { + var n int + err := db.QueryRowContext(ctx, `SELECT send_count FROM sender_outbox WHERE operation_id=?`, id).Scan(&n) + return err == nil && n > 0 + }) + first.cmd.Process.Kill() + if err := <-first.done; err == nil { + t.Fatal("expected real SIGKILL") + } + offline.Store(false) + second := start() + wait(func() bool { return get(second.url, "/jobs/"+id)["execution"] == "SUCCEEDED" }) + if post(second.url, payload, 202)["duplicate"] != true { + t.Fatal("original operation duplicated after restart") + } + post(second.url, bytes.ReplaceAll(payload, []byte("http-demo"), []byte("changed")), 409) + view := get(second.url, "/jobs/"+id) + result, ok := view["result"].(map[string]any) + if !ok || result["normalized_ref"] != "OBJECTS/HTTP-DEMO.TXT" { + t.Fatal(view) + } + var jobs, inboxRows, fks int + if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM sender_jobs WHERE operation_id=?`, id).Scan(&jobs); err != nil { + t.Fatal(err) + } + if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM receiver_inbox WHERE queue_id=(SELECT queue_id FROM sender_outbox WHERE operation_id=?)`, id).Scan(&inboxRows); err != nil { + t.Fatal(err) + } + if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&fks); err != nil || jobs != 1 || inboxRows != 1 || fks != 0 { + t.Fatal(jobs, inboxRows, fks, err) + } + // 先看到平台持久ACK，再发SIGTERM，验证通知不是只停留在业务内存或数据库。 + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var creds config + if json.Unmarshal(raw, &creds) != nil { + t.Fatal("credentials") + } + api, err := orbit.NewClient(base, orbit.Options{Token: creds.Token}) + if err != nil { + t.Fatal(err) + } + wait(func() bool { v, err := api.GetSubscription(ctx, sub); return err == nil && v.PendingCount == 0 }) + if err = second.cmd.Process.Signal(syscall.SIGTERM); err != nil { + t.Fatal(err) + } + select { + case err = <-second.done: + if err != nil { + t.Fatalf("SIGTERM failed %v %s", err, second.stderr.String()) + } + case <-ctx.Done(): + t.Fatal("shutdown timeout") + } + t.Log("Orbit unavailable: HTTP transaction committed; real SIGKILL/restart recovered original operation; Worker + persistent Inbox + ACK completed once; conflict rejected; SIGTERM exited cleanly; zero foreign keys") +} diff --git a/examples/http-service/cmd/server/lifecycle.go b/examples/http-service/cmd/server/lifecycle.go new file mode 100644 index 0000000..0a19a88 --- /dev/null +++ b/examples/http-service/cmd/server/lifecycle.go @@ -0,0 +1,99 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "sync/atomic" + "time" +) + +// component 不接管进程信号；宿主负责为每一类生命周期提供独立取消和等待。 +type component struct { + name string + run func(context.Context) error +} +type completion struct { + name string + err error +} + +// serve 在HTTP监听已成功后启动后台组件；返回前完成HTTP→发送/执行→结果消费的关闭顺序。 +func serve(parent context.Context, listener net.Listener, server *http.Server, ready *atomic.Bool, workers []component, consumer component, grace time.Duration) error { + workCtx, stopWork := context.WithCancel(context.Background()) + defer stopWork() + eventCtx, stopEvents := context.WithCancel(context.Background()) + defer stopEvents() + workDone := make(chan completion, len(workers)) + eventDone := make(chan completion, 1) + for _, c := range workers { + go func() { workDone <- completion{c.name, c.run(workCtx)} }() + } + go func() { eventDone <- completion{consumer.name, consumer.run(eventCtx)} }() + httpDone := make(chan error, 1) + ready.Store(true) + go func() { httpDone <- server.Serve(listener) }() + remaining := len(workers) + eventsPending := true + var result error + add := func(c completion) { + if c.err != nil { + result = errors.Join(result, fmt.Errorf("%s: %w", c.name, c.err)) + } + } + select { + case <-parent.Done(): + case err := <-httpDone: + if !errors.Is(err, http.ErrServerClosed) { + result = errors.Join(result, err) + } + case c := <-workDone: + remaining-- + if c.err == nil { + c.err = errors.New("unexpected component exit") + } + add(c) + case c := <-eventDone: + eventsPending = false + if c.err == nil { + c.err = errors.New("unexpected component exit") + } + add(c) + } + ready.Store(false) + // 用新的关闭上下文；进程信号的context此时可能已取消，不能用它调用Shutdown。 + shut, cancel := context.WithTimeout(context.Background(), grace) + if err := server.Shutdown(shut); err != nil { + result = errors.Join(result, fmt.Errorf("http shutdown: %w", err)) + _ = server.Close() + } + cancel() + stopWork() + deadline := time.NewTimer(grace) + defer deadline.Stop() + for remaining > 0 { + select { + case c := <-workDone: + remaining-- + add(c) + case <-deadline.C: + result = errors.Join(result, errors.New("background shutdown deadline exceeded")) + remaining = 0 + } + } + // 结果消费器最后停止。仍未投递的结果会在重启后由持久订阅恢复，不以本地空闲判断全局完成。 + stopEvents() + if eventsPending { + timer := time.NewTimer(grace) + defer timer.Stop() + select { + case c := <-eventDone: + add(c) + case <-timer.C: + result = errors.Join(result, errors.New("consumer shutdown deadline exceeded")) + } + } + return result +} diff --git a/examples/http-service/cmd/server/lifecycle_test.go b/examples/http-service/cmd/server/lifecycle_test.go new file mode 100644 index 0000000..356ff02 --- /dev/null +++ b/examples/http-service/cmd/server/lifecycle_test.go @@ -0,0 +1,90 @@ +package main + +import ( + "context" + "errors" + "net" + "net/http" + "sync/atomic" + "testing" + "time" +) + +func TestShutdownKeepsDependenciesUntilHTTPReturns(t *testing.T) { + entered := make(chan struct{}) + release := make(chan struct{}) + workerStopped := make(chan struct{}) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { close(entered); <-release; w.WriteHeader(200) })} + ready := new(atomic.Bool) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { + done <- serve(ctx, listener, server, ready, []component{{"worker", func(ctx context.Context) error { <-ctx.Done(); close(workerStopped); return nil }}}, component{"events", func(ctx context.Context) error { + <-ctx.Done() + select { + case <-workerStopped: + return nil + default: + return errors.New("consumer stopped before worker") + } + }}, time.Second) + }() + response := make(chan error, 1) + go func() { + r, e := http.Get("http://" + listener.Addr().String()) + if e == nil { + r.Body.Close() + } + response <- e + }() + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("HTTP not entered") + } + cancel() + select { + case <-workerStopped: + t.Fatal("worker stopped during HTTP transaction") + case <-time.After(20 * time.Millisecond): + } + if ready.Load() { + t.Fatal("readiness remained true during shutdown") + } + close(release) + if err = <-response; err != nil { + t.Fatal(err) + } + select { + case err = <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("shutdown blocked") + } +} +func TestBackgroundFailureStopsServiceAndPeers(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + boom := errors.New("fatal worker") + stopped := make(chan struct{}) + result := serve(context.Background(), listener, &http.Server{}, new(atomic.Bool), []component{{"worker", func(context.Context) error { return boom }}}, component{"consumer", func(ctx context.Context) error { <-ctx.Done(); close(stopped); return nil }}, time.Second) + if !errors.Is(result, boom) { + t.Fatal(result) + } + select { + case <-stopped: + default: + t.Fatal("consumer leaked") + } +} diff --git a/examples/http-service/cmd/server/main.go b/examples/http-service/cmd/server/main.go new file mode 100644 index 0000000..083d9db --- /dev/null +++ b/examples/http-service/cmd/server/main.go @@ -0,0 +1,234 @@ +// server 演示将 Orbit SDK 接入现有 HTTP 服务；平台仍独立部署。 +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/signal" + "strings" + "sync/atomic" + "syscall" + "time" + + outbox "git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql" + "git.yuntuo.hk/k2safe/orbit/examples/http-service/internal/inbox" + "git.yuntuo.hk/k2safe/orbit/examples/http-service/internal/jobs" + orbit "git.yuntuo.hk/k2safe/orbit/sdk/go" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +type config struct { + Token string `json:"token"` + Namespace string `json:"namespace_id"` + Queue string `json:"queue_id"` +} + +func client() (*orbit.Client, config, error) { + c := config{Token: os.Getenv("ORBIT_TOKEN"), Namespace: os.Getenv("ORBIT_NAMESPACE_ID"), Queue: os.Getenv("ORBIT_QUEUE_ID")} + if file := os.Getenv("ORBIT_CREDENTIALS_FILE"); file != "" { + raw, err := os.ReadFile(file) + if err != nil { + return nil, c, errors.New("cannot read credential file") + } + var saved config + if json.Unmarshal(raw, &saved) != nil { + return nil, c, errors.New("invalid credential file") + } + if c.Token == "" { + c.Token = saved.Token + } + if c.Namespace == "" { + c.Namespace = saved.Namespace + } + if c.Queue == "" { + c.Queue = saved.Queue + } + } + if c.Token == "" || c.Namespace == "" || c.Queue == "" { + return nil, c, errors.New("Orbit token, namespace and queue are required") + } + api, err := orbit.NewClient(os.Getenv("ORBIT_URL"), orbit.Options{Token: c.Token}) + return api, c, err +} +func run() error { + if len(os.Args) != 2 { + return errors.New("usage: server init|provision|serve") + } + command := os.Args[1] + if command != "init" && command != "provision" && command != "serve" { + return errors.New("unknown command") + } + process, signalStop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer signalStop() + startup, stop := context.WithTimeout(process, 10*time.Second) + defer stop() + if command == "provision" { + api, c, err := client() + if err != nil { + return err + } + // 首次使用专属队列前创建，键和内容固定；部署重试不可改名/换键。运行期间从配置复用ID。 + result, err := api.CreateSubscription(startup, "http-service-sub-"+c.Queue, orbit.CreateSubscriptionRequest{Name: "http-service-" + c.Queue, QueueID: c.Queue, MaxPending: 100, MaxAttempts: 20}) + if err != nil { + return err + } + fmt.Println(result.Subscription.ID) + return nil + } + db, err := outbox.Open(startup, os.Getenv("APP_MYSQL_DSN")) + if err != nil { + return err + } + // 所有组件完成关闭后才关闭共享连接池；初始化/构造/监听失败也走同一回收路径。 + defer db.Close() + if command == "init" { + if err = jobs.Init(startup, db); err != nil { + return err + } + if err = inbox.Init(startup, db); err != nil { + return err + } + fmt.Println("business_schema_initialized=true") + return nil + } + api, c, err := client() + if err != nil { + return err + } + store, err := jobs.New(db, c.Namespace, c.Queue) + if err != nil { + return err + } + // 尽早核验业务迁移；不会自动建表，也不依赖Orbit在线才能启动。 + for _, query := range []string{ + `SELECT namespace_id,queue_id,operation_id,state FROM sender_outbox LIMIT 0`, + `SELECT namespace_id,operation_id,source_ref,task_id FROM sender_jobs LIMIT 0`, + `SELECT consumer_name,event_id,event_digest FROM receiver_inbox LIMIT 0`, + `SELECT consumer_name,queue_id,task_id,last_revision,result_json FROM receiver_task_views LIMIT 0`, + } { + if _, err = db.ExecContext(startup, query); err != nil { + return errors.New("business schema missing") + } + } + receiver, err := inbox.New(db, "http-service", c.Queue) + if err != nil { + return err + } + worker, err := orbit.NewWorker(api, orbit.WorkerOptions{QueueID: c.Queue, Capacity: 4, ShutdownTimeout: 20 * time.Second, Handlers: []orbit.WorkerHandler{{Capability: orbit.WorkerCapability{Handler: "http.object.prepare", HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: prepare}}}) + if err != nil { + return err + } + consumer, err := orbit.NewEventConsumer(api, orbit.EventConsumerOptions{SubscriptionID: os.Getenv("ORBIT_SUBSCRIPTION_ID"), Capacity: 4, ShutdownTimeout: 10 * time.Second, Handle: receiver.Handle}) + if err != nil { + return err + } + ready := new(atomic.Bool) + mux := http.NewServeMux() + mux.HandleFunc("GET /health/live", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("live\n")) }) + mux.HandleFunc("GET /health/ready", func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond) + defer cancel() + if !ready.Load() || db.PingContext(ctx) != nil { + http.Error(w, "not ready", 503) + return + } + w.Write([]byte("ready\n")) + }) + mux.HandleFunc("POST /jobs", func(w http.ResponseWriter, r *http.Request) { + if !ready.Load() { + http.Error(w, "stopping", 503) + return + } + r.Body = http.MaxBytesReader(w, r.Body, 4096) + var in struct { + OperationID string `json:"operation_id"` + SourceRef string `json:"source_ref"` + } + d := json.NewDecoder(r.Body) + d.DisallowUnknownFields() + if d.Decode(&in) != nil || d.Decode(new(any)) != io.EOF { + http.Error(w, "invalid request", 400) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + payload, _ := json.Marshal(map[string]string{"source_ref": in.SourceRef}) + duplicate, err := store.Enqueue(ctx, in.OperationID, in.SourceRef, orbit.SubmitTaskRequest{QueueID: c.Queue, Handler: "http.object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: payload, BusinessKey: in.OperationID, TaskTimeoutMS: 86400000, RecoverySafety: orbit.RecoveryIdempotent, Retry: orbit.RetryPolicy{MaxAttempts: 3, AttemptTimeoutMS: 10000}}) + if err != nil { + httpFailure(w, err) + return + } + writeJSON(w, 202, map[string]any{"operation_id": in.OperationID, "local_committed": true, "duplicate": duplicate}) + }) + mux.HandleFunc("GET /jobs/{id}", func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + result, err := store.View(ctx, r.PathValue("id")) + if err != nil { + httpFailure(w, err) + return + } + writeJSON(w, 200, result) + }) + addr := os.Getenv("APP_HTTP_ADDR") + if addr == "" { + addr = "127.0.0.1:8090" + } + host, _, err := net.SplitHostPort(addr) + if err != nil || net.ParseIP(host) == nil || !net.ParseIP(host).IsLoopback() { + return errors.New("example requires a loopback APP_HTTP_ADDR; mount handlers behind your application authentication for production") + } + listener, err := net.Listen("tcp", addr) + if err != nil { + return errors.New("cannot listen on APP_HTTP_ADDR") + } + defer listener.Close() + server := &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 16 << 10} + fmt.Printf("http_listening=%s\n", listener.Addr()) + return serve(process, listener, server, ready, []component{{"outbox", func(ctx context.Context) error { + return store.Run(ctx, api, outbox.RunOptions{ShutdownTimeout: 20 * time.Second}) + }}, {"worker", worker.Run}}, component{"events", consumer.Run}, 25*time.Second) +} +func httpFailure(w http.ResponseWriter, err error) { + code, text := 503, "unconfirmed; retry original operation and content" + if errors.Is(err, outbox.ErrInvalid) { + code, text = 400, "invalid request" + } + if errors.Is(err, outbox.ErrConflict) { + code, text = 409, "operation content conflict" + } + if errors.Is(err, outbox.ErrNotFound) { + code, text = 404, "operation not found" + } + http.Error(w, text, code) +} +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} +func prepare(ctx context.Context, e orbit.WorkerExecution) (json.RawMessage, error) { + var in struct { + SourceRef string `json:"source_ref"` + } + if json.Unmarshal(e.Task.Payload, &in) != nil || in.SourceRef == "" { + return nil, &orbit.WorkerFailure{Code: "INVALID_SOURCE"} + } + if err := ctx.Err(); err != nil { + return nil, err + } + // 纯计算可重复；实际外部副作用必须由业务实现幂等或核对，不能照搬幂等声明。 + return json.Marshal(map[string]string{"source_ref": in.SourceRef, "normalized_ref": strings.ToUpper(in.SourceRef)}) +} diff --git a/examples/http-service/go.mod b/examples/http-service/go.mod new file mode 100644 index 0000000..970380a --- /dev/null +++ b/examples/http-service/go.mod @@ -0,0 +1,16 @@ +module git.yuntuo.hk/k2safe/orbit/examples/http-service + +go 1.27.1 + +require ( + git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql v0.0.0 + git.yuntuo.hk/k2safe/orbit/sdk/go v0.1.0 + github.com/go-sql-driver/mysql v1.10.1 +) + +require ( + filippo.io/edwards25519 v1.2.0 // indirect + github.com/coder/websocket v1.8.15 // indirect +) + +replace git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql => ../../adapters/outbox/mysql diff --git a/examples/http-service/go.sum b/examples/http-service/go.sum new file mode 100644 index 0000000..a74aa43 --- /dev/null +++ b/examples/http-service/go.sum @@ -0,0 +1,8 @@ +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +git.yuntuo.hk/k2safe/orbit/sdk/go v0.1.0 h1:OJp+U8dDNKjiXy4ZhO4MMq68Ek8TkjphUAqsjOOg77c= +git.yuntuo.hk/k2safe/orbit/sdk/go v0.1.0/go.mod h1:9KlMW7Gy+EGaDoEwr3wHHFY47mnAymSfYeQpMS7h4ag= +github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= +github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= +github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= +github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= diff --git a/examples/http-service/internal/inbox/model.go b/examples/http-service/internal/inbox/model.go new file mode 100644 index 0000000..2f708cc --- /dev/null +++ b/examples/http-service/internal/inbox/model.go @@ -0,0 +1,28 @@ +package inbox + +import "time" + +// Record 对应receiver_inbox，与业务视图同事务提交；存在记录才表示该事件处理已提交。 +type Record struct { + ConsumerName string // ConsumerName 固定用途名；进程重启及多实例必须复用。 + QueueID string // QueueID 固定来源队列，不能从连接会话推导。 + EventID string // EventID 稳定事件身份，不使用交付ID或代次。 + EventDigest []byte // EventDigest 32字节SHA256，防止同事件身份被换成不同内容。 + EventJSON []byte // EventJSON 规范化冻结事件，含业务数据、不含租约秘密。 + Applied bool // Applied 该事件是否推进业务视图；旧版本为false但仍可靠保存。 + ReceivedAt time.Time // ReceivedAt 数据库UTC受理时间，微秒精度。 +} + +// TaskView 对应receiver_task_views；这是接收方业务数据，不是Orbit任务权威状态。 +type TaskView struct { + ConsumerName string // ConsumerName 与Inbox相同的固定用途。 + QueueID string // QueueID 与Inbox相同的绑定队列。 + TaskID string // TaskID 原任务标识，业务视图主键的一部分。 + LastRevision uint64 // LastRevision 已应用的最大任务版本；0只允许在未提交事务的占位行中。 + LastEventID *string // LastEventID 最后推进视图的事件；仅占位状态可为空。 + LastState string // LastState 冻结结果状态；NEEDS_REVIEW不能作为成功或最终失败。 + ResultJSON []byte // ResultJSON 当前结果，nil表示来源未提供；数值保持精确。 + AppliedEvents uint64 // AppliedEvents 实际推进次数，重复及迟到旧版本不增加。 + CreatedAt time.Time // CreatedAt 视图首次创建的UTC时间，微秒精度。 + UpdatedAt time.Time // UpdatedAt 最后推进的UTC时间，微秒精度。 +} diff --git a/examples/http-service/internal/inbox/schema.sql b/examples/http-service/internal/inbox/schema.sql new file mode 100644 index 0000000..a2ca50f --- /dev/null +++ b/examples/http-service/internal/inbox/schema.sql @@ -0,0 +1,32 @@ +-- 接收方独立数据库的初始示例表；不属于Orbit平台迁移，不创建外键或级联删除。 +CREATE TABLE receiver_inbox ( + consumer_name VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定消费者用途；同一用途重启或多实例必须复用', + queue_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '接收方绑定的队列标识；隔离不同来源范围', + event_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '原事件标识；跨连接及交付代次保持不变', + event_digest BINARY(32) NOT NULL COMMENT '规范化冻结事件JSON的SHA256；同身份不同内容拒绝确认', + event_json MEDIUMTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '冻结事件及业务结果；包含业务数据须限制访问，不含租约证明', + applied BOOLEAN NOT NULL DEFAULT FALSE COMMENT '是否推进业务视图；迟到的旧版本只保留Inbox历史', + received_at DATETIME(6) NOT NULL COMMENT '本次业务事务受理事件的UTC时间，微秒精度', + PRIMARY KEY (consumer_name,queue_id,event_id), + CONSTRAINT ck_receiver_event_json CHECK (JSON_VALID(event_json)), + CONSTRAINT ck_receiver_applied CHECK (applied IN (0,1)) +) ENGINE=InnoDB COMMENT='事务Inbox；去重记录与业务视图一起提交，禁止独立预写成功标志'; + +CREATE TABLE receiver_task_views ( + consumer_name VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定消费者用途；与Inbox使用相同范围', + queue_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定队列来源；由处理器事务前验证归属', + task_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '原任务标识；每个任务建立一份版本化业务视图', + last_revision BIGINT UNSIGNED NOT NULL COMMENT '已应用最大任务版本；0仅为同事务中尚未更新的占位状态', + last_event_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL COMMENT '推进当前视图的事件；占位行为空，提交后的有效视图非空', + last_state VARCHAR(24) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '冻结任务结果状态；UNINITIALIZED只用于同事务内占位', + result_json MEDIUMTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL COMMENT '当前结果JSON；来源未提供结果时为空，保留精确数字', + applied_events BIGINT UNSIGNED NOT NULL COMMENT '实际推进视图的事件次数；重复和旧版本不增加', + created_at DATETIME(6) NOT NULL COMMENT '视图首次建立的UTC时间，微秒精度', + updated_at DATETIME(6) NOT NULL COMMENT '视图最后推进的UTC时间，微秒精度', + PRIMARY KEY (consumer_name,queue_id,task_id), + CONSTRAINT ck_receiver_revision CHECK ( + (last_revision=0 AND last_event_id IS NULL AND last_state='UNINITIALIZED' AND applied_events=0) + OR (last_revision>0 AND last_event_id IS NOT NULL AND applied_events>0 AND last_state IN ('SUCCEEDED','FAILED','CANCELLED','NEEDS_REVIEW')) + ), + CONSTRAINT ck_receiver_result CHECK (result_json IS NULL OR JSON_VALID(result_json)) +) ENGINE=InnoDB COMMENT='业务侧任务结果视图；锁定单任务行后仅接受更高版本，防止迟到结果倒退'; diff --git a/examples/http-service/internal/inbox/store.go b/examples/http-service/internal/inbox/store.go new file mode 100644 index 0000000..cb6dbf2 --- /dev/null +++ b/examples/http-service/internal/inbox/store.go @@ -0,0 +1,314 @@ +// Package inbox 演示业务端同事务去重和版本化结果视图，不是通用业务恰好一次框架。 +package inbox + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + _ "embed" + "encoding/json" + "errors" + "io" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" + + orbit "git.yuntuo.hk/k2safe/orbit/sdk/go" + "github.com/go-sql-driver/mysql" +) + +//go:embed schema.sql +var schema string + +var ( + ErrInvalidEvent = errors.New("receiver event or scope is invalid") + ErrEventConflict = errors.New("receiver event identity has conflicting content; investigate before acknowledgement") + ErrRevisionConflict = errors.New("receiver task revision has conflicting event identities") + ErrStorage = errors.New("receiver storage operation failed; event remains unacknowledged") + ErrCommitUnconfirmed = errors.New("receiver transaction commit is unconfirmed; reconcile original event on redelivery") + namePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`) + idPattern = regexp.MustCompile(`^[a-f0-9]{32}$`) +) + +type Store struct { + db *sql.DB + consumer, queue string +} +type Result struct { + Duplicate bool // Duplicate 当前事件已在相同消费者范围内提交，内容摘要也相同。 + Applied bool // Applied 原事务是否推进视图；重复命中时返回历史值，不表示再次推进。 +} + +// Open 仅连接接收方数据库；密码和原始DSN不进入错误输出，不自动创建或修改表。 +func Open(ctx context.Context, dsn string) (*sql.DB, error) { + cfg, err := mysql.ParseDSN(dsn) + if err != nil || cfg.DBName == "" { + return nil, errors.New("valid receiver MySQL DSN with explicit database is required") + } + cfg.ParseTime = true + cfg.Loc = time.UTC + cfg.MultiStatements = false + cfg.AllowAllFiles = false + cfg.Timeout = 5 * time.Second + cfg.ReadTimeout = 8 * time.Second + cfg.WriteTimeout = 8 * time.Second + if cfg.Params == nil { + cfg.Params = map[string]string{} + } + cfg.Params["time_zone"] = "'+00:00'" + connector, err := mysql.NewConnector(cfg) + if err != nil { + return nil, ErrStorage + } + db := sql.OpenDB(connector) + db.SetMaxOpenConns(8) + db.SetMaxIdleConns(4) + db.SetConnMaxLifetime(5 * time.Minute) + if err = db.PingContext(ctx); err != nil { + db.Close() + return nil, ErrStorage + } + return db, nil +} + +// Init 初始化专用空数据库。已有表会报错；升级已有业务数据库应使用接收方自己的迁移。 +func Init(ctx context.Context, db *sql.DB) error { + for _, statement := range strings.Split(schema, ";") { + if strings.TrimSpace(statement) == "" { + continue + } + if _, err := db.ExecContext(ctx, statement); err != nil { + return ErrStorage + } + } + return nil +} +func New(db *sql.DB, consumer, queue string) (*Store, error) { + if db == nil || !namePattern.MatchString(consumer) || !idPattern.MatchString(queue) { + return nil, ErrInvalidEvent + } + return &Store{db: db, consumer: consumer, queue: queue}, nil +} + +// Handle 可直接作为EventConsumer处理器；只有事务确认提交或命中相同已提交记录才返回nil。 +func (s *Store) Handle(ctx context.Context, event orbit.ResultEvent) error { + _, err := s.Apply(ctx, event) + return err +} + +// Apply 不含网络副作用。事务失败或COMMIT响应未知时不允许SDK发送ACK。 +func (s *Store) Apply(parent context.Context, event orbit.ResultEvent) (Result, error) { + ctx, stop := context.WithTimeout(parent, 5*time.Second) + defer stop() + raw, digest, err := s.document(event) + if err != nil { + return Result{}, err + } + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) + if err != nil { + return Result{}, ErrStorage + } + defer tx.Rollback() + _, err = tx.ExecContext(ctx, `INSERT INTO receiver_inbox(consumer_name,queue_id,event_id,event_digest,event_json,applied,received_at) VALUES(?,?,?,?,?,FALSE,UTC_TIMESTAMP(6))`, s.consumer, s.queue, event.ID, digest[:], raw) + if err != nil { + var duplicate *mysql.MySQLError + if !errors.As(err, &duplicate) || duplicate.Number != 1062 { + return Result{}, ErrStorage + } + // 唯一键冲突已等待胜出事务完成。结束本事务，再核对已提交记录；不能忽略其他SQL错误。 + if err = tx.Rollback(); err != nil { + return Result{}, ErrStorage + } + var saved []byte + var applied bool + if err = s.db.QueryRowContext(ctx, `SELECT event_digest,applied FROM receiver_inbox WHERE consumer_name=? AND queue_id=? AND event_id=?`, s.consumer, s.queue, event.ID).Scan(&saved, &applied); err != nil { + return Result{}, ErrStorage + } + if !bytes.Equal(saved, digest[:]) { + return Result{}, ErrEventConflict + } + return Result{Duplicate: true, Applied: applied}, nil + } + // 原子建立或取得单任务行锁，不依赖客户端RowsAffected选项判断新旧。 + _, err = tx.ExecContext(ctx, `INSERT INTO receiver_task_views(consumer_name,queue_id,task_id,last_revision,last_state,applied_events,created_at,updated_at) VALUES(?,?,?,0,'UNINITIALIZED',0,UTC_TIMESTAMP(6),UTC_TIMESTAMP(6)) ON DUPLICATE KEY UPDATE task_id=receiver_task_views.task_id`, s.consumer, s.queue, event.AggregateID) + if err != nil { + return Result{}, ErrStorage + } + var revision uint64 + if err = tx.QueryRowContext(ctx, `SELECT last_revision FROM receiver_task_views WHERE consumer_name=? AND queue_id=? AND task_id=? FOR UPDATE`, s.consumer, s.queue, event.AggregateID).Scan(&revision); err != nil { + return Result{}, ErrStorage + } + if event.AggregateRevision == revision { + return Result{}, ErrRevisionConflict + } + applied := event.AggregateRevision > revision + if applied { + var result any + if len(event.Snapshot.Result) > 0 { + result = []byte(event.Snapshot.Result) + } + _, err = tx.ExecContext(ctx, `UPDATE receiver_task_views SET last_revision=?,last_event_id=?,last_state=?,result_json=?,applied_events=applied_events+1,updated_at=UTC_TIMESTAMP(6) WHERE consumer_name=? AND queue_id=? AND task_id=?`, event.AggregateRevision, event.ID, string(event.Snapshot.State), result, s.consumer, s.queue, event.AggregateID) + if err != nil { + return Result{}, ErrStorage + } + if _, err = tx.ExecContext(ctx, `UPDATE receiver_inbox SET applied=TRUE WHERE consumer_name=? AND queue_id=? AND event_id=?`, s.consumer, s.queue, event.ID); err != nil { + return Result{}, ErrStorage + } + } + if err = tx.Commit(); err != nil { + return Result{}, ErrCommitUnconfirmed + } + return Result{Applied: applied}, nil +} + +func (s *Store) document(event orbit.ResultEvent) ([]byte, [32]byte, error) { + var none [32]byte + snapshot := event.Snapshot + if len(event.Details)+len(snapshot.Result) > 512<<10 { + return nil, none, ErrInvalidEvent + } + if !idPattern.MatchString(event.ID) || event.Sequence == 0 || event.AggregateKind != "task" || !idPattern.MatchString(event.AggregateID) || snapshot.TaskID != event.AggregateID || snapshot.QueueID != s.queue || event.AggregateRevision == 0 || event.AggregateRevision != snapshot.Revision || event.OccurredAt.IsZero() || snapshot.SchemaVersion != 1 || snapshot.CurrentAttemptID != nil && !idPattern.MatchString(*snapshot.CurrentAttemptID) { + return nil, none, ErrInvalidEvent + } + states := map[string]orbit.TaskState{"task.succeeded": orbit.TaskSucceeded, "task.failed": orbit.TaskFailed, "task.cancelled": orbit.TaskCancelled, "task.needs_review": orbit.TaskNeedsReview} + if state, ok := states[event.Type]; !ok || state != snapshot.State { + return nil, none, ErrInvalidEvent + } + if len(bytes.TrimSpace(event.Details)) == 0 || bytes.TrimSpace(event.Details)[0] != '{' { + return nil, none, ErrInvalidEvent + } + event.OccurredAt = event.OccurredAt.UTC() + var buffer bytes.Buffer + encoder := json.NewEncoder(&buffer) + encoder.SetEscapeHTML(false) + if encoder.Encode(event) != nil || buffer.Len() > 512<<10 { + return nil, none, ErrInvalidEvent + } + // 对象键与空白规范化，数字使用json.Number保留原精度及十进制表达。 + canonical, err := canonicalJSON(buffer.Bytes()) + if err != nil { + return nil, none, err + } + return canonical, sha256.Sum256(canonical), nil +} +func canonicalJSON(raw []byte) ([]byte, error) { + if !utf8.Valid(raw) || !json.Valid(raw) || !pairedUnicode(raw) { + return nil, ErrInvalidEvent + } + d := json.NewDecoder(bytes.NewReader(raw)) + d.UseNumber() + var value func(int) (any, error) + value = func(depth int) (any, error) { + if depth > 32 { + return nil, ErrInvalidEvent + } + token, err := d.Token() + if err != nil { + return nil, ErrInvalidEvent + } + switch token { + case json.Delim('{'): + out := map[string]any{} + for d.More() { + key, err := d.Token() + if err != nil { + return nil, ErrInvalidEvent + } + name, ok := key.(string) + if !ok { + return nil, ErrInvalidEvent + } + if _, found := out[name]; found { + return nil, ErrInvalidEvent + } + out[name], err = value(depth + 1) + if err != nil { + return nil, err + } + } + _, err = d.Token() + return out, err + case json.Delim('['): + out := []any{} + for d.More() { + item, err := value(depth + 1) + if err != nil { + return nil, err + } + out = append(out, item) + } + _, err = d.Token() + return out, err + default: + return token, nil + } + } + doc, err := value(0) + if err != nil { + return nil, ErrInvalidEvent + } + if _, err = d.Token(); err != io.EOF { + return nil, ErrInvalidEvent + } + var out bytes.Buffer + e := json.NewEncoder(&out) + e.SetEscapeHTML(false) + if e.Encode(doc) != nil { + return nil, ErrInvalidEvent + } + return bytes.TrimSuffix(out.Bytes(), []byte{'\n'}), nil +} + +// 不将未配对UTF-16转义静默转换成替代字符，避免摘要与实际业务文本失真。 +func pairedUnicode(raw []byte) bool { + inside := false + for i := 0; i < len(raw); i++ { + if raw[i] == '"' { + inside = !inside + continue + } + if !inside || raw[i] != '\\' { + continue + } + i++ + if i >= len(raw) { + return false + } + if raw[i] != 'u' { + continue + } + if i+4 >= len(raw) { + return false + } + n, err := strconv.ParseUint(string(raw[i+1:i+5]), 16, 16) + if err != nil { + return false + } + i += 4 + if n < 0xd800 || n > 0xdfff { + continue + } + if n > 0xdbff || i+6 >= len(raw) || raw[i+1] != '\\' || raw[i+2] != 'u' { + return false + } + low, err := strconv.ParseUint(string(raw[i+3:i+7]), 16, 16) + if err != nil || low < 0xdc00 || low > 0xdfff { + return false + } + i += 6 + } + return !inside +} + +// View 返回业务侧当前视图；不会从Orbit重新拼装结果。 +func (s *Store) View(ctx context.Context, task string) (TaskView, error) { + out := TaskView{ConsumerName: s.consumer, QueueID: s.queue, TaskID: task} + err := s.db.QueryRowContext(ctx, `SELECT last_revision,last_event_id,last_state,result_json,applied_events,created_at,updated_at FROM receiver_task_views WHERE consumer_name=? AND queue_id=? AND task_id=?`, s.consumer, s.queue, task).Scan(&out.LastRevision, &out.LastEventID, &out.LastState, &out.ResultJSON, &out.AppliedEvents, &out.CreatedAt, &out.UpdatedAt) + if err != nil { + return TaskView{}, ErrStorage + } + return out, nil +} diff --git a/examples/http-service/internal/inbox/store_test.go b/examples/http-service/internal/inbox/store_test.go new file mode 100644 index 0000000..545a2cc --- /dev/null +++ b/examples/http-service/internal/inbox/store_test.go @@ -0,0 +1,59 @@ +package inbox + +import ( + "bytes" + "encoding/json" + "errors" + "testing" + "time" + + orbit "git.yuntuo.hk/k2safe/orbit/sdk/go" +) + +func newID(t *testing.T) string { + t.Helper() + id, err := orbit.NewEventMessageID() + if err != nil { + t.Fatal(err) + } + return id +} +func testEvent(t *testing.T, queue string) orbit.ResultEvent { + t.Helper() + task := newID(t) + return orbit.ResultEvent{ID: newID(t), Sequence: 7, Type: "task.succeeded", AggregateKind: "task", AggregateID: task, AggregateRevision: 9007199254740993, OccurredAt: time.Now().UTC().Truncate(time.Microsecond), Details: json.RawMessage(`{"origin":"worker"}`), Snapshot: orbit.TaskResultSnapshot{SchemaVersion: 1, TaskID: task, QueueID: queue, State: orbit.TaskSucceeded, Revision: 9007199254740993, Result: json.RawMessage(`{"n":9007199254740993,"text":"<&>"}`)}} +} +func TestFrozenEventIdentity(t *testing.T) { + s := &Store{consumer: "result-view", queue: newID(t)} + e := testEvent(t, s.queue) + raw, a, err := s.document(e) + if err != nil || !bytes.Contains(raw, []byte("9007199254740993")) || !bytes.Contains(raw, []byte("<&>")) { + t.Fatal("precision changed", err) + } + e.Snapshot.Result = json.RawMessage(`{ "text": "<&>", "n": 9007199254740993 }`) + _, b, err := s.document(e) + if err != nil || a != b { + t.Fatal("equivalent formatting changed identity", err) + } + e.OccurredAt = e.OccurredAt.In(time.FixedZone("offset", 8*3600)) + _, b, err = s.document(e) + if err != nil || a != b { + t.Fatal("timezone representation changed event identity", err) + } + e.Snapshot.Result = json.RawMessage(`{"n":9007199254740994,"text":"<&>"}`) + _, b, err = s.document(e) + if err != nil || a == b { + t.Fatal("changed result not detected", err) + } + e.Snapshot.QueueID = newID(t) + if _, _, err = s.document(e); !errors.Is(err, ErrInvalidEvent) { + t.Fatal("wrong receiver scope accepted", err) + } +} +func TestCanonicalRejectsDuplicateAndInvalidJSON(t *testing.T) { + for _, raw := range []string{`{"a":1,"a":2}`, `{"a":null,"a":2}`, `{"a":1} {}`, `{"a":`, `{"a":"\ud800"}`, `{"a":"\udfff"}`, "\xff"} { + if _, err := canonicalJSON([]byte(raw)); !errors.Is(err, ErrInvalidEvent) { + t.Fatal("invalid document accepted") + } + } +} diff --git a/examples/http-service/internal/jobs/schema.sql b/examples/http-service/internal/jobs/schema.sql new file mode 100644 index 0000000..489af49 --- /dev/null +++ b/examples/http-service/internal/jobs/schema.sql @@ -0,0 +1,9 @@ +CREATE TABLE sender_jobs ( + namespace_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '与Outbox相同的命名空间范围', + operation_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '本地业务操作标识，与Outbox同事务创建', + source_ref VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL COMMENT '待处理对象引用，首次入库后固定，不存对象正文或凭据', + task_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL COMMENT '已确认的平台受理任务，为空不证明远端未受理', + created_at DATETIME(6) NOT NULL COMMENT '业务记录创建UTC时间，微秒精度', + updated_at DATETIME(6) NOT NULL COMMENT '关联任务最近变化UTC时间，微秒精度', + PRIMARY KEY (namespace_id,operation_id) +) ENGINE=InnoDB COMMENT='接入方示例业务记录，与待发请求及受理关联分别同事务保存'; diff --git a/examples/http-service/internal/jobs/store.go b/examples/http-service/internal/jobs/store.go new file mode 100644 index 0000000..42ff5c8 --- /dev/null +++ b/examples/http-service/internal/jobs/store.go @@ -0,0 +1,126 @@ +// Package outbox 演示在业务事务中使用公共 MySQL Outbox 适配包。 +package jobs + +import ( + "context" + "database/sql" + _ "embed" + "encoding/json" + "errors" + "strings" + "unicode" + "unicode/utf8" + + adapter "git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql" + orbit "git.yuntuo.hk/k2safe/orbit/sdk/go" +) + +//go:embed schema.sql +var schema string +var ( + ErrInvalid = adapter.ErrInvalid + ErrStorage = adapter.ErrStorage + ErrConflict = adapter.ErrConflict + ErrCommitUnconfirmed = adapter.ErrCommitUnconfirmed + ErrIdle = adapter.ErrIdle +) + +type Record = adapter.Record + +// Store 仅补充示例业务事务；发送、查询、租约由公共包负责。 +type Store struct { + *adapter.Store + db *sql.DB + namespace string +} + +var Open = adapter.Open + +func Init(ctx context.Context, db *sql.DB) error { + if err := adapter.Init(ctx, db); err != nil { + return err + } + if _, err := db.ExecContext(ctx, schema); err != nil { + return ErrStorage + } + return nil +} +func New(db *sql.DB, namespace, queue string) (*Store, error) { + s, err := adapter.New(db, namespace, queue, adapter.Options{OnAccepted: accepted}) + if err != nil { + return nil, err + } + return &Store{s, db, namespace}, nil +} +func accepted(ctx context.Context, tx *sql.Tx, r adapter.Record, receipt orbit.SubmitTaskResult) error { + var existing *string + if err := tx.QueryRowContext(ctx, `SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, r.NamespaceID, r.OperationID).Scan(&existing); err != nil { + return ErrStorage + } + if existing != nil && *existing != receipt.Task.ID { + return ErrConflict + } + _, err := tx.ExecContext(ctx, `UPDATE sender_jobs SET task_id=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, receipt.Task.ID, r.NamespaceID, r.OperationID) + if err != nil { + return ErrStorage + } + return nil +} +func (s *Store) Enqueue(ctx context.Context, id, source string, in orbit.SubmitTaskRequest) (bool, error) { + if !utf8.ValidString(source) || strings.TrimSpace(source) == "" || utf8.RuneCountInString(source) > 512 || strings.ContainsFunc(source, unicode.IsControl) { + return false, ErrInvalid + } + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) + if err != nil { + return false, ErrStorage + } + defer tx.Rollback() + duplicate, err := s.Store.EnqueueTx(ctx, tx, id, in) + if err != nil { + return false, err + } + if duplicate { + var saved string + if err = tx.QueryRowContext(ctx, `SELECT source_ref FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, s.namespace, id).Scan(&saved); err != nil { + return false, ErrStorage + } + if saved != source { + return false, ErrConflict + } + } else { + if _, err = tx.ExecContext(ctx, `INSERT INTO sender_jobs(namespace_id,operation_id,source_ref,created_at,updated_at) VALUES(?,?,?,UTC_TIMESTAMP(6),UTC_TIMESTAMP(6))`, s.namespace, id, source); err != nil { + return false, ErrStorage + } + } + if tx.Commit() != nil { + return false, ErrCommitUnconfirmed + } + return duplicate, nil +} + +// View 是业务HTTP的读取结果；Submission只表示提交阶段，Execution来自持久化Inbox。 +type View struct { + OperationID string `json:"operation_id"` // OperationID 调用方提供的稳定操作标识。 + Submission string `json:"submission"` // Submission 待发/发送中/已受理/需核查，不是执行结果。 + Code string `json:"code"` // Code 最近稳定的提交分类。 + TaskID *string `json:"task_id,omitempty"` // TaskID 已核对的平台任务，未确认时为空。 + Execution string `json:"execution,omitempty"` // Execution 业务Inbox最后收到的任务状态。 + Result json.RawMessage `json:"result,omitempty"` // Result 业务Inbox中的结果，尚未到达时为空。 +} + +func (s *Store) View(ctx context.Context, id string) (View, error) { + r, err := s.Status(ctx, id) + if err != nil { + return View{}, err + } + v := View{OperationID: id, Submission: r.State, Code: r.LastCode, TaskID: r.TaskID} + if r.TaskID != nil { + var result []byte + err = s.db.QueryRowContext(ctx, `SELECT last_state,result_json FROM receiver_task_views WHERE consumer_name='http-service' AND queue_id=? AND task_id=?`, r.QueueID, *r.TaskID).Scan(&v.Execution, &result) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return View{}, ErrStorage + } + v.Result = result + } + return v, nil +} diff --git a/examples/task-outbox/README.md b/examples/task-outbox/README.md index b209bf4..9a783aa 100644 --- a/examples/task-outbox/README.md +++ b/examples/task-outbox/README.md @@ -4,7 +4,7 @@ - [完整初始化、运行、恢复与接口说明](../../docs/examples/task-outbox.md) - [命令行程序](cmd/sender/main.go)：init、enqueue、send、status、demo。 -- [业务事务](internal/outbox/store.go)、[发送器](internal/outbox/sender.go)、[中文模型](internal/outbox/model.go)与[无外键表结构](internal/outbox/schema.sql)。 +- [业务事务](internal/outbox/store.go)、[发送器](../../adapters/outbox/mysql/sender.go)、[中文模型](internal/outbox/model.go)与[无外键表结构](internal/outbox/schema.sql)。 本模块是可运行的业务接入参考。核心SDK不引入MySQL驱动；生产业务应将本地事务接入自身业务模型、权限、审计与迁移，固定SDK版本并验证兼容性。本例的internal包不作为已发布通用SDK包承诺兼容。 @@ -17,3 +17,5 @@ GOWORK=off go build ./cmd/sender ``` 真实数据库、HTTP、COMMIT响应丢失、CLI及发送进程故障由仓库根目录的`make integration`统一执行，使用专门的一次性数据库。没有测试环境变量时Integration会跳过。 + +发送核心已抽为[公共 MySQL 适配包](../../adapters/outbox/mysql/README.md)；本项目仅保留示例业务事务。已有 sender_outbox / sender_jobs 表保持兼容。 diff --git a/examples/task-outbox/go.mod b/examples/task-outbox/go.mod index 7fcb21b..4830675 100644 --- a/examples/task-outbox/go.mod +++ b/examples/task-outbox/go.mod @@ -3,13 +3,16 @@ module git.yuntuo.hk/k2safe/orbit/examples/task-outbox go 1.27.1 require ( - git.yuntuo.hk/k2safe/orbit/sdk/go v0.0.0 - github.com/go-sql-driver/mysql v1.10.1 + git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql v0.0.0 + git.yuntuo.hk/k2safe/orbit/sdk/go v0.1.0 ) require ( filippo.io/edwards25519 v1.2.0 // indirect github.com/coder/websocket v1.8.15 // indirect + github.com/go-sql-driver/mysql v1.10.1 // indirect ) replace git.yuntuo.hk/k2safe/orbit/sdk/go => ../../sdk/go + +replace git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql => ../../adapters/outbox/mysql diff --git a/examples/task-outbox/internal/outbox/model.go b/examples/task-outbox/internal/outbox/model.go index a6358ae..bd556f9 100644 --- a/examples/task-outbox/internal/outbox/model.go +++ b/examples/task-outbox/internal/outbox/model.go @@ -1,26 +1,13 @@ package outbox -import "time" +import ( + "time" -// Record 是业务数据库中的待发命令；只确认平台受理，不能表示业务执行成功。 -type Record struct { - NamespaceID string // NamespaceID 固定目标命名空间，换凭据不能改变归属。 - QueueID string // QueueID 固定目标队列，发送前通过当前凭据核对归属。 - OperationID string // OperationID 调用方持久化的业务操作ID，同命名空间内唯一。 - IdempotencyKey string // IdempotencyKey 由操作ID派生的固定提交键，重报禁止换键。 - RequestJSON []byte // RequestJSON 冻结的任务提交正文，保留数字精度，不含凭据。 - RequestDigest []byte // RequestDigest 冻结正文的SHA256，发送前重新校验，非平台语义摘要。 - State string // State PENDING待发、SENDING领取中、ACCEPTED已受理、REVIEW需核查。 - SendCount uint32 // SendCount 持久化领取次数，包含领取后未发HTTP即退出的次数，上限100。 - LeaseEpoch uint64 // LeaseEpoch 每次领取递增；旧代次不能更新新代次的本地结果。 - WakeAt *time.Time // WakeAt PENDING的下次领取时间或SENDING的租约期限；终结状态为空，UTC微秒。 - SendDeadlineAt time.Time // SendDeadlineAt 首次入库后24小时停止新发送，短于平台90天幂等窗口。 - LastCode string // LastCode 最近一次稳定分类，不保存HTTP错误正文、凭据或业务秘密。 - TaskID *string // TaskID 仅ACCEPTED记录平台任务ID，其余状态为空。 - ReceiptJSON []byte // ReceiptJSON 仅ACCEPTED保存原受理回执；不是当前执行状态。 - CreatedAt time.Time // CreatedAt 业务事务首次提交命令的UTC时间，微秒精度。 - UpdatedAt time.Time // UpdatedAt 最近状态变化的UTC时间，不能替代租约代次。 -} + adapter "git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql" +) + +// Record 复用公共适配器的中文持久化模型。 +type Record = adapter.Record // Job 演示接入方业务记录，与Record原子创建，受理确认时再原子关联任务。 type Job struct { diff --git a/examples/task-outbox/internal/outbox/schema.sql b/examples/task-outbox/internal/outbox/schema.sql index 616830e..489af49 100644 --- a/examples/task-outbox/internal/outbox/schema.sql +++ b/examples/task-outbox/internal/outbox/schema.sql @@ -1,36 +1,3 @@ --- 业务方独立数据库的示例表，不属于平台迁移，不创建外键或级联删除。 -CREATE TABLE sender_outbox ( - namespace_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定目标命名空间，重启及换凭据不得改变', - queue_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定目标队列，发送前核对当前授权及命名空间', - operation_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '业务操作标识，同命名空间内唯一，调用方持久化后复用', - idempotency_key VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '冻结平台提交键，由业务操作标识派生，禁止换键补发', - request_json MEDIUMTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '冻结任务提交正文，保留精确数字，不存认证凭据', - request_digest BINARY(32) NOT NULL COMMENT '冻结正文SHA256，防止同身份不同内容，非平台摘要算法', - state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT 'PENDING待发，SENDING领取中，ACCEPTED已受理，REVIEW需核查', - send_count INT UNSIGNED NOT NULL COMMENT '领取次数，包含领取后崩溃的次数，最多100', - lease_epoch BIGINT UNSIGNED NOT NULL COMMENT '领取代次递增，旧发送器不能覆盖新代次结果', - wake_at DATETIME(6) NULL COMMENT '待发下次时间或发送租约期限，终结后为空，UTC微秒', - send_deadline_at DATETIME(6) NOT NULL COMMENT '首次入库24小时后的发送截止，短于平台幂等保留窗口', - last_code VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '最近稳定分类，不保存服务端错误正文或秘密', - task_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL COMMENT '已确认的平台任务标识，只在ACCEPTED非空', - receipt_json MEDIUMTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL COMMENT '原受理回执，仅ACCEPTED非空，不代表当前执行结果', - created_at DATETIME(6) NOT NULL COMMENT '业务事务首次创建命令UTC时间，微秒精度', - updated_at DATETIME(6) NOT NULL COMMENT '本地状态最近变化UTC时间，微秒精度', - PRIMARY KEY (namespace_id,operation_id), - UNIQUE KEY uq_sender_identity(namespace_id,idempotency_key), - KEY ix_sender_due(namespace_id,queue_id,wake_at,operation_id), - CONSTRAINT ck_sender_document CHECK (JSON_VALID(request_json)), - CONSTRAINT ck_sender_receipt CHECK (receipt_json IS NULL OR JSON_VALID(receipt_json)), - CONSTRAINT ck_sender_count CHECK (send_count<=100 AND lease_epoch=send_count), - CONSTRAINT ck_sender_state CHECK ( - (state IN ('PENDING','SENDING') AND wake_at IS NOT NULL AND task_id IS NULL AND receipt_json IS NULL) - OR (state='REVIEW' AND wake_at IS NULL AND task_id IS NULL AND receipt_json IS NULL) - OR (state='ACCEPTED' AND wake_at IS NULL AND task_id IS NOT NULL AND receipt_json IS NOT NULL) - ), - CONSTRAINT ck_sender_lease CHECK (state<>'SENDING' OR lease_epoch>0), - CONSTRAINT ck_sender_deadline CHECK (send_deadline_at>created_at AND send_deadline_at<=TIMESTAMPADD(HOUR,24,created_at)) -) ENGINE=InnoDB COMMENT='业务事务Outbox，持久化原请求与有限重发状态，不承担业务恰好一次'; - CREATE TABLE sender_jobs ( namespace_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '与Outbox相同的命名空间范围', operation_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '本地业务操作标识，与Outbox同事务创建', diff --git a/examples/task-outbox/internal/outbox/store.go b/examples/task-outbox/internal/outbox/store.go index 20bae7b..f45a3ec 100644 --- a/examples/task-outbox/internal/outbox/store.go +++ b/examples/task-outbox/internal/outbox/store.go @@ -1,321 +1,95 @@ -// Package outbox 演示接入方本地业务事务、稳定提交身份及有限发送恢复。 +// Package outbox 演示在业务事务中使用公共 MySQL Outbox 适配包。 package outbox import ( - "bytes" "context" - "crypto/sha256" "database/sql" _ "embed" - "encoding/json" - "errors" - "regexp" "strings" - "time" "unicode" "unicode/utf8" + adapter "git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql" orbit "git.yuntuo.hk/k2safe/orbit/sdk/go" - "github.com/go-sql-driver/mysql" ) //go:embed schema.sql var schema string - var ( - ErrInvalid = errors.New("outbox scope, operation or request is invalid") - ErrConflict = errors.New("outbox operation already has different content; keep the original identity") - ErrStorage = errors.New("outbox storage failed; reconcile the original operation") - ErrCommitUnconfirmed = errors.New("outbox commit is unconfirmed; reconcile the original operation") - ErrIdle = errors.New("no due outbox record") - ErrNotFound = errors.New("outbox operation not found in this scope") - ErrLeaseLost = errors.New("outbox lease was replaced; do not overwrite its result") - idPattern = regexp.MustCompile(`^[a-f0-9]{32}$`) + ErrInvalid = adapter.ErrInvalid + ErrStorage = adapter.ErrStorage + ErrConflict = adapter.ErrConflict + ErrCommitUnconfirmed = adapter.ErrCommitUnconfirmed + ErrIdle = adapter.ErrIdle ) -const maxSends = 100 -const recordColumns = `namespace_id,queue_id,operation_id,idempotency_key,request_json,request_digest,state,send_count,lease_epoch,wake_at,send_deadline_at,last_code,task_id,receipt_json,created_at,updated_at` - +// Store 仅补充示例业务事务；发送、查询、租约由公共包负责。 type Store struct { - db *sql.DB - namespace, queue string + *adapter.Store + db *sql.DB + namespace string } -// Open 仅连接业务方数据库，不创建表，不输出原始DSN或数据库错误。 -func Open(ctx context.Context, dsn string) (*sql.DB, error) { - cfg, err := mysql.ParseDSN(dsn) - if err != nil || cfg.DBName == "" { - return nil, ErrInvalid - } - cfg.ParseTime = true - cfg.Loc = time.UTC - cfg.MultiStatements = false - cfg.AllowAllFiles = false - cfg.Timeout, cfg.ReadTimeout, cfg.WriteTimeout = 5*time.Second, 8*time.Second, 8*time.Second - if cfg.Params == nil { - cfg.Params = map[string]string{} - } - cfg.Params["time_zone"] = "'+00:00'" - connector, err := mysql.NewConnector(cfg) - if err != nil { - return nil, ErrStorage - } - db := sql.OpenDB(connector) - db.SetMaxOpenConns(8) - db.SetMaxIdleConns(4) - db.SetConnMaxLifetime(5 * time.Minute) - if err = db.PingContext(ctx); err != nil { - db.Close() - return nil, ErrStorage - } - return db, nil -} +var Open = adapter.Open -// Init 仅用于专用空库，已有表会报错；生产接入使用业务方自己的迁移。 func Init(ctx context.Context, db *sql.DB) error { - for _, statement := range strings.Split(schema, ";") { - if strings.TrimSpace(statement) == "" { - continue - } - if _, err := db.ExecContext(ctx, statement); err != nil { - return ErrStorage - } + if err := adapter.Init(ctx, db); err != nil { + return err + } + if _, err := db.ExecContext(ctx, schema); err != nil { + return ErrStorage } return nil } func New(db *sql.DB, namespace, queue string) (*Store, error) { - if db == nil || !idPattern.MatchString(namespace) || !idPattern.MatchString(queue) { - return nil, ErrInvalid + s, err := adapter.New(db, namespace, queue, adapter.Options{OnAccepted: accepted}) + if err != nil { + return nil, err } - return &Store{db: db, namespace: namespace, queue: queue}, nil + return &Store{s, db, namespace}, nil } - -// document 冻结当前v1默认值及UTC表示；Outbox按文档字节去重，不冒充平台语义摘要。 -func (s *Store) document(in orbit.SubmitTaskRequest) ([]byte, error) { - // encoding/json会把非法UTF-8替换成U+FFFD；在冻结摘要前拒绝，不能静默改变业务标识。 - for _, value := range []string{in.QueueID, in.Handler, in.HandlerVersion, in.PayloadVersion, in.BusinessKey, in.ConcurrencyKey} { - if !utf8.ValidString(value) { - return nil, ErrInvalid - } +func accepted(ctx context.Context, tx *sql.Tx, r adapter.Record, receipt orbit.SubmitTaskResult) error { + var existing *string + if err := tx.QueryRowContext(ctx, `SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, r.NamespaceID, r.OperationID).Scan(&existing); err != nil { + return ErrStorage } - if in.QueueID != s.queue || in.Handler == "" || in.HandlerVersion == "" || in.PayloadVersion == "" || len(in.Payload) > 256<<10 || !utf8.Valid(in.Payload) || !json.Valid(in.Payload) || bytes.Equal(bytes.TrimSpace(in.Payload), []byte("null")) { - return nil, ErrInvalid + if existing != nil && *existing != receipt.Task.ID { + return ErrConflict } - if in.TaskTimeoutMS == 0 { - in.TaskTimeoutMS = 86400000 + _, err := tx.ExecContext(ctx, `UPDATE sender_jobs SET task_id=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, receipt.Task.ID, r.NamespaceID, r.OperationID) + if err != nil { + return ErrStorage } - if in.RecoverySafety == "" { - in.RecoverySafety = orbit.RecoveryReconcileOnly - } - if in.Retry.MaxAttempts == 0 { - in.Retry.MaxAttempts = 1 - } - if in.Retry.AttemptTimeoutMS == 0 { - in.Retry.AttemptTimeoutMS = 30000 - } - if in.Retry.MaxAttempts > 1 { - if in.Retry.InitialDelayMS == 0 { - in.Retry.InitialDelayMS = 1000 - } - if in.Retry.MaxDelayMS == 0 { - in.Retry.MaxDelayMS = 30000 - } - } - if in.AvailableAt != nil { - at := in.AvailableAt.UTC() - in.AvailableAt = &at - } - if in.StartDeadlineAt != nil { - at := in.StartDeadlineAt.UTC() - in.StartDeadlineAt = &at - } - raw, err := json.Marshal(in) - if err != nil || len(raw) > 512<<10 { - return nil, ErrInvalid - } - return raw, nil + return nil } - -// Enqueue 原子创建业务记录及Outbox。duplicate只表示该本地事务已经提交，不表示平台受理。 -// 调用方在第一次调用前持久化operationID；COMMIT回执未知时仍用原ID、原内容再次调用。 -func (s *Store) Enqueue(parent context.Context, operationID, sourceRef string, in orbit.SubmitTaskRequest) (duplicate bool, err error) { - if !idPattern.MatchString(operationID) || !utf8.ValidString(sourceRef) || strings.TrimSpace(sourceRef) == "" || utf8.RuneCountInString(sourceRef) > 512 || strings.ContainsFunc(sourceRef, unicode.IsControl) { +func (s *Store) Enqueue(ctx context.Context, id, source string, in orbit.SubmitTaskRequest) (bool, error) { + if !utf8.ValidString(source) || strings.TrimSpace(source) == "" || utf8.RuneCountInString(source) > 512 || strings.ContainsFunc(source, unicode.IsControl) { return false, ErrInvalid } - raw, err := s.document(in) - if err != nil { - return false, err - } - digest := sha256.Sum256(raw) - ctx, stop := context.WithTimeout(parent, 5*time.Second) - defer stop() tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) if err != nil { return false, ErrStorage } defer tx.Rollback() - _, err = tx.ExecContext(ctx, `INSERT INTO sender_outbox(namespace_id,queue_id,operation_id,idempotency_key,request_json,request_digest,state,send_count,lease_epoch,wake_at,send_deadline_at,last_code,created_at,updated_at) VALUES(?,?,?,?,?,?,'PENDING',0,0,UTC_TIMESTAMP(6),TIMESTAMPADD(HOUR,24,UTC_TIMESTAMP(6)),'',UTC_TIMESTAMP(6),UTC_TIMESTAMP(6))`, s.namespace, s.queue, operationID, "outbox-task-"+operationID, raw, digest[:]) + duplicate, err := s.Store.EnqueueTx(ctx, tx, id, in) if err != nil { - var mysqlErr *mysql.MySQLError - if !errors.As(err, &mysqlErr) || mysqlErr.Number != 1062 { + return false, err + } + if duplicate { + var saved string + if err = tx.QueryRowContext(ctx, `SELECT source_ref FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, s.namespace, id).Scan(&saved); err != nil { return false, ErrStorage } - if tx.Rollback() != nil { - return false, ErrStorage - } - var saved []byte - var queue, source string - err = s.db.QueryRowContext(ctx, `SELECT o.request_digest,o.queue_id,j.source_ref FROM sender_outbox o JOIN sender_jobs j ON j.namespace_id=o.namespace_id AND j.operation_id=o.operation_id WHERE o.namespace_id=? AND o.operation_id=?`, s.namespace, operationID).Scan(&saved, &queue, &source) - if err != nil { - return false, ErrStorage - } - if queue != s.queue || source != sourceRef || !bytes.Equal(saved, digest[:]) { + if saved != source { return false, ErrConflict } - return true, nil - } - // 锁序统一为Outbox→业务行，不能在业务行持锁时反向调用发送确认。 - if _, err = tx.ExecContext(ctx, `INSERT INTO sender_jobs(namespace_id,operation_id,source_ref,created_at,updated_at) VALUES(?,?,?,UTC_TIMESTAMP(6),UTC_TIMESTAMP(6))`, s.namespace, operationID, sourceRef); err != nil { - return false, ErrStorage + } else { + if _, err = tx.ExecContext(ctx, `INSERT INTO sender_jobs(namespace_id,operation_id,source_ref,created_at,updated_at) VALUES(?,?,?,UTC_TIMESTAMP(6),UTC_TIMESTAMP(6))`, s.namespace, id, source); err != nil { + return false, ErrStorage + } } if tx.Commit() != nil { return false, ErrCommitUnconfirmed } - return false, nil -} - -type scanner interface{ Scan(...any) error } - -func scanRecord(row scanner, r *Record, extra ...any) error { - args := []any{&r.NamespaceID, &r.QueueID, &r.OperationID, &r.IdempotencyKey, &r.RequestJSON, &r.RequestDigest, &r.State, &r.SendCount, &r.LeaseEpoch, &r.WakeAt, &r.SendDeadlineAt, &r.LastCode, &r.TaskID, &r.ReceiptJSON, &r.CreatedAt, &r.UpdatedAt} - return row.Scan(append(args, extra...)...) -} -func (s *Store) Status(parent context.Context, id string) (Record, error) { - if !idPattern.MatchString(id) { - return Record{}, ErrInvalid - } - ctx, stop := context.WithTimeout(parent, 5*time.Second) - defer stop() - var r Record - err := scanRecord(s.db.QueryRowContext(ctx, `SELECT `+recordColumns+` FROM sender_outbox WHERE namespace_id=? AND queue_id=? AND operation_id=?`, s.namespace, s.queue, id), &r) - if errors.Is(err, sql.ErrNoRows) { - return Record{}, ErrNotFound - } - if err != nil { - return Record{}, ErrStorage - } - return r, nil -} - -func (s *Store) claim(parent context.Context) (Record, error) { - ctx, stop := context.WithTimeout(parent, 5*time.Second) - defer stop() - tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) - if err != nil { - return Record{}, ErrStorage - } - defer tx.Rollback() - var r Record - var now time.Time - err = scanRecord(tx.QueryRowContext(ctx, `SELECT `+recordColumns+`,UTC_TIMESTAMP(6) FROM sender_outbox FORCE INDEX(ix_sender_due) WHERE namespace_id=? AND queue_id=? AND wake_at<=UTC_TIMESTAMP(6) ORDER BY wake_at,operation_id LIMIT 1 FOR UPDATE SKIP LOCKED`, s.namespace, s.queue), &r, &now) - if errors.Is(err, sql.ErrNoRows) { - return Record{}, ErrIdle - } - if err != nil { - return Record{}, ErrStorage - } - if r.State != "PENDING" && r.State != "SENDING" { - return Record{}, ErrStorage - } - if !now.Before(r.SendDeadlineAt) || r.SendCount >= maxSends { - r.State = "REVIEW" - r.WakeAt = nil - r.LastCode = "SEND_BUDGET_EXHAUSTED" - if !now.Before(r.SendDeadlineAt) { - r.LastCode = "SEND_DEADLINE" - } - } else { - r.State = "SENDING" - r.SendCount++ - r.LeaseEpoch++ - wake := now.Add(30 * time.Second) - if wake.After(r.SendDeadlineAt) { - wake = r.SendDeadlineAt - } - r.WakeAt = &wake - } - _, err = tx.ExecContext(ctx, `UPDATE sender_outbox SET state=?,send_count=?,lease_epoch=?,wake_at=?,last_code=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, r.State, r.SendCount, r.LeaseEpoch, r.WakeAt, r.LastCode, s.namespace, r.OperationID) - if err != nil { - return Record{}, ErrStorage - } - if tx.Commit() != nil { - return Record{}, ErrCommitUnconfirmed - } - return r, nil -} - -// finish 只接受最近代次；过期但尚未被替代的成功回执可落库，重复HTTP只复用原键。 -func (s *Store) finish(parent context.Context, claim Record, state, code string, delay time.Duration, receipt *orbit.SubmitTaskResult) error { - ctx, stop := context.WithTimeout(parent, 5*time.Second) - defer stop() - tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) - if err != nil { - return ErrStorage - } - defer tx.Rollback() - var current Record - var now time.Time - err = scanRecord(tx.QueryRowContext(ctx, `SELECT `+recordColumns+`,UTC_TIMESTAMP(6) FROM sender_outbox WHERE namespace_id=? AND queue_id=? AND operation_id=? FOR UPDATE`, s.namespace, s.queue, claim.OperationID), ¤t, &now) - if err != nil { - return ErrStorage - } - if current.State != "SENDING" || current.LeaseEpoch != claim.LeaseEpoch { - return ErrLeaseLost - } - var wake *time.Time - var task *string - var raw []byte - if state == "ACCEPTED" { - if receipt == nil || !validReceipt(current, *receipt) { - return ErrInvalid - } - raw, err = json.Marshal(receipt) - if err != nil || len(raw) > 1<<20 { - return ErrInvalid - } - task = &receipt.Task.ID - var existing *string - if err = tx.QueryRowContext(ctx, `SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, s.namespace, current.OperationID).Scan(&existing); err != nil { - return ErrStorage - } - if existing != nil && *existing != *task { - return ErrConflict - } - if _, err = tx.ExecContext(ctx, `UPDATE sender_jobs SET task_id=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, *task, s.namespace, current.OperationID); err != nil { - return ErrStorage - } - } else if state == "PENDING" { - if delay < time.Second || delay > time.Hour { - return ErrInvalid - } - if !now.Before(current.SendDeadlineAt) || current.SendCount >= maxSends { - state = "REVIEW" - code = "SEND_BUDGET_EXHAUSTED" - } else { - next := now.Add(delay) - if next.After(current.SendDeadlineAt) { - next = current.SendDeadlineAt - } - wake = &next - } - } else if state != "REVIEW" { - return ErrInvalid - } - if _, err = tx.ExecContext(ctx, `UPDATE sender_outbox SET state=?,last_code=?,wake_at=?,task_id=?,receipt_json=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, state, code, wake, task, raw, s.namespace, current.OperationID); err != nil { - return ErrStorage - } - if tx.Commit() != nil { - return ErrCommitUnconfirmed - } - return nil + return duplicate, nil } diff --git a/go.work b/go.work index 4ff7073..9afed52 100644 --- a/go.work +++ b/go.work @@ -1,6 +1,7 @@ go 1.27.1 use ( - . - ./sdk/go + . + ./adapters/outbox/mysql + ./sdk/go ) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 0000000..15616a8 --- /dev/null +++ b/go.work.sum @@ -0,0 +1 @@ +git.yuntuo.hk/k2safe/orbit/sdk/go v0.1.0/go.mod h1:9KlMW7Gy+EGaDoEwr3wHHFY47mnAymSfYeQpMS7h4ag= diff --git a/scripts/test-adapters.sh b/scripts/test-adapters.sh new file mode 100755 index 0000000..5d990ca --- /dev/null +++ b/scripts/test-adapters.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# 独立业务库 + 真实 Orbit 进程；仅操作本脚本创建的隔离容器和目录。 +set -eu +TASK_GO=${GO:-go} +TASK_NAME="orbit-adapters-test-$$" +TASK_DIR=$(mktemp -d "${TMPDIR:-/tmp}/orbit-adapters.XXXXXX") +TASK_PID= +cleanup() { + if [ -n "$TASK_PID" ]; then kill "$TASK_PID" >/dev/null 2>&1 || true; wait "$TASK_PID" 2>/dev/null || true; fi + docker stop "$TASK_NAME" >/dev/null 2>&1 || true + docker container rm "$TASK_NAME" >/dev/null 2>&1 || true + for TASK_FILE in "$TASK_DIR"/*; do if [ -f "$TASK_FILE" ]; then unlink "$TASK_FILE"; fi; done + rmdir "$TASK_DIR" +} +trap cleanup EXIT HUP INT TERM +umask 077 +TASK_PASSWORD=$(openssl rand -hex 24) +printf 'MYSQL_ROOT_PASSWORD=%s\nMYSQL_DATABASE=orbit_test\n' "$TASK_PASSWORD" > "$TASK_DIR/mysql.env" +docker run -d --name "$TASK_NAME" --env-file "$TASK_DIR/mysql.env" -p 127.0.0.1::3306 "${ORBIT_TEST_MYSQL_IMAGE:-mysql:8.4}" >/dev/null +TASK_READY=0 +for TASK_ATTEMPT in $(seq 1 60); do + if docker exec "$TASK_NAME" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=tcp -e "SELECT 1" >/dev/null 2>&1'; then TASK_READY=1; break; fi + sleep 1 +done +if [ "$TASK_READY" -ne 1 ]; then echo 'isolated MySQL startup failed' >&2; exit 1; fi +TASK_PORT=$(docker port "$TASK_NAME" 3306/tcp | sed 's/.*://') +export ORBIT_MYSQL_DSN="root:$TASK_PASSWORD@tcp(127.0.0.1:$TASK_PORT)/orbit_test" +"$TASK_GO" build -o "$TASK_DIR/orbit" ./cmd/orbit +"$TASK_GO" build -o "$TASK_DIR/admin" ./cmd/orbit-admin +"$TASK_GO" run ./cmd/orbit-migrate up +"$TASK_DIR/admin" bootstrap --application adapters-test --namespace default --environment test --queue sender --output "$TASK_DIR/sender.json" +"$TASK_DIR/admin" bootstrap --application http-example --namespace default --environment test --queue http --worker-access --event-access --output "$TASK_DIR/http.json" +ORBIT_HTTP_ADDR=127.0.0.1:0 "$TASK_DIR/orbit" >"$TASK_DIR/orbit.log" 2>&1 & +TASK_PID=$! +TASK_HTTP_PORT= +for TASK_ATTEMPT in $(seq 1 20); do + TASK_HTTP_PORT=$(sed -n 's/.*"address":"127.0.0.1:\([0-9]*\)".*/\1/p' "$TASK_DIR/orbit.log" | head -n 1) + if [ -n "$TASK_HTTP_PORT" ]; then break; fi + if ! kill -0 "$TASK_PID" 2>/dev/null; then echo 'Orbit startup failed' >&2; exit 1; fi + sleep 1 +done +if [ -z "$TASK_HTTP_PORT" ]; then echo 'Orbit listener not ready' >&2; exit 1; fi +docker exec "$TASK_NAME" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=tcp -e "CREATE DATABASE orbit_sender_test CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; CREATE DATABASE orbit_http_test CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin"' +export ORBIT_TEST_SENDER_DSN="root:$TASK_PASSWORD@tcp(127.0.0.1:$TASK_PORT)/orbit_sender_test" +export ORBIT_TEST_SENDER_URL="http://127.0.0.1:$TASK_HTTP_PORT" +export ORBIT_TEST_SENDER_CREDENTIALS_FILE="$TASK_DIR/sender.json" +(cd examples/task-outbox && GOWORK=off ORBIT_SENDER_DSN="$ORBIT_TEST_SENDER_DSN" "$TASK_GO" run ./cmd/sender init) +(cd adapters/outbox/mysql && GOWORK=off "$TASK_GO" test -race -count=1 -run Integration -v ./...) +export ORBIT_TEST_HTTP_DSN="root:$TASK_PASSWORD@tcp(127.0.0.1:$TASK_PORT)/orbit_http_test" +export ORBIT_TEST_HTTP_URL="http://127.0.0.1:$TASK_HTTP_PORT" +export ORBIT_TEST_HTTP_CREDENTIALS_FILE="$TASK_DIR/http.json" +(cd examples/http-service && GOWORK=off "$TASK_GO" test -race -count=1 -run Integration -v ./...) diff --git a/scripts/test-mysql.sh b/scripts/test-mysql.sh index 55d3cd6..b0a6b56 100755 --- a/scripts/test-mysql.sh +++ b/scripts/test-mysql.sh @@ -135,9 +135,15 @@ ORBIT_MYSQL_DSN="$ORBIT_TEST_MYSQL_DSN" "$TASK_TEST_DIR/orbit-admin" bootstrap - "$TASK_DOCKER" exec "$TASK_TEST_NAME" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=tcp -e "CREATE DATABASE orbit_sender_test CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin"' export ORBIT_TEST_SENDER_DSN="root:$TASK_DB_PASSWORD@tcp(127.0.0.1:$TASK_DB_PORT)/orbit_sender_test" (cd examples/task-outbox && GOWORK=off ORBIT_SENDER_DSN="$ORBIT_TEST_SENDER_DSN" "$TASK_GO" run ./cmd/sender init) -(cd examples/task-outbox && GOWORK=off ORBIT_TEST_SENDER_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_TEST_SENDER_CREDENTIALS_FILE="$TASK_TEST_DIR/sender.credentials.json" "$TASK_GO" test -race -count=1 -run Integration -v ./...) +(cd adapters/outbox/mysql && GOWORK=off ORBIT_TEST_SENDER_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_TEST_SENDER_CREDENTIALS_FILE="$TASK_TEST_DIR/sender.credentials.json" "$TASK_GO" test -race -count=1 -run Integration -v ./...) (cd examples/task-outbox && GOWORK=off ORBIT_SENDER_DSN="$ORBIT_TEST_SENDER_DSN" ORBIT_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_CREDENTIALS_FILE="$TASK_TEST_DIR/sender.credentials.json" "$TASK_GO" run ./cmd/sender demo) +# HTTP宿主完整链路：隔离业务库，真实进程退出/重启及结果订阅。 +ORBIT_MYSQL_DSN="$ORBIT_TEST_MYSQL_DSN" "$TASK_TEST_DIR/orbit-admin" bootstrap --application sdk-http --namespace default --environment test --queue http --worker-access --event-access --output "$TASK_TEST_DIR/http.credentials.json" +"$TASK_DOCKER" exec "$TASK_TEST_NAME" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=tcp -e "CREATE DATABASE orbit_http_test CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin"' +export ORBIT_TEST_HTTP_DSN="root:$TASK_DB_PASSWORD@tcp(127.0.0.1:$TASK_DB_PORT)/orbit_http_test" +(cd examples/http-service && GOWORK=off ORBIT_TEST_HTTP_URL="http://127.0.0.1:$TASK_HTTP_PORT" ORBIT_TEST_HTTP_CREDENTIALS_FILE="$TASK_TEST_DIR/http.credentials.json" "$TASK_GO" test -race -count=1 -run Integration -v ./...) + # 应用配额场景使用独立应用与两个命名空间，普通 Worker 凭据不扩权。 ORBIT_MYSQL_DSN="$ORBIT_TEST_MYSQL_DSN" "$TASK_TEST_DIR/orbit-admin" bootstrap --application sdk-app-resources --namespace first --environment test --queue first --resource-access --output "$TASK_TEST_DIR/app.credentials.json" TASK_APP_ID=$(sed -n 's/.*"application_id":"\([^"]*\)".*/\1/p' "$TASK_TEST_DIR/app.credentials.json")