feat: add reusable MySQL outbox and HTTP lifecycle example
This commit is contained in:
11
examples/http-service/README.md
Normal file
11
examples/http-service/README.md
Normal file
@@ -0,0 +1,11 @@
|
||||
# 在现有 HTTP 服务中使用 Orbit
|
||||
|
||||
可运行的 Go 服务:HTTP 业务事务 → 公共 MySQL Outbox → Orbit → 内嵌 Worker → WebSocket 结果 → 业务 Inbox。
|
||||
|
||||
- [完整准备、运行、API、故障与退出说明](../../docs/examples/http-service.md)
|
||||
- [业务路由与组件装配](cmd/server/main.go)
|
||||
- [生命周期顺序](cmd/server/lifecycle.go)
|
||||
- [业务事务](internal/jobs/store.go)
|
||||
- [业务 Inbox 示例实现](internal/inbox/store.go),沿用独立 result-inbox 示例的事件去重与版本约束
|
||||
|
||||
`init` 创建业务库表;`provision` 用稳定身份创建结果订阅;`serve` 使用配置中的订阅 ID,启动时无需 Orbit 在线。默认只监听回环地址 127.0.0.1:8090。接入已有服务时挂载业务路由,并接入原有鉴权、监控及生命周期。
|
||||
251
examples/http-service/cmd/server/integration_test.go
Normal file
251
examples/http-service/cmd/server/integration_test.go
Normal file
@@ -0,0 +1,251 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
outbox "git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql"
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func TestIntegrationHTTPServiceRestartAndResults(t *testing.T) {
|
||||
dsn, base, path := os.Getenv("ORBIT_TEST_HTTP_DSN"), os.Getenv("ORBIT_TEST_HTTP_URL"), os.Getenv("ORBIT_TEST_HTTP_CREDENTIALS_FILE")
|
||||
if dsn == "" || base == "" || path == "" {
|
||||
t.Skip("requires disposable business database and Orbit process")
|
||||
}
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil || cfg.DBName != "orbit_http_test" || !strings.HasPrefix(cfg.Addr, "127.0.0.1:") || !strings.HasPrefix(base, "http://127.0.0.1:") {
|
||||
t.Fatal("requires isolated loopback environment")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Second)
|
||||
defer cancel()
|
||||
db, err := outbox.Open(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
binary := filepath.Join(t.TempDir(), "server")
|
||||
build := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-race", "-o", binary, ".")
|
||||
if raw, err := build.CombinedOutput(); err != nil {
|
||||
t.Fatalf("build failed %v %s", err, raw)
|
||||
}
|
||||
env := []string{}
|
||||
for _, v := range os.Environ() {
|
||||
if !strings.HasPrefix(v, "ORBIT_") && !strings.HasPrefix(v, "APP_") {
|
||||
env = append(env, v)
|
||||
}
|
||||
}
|
||||
env = append(env, "APP_MYSQL_DSN="+dsn, "APP_HTTP_ADDR=127.0.0.1:0", "ORBIT_URL="+base, "ORBIT_CREDENTIALS_FILE="+path)
|
||||
call := func(args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.CommandContext(ctx, binary, args...)
|
||||
cmd.Env = env
|
||||
raw, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("command failed: %v %s", err, raw)
|
||||
}
|
||||
return strings.TrimSpace(string(raw))
|
||||
}
|
||||
call("init")
|
||||
sub := call("provision")
|
||||
if call("provision") != sub {
|
||||
t.Fatal("provision changed subscription")
|
||||
}
|
||||
env = append(env, "ORBIT_SUBSCRIPTION_ID="+sub)
|
||||
target, _ := url.Parse(base)
|
||||
proxy := httputil.NewSingleHostReverseProxy(target)
|
||||
var offline atomic.Bool
|
||||
offline.Store(true)
|
||||
gateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if offline.Load() {
|
||||
http.Error(w, "offline", 503)
|
||||
return
|
||||
}
|
||||
proxy.ServeHTTP(w, r)
|
||||
}))
|
||||
defer gateway.Close()
|
||||
for i, v := range env {
|
||||
if strings.HasPrefix(v, "ORBIT_URL=") {
|
||||
env[i] = "ORBIT_URL=" + gateway.URL
|
||||
}
|
||||
}
|
||||
type running struct {
|
||||
cmd *exec.Cmd
|
||||
done chan error
|
||||
url string
|
||||
stderr *bytes.Buffer
|
||||
}
|
||||
start := func() *running {
|
||||
t.Helper()
|
||||
cmd := exec.CommandContext(ctx, binary, "serve")
|
||||
cmd.Env = env
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stderr := new(bytes.Buffer)
|
||||
cmd.Stderr = stderr
|
||||
if err = cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
address := make(chan string, 1)
|
||||
go func() {
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
for scanner.Scan() {
|
||||
if value, ok := strings.CutPrefix(scanner.Text(), "http_listening="); ok {
|
||||
address <- "http://" + value
|
||||
}
|
||||
}
|
||||
close(address)
|
||||
}()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- cmd.Wait(); close(done) }()
|
||||
run := &running{cmd: cmd, done: done, stderr: stderr}
|
||||
t.Cleanup(func() { cmd.Process.Kill(); <-done })
|
||||
select {
|
||||
case run.url = <-address:
|
||||
if run.url == "" {
|
||||
<-done
|
||||
t.Fatalf("startup failed: %s", stderr.String())
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("startup timeout")
|
||||
}
|
||||
return run
|
||||
}
|
||||
httpClient := &http.Client{Timeout: 5 * time.Second}
|
||||
get := func(base, path string) map[string]any {
|
||||
t.Helper()
|
||||
r, err := httpClient.Get(base + path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Body.Close()
|
||||
if r.StatusCode != 200 {
|
||||
t.Fatalf("GET %s: %d", path, r.StatusCode)
|
||||
}
|
||||
var v map[string]any
|
||||
if json.NewDecoder(r.Body).Decode(&v) != nil {
|
||||
t.Fatal("invalid JSON")
|
||||
}
|
||||
return v
|
||||
}
|
||||
id, err := orbit.NewEventMessageID()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload := []byte(`{"operation_id":"` + id + `","source_ref":"objects/http-demo.txt"}`)
|
||||
post := func(base string, body []byte, status int) map[string]any {
|
||||
t.Helper()
|
||||
r, err := httpClient.Post(base+"/jobs", "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Body.Close()
|
||||
if r.StatusCode != status {
|
||||
raw, _ := io.ReadAll(r.Body)
|
||||
t.Fatalf("POST: %d %s", r.StatusCode, raw)
|
||||
}
|
||||
var out map[string]any
|
||||
if status == 202 {
|
||||
json.NewDecoder(r.Body).Decode(&out)
|
||||
}
|
||||
return out
|
||||
}
|
||||
wait := func(check func() bool) {
|
||||
t.Helper()
|
||||
for !check() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatal("condition timeout")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
first := start()
|
||||
wait(func() bool {
|
||||
r, err := httpClient.Get(first.url + "/health/ready")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
r.Body.Close()
|
||||
return r.StatusCode == 200
|
||||
})
|
||||
if v := post(first.url, payload, 202); v["local_committed"] != true || v["duplicate"] != false {
|
||||
t.Fatal(v)
|
||||
}
|
||||
wait(func() bool {
|
||||
var n int
|
||||
err := db.QueryRowContext(ctx, `SELECT send_count FROM sender_outbox WHERE operation_id=?`, id).Scan(&n)
|
||||
return err == nil && n > 0
|
||||
})
|
||||
first.cmd.Process.Kill()
|
||||
if err := <-first.done; err == nil {
|
||||
t.Fatal("expected real SIGKILL")
|
||||
}
|
||||
offline.Store(false)
|
||||
second := start()
|
||||
wait(func() bool { return get(second.url, "/jobs/"+id)["execution"] == "SUCCEEDED" })
|
||||
if post(second.url, payload, 202)["duplicate"] != true {
|
||||
t.Fatal("original operation duplicated after restart")
|
||||
}
|
||||
post(second.url, bytes.ReplaceAll(payload, []byte("http-demo"), []byte("changed")), 409)
|
||||
view := get(second.url, "/jobs/"+id)
|
||||
result, ok := view["result"].(map[string]any)
|
||||
if !ok || result["normalized_ref"] != "OBJECTS/HTTP-DEMO.TXT" {
|
||||
t.Fatal(view)
|
||||
}
|
||||
var jobs, inboxRows, fks int
|
||||
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM sender_jobs WHERE operation_id=?`, id).Scan(&jobs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM receiver_inbox WHERE queue_id=(SELECT queue_id FROM sender_outbox WHERE operation_id=?)`, id).Scan(&inboxRows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&fks); err != nil || jobs != 1 || inboxRows != 1 || fks != 0 {
|
||||
t.Fatal(jobs, inboxRows, fks, err)
|
||||
}
|
||||
// 先看到平台持久ACK,再发SIGTERM,验证通知不是只停留在业务内存或数据库。
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var creds config
|
||||
if json.Unmarshal(raw, &creds) != nil {
|
||||
t.Fatal("credentials")
|
||||
}
|
||||
api, err := orbit.NewClient(base, orbit.Options{Token: creds.Token})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wait(func() bool { v, err := api.GetSubscription(ctx, sub); return err == nil && v.PendingCount == 0 })
|
||||
if err = second.cmd.Process.Signal(syscall.SIGTERM); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case err = <-second.done:
|
||||
if err != nil {
|
||||
t.Fatalf("SIGTERM failed %v %s", err, second.stderr.String())
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("shutdown timeout")
|
||||
}
|
||||
t.Log("Orbit unavailable: HTTP transaction committed; real SIGKILL/restart recovered original operation; Worker + persistent Inbox + ACK completed once; conflict rejected; SIGTERM exited cleanly; zero foreign keys")
|
||||
}
|
||||
99
examples/http-service/cmd/server/lifecycle.go
Normal file
99
examples/http-service/cmd/server/lifecycle.go
Normal file
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// component 不接管进程信号;宿主负责为每一类生命周期提供独立取消和等待。
|
||||
type component struct {
|
||||
name string
|
||||
run func(context.Context) error
|
||||
}
|
||||
type completion struct {
|
||||
name string
|
||||
err error
|
||||
}
|
||||
|
||||
// serve 在HTTP监听已成功后启动后台组件;返回前完成HTTP→发送/执行→结果消费的关闭顺序。
|
||||
func serve(parent context.Context, listener net.Listener, server *http.Server, ready *atomic.Bool, workers []component, consumer component, grace time.Duration) error {
|
||||
workCtx, stopWork := context.WithCancel(context.Background())
|
||||
defer stopWork()
|
||||
eventCtx, stopEvents := context.WithCancel(context.Background())
|
||||
defer stopEvents()
|
||||
workDone := make(chan completion, len(workers))
|
||||
eventDone := make(chan completion, 1)
|
||||
for _, c := range workers {
|
||||
go func() { workDone <- completion{c.name, c.run(workCtx)} }()
|
||||
}
|
||||
go func() { eventDone <- completion{consumer.name, consumer.run(eventCtx)} }()
|
||||
httpDone := make(chan error, 1)
|
||||
ready.Store(true)
|
||||
go func() { httpDone <- server.Serve(listener) }()
|
||||
remaining := len(workers)
|
||||
eventsPending := true
|
||||
var result error
|
||||
add := func(c completion) {
|
||||
if c.err != nil {
|
||||
result = errors.Join(result, fmt.Errorf("%s: %w", c.name, c.err))
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-parent.Done():
|
||||
case err := <-httpDone:
|
||||
if !errors.Is(err, http.ErrServerClosed) {
|
||||
result = errors.Join(result, err)
|
||||
}
|
||||
case c := <-workDone:
|
||||
remaining--
|
||||
if c.err == nil {
|
||||
c.err = errors.New("unexpected component exit")
|
||||
}
|
||||
add(c)
|
||||
case c := <-eventDone:
|
||||
eventsPending = false
|
||||
if c.err == nil {
|
||||
c.err = errors.New("unexpected component exit")
|
||||
}
|
||||
add(c)
|
||||
}
|
||||
ready.Store(false)
|
||||
// 用新的关闭上下文;进程信号的context此时可能已取消,不能用它调用Shutdown。
|
||||
shut, cancel := context.WithTimeout(context.Background(), grace)
|
||||
if err := server.Shutdown(shut); err != nil {
|
||||
result = errors.Join(result, fmt.Errorf("http shutdown: %w", err))
|
||||
_ = server.Close()
|
||||
}
|
||||
cancel()
|
||||
stopWork()
|
||||
deadline := time.NewTimer(grace)
|
||||
defer deadline.Stop()
|
||||
for remaining > 0 {
|
||||
select {
|
||||
case c := <-workDone:
|
||||
remaining--
|
||||
add(c)
|
||||
case <-deadline.C:
|
||||
result = errors.Join(result, errors.New("background shutdown deadline exceeded"))
|
||||
remaining = 0
|
||||
}
|
||||
}
|
||||
// 结果消费器最后停止。仍未投递的结果会在重启后由持久订阅恢复,不以本地空闲判断全局完成。
|
||||
stopEvents()
|
||||
if eventsPending {
|
||||
timer := time.NewTimer(grace)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case c := <-eventDone:
|
||||
add(c)
|
||||
case <-timer.C:
|
||||
result = errors.Join(result, errors.New("consumer shutdown deadline exceeded"))
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
90
examples/http-service/cmd/server/lifecycle_test.go
Normal file
90
examples/http-service/cmd/server/lifecycle_test.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestShutdownKeepsDependenciesUntilHTTPReturns(t *testing.T) {
|
||||
entered := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
workerStopped := make(chan struct{})
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { close(entered); <-release; w.WriteHeader(200) })}
|
||||
ready := new(atomic.Bool)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- serve(ctx, listener, server, ready, []component{{"worker", func(ctx context.Context) error { <-ctx.Done(); close(workerStopped); return nil }}}, component{"events", func(ctx context.Context) error {
|
||||
<-ctx.Done()
|
||||
select {
|
||||
case <-workerStopped:
|
||||
return nil
|
||||
default:
|
||||
return errors.New("consumer stopped before worker")
|
||||
}
|
||||
}}, time.Second)
|
||||
}()
|
||||
response := make(chan error, 1)
|
||||
go func() {
|
||||
r, e := http.Get("http://" + listener.Addr().String())
|
||||
if e == nil {
|
||||
r.Body.Close()
|
||||
}
|
||||
response <- e
|
||||
}()
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("HTTP not entered")
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-workerStopped:
|
||||
t.Fatal("worker stopped during HTTP transaction")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
if ready.Load() {
|
||||
t.Fatal("readiness remained true during shutdown")
|
||||
}
|
||||
close(release)
|
||||
if err = <-response; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case err = <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("shutdown blocked")
|
||||
}
|
||||
}
|
||||
func TestBackgroundFailureStopsServiceAndPeers(t *testing.T) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
boom := errors.New("fatal worker")
|
||||
stopped := make(chan struct{})
|
||||
result := serve(context.Background(), listener, &http.Server{}, new(atomic.Bool), []component{{"worker", func(context.Context) error { return boom }}}, component{"consumer", func(ctx context.Context) error { <-ctx.Done(); close(stopped); return nil }}, time.Second)
|
||||
if !errors.Is(result, boom) {
|
||||
t.Fatal(result)
|
||||
}
|
||||
select {
|
||||
case <-stopped:
|
||||
default:
|
||||
t.Fatal("consumer leaked")
|
||||
}
|
||||
}
|
||||
234
examples/http-service/cmd/server/main.go
Normal file
234
examples/http-service/cmd/server/main.go
Normal file
@@ -0,0 +1,234 @@
|
||||
// server 演示将 Orbit SDK 接入现有 HTTP 服务;平台仍独立部署。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
outbox "git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql"
|
||||
"git.yuntuo.hk/k2safe/orbit/examples/http-service/internal/inbox"
|
||||
"git.yuntuo.hk/k2safe/orbit/examples/http-service/internal/jobs"
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
type config struct {
|
||||
Token string `json:"token"`
|
||||
Namespace string `json:"namespace_id"`
|
||||
Queue string `json:"queue_id"`
|
||||
}
|
||||
|
||||
func client() (*orbit.Client, config, error) {
|
||||
c := config{Token: os.Getenv("ORBIT_TOKEN"), Namespace: os.Getenv("ORBIT_NAMESPACE_ID"), Queue: os.Getenv("ORBIT_QUEUE_ID")}
|
||||
if file := os.Getenv("ORBIT_CREDENTIALS_FILE"); file != "" {
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, c, errors.New("cannot read credential file")
|
||||
}
|
||||
var saved config
|
||||
if json.Unmarshal(raw, &saved) != nil {
|
||||
return nil, c, errors.New("invalid credential file")
|
||||
}
|
||||
if c.Token == "" {
|
||||
c.Token = saved.Token
|
||||
}
|
||||
if c.Namespace == "" {
|
||||
c.Namespace = saved.Namespace
|
||||
}
|
||||
if c.Queue == "" {
|
||||
c.Queue = saved.Queue
|
||||
}
|
||||
}
|
||||
if c.Token == "" || c.Namespace == "" || c.Queue == "" {
|
||||
return nil, c, errors.New("Orbit token, namespace and queue are required")
|
||||
}
|
||||
api, err := orbit.NewClient(os.Getenv("ORBIT_URL"), orbit.Options{Token: c.Token})
|
||||
return api, c, err
|
||||
}
|
||||
func run() error {
|
||||
if len(os.Args) != 2 {
|
||||
return errors.New("usage: server init|provision|serve")
|
||||
}
|
||||
command := os.Args[1]
|
||||
if command != "init" && command != "provision" && command != "serve" {
|
||||
return errors.New("unknown command")
|
||||
}
|
||||
process, signalStop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer signalStop()
|
||||
startup, stop := context.WithTimeout(process, 10*time.Second)
|
||||
defer stop()
|
||||
if command == "provision" {
|
||||
api, c, err := client()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 首次使用专属队列前创建,键和内容固定;部署重试不可改名/换键。运行期间从配置复用ID。
|
||||
result, err := api.CreateSubscription(startup, "http-service-sub-"+c.Queue, orbit.CreateSubscriptionRequest{Name: "http-service-" + c.Queue, QueueID: c.Queue, MaxPending: 100, MaxAttempts: 20})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(result.Subscription.ID)
|
||||
return nil
|
||||
}
|
||||
db, err := outbox.Open(startup, os.Getenv("APP_MYSQL_DSN"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 所有组件完成关闭后才关闭共享连接池;初始化/构造/监听失败也走同一回收路径。
|
||||
defer db.Close()
|
||||
if command == "init" {
|
||||
if err = jobs.Init(startup, db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = inbox.Init(startup, db); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("business_schema_initialized=true")
|
||||
return nil
|
||||
}
|
||||
api, c, err := client()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
store, err := jobs.New(db, c.Namespace, c.Queue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 尽早核验业务迁移;不会自动建表,也不依赖Orbit在线才能启动。
|
||||
for _, query := range []string{
|
||||
`SELECT namespace_id,queue_id,operation_id,state FROM sender_outbox LIMIT 0`,
|
||||
`SELECT namespace_id,operation_id,source_ref,task_id FROM sender_jobs LIMIT 0`,
|
||||
`SELECT consumer_name,event_id,event_digest FROM receiver_inbox LIMIT 0`,
|
||||
`SELECT consumer_name,queue_id,task_id,last_revision,result_json FROM receiver_task_views LIMIT 0`,
|
||||
} {
|
||||
if _, err = db.ExecContext(startup, query); err != nil {
|
||||
return errors.New("business schema missing")
|
||||
}
|
||||
}
|
||||
receiver, err := inbox.New(db, "http-service", c.Queue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
worker, err := orbit.NewWorker(api, orbit.WorkerOptions{QueueID: c.Queue, Capacity: 4, ShutdownTimeout: 20 * time.Second, Handlers: []orbit.WorkerHandler{{Capability: orbit.WorkerCapability{Handler: "http.object.prepare", HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: prepare}}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
consumer, err := orbit.NewEventConsumer(api, orbit.EventConsumerOptions{SubscriptionID: os.Getenv("ORBIT_SUBSCRIPTION_ID"), Capacity: 4, ShutdownTimeout: 10 * time.Second, Handle: receiver.Handle})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ready := new(atomic.Bool)
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /health/live", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("live\n")) })
|
||||
mux.HandleFunc("GET /health/ready", func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond)
|
||||
defer cancel()
|
||||
if !ready.Load() || db.PingContext(ctx) != nil {
|
||||
http.Error(w, "not ready", 503)
|
||||
return
|
||||
}
|
||||
w.Write([]byte("ready\n"))
|
||||
})
|
||||
mux.HandleFunc("POST /jobs", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !ready.Load() {
|
||||
http.Error(w, "stopping", 503)
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 4096)
|
||||
var in struct {
|
||||
OperationID string `json:"operation_id"`
|
||||
SourceRef string `json:"source_ref"`
|
||||
}
|
||||
d := json.NewDecoder(r.Body)
|
||||
d.DisallowUnknownFields()
|
||||
if d.Decode(&in) != nil || d.Decode(new(any)) != io.EOF {
|
||||
http.Error(w, "invalid request", 400)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
payload, _ := json.Marshal(map[string]string{"source_ref": in.SourceRef})
|
||||
duplicate, err := store.Enqueue(ctx, in.OperationID, in.SourceRef, orbit.SubmitTaskRequest{QueueID: c.Queue, Handler: "http.object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: payload, BusinessKey: in.OperationID, TaskTimeoutMS: 86400000, RecoverySafety: orbit.RecoveryIdempotent, Retry: orbit.RetryPolicy{MaxAttempts: 3, AttemptTimeoutMS: 10000}})
|
||||
if err != nil {
|
||||
httpFailure(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 202, map[string]any{"operation_id": in.OperationID, "local_committed": true, "duplicate": duplicate})
|
||||
})
|
||||
mux.HandleFunc("GET /jobs/{id}", func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
result, err := store.View(ctx, r.PathValue("id"))
|
||||
if err != nil {
|
||||
httpFailure(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, result)
|
||||
})
|
||||
addr := os.Getenv("APP_HTTP_ADDR")
|
||||
if addr == "" {
|
||||
addr = "127.0.0.1:8090"
|
||||
}
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil || net.ParseIP(host) == nil || !net.ParseIP(host).IsLoopback() {
|
||||
return errors.New("example requires a loopback APP_HTTP_ADDR; mount handlers behind your application authentication for production")
|
||||
}
|
||||
listener, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
return errors.New("cannot listen on APP_HTTP_ADDR")
|
||||
}
|
||||
defer listener.Close()
|
||||
server := &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 16 << 10}
|
||||
fmt.Printf("http_listening=%s\n", listener.Addr())
|
||||
return serve(process, listener, server, ready, []component{{"outbox", func(ctx context.Context) error {
|
||||
return store.Run(ctx, api, outbox.RunOptions{ShutdownTimeout: 20 * time.Second})
|
||||
}}, {"worker", worker.Run}}, component{"events", consumer.Run}, 25*time.Second)
|
||||
}
|
||||
func httpFailure(w http.ResponseWriter, err error) {
|
||||
code, text := 503, "unconfirmed; retry original operation and content"
|
||||
if errors.Is(err, outbox.ErrInvalid) {
|
||||
code, text = 400, "invalid request"
|
||||
}
|
||||
if errors.Is(err, outbox.ErrConflict) {
|
||||
code, text = 409, "operation content conflict"
|
||||
}
|
||||
if errors.Is(err, outbox.ErrNotFound) {
|
||||
code, text = 404, "operation not found"
|
||||
}
|
||||
http.Error(w, text, code)
|
||||
}
|
||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
func prepare(ctx context.Context, e orbit.WorkerExecution) (json.RawMessage, error) {
|
||||
var in struct {
|
||||
SourceRef string `json:"source_ref"`
|
||||
}
|
||||
if json.Unmarshal(e.Task.Payload, &in) != nil || in.SourceRef == "" {
|
||||
return nil, &orbit.WorkerFailure{Code: "INVALID_SOURCE"}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// 纯计算可重复;实际外部副作用必须由业务实现幂等或核对,不能照搬幂等声明。
|
||||
return json.Marshal(map[string]string{"source_ref": in.SourceRef, "normalized_ref": strings.ToUpper(in.SourceRef)})
|
||||
}
|
||||
16
examples/http-service/go.mod
Normal file
16
examples/http-service/go.mod
Normal file
@@ -0,0 +1,16 @@
|
||||
module git.yuntuo.hk/k2safe/orbit/examples/http-service
|
||||
|
||||
go 1.27.1
|
||||
|
||||
require (
|
||||
git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql v0.0.0
|
||||
git.yuntuo.hk/k2safe/orbit/sdk/go v0.1.0
|
||||
github.com/go-sql-driver/mysql v1.10.1
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/coder/websocket v1.8.15 // indirect
|
||||
)
|
||||
|
||||
replace git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql => ../../adapters/outbox/mysql
|
||||
8
examples/http-service/go.sum
Normal file
8
examples/http-service/go.sum
Normal file
@@ -0,0 +1,8 @@
|
||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
git.yuntuo.hk/k2safe/orbit/sdk/go v0.1.0 h1:OJp+U8dDNKjiXy4ZhO4MMq68Ek8TkjphUAqsjOOg77c=
|
||||
git.yuntuo.hk/k2safe/orbit/sdk/go v0.1.0/go.mod h1:9KlMW7Gy+EGaDoEwr3wHHFY47mnAymSfYeQpMS7h4ag=
|
||||
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
|
||||
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
|
||||
github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME=
|
||||
github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
28
examples/http-service/internal/inbox/model.go
Normal file
28
examples/http-service/internal/inbox/model.go
Normal file
@@ -0,0 +1,28 @@
|
||||
package inbox
|
||||
|
||||
import "time"
|
||||
|
||||
// Record 对应receiver_inbox,与业务视图同事务提交;存在记录才表示该事件处理已提交。
|
||||
type Record struct {
|
||||
ConsumerName string // ConsumerName 固定用途名;进程重启及多实例必须复用。
|
||||
QueueID string // QueueID 固定来源队列,不能从连接会话推导。
|
||||
EventID string // EventID 稳定事件身份,不使用交付ID或代次。
|
||||
EventDigest []byte // EventDigest 32字节SHA256,防止同事件身份被换成不同内容。
|
||||
EventJSON []byte // EventJSON 规范化冻结事件,含业务数据、不含租约秘密。
|
||||
Applied bool // Applied 该事件是否推进业务视图;旧版本为false但仍可靠保存。
|
||||
ReceivedAt time.Time // ReceivedAt 数据库UTC受理时间,微秒精度。
|
||||
}
|
||||
|
||||
// TaskView 对应receiver_task_views;这是接收方业务数据,不是Orbit任务权威状态。
|
||||
type TaskView struct {
|
||||
ConsumerName string // ConsumerName 与Inbox相同的固定用途。
|
||||
QueueID string // QueueID 与Inbox相同的绑定队列。
|
||||
TaskID string // TaskID 原任务标识,业务视图主键的一部分。
|
||||
LastRevision uint64 // LastRevision 已应用的最大任务版本;0只允许在未提交事务的占位行中。
|
||||
LastEventID *string // LastEventID 最后推进视图的事件;仅占位状态可为空。
|
||||
LastState string // LastState 冻结结果状态;NEEDS_REVIEW不能作为成功或最终失败。
|
||||
ResultJSON []byte // ResultJSON 当前结果,nil表示来源未提供;数值保持精确。
|
||||
AppliedEvents uint64 // AppliedEvents 实际推进次数,重复及迟到旧版本不增加。
|
||||
CreatedAt time.Time // CreatedAt 视图首次创建的UTC时间,微秒精度。
|
||||
UpdatedAt time.Time // UpdatedAt 最后推进的UTC时间,微秒精度。
|
||||
}
|
||||
32
examples/http-service/internal/inbox/schema.sql
Normal file
32
examples/http-service/internal/inbox/schema.sql
Normal file
@@ -0,0 +1,32 @@
|
||||
-- 接收方独立数据库的初始示例表;不属于Orbit平台迁移,不创建外键或级联删除。
|
||||
CREATE TABLE receiver_inbox (
|
||||
consumer_name VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定消费者用途;同一用途重启或多实例必须复用',
|
||||
queue_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '接收方绑定的队列标识;隔离不同来源范围',
|
||||
event_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '原事件标识;跨连接及交付代次保持不变',
|
||||
event_digest BINARY(32) NOT NULL COMMENT '规范化冻结事件JSON的SHA256;同身份不同内容拒绝确认',
|
||||
event_json MEDIUMTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '冻结事件及业务结果;包含业务数据须限制访问,不含租约证明',
|
||||
applied BOOLEAN NOT NULL DEFAULT FALSE COMMENT '是否推进业务视图;迟到的旧版本只保留Inbox历史',
|
||||
received_at DATETIME(6) NOT NULL COMMENT '本次业务事务受理事件的UTC时间,微秒精度',
|
||||
PRIMARY KEY (consumer_name,queue_id,event_id),
|
||||
CONSTRAINT ck_receiver_event_json CHECK (JSON_VALID(event_json)),
|
||||
CONSTRAINT ck_receiver_applied CHECK (applied IN (0,1))
|
||||
) ENGINE=InnoDB COMMENT='事务Inbox;去重记录与业务视图一起提交,禁止独立预写成功标志';
|
||||
|
||||
CREATE TABLE receiver_task_views (
|
||||
consumer_name VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定消费者用途;与Inbox使用相同范围',
|
||||
queue_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定队列来源;由处理器事务前验证归属',
|
||||
task_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '原任务标识;每个任务建立一份版本化业务视图',
|
||||
last_revision BIGINT UNSIGNED NOT NULL COMMENT '已应用最大任务版本;0仅为同事务中尚未更新的占位状态',
|
||||
last_event_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL COMMENT '推进当前视图的事件;占位行为空,提交后的有效视图非空',
|
||||
last_state VARCHAR(24) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '冻结任务结果状态;UNINITIALIZED只用于同事务内占位',
|
||||
result_json MEDIUMTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL COMMENT '当前结果JSON;来源未提供结果时为空,保留精确数字',
|
||||
applied_events BIGINT UNSIGNED NOT NULL COMMENT '实际推进视图的事件次数;重复和旧版本不增加',
|
||||
created_at DATETIME(6) NOT NULL COMMENT '视图首次建立的UTC时间,微秒精度',
|
||||
updated_at DATETIME(6) NOT NULL COMMENT '视图最后推进的UTC时间,微秒精度',
|
||||
PRIMARY KEY (consumer_name,queue_id,task_id),
|
||||
CONSTRAINT ck_receiver_revision CHECK (
|
||||
(last_revision=0 AND last_event_id IS NULL AND last_state='UNINITIALIZED' AND applied_events=0)
|
||||
OR (last_revision>0 AND last_event_id IS NOT NULL AND applied_events>0 AND last_state IN ('SUCCEEDED','FAILED','CANCELLED','NEEDS_REVIEW'))
|
||||
),
|
||||
CONSTRAINT ck_receiver_result CHECK (result_json IS NULL OR JSON_VALID(result_json))
|
||||
) ENGINE=InnoDB COMMENT='业务侧任务结果视图;锁定单任务行后仅接受更高版本,防止迟到结果倒退';
|
||||
314
examples/http-service/internal/inbox/store.go
Normal file
314
examples/http-service/internal/inbox/store.go
Normal file
@@ -0,0 +1,314 @@
|
||||
// Package inbox 演示业务端同事务去重和版本化结果视图,不是通用业务恰好一次框架。
|
||||
package inbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
//go:embed schema.sql
|
||||
var schema string
|
||||
|
||||
var (
|
||||
ErrInvalidEvent = errors.New("receiver event or scope is invalid")
|
||||
ErrEventConflict = errors.New("receiver event identity has conflicting content; investigate before acknowledgement")
|
||||
ErrRevisionConflict = errors.New("receiver task revision has conflicting event identities")
|
||||
ErrStorage = errors.New("receiver storage operation failed; event remains unacknowledged")
|
||||
ErrCommitUnconfirmed = errors.New("receiver transaction commit is unconfirmed; reconcile original event on redelivery")
|
||||
namePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`)
|
||||
idPattern = regexp.MustCompile(`^[a-f0-9]{32}$`)
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
consumer, queue string
|
||||
}
|
||||
type Result struct {
|
||||
Duplicate bool // Duplicate 当前事件已在相同消费者范围内提交,内容摘要也相同。
|
||||
Applied bool // Applied 原事务是否推进视图;重复命中时返回历史值,不表示再次推进。
|
||||
}
|
||||
|
||||
// Open 仅连接接收方数据库;密码和原始DSN不进入错误输出,不自动创建或修改表。
|
||||
func Open(ctx context.Context, dsn string) (*sql.DB, error) {
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil || cfg.DBName == "" {
|
||||
return nil, errors.New("valid receiver MySQL DSN with explicit database is required")
|
||||
}
|
||||
cfg.ParseTime = true
|
||||
cfg.Loc = time.UTC
|
||||
cfg.MultiStatements = false
|
||||
cfg.AllowAllFiles = false
|
||||
cfg.Timeout = 5 * time.Second
|
||||
cfg.ReadTimeout = 8 * time.Second
|
||||
cfg.WriteTimeout = 8 * time.Second
|
||||
if cfg.Params == nil {
|
||||
cfg.Params = map[string]string{}
|
||||
}
|
||||
cfg.Params["time_zone"] = "'+00:00'"
|
||||
connector, err := mysql.NewConnector(cfg)
|
||||
if err != nil {
|
||||
return nil, ErrStorage
|
||||
}
|
||||
db := sql.OpenDB(connector)
|
||||
db.SetMaxOpenConns(8)
|
||||
db.SetMaxIdleConns(4)
|
||||
db.SetConnMaxLifetime(5 * time.Minute)
|
||||
if err = db.PingContext(ctx); err != nil {
|
||||
db.Close()
|
||||
return nil, ErrStorage
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// Init 初始化专用空数据库。已有表会报错;升级已有业务数据库应使用接收方自己的迁移。
|
||||
func Init(ctx context.Context, db *sql.DB) error {
|
||||
for _, statement := range strings.Split(schema, ";") {
|
||||
if strings.TrimSpace(statement) == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, statement); err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func New(db *sql.DB, consumer, queue string) (*Store, error) {
|
||||
if db == nil || !namePattern.MatchString(consumer) || !idPattern.MatchString(queue) {
|
||||
return nil, ErrInvalidEvent
|
||||
}
|
||||
return &Store{db: db, consumer: consumer, queue: queue}, nil
|
||||
}
|
||||
|
||||
// Handle 可直接作为EventConsumer处理器;只有事务确认提交或命中相同已提交记录才返回nil。
|
||||
func (s *Store) Handle(ctx context.Context, event orbit.ResultEvent) error {
|
||||
_, err := s.Apply(ctx, event)
|
||||
return err
|
||||
}
|
||||
|
||||
// Apply 不含网络副作用。事务失败或COMMIT响应未知时不允许SDK发送ACK。
|
||||
func (s *Store) Apply(parent context.Context, event orbit.ResultEvent) (Result, error) {
|
||||
ctx, stop := context.WithTimeout(parent, 5*time.Second)
|
||||
defer stop()
|
||||
raw, digest, err := s.document(event)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
return Result{}, ErrStorage
|
||||
}
|
||||
defer tx.Rollback()
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO receiver_inbox(consumer_name,queue_id,event_id,event_digest,event_json,applied,received_at) VALUES(?,?,?,?,?,FALSE,UTC_TIMESTAMP(6))`, s.consumer, s.queue, event.ID, digest[:], raw)
|
||||
if err != nil {
|
||||
var duplicate *mysql.MySQLError
|
||||
if !errors.As(err, &duplicate) || duplicate.Number != 1062 {
|
||||
return Result{}, ErrStorage
|
||||
}
|
||||
// 唯一键冲突已等待胜出事务完成。结束本事务,再核对已提交记录;不能忽略其他SQL错误。
|
||||
if err = tx.Rollback(); err != nil {
|
||||
return Result{}, ErrStorage
|
||||
}
|
||||
var saved []byte
|
||||
var applied bool
|
||||
if err = s.db.QueryRowContext(ctx, `SELECT event_digest,applied FROM receiver_inbox WHERE consumer_name=? AND queue_id=? AND event_id=?`, s.consumer, s.queue, event.ID).Scan(&saved, &applied); err != nil {
|
||||
return Result{}, ErrStorage
|
||||
}
|
||||
if !bytes.Equal(saved, digest[:]) {
|
||||
return Result{}, ErrEventConflict
|
||||
}
|
||||
return Result{Duplicate: true, Applied: applied}, nil
|
||||
}
|
||||
// 原子建立或取得单任务行锁,不依赖客户端RowsAffected选项判断新旧。
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO receiver_task_views(consumer_name,queue_id,task_id,last_revision,last_state,applied_events,created_at,updated_at) VALUES(?,?,?,0,'UNINITIALIZED',0,UTC_TIMESTAMP(6),UTC_TIMESTAMP(6)) ON DUPLICATE KEY UPDATE task_id=receiver_task_views.task_id`, s.consumer, s.queue, event.AggregateID)
|
||||
if err != nil {
|
||||
return Result{}, ErrStorage
|
||||
}
|
||||
var revision uint64
|
||||
if err = tx.QueryRowContext(ctx, `SELECT last_revision FROM receiver_task_views WHERE consumer_name=? AND queue_id=? AND task_id=? FOR UPDATE`, s.consumer, s.queue, event.AggregateID).Scan(&revision); err != nil {
|
||||
return Result{}, ErrStorage
|
||||
}
|
||||
if event.AggregateRevision == revision {
|
||||
return Result{}, ErrRevisionConflict
|
||||
}
|
||||
applied := event.AggregateRevision > revision
|
||||
if applied {
|
||||
var result any
|
||||
if len(event.Snapshot.Result) > 0 {
|
||||
result = []byte(event.Snapshot.Result)
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE receiver_task_views SET last_revision=?,last_event_id=?,last_state=?,result_json=?,applied_events=applied_events+1,updated_at=UTC_TIMESTAMP(6) WHERE consumer_name=? AND queue_id=? AND task_id=?`, event.AggregateRevision, event.ID, string(event.Snapshot.State), result, s.consumer, s.queue, event.AggregateID)
|
||||
if err != nil {
|
||||
return Result{}, ErrStorage
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE receiver_inbox SET applied=TRUE WHERE consumer_name=? AND queue_id=? AND event_id=?`, s.consumer, s.queue, event.ID); err != nil {
|
||||
return Result{}, ErrStorage
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return Result{}, ErrCommitUnconfirmed
|
||||
}
|
||||
return Result{Applied: applied}, nil
|
||||
}
|
||||
|
||||
func (s *Store) document(event orbit.ResultEvent) ([]byte, [32]byte, error) {
|
||||
var none [32]byte
|
||||
snapshot := event.Snapshot
|
||||
if len(event.Details)+len(snapshot.Result) > 512<<10 {
|
||||
return nil, none, ErrInvalidEvent
|
||||
}
|
||||
if !idPattern.MatchString(event.ID) || event.Sequence == 0 || event.AggregateKind != "task" || !idPattern.MatchString(event.AggregateID) || snapshot.TaskID != event.AggregateID || snapshot.QueueID != s.queue || event.AggregateRevision == 0 || event.AggregateRevision != snapshot.Revision || event.OccurredAt.IsZero() || snapshot.SchemaVersion != 1 || snapshot.CurrentAttemptID != nil && !idPattern.MatchString(*snapshot.CurrentAttemptID) {
|
||||
return nil, none, ErrInvalidEvent
|
||||
}
|
||||
states := map[string]orbit.TaskState{"task.succeeded": orbit.TaskSucceeded, "task.failed": orbit.TaskFailed, "task.cancelled": orbit.TaskCancelled, "task.needs_review": orbit.TaskNeedsReview}
|
||||
if state, ok := states[event.Type]; !ok || state != snapshot.State {
|
||||
return nil, none, ErrInvalidEvent
|
||||
}
|
||||
if len(bytes.TrimSpace(event.Details)) == 0 || bytes.TrimSpace(event.Details)[0] != '{' {
|
||||
return nil, none, ErrInvalidEvent
|
||||
}
|
||||
event.OccurredAt = event.OccurredAt.UTC()
|
||||
var buffer bytes.Buffer
|
||||
encoder := json.NewEncoder(&buffer)
|
||||
encoder.SetEscapeHTML(false)
|
||||
if encoder.Encode(event) != nil || buffer.Len() > 512<<10 {
|
||||
return nil, none, ErrInvalidEvent
|
||||
}
|
||||
// 对象键与空白规范化,数字使用json.Number保留原精度及十进制表达。
|
||||
canonical, err := canonicalJSON(buffer.Bytes())
|
||||
if err != nil {
|
||||
return nil, none, err
|
||||
}
|
||||
return canonical, sha256.Sum256(canonical), nil
|
||||
}
|
||||
func canonicalJSON(raw []byte) ([]byte, error) {
|
||||
if !utf8.Valid(raw) || !json.Valid(raw) || !pairedUnicode(raw) {
|
||||
return nil, ErrInvalidEvent
|
||||
}
|
||||
d := json.NewDecoder(bytes.NewReader(raw))
|
||||
d.UseNumber()
|
||||
var value func(int) (any, error)
|
||||
value = func(depth int) (any, error) {
|
||||
if depth > 32 {
|
||||
return nil, ErrInvalidEvent
|
||||
}
|
||||
token, err := d.Token()
|
||||
if err != nil {
|
||||
return nil, ErrInvalidEvent
|
||||
}
|
||||
switch token {
|
||||
case json.Delim('{'):
|
||||
out := map[string]any{}
|
||||
for d.More() {
|
||||
key, err := d.Token()
|
||||
if err != nil {
|
||||
return nil, ErrInvalidEvent
|
||||
}
|
||||
name, ok := key.(string)
|
||||
if !ok {
|
||||
return nil, ErrInvalidEvent
|
||||
}
|
||||
if _, found := out[name]; found {
|
||||
return nil, ErrInvalidEvent
|
||||
}
|
||||
out[name], err = value(depth + 1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
_, err = d.Token()
|
||||
return out, err
|
||||
case json.Delim('['):
|
||||
out := []any{}
|
||||
for d.More() {
|
||||
item, err := value(depth + 1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
_, err = d.Token()
|
||||
return out, err
|
||||
default:
|
||||
return token, nil
|
||||
}
|
||||
}
|
||||
doc, err := value(0)
|
||||
if err != nil {
|
||||
return nil, ErrInvalidEvent
|
||||
}
|
||||
if _, err = d.Token(); err != io.EOF {
|
||||
return nil, ErrInvalidEvent
|
||||
}
|
||||
var out bytes.Buffer
|
||||
e := json.NewEncoder(&out)
|
||||
e.SetEscapeHTML(false)
|
||||
if e.Encode(doc) != nil {
|
||||
return nil, ErrInvalidEvent
|
||||
}
|
||||
return bytes.TrimSuffix(out.Bytes(), []byte{'\n'}), nil
|
||||
}
|
||||
|
||||
// 不将未配对UTF-16转义静默转换成替代字符,避免摘要与实际业务文本失真。
|
||||
func pairedUnicode(raw []byte) bool {
|
||||
inside := false
|
||||
for i := 0; i < len(raw); i++ {
|
||||
if raw[i] == '"' {
|
||||
inside = !inside
|
||||
continue
|
||||
}
|
||||
if !inside || raw[i] != '\\' {
|
||||
continue
|
||||
}
|
||||
i++
|
||||
if i >= len(raw) {
|
||||
return false
|
||||
}
|
||||
if raw[i] != 'u' {
|
||||
continue
|
||||
}
|
||||
if i+4 >= len(raw) {
|
||||
return false
|
||||
}
|
||||
n, err := strconv.ParseUint(string(raw[i+1:i+5]), 16, 16)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
i += 4
|
||||
if n < 0xd800 || n > 0xdfff {
|
||||
continue
|
||||
}
|
||||
if n > 0xdbff || i+6 >= len(raw) || raw[i+1] != '\\' || raw[i+2] != 'u' {
|
||||
return false
|
||||
}
|
||||
low, err := strconv.ParseUint(string(raw[i+3:i+7]), 16, 16)
|
||||
if err != nil || low < 0xdc00 || low > 0xdfff {
|
||||
return false
|
||||
}
|
||||
i += 6
|
||||
}
|
||||
return !inside
|
||||
}
|
||||
|
||||
// View 返回业务侧当前视图;不会从Orbit重新拼装结果。
|
||||
func (s *Store) View(ctx context.Context, task string) (TaskView, error) {
|
||||
out := TaskView{ConsumerName: s.consumer, QueueID: s.queue, TaskID: task}
|
||||
err := s.db.QueryRowContext(ctx, `SELECT last_revision,last_event_id,last_state,result_json,applied_events,created_at,updated_at FROM receiver_task_views WHERE consumer_name=? AND queue_id=? AND task_id=?`, s.consumer, s.queue, task).Scan(&out.LastRevision, &out.LastEventID, &out.LastState, &out.ResultJSON, &out.AppliedEvents, &out.CreatedAt, &out.UpdatedAt)
|
||||
if err != nil {
|
||||
return TaskView{}, ErrStorage
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
59
examples/http-service/internal/inbox/store_test.go
Normal file
59
examples/http-service/internal/inbox/store_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package inbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
func newID(t *testing.T) string {
|
||||
t.Helper()
|
||||
id, err := orbit.NewEventMessageID()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
func testEvent(t *testing.T, queue string) orbit.ResultEvent {
|
||||
t.Helper()
|
||||
task := newID(t)
|
||||
return orbit.ResultEvent{ID: newID(t), Sequence: 7, Type: "task.succeeded", AggregateKind: "task", AggregateID: task, AggregateRevision: 9007199254740993, OccurredAt: time.Now().UTC().Truncate(time.Microsecond), Details: json.RawMessage(`{"origin":"worker"}`), Snapshot: orbit.TaskResultSnapshot{SchemaVersion: 1, TaskID: task, QueueID: queue, State: orbit.TaskSucceeded, Revision: 9007199254740993, Result: json.RawMessage(`{"n":9007199254740993,"text":"<&>"}`)}}
|
||||
}
|
||||
func TestFrozenEventIdentity(t *testing.T) {
|
||||
s := &Store{consumer: "result-view", queue: newID(t)}
|
||||
e := testEvent(t, s.queue)
|
||||
raw, a, err := s.document(e)
|
||||
if err != nil || !bytes.Contains(raw, []byte("9007199254740993")) || !bytes.Contains(raw, []byte("<&>")) {
|
||||
t.Fatal("precision changed", err)
|
||||
}
|
||||
e.Snapshot.Result = json.RawMessage(`{ "text": "<&>", "n": 9007199254740993 }`)
|
||||
_, b, err := s.document(e)
|
||||
if err != nil || a != b {
|
||||
t.Fatal("equivalent formatting changed identity", err)
|
||||
}
|
||||
e.OccurredAt = e.OccurredAt.In(time.FixedZone("offset", 8*3600))
|
||||
_, b, err = s.document(e)
|
||||
if err != nil || a != b {
|
||||
t.Fatal("timezone representation changed event identity", err)
|
||||
}
|
||||
e.Snapshot.Result = json.RawMessage(`{"n":9007199254740994,"text":"<&>"}`)
|
||||
_, b, err = s.document(e)
|
||||
if err != nil || a == b {
|
||||
t.Fatal("changed result not detected", err)
|
||||
}
|
||||
e.Snapshot.QueueID = newID(t)
|
||||
if _, _, err = s.document(e); !errors.Is(err, ErrInvalidEvent) {
|
||||
t.Fatal("wrong receiver scope accepted", err)
|
||||
}
|
||||
}
|
||||
func TestCanonicalRejectsDuplicateAndInvalidJSON(t *testing.T) {
|
||||
for _, raw := range []string{`{"a":1,"a":2}`, `{"a":null,"a":2}`, `{"a":1} {}`, `{"a":`, `{"a":"\ud800"}`, `{"a":"\udfff"}`, "\xff"} {
|
||||
if _, err := canonicalJSON([]byte(raw)); !errors.Is(err, ErrInvalidEvent) {
|
||||
t.Fatal("invalid document accepted")
|
||||
}
|
||||
}
|
||||
}
|
||||
9
examples/http-service/internal/jobs/schema.sql
Normal file
9
examples/http-service/internal/jobs/schema.sql
Normal file
@@ -0,0 +1,9 @@
|
||||
CREATE TABLE sender_jobs (
|
||||
namespace_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '与Outbox相同的命名空间范围',
|
||||
operation_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '本地业务操作标识,与Outbox同事务创建',
|
||||
source_ref VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL COMMENT '待处理对象引用,首次入库后固定,不存对象正文或凭据',
|
||||
task_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL COMMENT '已确认的平台受理任务,为空不证明远端未受理',
|
||||
created_at DATETIME(6) NOT NULL COMMENT '业务记录创建UTC时间,微秒精度',
|
||||
updated_at DATETIME(6) NOT NULL COMMENT '关联任务最近变化UTC时间,微秒精度',
|
||||
PRIMARY KEY (namespace_id,operation_id)
|
||||
) ENGINE=InnoDB COMMENT='接入方示例业务记录,与待发请求及受理关联分别同事务保存';
|
||||
126
examples/http-service/internal/jobs/store.go
Normal file
126
examples/http-service/internal/jobs/store.go
Normal file
@@ -0,0 +1,126 @@
|
||||
// Package outbox 演示在业务事务中使用公共 MySQL Outbox 适配包。
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
adapter "git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql"
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
//go:embed schema.sql
|
||||
var schema string
|
||||
var (
|
||||
ErrInvalid = adapter.ErrInvalid
|
||||
ErrStorage = adapter.ErrStorage
|
||||
ErrConflict = adapter.ErrConflict
|
||||
ErrCommitUnconfirmed = adapter.ErrCommitUnconfirmed
|
||||
ErrIdle = adapter.ErrIdle
|
||||
)
|
||||
|
||||
type Record = adapter.Record
|
||||
|
||||
// Store 仅补充示例业务事务;发送、查询、租约由公共包负责。
|
||||
type Store struct {
|
||||
*adapter.Store
|
||||
db *sql.DB
|
||||
namespace string
|
||||
}
|
||||
|
||||
var Open = adapter.Open
|
||||
|
||||
func Init(ctx context.Context, db *sql.DB) error {
|
||||
if err := adapter.Init(ctx, db); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, schema); err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func New(db *sql.DB, namespace, queue string) (*Store, error) {
|
||||
s, err := adapter.New(db, namespace, queue, adapter.Options{OnAccepted: accepted})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Store{s, db, namespace}, nil
|
||||
}
|
||||
func accepted(ctx context.Context, tx *sql.Tx, r adapter.Record, receipt orbit.SubmitTaskResult) error {
|
||||
var existing *string
|
||||
if err := tx.QueryRowContext(ctx, `SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, r.NamespaceID, r.OperationID).Scan(&existing); err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
if existing != nil && *existing != receipt.Task.ID {
|
||||
return ErrConflict
|
||||
}
|
||||
_, err := tx.ExecContext(ctx, `UPDATE sender_jobs SET task_id=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, receipt.Task.ID, r.NamespaceID, r.OperationID)
|
||||
if err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *Store) Enqueue(ctx context.Context, id, source string, in orbit.SubmitTaskRequest) (bool, error) {
|
||||
if !utf8.ValidString(source) || strings.TrimSpace(source) == "" || utf8.RuneCountInString(source) > 512 || strings.ContainsFunc(source, unicode.IsControl) {
|
||||
return false, ErrInvalid
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
return false, ErrStorage
|
||||
}
|
||||
defer tx.Rollback()
|
||||
duplicate, err := s.Store.EnqueueTx(ctx, tx, id, in)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if duplicate {
|
||||
var saved string
|
||||
if err = tx.QueryRowContext(ctx, `SELECT source_ref FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, s.namespace, id).Scan(&saved); err != nil {
|
||||
return false, ErrStorage
|
||||
}
|
||||
if saved != source {
|
||||
return false, ErrConflict
|
||||
}
|
||||
} else {
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO sender_jobs(namespace_id,operation_id,source_ref,created_at,updated_at) VALUES(?,?,?,UTC_TIMESTAMP(6),UTC_TIMESTAMP(6))`, s.namespace, id, source); err != nil {
|
||||
return false, ErrStorage
|
||||
}
|
||||
}
|
||||
if tx.Commit() != nil {
|
||||
return false, ErrCommitUnconfirmed
|
||||
}
|
||||
return duplicate, nil
|
||||
}
|
||||
|
||||
// View 是业务HTTP的读取结果;Submission只表示提交阶段,Execution来自持久化Inbox。
|
||||
type View struct {
|
||||
OperationID string `json:"operation_id"` // OperationID 调用方提供的稳定操作标识。
|
||||
Submission string `json:"submission"` // Submission 待发/发送中/已受理/需核查,不是执行结果。
|
||||
Code string `json:"code"` // Code 最近稳定的提交分类。
|
||||
TaskID *string `json:"task_id,omitempty"` // TaskID 已核对的平台任务,未确认时为空。
|
||||
Execution string `json:"execution,omitempty"` // Execution 业务Inbox最后收到的任务状态。
|
||||
Result json.RawMessage `json:"result,omitempty"` // Result 业务Inbox中的结果,尚未到达时为空。
|
||||
}
|
||||
|
||||
func (s *Store) View(ctx context.Context, id string) (View, error) {
|
||||
r, err := s.Status(ctx, id)
|
||||
if err != nil {
|
||||
return View{}, err
|
||||
}
|
||||
v := View{OperationID: id, Submission: r.State, Code: r.LastCode, TaskID: r.TaskID}
|
||||
if r.TaskID != nil {
|
||||
var result []byte
|
||||
err = s.db.QueryRowContext(ctx, `SELECT last_state,result_json FROM receiver_task_views WHERE consumer_name='http-service' AND queue_id=? AND task_id=?`, r.QueueID, *r.TaskID).Scan(&v.Execution, &result)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return View{}, ErrStorage
|
||||
}
|
||||
v.Result = result
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
- [完整初始化、运行、恢复与接口说明](../../docs/examples/task-outbox.md)
|
||||
- [命令行程序](cmd/sender/main.go):init、enqueue、send、status、demo。
|
||||
- [业务事务](internal/outbox/store.go)、[发送器](internal/outbox/sender.go)、[中文模型](internal/outbox/model.go)与[无外键表结构](internal/outbox/schema.sql)。
|
||||
- [业务事务](internal/outbox/store.go)、[发送器](../../adapters/outbox/mysql/sender.go)、[中文模型](internal/outbox/model.go)与[无外键表结构](internal/outbox/schema.sql)。
|
||||
|
||||
本模块是可运行的业务接入参考。核心SDK不引入MySQL驱动;生产业务应将本地事务接入自身业务模型、权限、审计与迁移,固定SDK版本并验证兼容性。本例的internal包不作为已发布通用SDK包承诺兼容。
|
||||
|
||||
@@ -17,3 +17,5 @@ GOWORK=off go build ./cmd/sender
|
||||
```
|
||||
|
||||
真实数据库、HTTP、COMMIT响应丢失、CLI及发送进程故障由仓库根目录的`make integration`统一执行,使用专门的一次性数据库。没有测试环境变量时Integration会跳过。
|
||||
|
||||
发送核心已抽为[公共 MySQL 适配包](../../adapters/outbox/mysql/README.md);本项目仅保留示例业务事务。已有 sender_outbox / sender_jobs 表保持兼容。
|
||||
|
||||
@@ -3,13 +3,16 @@ module git.yuntuo.hk/k2safe/orbit/examples/task-outbox
|
||||
go 1.27.1
|
||||
|
||||
require (
|
||||
git.yuntuo.hk/k2safe/orbit/sdk/go v0.0.0
|
||||
github.com/go-sql-driver/mysql v1.10.1
|
||||
git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql v0.0.0
|
||||
git.yuntuo.hk/k2safe/orbit/sdk/go v0.1.0
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/coder/websocket v1.8.15 // indirect
|
||||
github.com/go-sql-driver/mysql v1.10.1 // indirect
|
||||
)
|
||||
|
||||
replace git.yuntuo.hk/k2safe/orbit/sdk/go => ../../sdk/go
|
||||
|
||||
replace git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql => ../../adapters/outbox/mysql
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIntegrationSenderCLI(t *testing.T) {
|
||||
s, api, base, _ := realStore(t)
|
||||
ctx, stop := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer stop()
|
||||
binary := filepath.Join(t.TempDir(), "sender")
|
||||
build := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "../../cmd/sender")
|
||||
if raw, err := build.CombinedOutput(); err != nil {
|
||||
t.Fatalf("build sender: %v %s", err, raw)
|
||||
}
|
||||
values := map[string]string{"ORBIT_SENDER_DSN": os.Getenv("ORBIT_TEST_SENDER_DSN"), "ORBIT_URL": base, "ORBIT_CREDENTIALS_FILE": os.Getenv("ORBIT_TEST_SENDER_CREDENTIALS_FILE"), "ORBIT_NAMESPACE_ID": s.namespace, "ORBIT_QUEUE_ID": s.queue, "ORBIT_TOKEN": ""}
|
||||
env := []string{}
|
||||
for _, entry := range os.Environ() {
|
||||
key, _, _ := strings.Cut(entry, "=")
|
||||
if _, ok := values[key]; !ok {
|
||||
env = append(env, entry)
|
||||
}
|
||||
}
|
||||
for key, value := range values {
|
||||
env = append(env, key+"="+value)
|
||||
}
|
||||
call := func(args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.CommandContext(ctx, binary, args...)
|
||||
cmd.Env = env
|
||||
raw, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("sender command failed: %v %s", err, raw)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
id := testID(t)
|
||||
if !strings.Contains(call("enqueue", id, "objects/cli.txt"), "duplicate=false") || !strings.Contains(call("enqueue", id, "objects/cli.txt"), "duplicate=true") || !strings.Contains(call("status", id), "state=PENDING") {
|
||||
t.Fatal("CLI local transaction mismatch")
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, binary, "send")
|
||||
cmd.Env = env
|
||||
cmd.Stdout = io.Discard
|
||||
cmd.Stderr = io.Discard
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := make(chan struct{})
|
||||
var waitErr error
|
||||
go func() { waitErr = cmd.Wait(); close(done) }()
|
||||
t.Cleanup(func() {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
default:
|
||||
}
|
||||
cmd.Process.Kill()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("owned sender CLI did not exit")
|
||||
}
|
||||
})
|
||||
var r Record
|
||||
for {
|
||||
r = status(t, s, id)
|
||||
if r.State == "ACCEPTED" {
|
||||
break
|
||||
}
|
||||
if r.State == "REVIEW" {
|
||||
t.Fatal("CLI needs review", r.LastCode)
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("sender exited before acceptance", waitErr)
|
||||
case <-ctx.Done():
|
||||
t.Fatal("CLI sender timeout")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
if err := cmd.Process.Signal(os.Interrupt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
if waitErr != nil {
|
||||
t.Fatal("sender did not stop gracefully", waitErr)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("sender shutdown timeout")
|
||||
}
|
||||
if !strings.Contains(call("status", id), "state=ACCEPTED") || countRows(t, s, "sender_jobs") != 1 || r.SendCount != 1 {
|
||||
t.Fatal("CLI acceptance did not persist")
|
||||
}
|
||||
cancelAccepted(t, api, *r.TaskID)
|
||||
t.Log("built CLI enqueue duplicate/status/send/graceful stop exercised against real business DB and Orbit")
|
||||
}
|
||||
@@ -1,226 +0,0 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// commitLossGateway 只用于无TLS的隔离MySQL回归;捕获COMMIT的真实成功包后丢弃并断连。
|
||||
// 不伪造SQL结果,也不修改数据库中的提交状态。
|
||||
type commitLossGateway struct {
|
||||
listener net.Listener
|
||||
dropped chan struct{}
|
||||
armed atomic.Bool
|
||||
mu sync.Mutex
|
||||
closing bool
|
||||
connections map[net.Conn]struct{}
|
||||
wg sync.WaitGroup
|
||||
accepted chan struct{}
|
||||
}
|
||||
|
||||
func newCommitLossGateway(t *testing.T, target string) *commitLossGateway {
|
||||
t.Helper()
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := &commitLossGateway{listener: listener, dropped: make(chan struct{}, 1), connections: map[net.Conn]struct{}{}, accepted: make(chan struct{})}
|
||||
go func() {
|
||||
defer close(g.accepted)
|
||||
for {
|
||||
down, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
g.wg.Add(1)
|
||||
go func() {
|
||||
defer g.wg.Done()
|
||||
defer down.Close()
|
||||
up, err := net.DialTimeout("tcp", target, 3*time.Second)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer up.Close()
|
||||
g.mu.Lock()
|
||||
if g.closing {
|
||||
g.mu.Unlock()
|
||||
return
|
||||
}
|
||||
g.connections[down] = struct{}{}
|
||||
g.connections[up] = struct{}{}
|
||||
g.mu.Unlock()
|
||||
defer func() { g.mu.Lock(); delete(g.connections, down); delete(g.connections, up); g.mu.Unlock() }()
|
||||
var committing atomic.Bool
|
||||
done := make(chan struct{}, 2)
|
||||
go func() {
|
||||
defer func() { done <- struct{}{} }()
|
||||
for {
|
||||
packet, err := readMySQLPacket(down)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if packet[3] == 0 && len(packet) > 4 && packet[4] == 3 {
|
||||
committing.Store(strings.EqualFold(strings.TrimSpace(string(packet[5:])), "COMMIT"))
|
||||
}
|
||||
if writeMySQLPacket(up, packet) != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer func() { done <- struct{}{} }()
|
||||
for {
|
||||
packet, err := readMySQLPacket(up)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if packet[3] == 1 && committing.Swap(false) && len(packet) > 4 && packet[4] == 0 && g.armed.CompareAndSwap(true, false) {
|
||||
select {
|
||||
case g.dropped <- struct{}{}:
|
||||
default:
|
||||
t.Error("unconsumed COMMIT fault marker")
|
||||
}
|
||||
return
|
||||
}
|
||||
if writeMySQLPacket(down, packet) != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
<-done
|
||||
down.Close()
|
||||
up.Close()
|
||||
<-done
|
||||
}()
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
listener.Close()
|
||||
<-g.accepted
|
||||
g.mu.Lock()
|
||||
g.closing = true
|
||||
for c := range g.connections {
|
||||
c.Close()
|
||||
}
|
||||
g.mu.Unlock()
|
||||
g.wg.Wait()
|
||||
})
|
||||
return g
|
||||
}
|
||||
func readMySQLPacket(r io.Reader) ([]byte, error) {
|
||||
var header [4]byte
|
||||
if _, err := io.ReadFull(r, header[:]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
size := int(binary.LittleEndian.Uint32(header[:]) & 0x00ffffff)
|
||||
packet := make([]byte, 4+size)
|
||||
copy(packet, header[:])
|
||||
_, err := io.ReadFull(r, packet[4:])
|
||||
return packet, err
|
||||
}
|
||||
func writeMySQLPacket(w io.Writer, b []byte) error {
|
||||
for len(b) > 0 {
|
||||
n, err := w.Write(b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n == 0 {
|
||||
return io.ErrShortWrite
|
||||
}
|
||||
b = b[n:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestIntegrationSenderCommitResponseLoss(t *testing.T) {
|
||||
direct, dsn := integrationDB(t)
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil || cfg.TLSConfig != "" {
|
||||
t.Fatal("fault requires isolated plaintext MySQL")
|
||||
}
|
||||
gateway := newCommitLossGateway(t, cfg.Addr)
|
||||
cfg.Addr = gateway.listener.Addr().String()
|
||||
ctx, stop := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer stop()
|
||||
proxied, err := Open(ctx, cfg.FormatDSN())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer proxied.Close()
|
||||
for _, phase := range []string{"enqueue", "claim", "confirm"} {
|
||||
t.Run(phase, func(t *testing.T) {
|
||||
s := testStore(t, proxied)
|
||||
fresh, _ := New(direct, s.namespace, s.queue)
|
||||
id := testID(t)
|
||||
in := testRequest(s.queue)
|
||||
if phase == "enqueue" {
|
||||
gateway.armed.Store(true)
|
||||
if _, err = s.Enqueue(ctx, id, "objects/test", in); !errors.Is(err, ErrCommitUnconfirmed) {
|
||||
t.Fatal("enqueue reported confirmed", err)
|
||||
}
|
||||
dup, e := fresh.Enqueue(ctx, id, "objects/test", in)
|
||||
if e != nil || !dup || countRows(t, fresh, "sender_jobs") != 1 {
|
||||
t.Fatal("local commit not reconciled", e)
|
||||
}
|
||||
} else {
|
||||
if _, err = s.Enqueue(ctx, id, "objects/test", in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if phase == "claim" {
|
||||
gateway.armed.Store(true)
|
||||
}
|
||||
claim, e := s.claim(ctx)
|
||||
if phase == "claim" {
|
||||
if !errors.Is(e, ErrCommitUnconfirmed) {
|
||||
t.Fatal("claim reported confirmed", e)
|
||||
}
|
||||
r := status(t, fresh, id)
|
||||
if r.State != "SENDING" || r.SendCount != 1 {
|
||||
t.Fatal("claim was not actually committed")
|
||||
}
|
||||
execSQL(t, direct, `UPDATE sender_outbox SET wake_at=TIMESTAMPADD(SECOND,-1,UTC_TIMESTAMP(6)) WHERE namespace_id=? AND operation_id=?`, s.namespace, id)
|
||||
r, e = fresh.claim(ctx)
|
||||
if e != nil || r.SendCount != 2 || r.IdempotencyKey != "outbox-task-"+id {
|
||||
t.Fatal("unknown claim did not recover", e)
|
||||
}
|
||||
} else {
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
receipt := fakeReceipt(t, s)
|
||||
gateway.armed.Store(true)
|
||||
if e = s.finish(ctx, claim, "ACCEPTED", "ACCEPTED", 0, &receipt); !errors.Is(e, ErrCommitUnconfirmed) {
|
||||
t.Fatal("confirmation reported confirmed", e)
|
||||
}
|
||||
r := status(t, fresh, id)
|
||||
if r.State != "ACCEPTED" || r.TaskID == nil || *r.TaskID != receipt.Task.ID {
|
||||
t.Fatal("confirmed commit not visible")
|
||||
}
|
||||
if _, e = fresh.claim(ctx); !errors.Is(e, ErrIdle) {
|
||||
t.Fatal("confirmed item eligible for resend", e)
|
||||
}
|
||||
var task string
|
||||
if e = direct.QueryRowContext(ctx, `SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=?`, s.namespace, id).Scan(&task); e != nil || task != receipt.Task.ID {
|
||||
t.Fatal("business confirmation not atomic", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-gateway.dropped:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("actual COMMIT OK not intercepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Log("real COMMIT OK discarded for business enqueue, sender claim and local acceptance; fresh stores reconciled persisted identity and state")
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
func realClient(t *testing.T) (*orbit.Client, string, string) {
|
||||
t.Helper()
|
||||
base, path := os.Getenv("ORBIT_TEST_SENDER_URL"), os.Getenv("ORBIT_TEST_SENDER_CREDENTIALS_FILE")
|
||||
if base == "" || path == "" {
|
||||
t.Skip("requires isolated Orbit process")
|
||||
}
|
||||
if !strings.HasPrefix(base, "http://127.0.0.1:") {
|
||||
t.Fatal("sender HTTP test requires loopback")
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal("cannot read sender credential")
|
||||
}
|
||||
var cred struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
if json.Unmarshal(raw, &cred) != nil || cred.Token == "" {
|
||||
t.Fatal("invalid sender credential")
|
||||
}
|
||||
api, err := orbit.NewClient(base, orbit.Options{Token: cred.Token})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return api, base, cred.Token
|
||||
}
|
||||
func realStore(t *testing.T) (*Store, *orbit.Client, string, string) {
|
||||
t.Helper()
|
||||
db, _ := integrationDB(t)
|
||||
api, base, token := realClient(t)
|
||||
ctx, stop := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer stop()
|
||||
id := testID(t)
|
||||
queue, err := api.CreateQueue(ctx, "outbox-test-"+id, orbit.CreateQueueRequest{Name: "outbox-" + id, MaxConcurrency: 1, RatePerSecond: 10})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := New(db, queue.Queue.NamespaceID, queue.Queue.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s, api, base, token
|
||||
}
|
||||
func cancelAccepted(t *testing.T, api *orbit.Client, id string) {
|
||||
t.Helper()
|
||||
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer stop()
|
||||
task, err := api.GetTask(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = api.CancelTask(ctx, id, "outbox-test-cancel-"+id, orbit.CancelTaskRequest{ExpectedRevision: task.Revision, Reason: "outbox regression completed"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationSenderHTTPReceiptLoss(t *testing.T) {
|
||||
s, api, base, token := realStore(t)
|
||||
id := enqueue(t, s)
|
||||
target, _ := url.Parse(base)
|
||||
var drop atomic.Bool
|
||||
drop.Store(true)
|
||||
var posts atomic.Int32
|
||||
committed := make(chan orbit.SubmitTaskResult, 1)
|
||||
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
up := r.Clone(r.Context())
|
||||
up.URL.Scheme = target.Scheme
|
||||
up.URL.Host = target.Host
|
||||
up.Host = target.Host
|
||||
up.RequestURI = ""
|
||||
resp, err := http.DefaultTransport.RoundTrip(up)
|
||||
if err != nil {
|
||||
http.Error(w, "upstream unavailable", 502)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, (1<<20)+1))
|
||||
if err != nil {
|
||||
http.Error(w, "upstream read failed", 502)
|
||||
return
|
||||
}
|
||||
if r.Method == "POST" && r.URL.Path == "/v1/tasks" {
|
||||
posts.Add(1)
|
||||
if r.Header.Get("Idempotency-Key") != "outbox-task-"+id {
|
||||
t.Error("original key changed")
|
||||
}
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 && drop.CompareAndSwap(true, false) {
|
||||
var envelope struct {
|
||||
Data orbit.SubmitTaskResult `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(raw, &envelope) != nil || !idPattern.MatchString(envelope.Data.Task.ID) {
|
||||
t.Error("invalid real receipt")
|
||||
http.Error(w, "bad receipt", 502)
|
||||
return
|
||||
}
|
||||
committed <- envelope.Data
|
||||
// 先发送成功响应头与不完整正文,避免Transport把无响应头的断线当成可透明重放。
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(raw)))
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
w.Write(raw[:1])
|
||||
w.(http.Flusher).Flush()
|
||||
conn, _, err := w.(http.Hijacker).Hijack()
|
||||
if err == nil {
|
||||
conn.Close()
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
for key, values := range resp.Header {
|
||||
w.Header()[key] = values
|
||||
}
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
w.Write(raw)
|
||||
}))
|
||||
defer proxy.Close()
|
||||
through, err := orbit.NewClient(proxy.URL, orbit.Options{Token: token})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, stop := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer stop()
|
||||
r, err := s.SendOnce(ctx, through)
|
||||
if err != nil || r.State != "PENDING" || r.LastCode != "UNCONFIRMED" {
|
||||
t.Fatal("unknown remote commit classified as done", r.State, err)
|
||||
}
|
||||
var original orbit.SubmitTaskResult
|
||||
select {
|
||||
case original = <-committed:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("real accepted receipt not intercepted")
|
||||
}
|
||||
if _, err = api.GetTask(ctx, original.Task.ID); err != nil {
|
||||
t.Fatal("remote was not actually committed", err)
|
||||
}
|
||||
execSQL(t, s.db, `UPDATE sender_outbox SET wake_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, s.namespace, id)
|
||||
r, err = s.SendOnce(ctx, through)
|
||||
if err != nil || r.State != "ACCEPTED" || r.TaskID == nil || *r.TaskID != original.Task.ID || r.SendCount != 2 {
|
||||
t.Fatal("original identity not recovered", r.State, err)
|
||||
}
|
||||
var receipt orbit.SubmitTaskResult
|
||||
if json.Unmarshal(r.ReceiptJSON, &receipt) != nil || !receipt.Deduplicated || !strings.Contains(string(receipt.Task.Payload), "9007199254740993") {
|
||||
t.Fatal("replayed receipt or precision lost")
|
||||
}
|
||||
page, err := api.ListTasks(ctx, orbit.ListTasksRequest{QueueID: s.queue})
|
||||
if err != nil || len(page.Items) != 1 || posts.Load() != 2 {
|
||||
t.Fatal("remote duplicate task", len(page.Items), err)
|
||||
}
|
||||
// 相同凭据搭配错误的持久化目标命名空间,必须在提交前停止。
|
||||
wrong, _ := New(s.db, testID(t), s.queue)
|
||||
wrongID := enqueue(t, wrong)
|
||||
r, err = wrong.SendOnce(ctx, through)
|
||||
if err != nil || r.OperationID != wrongID || r.State != "REVIEW" || r.LastCode != "TARGET_MISMATCH" || posts.Load() != 2 {
|
||||
t.Fatal("wrong namespace submitted", r.State, err)
|
||||
}
|
||||
cancelAccepted(t, api, original.Task.ID)
|
||||
t.Log("real HTTP receipt discarded after Orbit commit; original key replay recovered exactly one task; wrong namespace never posted")
|
||||
}
|
||||
@@ -1,26 +1,13 @@
|
||||
package outbox
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"time"
|
||||
|
||||
// Record 是业务数据库中的待发命令;只确认平台受理,不能表示业务执行成功。
|
||||
type Record struct {
|
||||
NamespaceID string // NamespaceID 固定目标命名空间,换凭据不能改变归属。
|
||||
QueueID string // QueueID 固定目标队列,发送前通过当前凭据核对归属。
|
||||
OperationID string // OperationID 调用方持久化的业务操作ID,同命名空间内唯一。
|
||||
IdempotencyKey string // IdempotencyKey 由操作ID派生的固定提交键,重报禁止换键。
|
||||
RequestJSON []byte // RequestJSON 冻结的任务提交正文,保留数字精度,不含凭据。
|
||||
RequestDigest []byte // RequestDigest 冻结正文的SHA256,发送前重新校验,非平台语义摘要。
|
||||
State string // State PENDING待发、SENDING领取中、ACCEPTED已受理、REVIEW需核查。
|
||||
SendCount uint32 // SendCount 持久化领取次数,包含领取后未发HTTP即退出的次数,上限100。
|
||||
LeaseEpoch uint64 // LeaseEpoch 每次领取递增;旧代次不能更新新代次的本地结果。
|
||||
WakeAt *time.Time // WakeAt PENDING的下次领取时间或SENDING的租约期限;终结状态为空,UTC微秒。
|
||||
SendDeadlineAt time.Time // SendDeadlineAt 首次入库后24小时停止新发送,短于平台90天幂等窗口。
|
||||
LastCode string // LastCode 最近一次稳定分类,不保存HTTP错误正文、凭据或业务秘密。
|
||||
TaskID *string // TaskID 仅ACCEPTED记录平台任务ID,其余状态为空。
|
||||
ReceiptJSON []byte // ReceiptJSON 仅ACCEPTED保存原受理回执;不是当前执行状态。
|
||||
CreatedAt time.Time // CreatedAt 业务事务首次提交命令的UTC时间,微秒精度。
|
||||
UpdatedAt time.Time // UpdatedAt 最近状态变化的UTC时间,不能替代租约代次。
|
||||
}
|
||||
adapter "git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql"
|
||||
)
|
||||
|
||||
// Record 复用公共适配器的中文持久化模型。
|
||||
type Record = adapter.Record
|
||||
|
||||
// Job 演示接入方业务记录,与Record原子创建,受理确认时再原子关联任务。
|
||||
type Job struct {
|
||||
|
||||
@@ -1,239 +0,0 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
type roundTripper func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripper) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
type senderMarker struct {
|
||||
Phase string `json:"phase"`
|
||||
TaskID string `json:"task_id"`
|
||||
Sends uint32 `json:"sends"`
|
||||
Duplicate bool `json:"duplicate"`
|
||||
}
|
||||
|
||||
func emitMarker(m senderMarker) {
|
||||
raw, _ := json.Marshal(m)
|
||||
fmt.Println("ORBIT_SENDER_TEST " + string(raw))
|
||||
}
|
||||
|
||||
// 只存在于测试二进制,使用正式Store和真实HTTP;屏障位于平台提交后、本地确认前。
|
||||
func TestSenderProcessHelper(t *testing.T) {
|
||||
mode := os.Getenv("ORBIT_TEST_SENDER_CHILD")
|
||||
if mode == "" {
|
||||
t.Skip("subprocess helper")
|
||||
}
|
||||
if mode != "hold_after_remote_commit" && mode != "recover" {
|
||||
t.Fatal("invalid child mode")
|
||||
}
|
||||
db, _ := integrationDB(t)
|
||||
_, base, token := realClient(t)
|
||||
s, err := New(db, os.Getenv("ORBIT_TEST_SENDER_NAMESPACE"), os.Getenv("ORBIT_TEST_SENDER_QUEUE"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options := orbit.Options{Token: token}
|
||||
if mode == "hold_after_remote_commit" {
|
||||
options.HTTPClient = &http.Client{Timeout: 10 * time.Second, Transport: roundTripper(func(r *http.Request) (*http.Response, error) {
|
||||
resp, err := http.DefaultTransport.RoundTrip(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.Method != "POST" || r.URL.Path != "/v1/tasks" {
|
||||
return resp, nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var envelope struct {
|
||||
Data orbit.SubmitTaskResult `json:"data"`
|
||||
}
|
||||
if resp.StatusCode != 201 || json.Unmarshal(raw, &envelope) != nil || !idPattern.MatchString(envelope.Data.Task.ID) {
|
||||
return nil, errors.New("child did not receive actual first acceptance")
|
||||
}
|
||||
emitMarker(senderMarker{Phase: "remote_committed", TaskID: envelope.Data.Task.ID})
|
||||
<-r.Context().Done()
|
||||
return nil, r.Context().Err()
|
||||
})}
|
||||
}
|
||||
api, err := orbit.NewClient(base, options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, stop := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer stop()
|
||||
for ctx.Err() == nil {
|
||||
r, err := s.SendOnce(ctx, api)
|
||||
if err == nil {
|
||||
if r.State != "ACCEPTED" {
|
||||
t.Fatal("child did not accept", r.State)
|
||||
}
|
||||
var receipt orbit.SubmitTaskResult
|
||||
if json.Unmarshal(r.ReceiptJSON, &receipt) != nil || !receipt.Deduplicated {
|
||||
t.Fatal("child did not recover original receipt")
|
||||
}
|
||||
emitMarker(senderMarker{Phase: "accepted", TaskID: *r.TaskID, Sends: r.SendCount, Duplicate: receipt.Deduplicated})
|
||||
return
|
||||
}
|
||||
if !errors.Is(err, ErrIdle) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
timer := time.NewTimer(100 * time.Millisecond)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
t.Fatal("child sender timed out")
|
||||
}
|
||||
|
||||
type senderChild struct {
|
||||
cmd *exec.Cmd
|
||||
done chan struct{}
|
||||
err error
|
||||
markers chan senderMarker
|
||||
}
|
||||
|
||||
func startSender(t *testing.T, s *Store, mode string) *senderChild {
|
||||
t.Helper()
|
||||
binary, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := exec.Command(binary, "-test.run=^TestSenderProcessHelper$")
|
||||
values := map[string]string{"ORBIT_TEST_SENDER_CHILD": mode, "ORBIT_TEST_SENDER_NAMESPACE": s.namespace, "ORBIT_TEST_SENDER_QUEUE": s.queue}
|
||||
for _, entry := range os.Environ() {
|
||||
key, _, _ := strings.Cut(entry, "=")
|
||||
if _, ok := values[key]; !ok {
|
||||
cmd.Env = append(cmd.Env, entry)
|
||||
}
|
||||
}
|
||||
for key, value := range values {
|
||||
cmd.Env = append(cmd.Env, key+"="+value)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd.Stderr = io.Discard
|
||||
p := &senderChild{cmd: cmd, done: make(chan struct{}), markers: make(chan senderMarker, 4)}
|
||||
if err = cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go func() {
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
for scanner.Scan() {
|
||||
if raw, ok := strings.CutPrefix(scanner.Text(), "ORBIT_SENDER_TEST "); ok {
|
||||
var m senderMarker
|
||||
if json.Unmarshal([]byte(raw), &m) == nil {
|
||||
p.markers <- m
|
||||
}
|
||||
}
|
||||
}
|
||||
p.err = cmd.Wait()
|
||||
close(p.done)
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
select {
|
||||
case <-p.done:
|
||||
return
|
||||
default:
|
||||
}
|
||||
_ = cmd.Process.Kill()
|
||||
select {
|
||||
case <-p.done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("owned sender child did not exit")
|
||||
}
|
||||
})
|
||||
return p
|
||||
}
|
||||
func (p *senderChild) marker(t *testing.T, ctx context.Context, phase string) senderMarker {
|
||||
t.Helper()
|
||||
for {
|
||||
select {
|
||||
case m := <-p.markers:
|
||||
if m.Phase == phase {
|
||||
return m
|
||||
}
|
||||
case <-p.done:
|
||||
select {
|
||||
case m := <-p.markers:
|
||||
if m.Phase == phase {
|
||||
return m
|
||||
}
|
||||
default:
|
||||
}
|
||||
t.Fatal("sender exited without expected marker", p.err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal("sender marker timeout")
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestIntegrationSenderProcessCommitRecovery(t *testing.T) {
|
||||
s, api, _, _ := realStore(t)
|
||||
id := enqueue(t, s)
|
||||
ctx, stop := context.WithTimeout(context.Background(), 50*time.Second)
|
||||
defer stop()
|
||||
first := startSender(t, s, "hold_after_remote_commit")
|
||||
committed := first.marker(t, ctx, "remote_committed")
|
||||
if _, err := api.GetTask(ctx, committed.TaskID); err != nil {
|
||||
t.Fatal("remote commit not confirmed", err)
|
||||
}
|
||||
r := status(t, s, id)
|
||||
if r.State != "SENDING" || r.TaskID != nil || r.SendCount != 1 {
|
||||
t.Fatal("local receipt already confirmed")
|
||||
}
|
||||
if err := first.cmd.Process.Kill(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-first.done:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("sender kill not observed")
|
||||
}
|
||||
state, ok := first.cmd.ProcessState.Sys().(syscall.WaitStatus)
|
||||
if !ok || !state.Signaled() || state.Signal() != syscall.SIGKILL {
|
||||
t.Fatal("sender was not killed", first.err)
|
||||
}
|
||||
restarted := startSender(t, s, "recover")
|
||||
result := restarted.marker(t, ctx, "accepted")
|
||||
select {
|
||||
case <-restarted.done:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("replacement sender not exited")
|
||||
}
|
||||
if restarted.err != nil || result.TaskID != committed.TaskID || result.Sends != 2 || !result.Duplicate {
|
||||
t.Fatal("replacement did not recover original receipt", restarted.err)
|
||||
}
|
||||
r = status(t, s, id)
|
||||
if r.State != "ACCEPTED" || r.TaskID == nil || *r.TaskID != committed.TaskID {
|
||||
t.Fatal("durable local acceptance missing")
|
||||
}
|
||||
page, err := api.ListTasks(ctx, orbit.ListTasksRequest{QueueID: s.queue})
|
||||
if err != nil || len(page.Items) != 1 || countRows(t, s, "sender_jobs") != 1 {
|
||||
t.Fatal("restart duplicated remote task or business record", err)
|
||||
}
|
||||
cancelAccepted(t, api, committed.TaskID)
|
||||
t.Log("sender SIGKILL observed after actual Orbit commit, before local confirmation; new process waited for natural lease expiry and recovered one task using original key")
|
||||
}
|
||||
@@ -1,36 +1,3 @@
|
||||
-- 业务方独立数据库的示例表,不属于平台迁移,不创建外键或级联删除。
|
||||
CREATE TABLE sender_outbox (
|
||||
namespace_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定目标命名空间,重启及换凭据不得改变',
|
||||
queue_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定目标队列,发送前核对当前授权及命名空间',
|
||||
operation_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '业务操作标识,同命名空间内唯一,调用方持久化后复用',
|
||||
idempotency_key VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '冻结平台提交键,由业务操作标识派生,禁止换键补发',
|
||||
request_json MEDIUMTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '冻结任务提交正文,保留精确数字,不存认证凭据',
|
||||
request_digest BINARY(32) NOT NULL COMMENT '冻结正文SHA256,防止同身份不同内容,非平台摘要算法',
|
||||
state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT 'PENDING待发,SENDING领取中,ACCEPTED已受理,REVIEW需核查',
|
||||
send_count INT UNSIGNED NOT NULL COMMENT '领取次数,包含领取后崩溃的次数,最多100',
|
||||
lease_epoch BIGINT UNSIGNED NOT NULL COMMENT '领取代次递增,旧发送器不能覆盖新代次结果',
|
||||
wake_at DATETIME(6) NULL COMMENT '待发下次时间或发送租约期限,终结后为空,UTC微秒',
|
||||
send_deadline_at DATETIME(6) NOT NULL COMMENT '首次入库24小时后的发送截止,短于平台幂等保留窗口',
|
||||
last_code VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '最近稳定分类,不保存服务端错误正文或秘密',
|
||||
task_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL COMMENT '已确认的平台任务标识,只在ACCEPTED非空',
|
||||
receipt_json MEDIUMTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL COMMENT '原受理回执,仅ACCEPTED非空,不代表当前执行结果',
|
||||
created_at DATETIME(6) NOT NULL COMMENT '业务事务首次创建命令UTC时间,微秒精度',
|
||||
updated_at DATETIME(6) NOT NULL COMMENT '本地状态最近变化UTC时间,微秒精度',
|
||||
PRIMARY KEY (namespace_id,operation_id),
|
||||
UNIQUE KEY uq_sender_identity(namespace_id,idempotency_key),
|
||||
KEY ix_sender_due(namespace_id,queue_id,wake_at,operation_id),
|
||||
CONSTRAINT ck_sender_document CHECK (JSON_VALID(request_json)),
|
||||
CONSTRAINT ck_sender_receipt CHECK (receipt_json IS NULL OR JSON_VALID(receipt_json)),
|
||||
CONSTRAINT ck_sender_count CHECK (send_count<=100 AND lease_epoch=send_count),
|
||||
CONSTRAINT ck_sender_state CHECK (
|
||||
(state IN ('PENDING','SENDING') AND wake_at IS NOT NULL AND task_id IS NULL AND receipt_json IS NULL)
|
||||
OR (state='REVIEW' AND wake_at IS NULL AND task_id IS NULL AND receipt_json IS NULL)
|
||||
OR (state='ACCEPTED' AND wake_at IS NULL AND task_id IS NOT NULL AND receipt_json IS NOT NULL)
|
||||
),
|
||||
CONSTRAINT ck_sender_lease CHECK (state<>'SENDING' OR lease_epoch>0),
|
||||
CONSTRAINT ck_sender_deadline CHECK (send_deadline_at>created_at AND send_deadline_at<=TIMESTAMPADD(HOUR,24,created_at))
|
||||
) ENGINE=InnoDB COMMENT='业务事务Outbox,持久化原请求与有限重发状态,不承担业务恰好一次';
|
||||
|
||||
CREATE TABLE sender_jobs (
|
||||
namespace_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '与Outbox相同的命名空间范围',
|
||||
operation_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '本地业务操作标识,与Outbox同事务创建',
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math/rand/v2"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
// SendOnce 有界领取并发送一项;网络交换在数据库事务外,ErrIdle表示本次无到期项。
|
||||
// 返回记录只包含本地发送状态;ACCEPTED不是业务执行成功,REVIEW必须人工核查原身份。
|
||||
func (s *Store) SendOnce(parent context.Context, api *orbit.Client) (Record, error) {
|
||||
if api == nil {
|
||||
return Record{}, ErrInvalid
|
||||
}
|
||||
// 整个网络阶段从领取前即受单调时钟15秒约束,短于30秒租约;暂停进程后不会获得新发送预算。
|
||||
work, stop := context.WithTimeout(parent, 15*time.Second)
|
||||
defer stop()
|
||||
r, err := s.claim(work)
|
||||
if err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
if r.State == "REVIEW" {
|
||||
return s.Status(parent, r.OperationID)
|
||||
}
|
||||
state, code, delay, receipt := s.deliver(work, api, r)
|
||||
// 网络超时不能阻止保存未知结果分类,收尾仍服从调用方总上下文且最多5秒。
|
||||
if err = s.finish(parent, r, state, code, delay, receipt); err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
return s.Status(parent, r.OperationID)
|
||||
}
|
||||
|
||||
func (s *Store) deliver(ctx context.Context, api *orbit.Client, r Record) (string, string, time.Duration, *orbit.SubmitTaskResult) {
|
||||
digest := sha256.Sum256(r.RequestJSON)
|
||||
if !bytes.Equal(digest[:], r.RequestDigest) || r.NamespaceID != s.namespace || r.QueueID != s.queue || r.IdempotencyKey != "outbox-task-"+r.OperationID {
|
||||
return "REVIEW", "INVALID_DOCUMENT", 0, nil
|
||||
}
|
||||
var in orbit.SubmitTaskRequest
|
||||
decoder := json.NewDecoder(bytes.NewReader(r.RequestJSON))
|
||||
decoder.DisallowUnknownFields()
|
||||
if decoder.Decode(&in) != nil || in.QueueID != s.queue {
|
||||
return "REVIEW", "INVALID_DOCUMENT", 0, nil
|
||||
}
|
||||
// 先通过当前凭据读取固定队列;不靠配置声称命名空间正确,不向错误环境盲发命令。
|
||||
queue, err := api.GetQueue(ctx, s.queue)
|
||||
if err != nil {
|
||||
state, code, delay := classify(err, r.SendCount)
|
||||
return state, code, delay, nil
|
||||
}
|
||||
if queue.ID != s.queue || queue.NamespaceID != s.namespace {
|
||||
return "REVIEW", "TARGET_MISMATCH", 0, nil
|
||||
}
|
||||
// 数据库时间决定剩余截止预算;额外查询不授予新租约,换代或临近截止时不再开始HTTP。
|
||||
var remaining int64
|
||||
err = s.db.QueryRowContext(ctx, `SELECT TIMESTAMPDIFF(MICROSECOND,UTC_TIMESTAMP(6),LEAST(wake_at,send_deadline_at)) FROM sender_outbox WHERE namespace_id=? AND queue_id=? AND operation_id=? AND state='SENDING' AND lease_epoch=?`, s.namespace, s.queue, r.OperationID, r.LeaseEpoch).Scan(&remaining)
|
||||
if err != nil || remaining <= 0 {
|
||||
return "PENDING", "UNCONFIRMED", time.Second, nil
|
||||
}
|
||||
send, stop := context.WithTimeout(ctx, min(10*time.Second, time.Duration(remaining)*time.Microsecond))
|
||||
defer stop()
|
||||
result, err := api.SubmitTask(send, r.IdempotencyKey, in)
|
||||
if err != nil {
|
||||
state, code, delay := classify(err, r.SendCount)
|
||||
return state, code, delay, nil
|
||||
}
|
||||
if !validReceipt(r, result) {
|
||||
return "REVIEW", "INVALID_RECEIPT", 0, nil
|
||||
}
|
||||
return "ACCEPTED", "ACCEPTED", 0, &result
|
||||
}
|
||||
|
||||
func validReceipt(r Record, receipt orbit.SubmitTaskResult) bool {
|
||||
t := receipt.Task
|
||||
return idPattern.MatchString(t.ID) && t.NamespaceID == r.NamespaceID && t.QueueID == r.QueueID && t.Revision > 0
|
||||
}
|
||||
|
||||
func classify(err error, count uint32) (state, code string, delay time.Duration) {
|
||||
state, code = "PENDING", "UNCONFIRMED"
|
||||
delay = min(time.Minute, time.Second*time.Duration(uint64(1)<<min(max(count, 1)-1, 6))) + time.Duration(rand.IntN(1000))*time.Millisecond
|
||||
var api *orbit.APIError
|
||||
if errors.As(err, &api) {
|
||||
if api.StatusCode == 429 {
|
||||
code = "BACKPRESSURE"
|
||||
delay = max(delay, api.RetryAfter)
|
||||
} else if api.StatusCode < 500 && api.StatusCode != 408 {
|
||||
state, code = "REVIEW", "HTTP_REJECTED"
|
||||
if api.StatusCode == 401 || api.StatusCode == 403 {
|
||||
code = "AUTHORIZATION"
|
||||
}
|
||||
if api.Code == "IDEMPOTENCY_CONFLICT" {
|
||||
code = "IDEMPOTENCY_CONFLICT"
|
||||
}
|
||||
}
|
||||
} else {
|
||||
var httpErr *orbit.HTTPError
|
||||
if errors.As(err, &httpErr) && httpErr.StatusCode < 500 && httpErr.StatusCode != 408 && httpErr.StatusCode != 429 {
|
||||
state, code = "REVIEW", "HTTP_REJECTED"
|
||||
}
|
||||
}
|
||||
return state, code, min(delay, time.Hour)
|
||||
}
|
||||
@@ -1,321 +1,95 @@
|
||||
// Package outbox 演示接入方本地业务事务、稳定提交身份及有限发送恢复。
|
||||
// Package outbox 演示在业务事务中使用公共 MySQL Outbox 适配包。
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
adapter "git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql"
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
//go:embed schema.sql
|
||||
var schema string
|
||||
|
||||
var (
|
||||
ErrInvalid = errors.New("outbox scope, operation or request is invalid")
|
||||
ErrConflict = errors.New("outbox operation already has different content; keep the original identity")
|
||||
ErrStorage = errors.New("outbox storage failed; reconcile the original operation")
|
||||
ErrCommitUnconfirmed = errors.New("outbox commit is unconfirmed; reconcile the original operation")
|
||||
ErrIdle = errors.New("no due outbox record")
|
||||
ErrNotFound = errors.New("outbox operation not found in this scope")
|
||||
ErrLeaseLost = errors.New("outbox lease was replaced; do not overwrite its result")
|
||||
idPattern = regexp.MustCompile(`^[a-f0-9]{32}$`)
|
||||
ErrInvalid = adapter.ErrInvalid
|
||||
ErrStorage = adapter.ErrStorage
|
||||
ErrConflict = adapter.ErrConflict
|
||||
ErrCommitUnconfirmed = adapter.ErrCommitUnconfirmed
|
||||
ErrIdle = adapter.ErrIdle
|
||||
)
|
||||
|
||||
const maxSends = 100
|
||||
const recordColumns = `namespace_id,queue_id,operation_id,idempotency_key,request_json,request_digest,state,send_count,lease_epoch,wake_at,send_deadline_at,last_code,task_id,receipt_json,created_at,updated_at`
|
||||
|
||||
// Store 仅补充示例业务事务;发送、查询、租约由公共包负责。
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
namespace, queue string
|
||||
*adapter.Store
|
||||
db *sql.DB
|
||||
namespace string
|
||||
}
|
||||
|
||||
// Open 仅连接业务方数据库,不创建表,不输出原始DSN或数据库错误。
|
||||
func Open(ctx context.Context, dsn string) (*sql.DB, error) {
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil || cfg.DBName == "" {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
cfg.ParseTime = true
|
||||
cfg.Loc = time.UTC
|
||||
cfg.MultiStatements = false
|
||||
cfg.AllowAllFiles = false
|
||||
cfg.Timeout, cfg.ReadTimeout, cfg.WriteTimeout = 5*time.Second, 8*time.Second, 8*time.Second
|
||||
if cfg.Params == nil {
|
||||
cfg.Params = map[string]string{}
|
||||
}
|
||||
cfg.Params["time_zone"] = "'+00:00'"
|
||||
connector, err := mysql.NewConnector(cfg)
|
||||
if err != nil {
|
||||
return nil, ErrStorage
|
||||
}
|
||||
db := sql.OpenDB(connector)
|
||||
db.SetMaxOpenConns(8)
|
||||
db.SetMaxIdleConns(4)
|
||||
db.SetConnMaxLifetime(5 * time.Minute)
|
||||
if err = db.PingContext(ctx); err != nil {
|
||||
db.Close()
|
||||
return nil, ErrStorage
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
var Open = adapter.Open
|
||||
|
||||
// Init 仅用于专用空库,已有表会报错;生产接入使用业务方自己的迁移。
|
||||
func Init(ctx context.Context, db *sql.DB) error {
|
||||
for _, statement := range strings.Split(schema, ";") {
|
||||
if strings.TrimSpace(statement) == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, statement); err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
if err := adapter.Init(ctx, db); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, schema); err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func New(db *sql.DB, namespace, queue string) (*Store, error) {
|
||||
if db == nil || !idPattern.MatchString(namespace) || !idPattern.MatchString(queue) {
|
||||
return nil, ErrInvalid
|
||||
s, err := adapter.New(db, namespace, queue, adapter.Options{OnAccepted: accepted})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Store{db: db, namespace: namespace, queue: queue}, nil
|
||||
return &Store{s, db, namespace}, nil
|
||||
}
|
||||
|
||||
// document 冻结当前v1默认值及UTC表示;Outbox按文档字节去重,不冒充平台语义摘要。
|
||||
func (s *Store) document(in orbit.SubmitTaskRequest) ([]byte, error) {
|
||||
// encoding/json会把非法UTF-8替换成U+FFFD;在冻结摘要前拒绝,不能静默改变业务标识。
|
||||
for _, value := range []string{in.QueueID, in.Handler, in.HandlerVersion, in.PayloadVersion, in.BusinessKey, in.ConcurrencyKey} {
|
||||
if !utf8.ValidString(value) {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
func accepted(ctx context.Context, tx *sql.Tx, r adapter.Record, receipt orbit.SubmitTaskResult) error {
|
||||
var existing *string
|
||||
if err := tx.QueryRowContext(ctx, `SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, r.NamespaceID, r.OperationID).Scan(&existing); err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
if in.QueueID != s.queue || in.Handler == "" || in.HandlerVersion == "" || in.PayloadVersion == "" || len(in.Payload) > 256<<10 || !utf8.Valid(in.Payload) || !json.Valid(in.Payload) || bytes.Equal(bytes.TrimSpace(in.Payload), []byte("null")) {
|
||||
return nil, ErrInvalid
|
||||
if existing != nil && *existing != receipt.Task.ID {
|
||||
return ErrConflict
|
||||
}
|
||||
if in.TaskTimeoutMS == 0 {
|
||||
in.TaskTimeoutMS = 86400000
|
||||
_, err := tx.ExecContext(ctx, `UPDATE sender_jobs SET task_id=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, receipt.Task.ID, r.NamespaceID, r.OperationID)
|
||||
if err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
if in.RecoverySafety == "" {
|
||||
in.RecoverySafety = orbit.RecoveryReconcileOnly
|
||||
}
|
||||
if in.Retry.MaxAttempts == 0 {
|
||||
in.Retry.MaxAttempts = 1
|
||||
}
|
||||
if in.Retry.AttemptTimeoutMS == 0 {
|
||||
in.Retry.AttemptTimeoutMS = 30000
|
||||
}
|
||||
if in.Retry.MaxAttempts > 1 {
|
||||
if in.Retry.InitialDelayMS == 0 {
|
||||
in.Retry.InitialDelayMS = 1000
|
||||
}
|
||||
if in.Retry.MaxDelayMS == 0 {
|
||||
in.Retry.MaxDelayMS = 30000
|
||||
}
|
||||
}
|
||||
if in.AvailableAt != nil {
|
||||
at := in.AvailableAt.UTC()
|
||||
in.AvailableAt = &at
|
||||
}
|
||||
if in.StartDeadlineAt != nil {
|
||||
at := in.StartDeadlineAt.UTC()
|
||||
in.StartDeadlineAt = &at
|
||||
}
|
||||
raw, err := json.Marshal(in)
|
||||
if err != nil || len(raw) > 512<<10 {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
return raw, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// Enqueue 原子创建业务记录及Outbox。duplicate只表示该本地事务已经提交,不表示平台受理。
|
||||
// 调用方在第一次调用前持久化operationID;COMMIT回执未知时仍用原ID、原内容再次调用。
|
||||
func (s *Store) Enqueue(parent context.Context, operationID, sourceRef string, in orbit.SubmitTaskRequest) (duplicate bool, err error) {
|
||||
if !idPattern.MatchString(operationID) || !utf8.ValidString(sourceRef) || strings.TrimSpace(sourceRef) == "" || utf8.RuneCountInString(sourceRef) > 512 || strings.ContainsFunc(sourceRef, unicode.IsControl) {
|
||||
func (s *Store) Enqueue(ctx context.Context, id, source string, in orbit.SubmitTaskRequest) (bool, error) {
|
||||
if !utf8.ValidString(source) || strings.TrimSpace(source) == "" || utf8.RuneCountInString(source) > 512 || strings.ContainsFunc(source, unicode.IsControl) {
|
||||
return false, ErrInvalid
|
||||
}
|
||||
raw, err := s.document(in)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
digest := sha256.Sum256(raw)
|
||||
ctx, stop := context.WithTimeout(parent, 5*time.Second)
|
||||
defer stop()
|
||||
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
return false, ErrStorage
|
||||
}
|
||||
defer tx.Rollback()
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO sender_outbox(namespace_id,queue_id,operation_id,idempotency_key,request_json,request_digest,state,send_count,lease_epoch,wake_at,send_deadline_at,last_code,created_at,updated_at) VALUES(?,?,?,?,?,?,'PENDING',0,0,UTC_TIMESTAMP(6),TIMESTAMPADD(HOUR,24,UTC_TIMESTAMP(6)),'',UTC_TIMESTAMP(6),UTC_TIMESTAMP(6))`, s.namespace, s.queue, operationID, "outbox-task-"+operationID, raw, digest[:])
|
||||
duplicate, err := s.Store.EnqueueTx(ctx, tx, id, in)
|
||||
if err != nil {
|
||||
var mysqlErr *mysql.MySQLError
|
||||
if !errors.As(err, &mysqlErr) || mysqlErr.Number != 1062 {
|
||||
return false, err
|
||||
}
|
||||
if duplicate {
|
||||
var saved string
|
||||
if err = tx.QueryRowContext(ctx, `SELECT source_ref FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, s.namespace, id).Scan(&saved); err != nil {
|
||||
return false, ErrStorage
|
||||
}
|
||||
if tx.Rollback() != nil {
|
||||
return false, ErrStorage
|
||||
}
|
||||
var saved []byte
|
||||
var queue, source string
|
||||
err = s.db.QueryRowContext(ctx, `SELECT o.request_digest,o.queue_id,j.source_ref FROM sender_outbox o JOIN sender_jobs j ON j.namespace_id=o.namespace_id AND j.operation_id=o.operation_id WHERE o.namespace_id=? AND o.operation_id=?`, s.namespace, operationID).Scan(&saved, &queue, &source)
|
||||
if err != nil {
|
||||
return false, ErrStorage
|
||||
}
|
||||
if queue != s.queue || source != sourceRef || !bytes.Equal(saved, digest[:]) {
|
||||
if saved != source {
|
||||
return false, ErrConflict
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
// 锁序统一为Outbox→业务行,不能在业务行持锁时反向调用发送确认。
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO sender_jobs(namespace_id,operation_id,source_ref,created_at,updated_at) VALUES(?,?,?,UTC_TIMESTAMP(6),UTC_TIMESTAMP(6))`, s.namespace, operationID, sourceRef); err != nil {
|
||||
return false, ErrStorage
|
||||
} else {
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO sender_jobs(namespace_id,operation_id,source_ref,created_at,updated_at) VALUES(?,?,?,UTC_TIMESTAMP(6),UTC_TIMESTAMP(6))`, s.namespace, id, source); err != nil {
|
||||
return false, ErrStorage
|
||||
}
|
||||
}
|
||||
if tx.Commit() != nil {
|
||||
return false, ErrCommitUnconfirmed
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
type scanner interface{ Scan(...any) error }
|
||||
|
||||
func scanRecord(row scanner, r *Record, extra ...any) error {
|
||||
args := []any{&r.NamespaceID, &r.QueueID, &r.OperationID, &r.IdempotencyKey, &r.RequestJSON, &r.RequestDigest, &r.State, &r.SendCount, &r.LeaseEpoch, &r.WakeAt, &r.SendDeadlineAt, &r.LastCode, &r.TaskID, &r.ReceiptJSON, &r.CreatedAt, &r.UpdatedAt}
|
||||
return row.Scan(append(args, extra...)...)
|
||||
}
|
||||
func (s *Store) Status(parent context.Context, id string) (Record, error) {
|
||||
if !idPattern.MatchString(id) {
|
||||
return Record{}, ErrInvalid
|
||||
}
|
||||
ctx, stop := context.WithTimeout(parent, 5*time.Second)
|
||||
defer stop()
|
||||
var r Record
|
||||
err := scanRecord(s.db.QueryRowContext(ctx, `SELECT `+recordColumns+` FROM sender_outbox WHERE namespace_id=? AND queue_id=? AND operation_id=?`, s.namespace, s.queue, id), &r)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Record{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return Record{}, ErrStorage
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (s *Store) claim(parent context.Context) (Record, error) {
|
||||
ctx, stop := context.WithTimeout(parent, 5*time.Second)
|
||||
defer stop()
|
||||
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
return Record{}, ErrStorage
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var r Record
|
||||
var now time.Time
|
||||
err = scanRecord(tx.QueryRowContext(ctx, `SELECT `+recordColumns+`,UTC_TIMESTAMP(6) FROM sender_outbox FORCE INDEX(ix_sender_due) WHERE namespace_id=? AND queue_id=? AND wake_at<=UTC_TIMESTAMP(6) ORDER BY wake_at,operation_id LIMIT 1 FOR UPDATE SKIP LOCKED`, s.namespace, s.queue), &r, &now)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Record{}, ErrIdle
|
||||
}
|
||||
if err != nil {
|
||||
return Record{}, ErrStorage
|
||||
}
|
||||
if r.State != "PENDING" && r.State != "SENDING" {
|
||||
return Record{}, ErrStorage
|
||||
}
|
||||
if !now.Before(r.SendDeadlineAt) || r.SendCount >= maxSends {
|
||||
r.State = "REVIEW"
|
||||
r.WakeAt = nil
|
||||
r.LastCode = "SEND_BUDGET_EXHAUSTED"
|
||||
if !now.Before(r.SendDeadlineAt) {
|
||||
r.LastCode = "SEND_DEADLINE"
|
||||
}
|
||||
} else {
|
||||
r.State = "SENDING"
|
||||
r.SendCount++
|
||||
r.LeaseEpoch++
|
||||
wake := now.Add(30 * time.Second)
|
||||
if wake.After(r.SendDeadlineAt) {
|
||||
wake = r.SendDeadlineAt
|
||||
}
|
||||
r.WakeAt = &wake
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE sender_outbox SET state=?,send_count=?,lease_epoch=?,wake_at=?,last_code=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, r.State, r.SendCount, r.LeaseEpoch, r.WakeAt, r.LastCode, s.namespace, r.OperationID)
|
||||
if err != nil {
|
||||
return Record{}, ErrStorage
|
||||
}
|
||||
if tx.Commit() != nil {
|
||||
return Record{}, ErrCommitUnconfirmed
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// finish 只接受最近代次;过期但尚未被替代的成功回执可落库,重复HTTP只复用原键。
|
||||
func (s *Store) finish(parent context.Context, claim Record, state, code string, delay time.Duration, receipt *orbit.SubmitTaskResult) error {
|
||||
ctx, stop := context.WithTimeout(parent, 5*time.Second)
|
||||
defer stop()
|
||||
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var current Record
|
||||
var now time.Time
|
||||
err = scanRecord(tx.QueryRowContext(ctx, `SELECT `+recordColumns+`,UTC_TIMESTAMP(6) FROM sender_outbox WHERE namespace_id=? AND queue_id=? AND operation_id=? FOR UPDATE`, s.namespace, s.queue, claim.OperationID), ¤t, &now)
|
||||
if err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
if current.State != "SENDING" || current.LeaseEpoch != claim.LeaseEpoch {
|
||||
return ErrLeaseLost
|
||||
}
|
||||
var wake *time.Time
|
||||
var task *string
|
||||
var raw []byte
|
||||
if state == "ACCEPTED" {
|
||||
if receipt == nil || !validReceipt(current, *receipt) {
|
||||
return ErrInvalid
|
||||
}
|
||||
raw, err = json.Marshal(receipt)
|
||||
if err != nil || len(raw) > 1<<20 {
|
||||
return ErrInvalid
|
||||
}
|
||||
task = &receipt.Task.ID
|
||||
var existing *string
|
||||
if err = tx.QueryRowContext(ctx, `SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, s.namespace, current.OperationID).Scan(&existing); err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
if existing != nil && *existing != *task {
|
||||
return ErrConflict
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE sender_jobs SET task_id=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, *task, s.namespace, current.OperationID); err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
} else if state == "PENDING" {
|
||||
if delay < time.Second || delay > time.Hour {
|
||||
return ErrInvalid
|
||||
}
|
||||
if !now.Before(current.SendDeadlineAt) || current.SendCount >= maxSends {
|
||||
state = "REVIEW"
|
||||
code = "SEND_BUDGET_EXHAUSTED"
|
||||
} else {
|
||||
next := now.Add(delay)
|
||||
if next.After(current.SendDeadlineAt) {
|
||||
next = current.SendDeadlineAt
|
||||
}
|
||||
wake = &next
|
||||
}
|
||||
} else if state != "REVIEW" {
|
||||
return ErrInvalid
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE sender_outbox SET state=?,last_code=?,wake_at=?,task_id=?,receipt_json=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, state, code, wake, task, raw, s.namespace, current.OperationID); err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
if tx.Commit() != nil {
|
||||
return ErrCommitUnconfirmed
|
||||
}
|
||||
return nil
|
||||
return duplicate, nil
|
||||
}
|
||||
|
||||
@@ -1,361 +0,0 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func integrationDB(t *testing.T) (*sql.DB, string) {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("ORBIT_TEST_SENDER_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires isolated sender database")
|
||||
}
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil || cfg.DBName != "orbit_sender_test" || !strings.HasPrefix(cfg.Addr, "127.0.0.1:") {
|
||||
t.Fatal("sender tests require disposable loopback database")
|
||||
}
|
||||
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer stop()
|
||||
db, err := Open(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
return db, dsn
|
||||
}
|
||||
func testStore(t *testing.T, db *sql.DB) *Store {
|
||||
t.Helper()
|
||||
s, err := New(db, testID(t), testID(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
func enqueue(t *testing.T, s *Store) string {
|
||||
t.Helper()
|
||||
id := testID(t)
|
||||
if _, err := s.Enqueue(context.Background(), id, "objects/test", testRequest(s.queue)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
func status(t *testing.T, s *Store, id string) Record {
|
||||
t.Helper()
|
||||
r, err := s.Status(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
func execSQL(t *testing.T, db *sql.DB, q string, args ...any) {
|
||||
t.Helper()
|
||||
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer stop()
|
||||
if _, err := db.ExecContext(ctx, q, args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
func countRows(t *testing.T, s *Store, table string) int {
|
||||
t.Helper()
|
||||
if table != "sender_jobs" && table != "sender_outbox" {
|
||||
t.Fatal("invalid test table")
|
||||
}
|
||||
var n int
|
||||
query := `SELECT COUNT(*) FROM sender_outbox WHERE namespace_id=? AND queue_id=?`
|
||||
if table == "sender_jobs" {
|
||||
query = `SELECT COUNT(*) FROM sender_jobs j JOIN sender_outbox o ON o.namespace_id=j.namespace_id AND o.operation_id=j.operation_id WHERE o.namespace_id=? AND o.queue_id=?`
|
||||
}
|
||||
if err := s.db.QueryRow(query, s.namespace, s.queue).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
func fakeReceipt(t *testing.T, s *Store) orbit.SubmitTaskResult {
|
||||
return orbit.SubmitTaskResult{Task: orbit.Task{ID: testID(t), NamespaceID: s.namespace, QueueID: s.queue, Revision: 1, State: orbit.TaskWaiting}}
|
||||
}
|
||||
|
||||
func TestIntegrationSenderSchema(t *testing.T) {
|
||||
db, _ := integrationDB(t)
|
||||
var fk int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&fk); err != nil || fk != 0 {
|
||||
t.Fatal(fk, err)
|
||||
}
|
||||
rows, err := db.Query(`SELECT COLUMN_COMMENT FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('sender_outbox','sender_jobs')`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
n := 0
|
||||
for rows.Next() {
|
||||
var c string
|
||||
if rows.Scan(&c) != nil || !strings.ContainsFunc(c, func(r rune) bool { return unicode.Is(unicode.Han, r) }) {
|
||||
t.Fatal("missing Chinese column comment")
|
||||
}
|
||||
n++
|
||||
}
|
||||
if rows.Err() != nil || n != 22 {
|
||||
t.Fatal(n, rows.Err())
|
||||
}
|
||||
rows.Close()
|
||||
s := testStore(t, db)
|
||||
id := enqueue(t, s)
|
||||
for _, assignment := range []string{"state='ACCEPTED'", "send_count=101,lease_epoch=101", "request_json='not json'", "send_deadline_at=TIMESTAMPADD(HOUR,48,created_at)"} {
|
||||
_, err = db.Exec(`UPDATE sender_outbox SET `+assignment+` WHERE namespace_id=? AND operation_id=?`, s.namespace, id)
|
||||
var rejected *mysql.MySQLError
|
||||
if !errors.As(err, &rejected) || rejected.Number != 3819 {
|
||||
t.Fatal("invalid persisted state did not violate CHECK", assignment, err)
|
||||
}
|
||||
}
|
||||
var plan string
|
||||
if err = db.QueryRow(`EXPLAIN FORMAT=JSON SELECT operation_id FROM sender_outbox FORCE INDEX(ix_sender_due) WHERE namespace_id=? AND queue_id=? AND wake_at<=UTC_TIMESTAMP(6) ORDER BY wake_at,operation_id LIMIT 1 FOR UPDATE SKIP LOCKED`, s.namespace, s.queue).Scan(&plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(plan, `"key": "ix_sender_due"`) || strings.Contains(plan, `"using_filesort": true`) {
|
||||
t.Fatal("sender due query lost ordered index", plan)
|
||||
}
|
||||
t.Log("two business tables, 22 Chinese-commented columns, zero foreign keys")
|
||||
}
|
||||
func TestIntegrationSenderConcurrentEnqueueAndConflict(t *testing.T) {
|
||||
db, _ := integrationDB(t)
|
||||
s := testStore(t, db)
|
||||
id := testID(t)
|
||||
in := testRequest(s.queue)
|
||||
invalid := in
|
||||
invalid.BusinessKey = "operation-\xff"
|
||||
if _, err := s.Enqueue(context.Background(), id, "objects/test", invalid); !errors.Is(err, ErrInvalid) || countRows(t, s, "sender_jobs") != 0 || countRows(t, s, "sender_outbox") != 0 {
|
||||
t.Fatal("invalid UTF-8 changed business state", err)
|
||||
}
|
||||
type outcome struct {
|
||||
duplicate bool
|
||||
err error
|
||||
}
|
||||
results := make(chan outcome, 8)
|
||||
var wg sync.WaitGroup
|
||||
for range 8 {
|
||||
wg.Go(func() {
|
||||
dup, err := s.Enqueue(context.Background(), id, "objects/test", in)
|
||||
results <- outcome{dup, err}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
duplicates := 0
|
||||
for r := range results {
|
||||
if r.err != nil {
|
||||
t.Fatal(r.err)
|
||||
}
|
||||
if r.duplicate {
|
||||
duplicates++
|
||||
}
|
||||
}
|
||||
if duplicates != 7 || countRows(t, s, "sender_jobs") != 1 || countRows(t, s, "sender_outbox") != 1 {
|
||||
t.Fatal("duplicate business transaction", duplicates)
|
||||
}
|
||||
if _, err := s.Enqueue(context.Background(), id, "objects/other", in); !errors.Is(err, ErrConflict) {
|
||||
t.Fatal("business identity conflict lost", err)
|
||||
}
|
||||
in.Payload = json.RawMessage(`{"sequence":9007199254740992}`)
|
||||
if _, err := s.Enqueue(context.Background(), id, "objects/test", in); !errors.Is(err, ErrConflict) {
|
||||
t.Fatal("request content conflict lost", err)
|
||||
}
|
||||
other, _ := New(db, s.namespace, testID(t))
|
||||
if _, err := other.Status(context.Background(), id); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatal("cross queue record returned", err)
|
||||
}
|
||||
if _, err := other.Enqueue(context.Background(), id, "objects/test", testRequest(other.queue)); !errors.Is(err, ErrConflict) {
|
||||
t.Fatal("operation was moved to another queue", err)
|
||||
}
|
||||
if !strings.Contains(string(status(t, s, id).RequestJSON), "9007199254740993") {
|
||||
t.Fatal("large integer rounded")
|
||||
}
|
||||
}
|
||||
func TestIntegrationSenderBusinessRollbackAndConfirmationAtomicity(t *testing.T) {
|
||||
db, _ := integrationDB(t)
|
||||
s := testStore(t, db)
|
||||
id := testID(t)
|
||||
execSQL(t, db, `CREATE TRIGGER sender_insert_fail BEFORE INSERT ON sender_jobs FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected business insert failure'`)
|
||||
t.Cleanup(func() {
|
||||
db.Exec(`DROP TRIGGER IF EXISTS sender_insert_fail`)
|
||||
db.Exec(`DROP TRIGGER IF EXISTS sender_update_fail`)
|
||||
})
|
||||
if _, err := s.Enqueue(context.Background(), id, "objects/test", testRequest(s.queue)); !errors.Is(err, ErrStorage) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if countRows(t, s, "sender_outbox") != 0 || countRows(t, s, "sender_jobs") != 0 {
|
||||
t.Fatal("business rollback left a request")
|
||||
}
|
||||
execSQL(t, db, `DROP TRIGGER sender_insert_fail`)
|
||||
if _, err := s.Enqueue(context.Background(), id, "objects/test", testRequest(s.queue)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claim, err := s.claim(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
receipt := fakeReceipt(t, s)
|
||||
execSQL(t, db, `CREATE TRIGGER sender_update_fail BEFORE UPDATE ON sender_jobs FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected business confirmation failure'`)
|
||||
if err = s.finish(context.Background(), claim, "ACCEPTED", "ACCEPTED", 0, &receipt); !errors.Is(err, ErrStorage) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r := status(t, s, id); r.State != "SENDING" || r.TaskID != nil {
|
||||
t.Fatal("partial confirmation committed")
|
||||
}
|
||||
execSQL(t, db, `DROP TRIGGER sender_update_fail`)
|
||||
if err = s.finish(context.Background(), claim, "ACCEPTED", "ACCEPTED", 0, &receipt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var task string
|
||||
if err = db.QueryRow(`SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=?`, s.namespace, id).Scan(&task); err != nil || task != receipt.Task.ID {
|
||||
t.Fatal("business confirmation missing", err)
|
||||
}
|
||||
}
|
||||
func TestIntegrationSenderConcurrentClaimsAndStaleGeneration(t *testing.T) {
|
||||
db, _ := integrationDB(t)
|
||||
s := testStore(t, db)
|
||||
for range 20 {
|
||||
enqueue(t, s)
|
||||
}
|
||||
claims := make(chan Record, 20)
|
||||
errs := make(chan error, 8)
|
||||
var wg sync.WaitGroup
|
||||
for range 8 {
|
||||
wg.Go(func() {
|
||||
for {
|
||||
r, err := s.claim(context.Background())
|
||||
if errors.Is(err, ErrIdle) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
errs <- err
|
||||
return
|
||||
}
|
||||
claims <- r
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
close(claims)
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var first Record
|
||||
for r := range claims {
|
||||
if seen[r.OperationID] || r.SendCount != 1 {
|
||||
t.Fatal("duplicate claim")
|
||||
}
|
||||
seen[r.OperationID] = true
|
||||
first = r
|
||||
}
|
||||
if len(seen) != 20 {
|
||||
t.Fatal("unclaimed records", len(seen))
|
||||
}
|
||||
execSQL(t, db, `UPDATE sender_outbox SET wake_at=TIMESTAMPADD(SECOND,-1,UTC_TIMESTAMP(6)) WHERE namespace_id=? AND operation_id=?`, s.namespace, first.OperationID)
|
||||
replacement, err := s.claim(context.Background())
|
||||
if err != nil || replacement.LeaseEpoch != 2 || replacement.IdempotencyKey != first.IdempotencyKey {
|
||||
t.Fatal(replacement, err)
|
||||
}
|
||||
receipt := fakeReceipt(t, s)
|
||||
if err = s.finish(context.Background(), first, "ACCEPTED", "ACCEPTED", 0, &receipt); !errors.Is(err, ErrLeaseLost) {
|
||||
t.Fatal("stale sender overwrote result", err)
|
||||
}
|
||||
if err = s.finish(context.Background(), replacement, "ACCEPTED", "ACCEPTED", 0, &receipt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
func TestIntegrationSenderHTTPClassificationAndBudgets(t *testing.T) {
|
||||
db, _ := integrationDB(t)
|
||||
for _, mode := range []string{"success", "backpressure", "forbidden", "conflict", "unknown", "wrong_target", "invalid_receipt", "corrupt_request", "exhausted", "last_attempt", "deadline"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
s := testStore(t, db)
|
||||
id := enqueue(t, s)
|
||||
var posts atomic.Int32
|
||||
if mode == "corrupt_request" {
|
||||
execSQL(t, db, `UPDATE sender_outbox SET request_json='{}' WHERE namespace_id=?`, s.namespace)
|
||||
}
|
||||
if mode == "exhausted" {
|
||||
execSQL(t, db, `UPDATE sender_outbox SET send_count=100,lease_epoch=100 WHERE namespace_id=?`, s.namespace)
|
||||
}
|
||||
if mode == "last_attempt" {
|
||||
execSQL(t, db, `UPDATE sender_outbox SET send_count=99,lease_epoch=99 WHERE namespace_id=?`, s.namespace)
|
||||
}
|
||||
if mode == "deadline" {
|
||||
execSQL(t, db, `UPDATE sender_outbox SET send_deadline_at=TIMESTAMPADD(MICROSECOND,1,created_at) WHERE namespace_id=?`, s.namespace)
|
||||
}
|
||||
receipt := fakeReceipt(t, s)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == "GET" {
|
||||
ns := s.namespace
|
||||
if mode == "wrong_target" {
|
||||
ns = testID(t)
|
||||
}
|
||||
json.NewEncoder(w).Encode(map[string]any{"request_id": id, "data": orbit.Queue{ID: s.queue, NamespaceID: ns}})
|
||||
return
|
||||
}
|
||||
posts.Add(1)
|
||||
if r.Header.Get("Idempotency-Key") != "outbox-task-"+id {
|
||||
t.Error("wrong submission identity")
|
||||
}
|
||||
switch mode {
|
||||
case "backpressure":
|
||||
w.Header().Set("Retry-After", "30")
|
||||
w.WriteHeader(429)
|
||||
json.NewEncoder(w).Encode(map[string]any{"request_id": id, "error": map[string]string{"code": "QUEUE_FULL", "message": "full"}})
|
||||
case "forbidden", "conflict":
|
||||
code, status := "PERMISSION_DENIED", 403
|
||||
if mode == "conflict" {
|
||||
code, status = "IDEMPOTENCY_CONFLICT", 409
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
json.NewEncoder(w).Encode(map[string]any{"request_id": id, "error": map[string]string{"code": code, "message": "rejected"}})
|
||||
case "unknown", "last_attempt":
|
||||
w.WriteHeader(502)
|
||||
default:
|
||||
reply := receipt
|
||||
if mode == "invalid_receipt" {
|
||||
reply.Task.NamespaceID = testID(t)
|
||||
}
|
||||
json.NewEncoder(w).Encode(map[string]any{"request_id": id, "data": reply})
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
api, _ := orbit.NewClient(server.URL, orbit.Options{Token: "test-token"})
|
||||
r, err := s.SendOnce(context.Background(), api)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "REVIEW"
|
||||
if mode == "success" {
|
||||
want = "ACCEPTED"
|
||||
}
|
||||
if mode == "backpressure" || mode == "unknown" {
|
||||
want = "PENDING"
|
||||
}
|
||||
if r.State != want {
|
||||
t.Fatal(mode, r.State, r.LastCode)
|
||||
}
|
||||
if (mode == "wrong_target" || mode == "corrupt_request" || mode == "exhausted" || mode == "deadline") && posts.Load() != 0 {
|
||||
t.Fatal("unsafe HTTP sent")
|
||||
}
|
||||
if mode == "backpressure" && (r.WakeAt == nil || time.Until(*r.WakeAt) < 29*time.Second) {
|
||||
t.Fatal("Retry-After was ignored")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
func testID(t *testing.T) string {
|
||||
t.Helper()
|
||||
id, err := orbit.NewEventMessageID()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
func testRequest(queue string) orbit.SubmitTaskRequest {
|
||||
return orbit.SubmitTaskRequest{QueueID: queue, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"sequence":9007199254740993,"ref":"objects/test"}`), TaskTimeoutMS: 60000}
|
||||
}
|
||||
func TestFrozenDocumentAndFailureClassification(t *testing.T) {
|
||||
s := &Store{namespace: testID(t), queue: testID(t)}
|
||||
in := testRequest(s.queue)
|
||||
in.TaskTimeoutMS = 0
|
||||
in.DelayMS = 1500
|
||||
raw, err := s.document(in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var frozen orbit.SubmitTaskRequest
|
||||
if json.Unmarshal(raw, &frozen) != nil || frozen.TaskTimeoutMS != 86400000 || frozen.DelayMS != 1500 || frozen.AvailableAt != nil || frozen.Retry.MaxAttempts != 1 || frozen.Retry.AttemptTimeoutMS != 30000 || !bytes.Contains(frozen.Payload, []byte("9007199254740993")) {
|
||||
t.Fatal("frozen defaults or precision changed")
|
||||
}
|
||||
in.QueueID = testID(t)
|
||||
if _, err = s.document(in); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatal("cross queue request frozen")
|
||||
}
|
||||
in = testRequest(s.queue)
|
||||
in.BusinessKey = "operation-\xff"
|
||||
if _, err = s.document(in); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatal("invalid UTF-8 was silently rewritten")
|
||||
}
|
||||
for _, test := range []struct {
|
||||
err error
|
||||
state, code string
|
||||
}{
|
||||
{context.DeadlineExceeded, "PENDING", "UNCONFIRMED"},
|
||||
{&orbit.APIError{StatusCode: 503, Code: "UNAVAILABLE"}, "PENDING", "UNCONFIRMED"},
|
||||
{&orbit.APIError{StatusCode: 429, RetryAfter: 30 * time.Second}, "PENDING", "BACKPRESSURE"},
|
||||
{&orbit.APIError{StatusCode: 403}, "REVIEW", "AUTHORIZATION"},
|
||||
{&orbit.APIError{StatusCode: 409, Code: "IDEMPOTENCY_CONFLICT"}, "REVIEW", "IDEMPOTENCY_CONFLICT"},
|
||||
{&orbit.HTTPError{StatusCode: 302}, "REVIEW", "HTTP_REJECTED"},
|
||||
{&orbit.HTTPError{StatusCode: 502}, "PENDING", "UNCONFIRMED"},
|
||||
} {
|
||||
state, code, delay := classify(test.err, 100)
|
||||
if state != test.state || code != test.code || delay < time.Second || delay > time.Hour {
|
||||
t.Fatal(state, code, delay)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user