feat: add durable business task outbox example and fault recovery
This commit is contained in:
19
examples/task-outbox/README.md
Normal file
19
examples/task-outbox/README.md
Normal file
@@ -0,0 +1,19 @@
|
||||
# 业务事务 Outbox 示例
|
||||
|
||||
独立Go module,使用业务方自己的MySQL保存业务记录与待发请求,再通过独立Orbit SDK提交任务。平台与业务库分别提交,通过原幂等身份衔接恢复。
|
||||
|
||||
- [完整初始化、运行、恢复与接口说明](../../docs/examples/task-outbox.md)
|
||||
- [命令行程序](cmd/sender/main.go):init、enqueue、send、status、demo。
|
||||
- [业务事务](internal/outbox/store.go)、[发送器](internal/outbox/sender.go)、[中文模型](internal/outbox/model.go)与[无外键表结构](internal/outbox/schema.sql)。
|
||||
|
||||
本模块是可运行的业务接入参考。核心SDK不引入MySQL驱动;生产业务应将本地事务接入自身业务模型、权限、审计与迁移,固定SDK版本并验证兼容性。本例的internal包不作为已发布通用SDK包承诺兼容。
|
||||
|
||||
从本目录运行基础检查:
|
||||
|
||||
```sh
|
||||
GOWORK=off go test -race ./...
|
||||
GOWORK=off go vet ./...
|
||||
GOWORK=off go build ./cmd/sender
|
||||
```
|
||||
|
||||
真实数据库、HTTP、COMMIT响应丢失、CLI及发送进程故障由仓库根目录的`make integration`统一执行,使用专门的一次性数据库。没有测试环境变量时Integration会跳过。
|
||||
192
examples/task-outbox/cmd/sender/main.go
Normal file
192
examples/task-outbox/cmd/sender/main.go
Normal file
@@ -0,0 +1,192 @@
|
||||
// sender 演示业务事务Outbox与可重启的有限发送器。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/examples/task-outbox/internal/outbox"
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
func run() error {
|
||||
if len(os.Args) < 2 {
|
||||
return errors.New("usage: sender init|enqueue <operation_id> <source_ref>|send|status <operation_id>|demo")
|
||||
}
|
||||
command := os.Args[1]
|
||||
if !(command == "init" && len(os.Args) == 2 || command == "send" && len(os.Args) == 2 || command == "demo" && len(os.Args) == 2 || command == "enqueue" && len(os.Args) == 4 || command == "status" && len(os.Args) == 3) {
|
||||
return errors.New("invalid sender command or arguments")
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
open, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
db, err := outbox.Open(open, os.Getenv("ORBIT_SENDER_DSN"))
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
if command == "init" {
|
||||
bounded, done := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer done()
|
||||
if err = outbox.Init(bounded, db); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("sender_schema_initialized=true foreign_keys=0")
|
||||
return nil
|
||||
}
|
||||
api, ns, queue, err := client()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
store, err := outbox.New(db, ns, queue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch command {
|
||||
case "enqueue":
|
||||
id, source := os.Args[2], os.Args[3]
|
||||
request := request(queue, id, source)
|
||||
duplicate, err := store.Enqueue(ctx, id, source, request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("operation=%s local_committed=true duplicate=%t\n", id, duplicate)
|
||||
return nil
|
||||
case "status":
|
||||
r, err := store.Status(ctx, os.Args[2])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printRecord(r)
|
||||
return nil
|
||||
case "demo":
|
||||
return demo(ctx, store, api, queue)
|
||||
case "send":
|
||||
for ctx.Err() == nil {
|
||||
r, err := store.SendOnce(ctx, api)
|
||||
wait := 500 * time.Millisecond
|
||||
if err == nil {
|
||||
printRecord(r)
|
||||
} else if !errors.Is(err, outbox.ErrIdle) {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
wait = 2 * time.Second
|
||||
}
|
||||
timer := time.NewTimer(wait)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func client() (*orbit.Client, string, string, error) {
|
||||
token, ns, queue := os.Getenv("ORBIT_TOKEN"), os.Getenv("ORBIT_NAMESPACE_ID"), os.Getenv("ORBIT_QUEUE_ID")
|
||||
if path := os.Getenv("ORBIT_CREDENTIALS_FILE"); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, "", "", errors.New("cannot read sender credential file")
|
||||
}
|
||||
var c struct {
|
||||
Token string `json:"token"`
|
||||
NamespaceID string `json:"namespace_id"`
|
||||
QueueID string `json:"queue_id"`
|
||||
}
|
||||
if json.Unmarshal(raw, &c) != nil {
|
||||
return nil, "", "", errors.New("invalid sender credential file")
|
||||
}
|
||||
if token == "" {
|
||||
token = c.Token
|
||||
}
|
||||
if ns == "" {
|
||||
ns = c.NamespaceID
|
||||
}
|
||||
if queue == "" {
|
||||
queue = c.QueueID
|
||||
}
|
||||
}
|
||||
if token == "" {
|
||||
return nil, "", "", errors.New("sender credential is required")
|
||||
}
|
||||
c, err := orbit.NewClient(os.Getenv("ORBIT_URL"), orbit.Options{Token: token})
|
||||
return c, ns, queue, err
|
||||
}
|
||||
func request(queue, id, source string) orbit.SubmitTaskRequest {
|
||||
payload, _ := json.Marshal(struct {
|
||||
SourceRef string `json:"source_ref"`
|
||||
Sequence uint64 `json:"sequence"`
|
||||
}{source, 9007199254740993})
|
||||
return orbit.SubmitTaskRequest{QueueID: queue, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: payload, BusinessKey: id, TaskTimeoutMS: 60000}
|
||||
}
|
||||
func printRecord(r outbox.Record) {
|
||||
task := ""
|
||||
if r.TaskID != nil {
|
||||
task = *r.TaskID
|
||||
}
|
||||
fmt.Printf("operation=%s state=%s sends=%d code=%s task=%s\n", r.OperationID, r.State, r.SendCount, r.LastCode, task)
|
||||
}
|
||||
func demo(parent context.Context, s *outbox.Store, api *orbit.Client, queue string) error {
|
||||
ctx, stop := context.WithTimeout(parent, 20*time.Second)
|
||||
defer stop()
|
||||
id, err := orbit.NewEventMessageID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 演示每次新建一个操作;真实接入必须先由业务方保存ID再调用Enqueue。
|
||||
fmt.Printf("demo_operation=%s\n", id)
|
||||
in := request(queue, id, "objects/outbox-demo.txt")
|
||||
if _, err = s.Enqueue(ctx, id, "objects/outbox-demo.txt", in); err != nil {
|
||||
return err
|
||||
}
|
||||
duplicate, err := s.Enqueue(ctx, id, "objects/outbox-demo.txt", in)
|
||||
if err != nil || !duplicate {
|
||||
return errors.New("outbox local duplicate was not recovered")
|
||||
}
|
||||
for {
|
||||
r, err := s.Status(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r.State == "REVIEW" {
|
||||
return errors.New("outbox demo needs review using its printed operation identity")
|
||||
}
|
||||
if r.State == "ACCEPTED" {
|
||||
task, err := api.GetTask(ctx, *r.TaskID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = api.CancelTask(ctx, task.ID, "outbox-demo-cancel-"+id, orbit.CancelTaskRequest{ExpectedRevision: task.Revision, Reason: "outbox submission demo completed"}); err != nil {
|
||||
return err
|
||||
}
|
||||
printRecord(r)
|
||||
fmt.Println("local_duplicate=true accepted_once=true execution_success_not_implied=true")
|
||||
return nil
|
||||
}
|
||||
_, err = s.SendOnce(ctx, api)
|
||||
if err != nil && !errors.Is(err, outbox.ErrIdle) {
|
||||
return err
|
||||
}
|
||||
timer := time.NewTimer(100 * time.Millisecond)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
15
examples/task-outbox/go.mod
Normal file
15
examples/task-outbox/go.mod
Normal file
@@ -0,0 +1,15 @@
|
||||
module git.yuntuo.hk/k2safe/orbit/examples/task-outbox
|
||||
|
||||
go 1.27.1
|
||||
|
||||
require (
|
||||
git.yuntuo.hk/k2safe/orbit/sdk/go v0.0.0
|
||||
github.com/go-sql-driver/mysql v1.10.1
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/coder/websocket v1.8.15 // indirect
|
||||
)
|
||||
|
||||
replace git.yuntuo.hk/k2safe/orbit/sdk/go => ../../sdk/go
|
||||
6
examples/task-outbox/go.sum
Normal file
6
examples/task-outbox/go.sum
Normal file
@@ -0,0 +1,6 @@
|
||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
|
||||
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
|
||||
github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME=
|
||||
github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
105
examples/task-outbox/internal/outbox/cli_integration_test.go
Normal file
105
examples/task-outbox/internal/outbox/cli_integration_test.go
Normal file
@@ -0,0 +1,105 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIntegrationSenderCLI(t *testing.T) {
|
||||
s, api, base, _ := realStore(t)
|
||||
ctx, stop := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer stop()
|
||||
binary := filepath.Join(t.TempDir(), "sender")
|
||||
build := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "../../cmd/sender")
|
||||
if raw, err := build.CombinedOutput(); err != nil {
|
||||
t.Fatalf("build sender: %v %s", err, raw)
|
||||
}
|
||||
values := map[string]string{"ORBIT_SENDER_DSN": os.Getenv("ORBIT_TEST_SENDER_DSN"), "ORBIT_URL": base, "ORBIT_CREDENTIALS_FILE": os.Getenv("ORBIT_TEST_SENDER_CREDENTIALS_FILE"), "ORBIT_NAMESPACE_ID": s.namespace, "ORBIT_QUEUE_ID": s.queue, "ORBIT_TOKEN": ""}
|
||||
env := []string{}
|
||||
for _, entry := range os.Environ() {
|
||||
key, _, _ := strings.Cut(entry, "=")
|
||||
if _, ok := values[key]; !ok {
|
||||
env = append(env, entry)
|
||||
}
|
||||
}
|
||||
for key, value := range values {
|
||||
env = append(env, key+"="+value)
|
||||
}
|
||||
call := func(args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.CommandContext(ctx, binary, args...)
|
||||
cmd.Env = env
|
||||
raw, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("sender command failed: %v %s", err, raw)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
id := testID(t)
|
||||
if !strings.Contains(call("enqueue", id, "objects/cli.txt"), "duplicate=false") || !strings.Contains(call("enqueue", id, "objects/cli.txt"), "duplicate=true") || !strings.Contains(call("status", id), "state=PENDING") {
|
||||
t.Fatal("CLI local transaction mismatch")
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, binary, "send")
|
||||
cmd.Env = env
|
||||
cmd.Stdout = io.Discard
|
||||
cmd.Stderr = io.Discard
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := make(chan struct{})
|
||||
var waitErr error
|
||||
go func() { waitErr = cmd.Wait(); close(done) }()
|
||||
t.Cleanup(func() {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
default:
|
||||
}
|
||||
cmd.Process.Kill()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("owned sender CLI did not exit")
|
||||
}
|
||||
})
|
||||
var r Record
|
||||
for {
|
||||
r = status(t, s, id)
|
||||
if r.State == "ACCEPTED" {
|
||||
break
|
||||
}
|
||||
if r.State == "REVIEW" {
|
||||
t.Fatal("CLI needs review", r.LastCode)
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("sender exited before acceptance", waitErr)
|
||||
case <-ctx.Done():
|
||||
t.Fatal("CLI sender timeout")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
if err := cmd.Process.Signal(os.Interrupt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
if waitErr != nil {
|
||||
t.Fatal("sender did not stop gracefully", waitErr)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("sender shutdown timeout")
|
||||
}
|
||||
if !strings.Contains(call("status", id), "state=ACCEPTED") || countRows(t, s, "sender_jobs") != 1 || r.SendCount != 1 {
|
||||
t.Fatal("CLI acceptance did not persist")
|
||||
}
|
||||
cancelAccepted(t, api, *r.TaskID)
|
||||
t.Log("built CLI enqueue duplicate/status/send/graceful stop exercised against real business DB and Orbit")
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// commitLossGateway 只用于无TLS的隔离MySQL回归;捕获COMMIT的真实成功包后丢弃并断连。
|
||||
// 不伪造SQL结果,也不修改数据库中的提交状态。
|
||||
type commitLossGateway struct {
|
||||
listener net.Listener
|
||||
dropped chan struct{}
|
||||
armed atomic.Bool
|
||||
mu sync.Mutex
|
||||
closing bool
|
||||
connections map[net.Conn]struct{}
|
||||
wg sync.WaitGroup
|
||||
accepted chan struct{}
|
||||
}
|
||||
|
||||
func newCommitLossGateway(t *testing.T, target string) *commitLossGateway {
|
||||
t.Helper()
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := &commitLossGateway{listener: listener, dropped: make(chan struct{}, 1), connections: map[net.Conn]struct{}{}, accepted: make(chan struct{})}
|
||||
go func() {
|
||||
defer close(g.accepted)
|
||||
for {
|
||||
down, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
g.wg.Add(1)
|
||||
go func() {
|
||||
defer g.wg.Done()
|
||||
defer down.Close()
|
||||
up, err := net.DialTimeout("tcp", target, 3*time.Second)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer up.Close()
|
||||
g.mu.Lock()
|
||||
if g.closing {
|
||||
g.mu.Unlock()
|
||||
return
|
||||
}
|
||||
g.connections[down] = struct{}{}
|
||||
g.connections[up] = struct{}{}
|
||||
g.mu.Unlock()
|
||||
defer func() { g.mu.Lock(); delete(g.connections, down); delete(g.connections, up); g.mu.Unlock() }()
|
||||
var committing atomic.Bool
|
||||
done := make(chan struct{}, 2)
|
||||
go func() {
|
||||
defer func() { done <- struct{}{} }()
|
||||
for {
|
||||
packet, err := readMySQLPacket(down)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if packet[3] == 0 && len(packet) > 4 && packet[4] == 3 {
|
||||
committing.Store(strings.EqualFold(strings.TrimSpace(string(packet[5:])), "COMMIT"))
|
||||
}
|
||||
if writeMySQLPacket(up, packet) != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer func() { done <- struct{}{} }()
|
||||
for {
|
||||
packet, err := readMySQLPacket(up)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if packet[3] == 1 && committing.Swap(false) && len(packet) > 4 && packet[4] == 0 && g.armed.CompareAndSwap(true, false) {
|
||||
select {
|
||||
case g.dropped <- struct{}{}:
|
||||
default:
|
||||
t.Error("unconsumed COMMIT fault marker")
|
||||
}
|
||||
return
|
||||
}
|
||||
if writeMySQLPacket(down, packet) != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
<-done
|
||||
down.Close()
|
||||
up.Close()
|
||||
<-done
|
||||
}()
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
listener.Close()
|
||||
<-g.accepted
|
||||
g.mu.Lock()
|
||||
g.closing = true
|
||||
for c := range g.connections {
|
||||
c.Close()
|
||||
}
|
||||
g.mu.Unlock()
|
||||
g.wg.Wait()
|
||||
})
|
||||
return g
|
||||
}
|
||||
func readMySQLPacket(r io.Reader) ([]byte, error) {
|
||||
var header [4]byte
|
||||
if _, err := io.ReadFull(r, header[:]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
size := int(binary.LittleEndian.Uint32(header[:]) & 0x00ffffff)
|
||||
packet := make([]byte, 4+size)
|
||||
copy(packet, header[:])
|
||||
_, err := io.ReadFull(r, packet[4:])
|
||||
return packet, err
|
||||
}
|
||||
func writeMySQLPacket(w io.Writer, b []byte) error {
|
||||
for len(b) > 0 {
|
||||
n, err := w.Write(b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n == 0 {
|
||||
return io.ErrShortWrite
|
||||
}
|
||||
b = b[n:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestIntegrationSenderCommitResponseLoss(t *testing.T) {
|
||||
direct, dsn := integrationDB(t)
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil || cfg.TLSConfig != "" {
|
||||
t.Fatal("fault requires isolated plaintext MySQL")
|
||||
}
|
||||
gateway := newCommitLossGateway(t, cfg.Addr)
|
||||
cfg.Addr = gateway.listener.Addr().String()
|
||||
ctx, stop := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer stop()
|
||||
proxied, err := Open(ctx, cfg.FormatDSN())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer proxied.Close()
|
||||
for _, phase := range []string{"enqueue", "claim", "confirm"} {
|
||||
t.Run(phase, func(t *testing.T) {
|
||||
s := testStore(t, proxied)
|
||||
fresh, _ := New(direct, s.namespace, s.queue)
|
||||
id := testID(t)
|
||||
in := testRequest(s.queue)
|
||||
if phase == "enqueue" {
|
||||
gateway.armed.Store(true)
|
||||
if _, err = s.Enqueue(ctx, id, "objects/test", in); !errors.Is(err, ErrCommitUnconfirmed) {
|
||||
t.Fatal("enqueue reported confirmed", err)
|
||||
}
|
||||
dup, e := fresh.Enqueue(ctx, id, "objects/test", in)
|
||||
if e != nil || !dup || countRows(t, fresh, "sender_jobs") != 1 {
|
||||
t.Fatal("local commit not reconciled", e)
|
||||
}
|
||||
} else {
|
||||
if _, err = s.Enqueue(ctx, id, "objects/test", in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if phase == "claim" {
|
||||
gateway.armed.Store(true)
|
||||
}
|
||||
claim, e := s.claim(ctx)
|
||||
if phase == "claim" {
|
||||
if !errors.Is(e, ErrCommitUnconfirmed) {
|
||||
t.Fatal("claim reported confirmed", e)
|
||||
}
|
||||
r := status(t, fresh, id)
|
||||
if r.State != "SENDING" || r.SendCount != 1 {
|
||||
t.Fatal("claim was not actually committed")
|
||||
}
|
||||
execSQL(t, direct, `UPDATE sender_outbox SET wake_at=TIMESTAMPADD(SECOND,-1,UTC_TIMESTAMP(6)) WHERE namespace_id=? AND operation_id=?`, s.namespace, id)
|
||||
r, e = fresh.claim(ctx)
|
||||
if e != nil || r.SendCount != 2 || r.IdempotencyKey != "outbox-task-"+id {
|
||||
t.Fatal("unknown claim did not recover", e)
|
||||
}
|
||||
} else {
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
receipt := fakeReceipt(t, s)
|
||||
gateway.armed.Store(true)
|
||||
if e = s.finish(ctx, claim, "ACCEPTED", "ACCEPTED", 0, &receipt); !errors.Is(e, ErrCommitUnconfirmed) {
|
||||
t.Fatal("confirmation reported confirmed", e)
|
||||
}
|
||||
r := status(t, fresh, id)
|
||||
if r.State != "ACCEPTED" || r.TaskID == nil || *r.TaskID != receipt.Task.ID {
|
||||
t.Fatal("confirmed commit not visible")
|
||||
}
|
||||
if _, e = fresh.claim(ctx); !errors.Is(e, ErrIdle) {
|
||||
t.Fatal("confirmed item eligible for resend", e)
|
||||
}
|
||||
var task string
|
||||
if e = direct.QueryRowContext(ctx, `SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=?`, s.namespace, id).Scan(&task); e != nil || task != receipt.Task.ID {
|
||||
t.Fatal("business confirmation not atomic", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-gateway.dropped:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("actual COMMIT OK not intercepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Log("real COMMIT OK discarded for business enqueue, sender claim and local acceptance; fresh stores reconciled persisted identity and state")
|
||||
}
|
||||
176
examples/task-outbox/internal/outbox/http_integration_test.go
Normal file
176
examples/task-outbox/internal/outbox/http_integration_test.go
Normal file
@@ -0,0 +1,176 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
func realClient(t *testing.T) (*orbit.Client, string, string) {
|
||||
t.Helper()
|
||||
base, path := os.Getenv("ORBIT_TEST_SENDER_URL"), os.Getenv("ORBIT_TEST_SENDER_CREDENTIALS_FILE")
|
||||
if base == "" || path == "" {
|
||||
t.Skip("requires isolated Orbit process")
|
||||
}
|
||||
if !strings.HasPrefix(base, "http://127.0.0.1:") {
|
||||
t.Fatal("sender HTTP test requires loopback")
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal("cannot read sender credential")
|
||||
}
|
||||
var cred struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
if json.Unmarshal(raw, &cred) != nil || cred.Token == "" {
|
||||
t.Fatal("invalid sender credential")
|
||||
}
|
||||
api, err := orbit.NewClient(base, orbit.Options{Token: cred.Token})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return api, base, cred.Token
|
||||
}
|
||||
func realStore(t *testing.T) (*Store, *orbit.Client, string, string) {
|
||||
t.Helper()
|
||||
db, _ := integrationDB(t)
|
||||
api, base, token := realClient(t)
|
||||
ctx, stop := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer stop()
|
||||
id := testID(t)
|
||||
queue, err := api.CreateQueue(ctx, "outbox-test-"+id, orbit.CreateQueueRequest{Name: "outbox-" + id, MaxConcurrency: 1, RatePerSecond: 10})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := New(db, queue.Queue.NamespaceID, queue.Queue.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s, api, base, token
|
||||
}
|
||||
func cancelAccepted(t *testing.T, api *orbit.Client, id string) {
|
||||
t.Helper()
|
||||
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer stop()
|
||||
task, err := api.GetTask(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = api.CancelTask(ctx, id, "outbox-test-cancel-"+id, orbit.CancelTaskRequest{ExpectedRevision: task.Revision, Reason: "outbox regression completed"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationSenderHTTPReceiptLoss(t *testing.T) {
|
||||
s, api, base, token := realStore(t)
|
||||
id := enqueue(t, s)
|
||||
target, _ := url.Parse(base)
|
||||
var drop atomic.Bool
|
||||
drop.Store(true)
|
||||
var posts atomic.Int32
|
||||
committed := make(chan orbit.SubmitTaskResult, 1)
|
||||
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
up := r.Clone(r.Context())
|
||||
up.URL.Scheme = target.Scheme
|
||||
up.URL.Host = target.Host
|
||||
up.Host = target.Host
|
||||
up.RequestURI = ""
|
||||
resp, err := http.DefaultTransport.RoundTrip(up)
|
||||
if err != nil {
|
||||
http.Error(w, "upstream unavailable", 502)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, (1<<20)+1))
|
||||
if err != nil {
|
||||
http.Error(w, "upstream read failed", 502)
|
||||
return
|
||||
}
|
||||
if r.Method == "POST" && r.URL.Path == "/v1/tasks" {
|
||||
posts.Add(1)
|
||||
if r.Header.Get("Idempotency-Key") != "outbox-task-"+id {
|
||||
t.Error("original key changed")
|
||||
}
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 && drop.CompareAndSwap(true, false) {
|
||||
var envelope struct {
|
||||
Data orbit.SubmitTaskResult `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(raw, &envelope) != nil || !idPattern.MatchString(envelope.Data.Task.ID) {
|
||||
t.Error("invalid real receipt")
|
||||
http.Error(w, "bad receipt", 502)
|
||||
return
|
||||
}
|
||||
committed <- envelope.Data
|
||||
// 先发送成功响应头与不完整正文,避免Transport把无响应头的断线当成可透明重放。
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(raw)))
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
w.Write(raw[:1])
|
||||
w.(http.Flusher).Flush()
|
||||
conn, _, err := w.(http.Hijacker).Hijack()
|
||||
if err == nil {
|
||||
conn.Close()
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
for key, values := range resp.Header {
|
||||
w.Header()[key] = values
|
||||
}
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
w.Write(raw)
|
||||
}))
|
||||
defer proxy.Close()
|
||||
through, err := orbit.NewClient(proxy.URL, orbit.Options{Token: token})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, stop := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer stop()
|
||||
r, err := s.SendOnce(ctx, through)
|
||||
if err != nil || r.State != "PENDING" || r.LastCode != "UNCONFIRMED" {
|
||||
t.Fatal("unknown remote commit classified as done", r.State, err)
|
||||
}
|
||||
var original orbit.SubmitTaskResult
|
||||
select {
|
||||
case original = <-committed:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("real accepted receipt not intercepted")
|
||||
}
|
||||
if _, err = api.GetTask(ctx, original.Task.ID); err != nil {
|
||||
t.Fatal("remote was not actually committed", err)
|
||||
}
|
||||
execSQL(t, s.db, `UPDATE sender_outbox SET wake_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, s.namespace, id)
|
||||
r, err = s.SendOnce(ctx, through)
|
||||
if err != nil || r.State != "ACCEPTED" || r.TaskID == nil || *r.TaskID != original.Task.ID || r.SendCount != 2 {
|
||||
t.Fatal("original identity not recovered", r.State, err)
|
||||
}
|
||||
var receipt orbit.SubmitTaskResult
|
||||
if json.Unmarshal(r.ReceiptJSON, &receipt) != nil || !receipt.Deduplicated || !strings.Contains(string(receipt.Task.Payload), "9007199254740993") {
|
||||
t.Fatal("replayed receipt or precision lost")
|
||||
}
|
||||
page, err := api.ListTasks(ctx, orbit.ListTasksRequest{QueueID: s.queue})
|
||||
if err != nil || len(page.Items) != 1 || posts.Load() != 2 {
|
||||
t.Fatal("remote duplicate task", len(page.Items), err)
|
||||
}
|
||||
// 相同凭据搭配错误的持久化目标命名空间,必须在提交前停止。
|
||||
wrong, _ := New(s.db, testID(t), s.queue)
|
||||
wrongID := enqueue(t, wrong)
|
||||
r, err = wrong.SendOnce(ctx, through)
|
||||
if err != nil || r.OperationID != wrongID || r.State != "REVIEW" || r.LastCode != "TARGET_MISMATCH" || posts.Load() != 2 {
|
||||
t.Fatal("wrong namespace submitted", r.State, err)
|
||||
}
|
||||
cancelAccepted(t, api, original.Task.ID)
|
||||
t.Log("real HTTP receipt discarded after Orbit commit; original key replay recovered exactly one task; wrong namespace never posted")
|
||||
}
|
||||
33
examples/task-outbox/internal/outbox/model.go
Normal file
33
examples/task-outbox/internal/outbox/model.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package outbox
|
||||
|
||||
import "time"
|
||||
|
||||
// Record 是业务数据库中的待发命令;只确认平台受理,不能表示业务执行成功。
|
||||
type Record struct {
|
||||
NamespaceID string // NamespaceID 固定目标命名空间,换凭据不能改变归属。
|
||||
QueueID string // QueueID 固定目标队列,发送前通过当前凭据核对归属。
|
||||
OperationID string // OperationID 调用方持久化的业务操作ID,同命名空间内唯一。
|
||||
IdempotencyKey string // IdempotencyKey 由操作ID派生的固定提交键,重报禁止换键。
|
||||
RequestJSON []byte // RequestJSON 冻结的任务提交正文,保留数字精度,不含凭据。
|
||||
RequestDigest []byte // RequestDigest 冻结正文的SHA256,发送前重新校验,非平台语义摘要。
|
||||
State string // State PENDING待发、SENDING领取中、ACCEPTED已受理、REVIEW需核查。
|
||||
SendCount uint32 // SendCount 持久化领取次数,包含领取后未发HTTP即退出的次数,上限100。
|
||||
LeaseEpoch uint64 // LeaseEpoch 每次领取递增;旧代次不能更新新代次的本地结果。
|
||||
WakeAt *time.Time // WakeAt PENDING的下次领取时间或SENDING的租约期限;终结状态为空,UTC微秒。
|
||||
SendDeadlineAt time.Time // SendDeadlineAt 首次入库后24小时停止新发送,短于平台90天幂等窗口。
|
||||
LastCode string // LastCode 最近一次稳定分类,不保存HTTP错误正文、凭据或业务秘密。
|
||||
TaskID *string // TaskID 仅ACCEPTED记录平台任务ID,其余状态为空。
|
||||
ReceiptJSON []byte // ReceiptJSON 仅ACCEPTED保存原受理回执;不是当前执行状态。
|
||||
CreatedAt time.Time // CreatedAt 业务事务首次提交命令的UTC时间,微秒精度。
|
||||
UpdatedAt time.Time // UpdatedAt 最近状态变化的UTC时间,不能替代租约代次。
|
||||
}
|
||||
|
||||
// Job 演示接入方业务记录,与Record原子创建,受理确认时再原子关联任务。
|
||||
type Job struct {
|
||||
NamespaceID string // NamespaceID 与待发命令使用相同命名空间范围。
|
||||
OperationID string // OperationID 业务操作标识,与同事务Outbox一一对应。
|
||||
SourceRef string // SourceRef 待处理对象引用,不含对象正文;首次创建后固定。
|
||||
TaskID *string // TaskID 已确认的平台受理ID;为空不证明平台尚未受理。
|
||||
CreatedAt time.Time // CreatedAt 业务记录首次创建UTC时间,微秒精度。
|
||||
UpdatedAt time.Time // UpdatedAt 关联平台任务时的UTC时间,微秒精度。
|
||||
}
|
||||
239
examples/task-outbox/internal/outbox/process_integration_test.go
Normal file
239
examples/task-outbox/internal/outbox/process_integration_test.go
Normal file
@@ -0,0 +1,239 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
type roundTripper func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripper) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
type senderMarker struct {
|
||||
Phase string `json:"phase"`
|
||||
TaskID string `json:"task_id"`
|
||||
Sends uint32 `json:"sends"`
|
||||
Duplicate bool `json:"duplicate"`
|
||||
}
|
||||
|
||||
func emitMarker(m senderMarker) {
|
||||
raw, _ := json.Marshal(m)
|
||||
fmt.Println("ORBIT_SENDER_TEST " + string(raw))
|
||||
}
|
||||
|
||||
// 只存在于测试二进制,使用正式Store和真实HTTP;屏障位于平台提交后、本地确认前。
|
||||
func TestSenderProcessHelper(t *testing.T) {
|
||||
mode := os.Getenv("ORBIT_TEST_SENDER_CHILD")
|
||||
if mode == "" {
|
||||
t.Skip("subprocess helper")
|
||||
}
|
||||
if mode != "hold_after_remote_commit" && mode != "recover" {
|
||||
t.Fatal("invalid child mode")
|
||||
}
|
||||
db, _ := integrationDB(t)
|
||||
_, base, token := realClient(t)
|
||||
s, err := New(db, os.Getenv("ORBIT_TEST_SENDER_NAMESPACE"), os.Getenv("ORBIT_TEST_SENDER_QUEUE"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options := orbit.Options{Token: token}
|
||||
if mode == "hold_after_remote_commit" {
|
||||
options.HTTPClient = &http.Client{Timeout: 10 * time.Second, Transport: roundTripper(func(r *http.Request) (*http.Response, error) {
|
||||
resp, err := http.DefaultTransport.RoundTrip(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.Method != "POST" || r.URL.Path != "/v1/tasks" {
|
||||
return resp, nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var envelope struct {
|
||||
Data orbit.SubmitTaskResult `json:"data"`
|
||||
}
|
||||
if resp.StatusCode != 201 || json.Unmarshal(raw, &envelope) != nil || !idPattern.MatchString(envelope.Data.Task.ID) {
|
||||
return nil, errors.New("child did not receive actual first acceptance")
|
||||
}
|
||||
emitMarker(senderMarker{Phase: "remote_committed", TaskID: envelope.Data.Task.ID})
|
||||
<-r.Context().Done()
|
||||
return nil, r.Context().Err()
|
||||
})}
|
||||
}
|
||||
api, err := orbit.NewClient(base, options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, stop := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer stop()
|
||||
for ctx.Err() == nil {
|
||||
r, err := s.SendOnce(ctx, api)
|
||||
if err == nil {
|
||||
if r.State != "ACCEPTED" {
|
||||
t.Fatal("child did not accept", r.State)
|
||||
}
|
||||
var receipt orbit.SubmitTaskResult
|
||||
if json.Unmarshal(r.ReceiptJSON, &receipt) != nil || !receipt.Deduplicated {
|
||||
t.Fatal("child did not recover original receipt")
|
||||
}
|
||||
emitMarker(senderMarker{Phase: "accepted", TaskID: *r.TaskID, Sends: r.SendCount, Duplicate: receipt.Deduplicated})
|
||||
return
|
||||
}
|
||||
if !errors.Is(err, ErrIdle) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
timer := time.NewTimer(100 * time.Millisecond)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
t.Fatal("child sender timed out")
|
||||
}
|
||||
|
||||
type senderChild struct {
|
||||
cmd *exec.Cmd
|
||||
done chan struct{}
|
||||
err error
|
||||
markers chan senderMarker
|
||||
}
|
||||
|
||||
func startSender(t *testing.T, s *Store, mode string) *senderChild {
|
||||
t.Helper()
|
||||
binary, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := exec.Command(binary, "-test.run=^TestSenderProcessHelper$")
|
||||
values := map[string]string{"ORBIT_TEST_SENDER_CHILD": mode, "ORBIT_TEST_SENDER_NAMESPACE": s.namespace, "ORBIT_TEST_SENDER_QUEUE": s.queue}
|
||||
for _, entry := range os.Environ() {
|
||||
key, _, _ := strings.Cut(entry, "=")
|
||||
if _, ok := values[key]; !ok {
|
||||
cmd.Env = append(cmd.Env, entry)
|
||||
}
|
||||
}
|
||||
for key, value := range values {
|
||||
cmd.Env = append(cmd.Env, key+"="+value)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd.Stderr = io.Discard
|
||||
p := &senderChild{cmd: cmd, done: make(chan struct{}), markers: make(chan senderMarker, 4)}
|
||||
if err = cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go func() {
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
for scanner.Scan() {
|
||||
if raw, ok := strings.CutPrefix(scanner.Text(), "ORBIT_SENDER_TEST "); ok {
|
||||
var m senderMarker
|
||||
if json.Unmarshal([]byte(raw), &m) == nil {
|
||||
p.markers <- m
|
||||
}
|
||||
}
|
||||
}
|
||||
p.err = cmd.Wait()
|
||||
close(p.done)
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
select {
|
||||
case <-p.done:
|
||||
return
|
||||
default:
|
||||
}
|
||||
_ = cmd.Process.Kill()
|
||||
select {
|
||||
case <-p.done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("owned sender child did not exit")
|
||||
}
|
||||
})
|
||||
return p
|
||||
}
|
||||
func (p *senderChild) marker(t *testing.T, ctx context.Context, phase string) senderMarker {
|
||||
t.Helper()
|
||||
for {
|
||||
select {
|
||||
case m := <-p.markers:
|
||||
if m.Phase == phase {
|
||||
return m
|
||||
}
|
||||
case <-p.done:
|
||||
select {
|
||||
case m := <-p.markers:
|
||||
if m.Phase == phase {
|
||||
return m
|
||||
}
|
||||
default:
|
||||
}
|
||||
t.Fatal("sender exited without expected marker", p.err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal("sender marker timeout")
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestIntegrationSenderProcessCommitRecovery(t *testing.T) {
|
||||
s, api, _, _ := realStore(t)
|
||||
id := enqueue(t, s)
|
||||
ctx, stop := context.WithTimeout(context.Background(), 50*time.Second)
|
||||
defer stop()
|
||||
first := startSender(t, s, "hold_after_remote_commit")
|
||||
committed := first.marker(t, ctx, "remote_committed")
|
||||
if _, err := api.GetTask(ctx, committed.TaskID); err != nil {
|
||||
t.Fatal("remote commit not confirmed", err)
|
||||
}
|
||||
r := status(t, s, id)
|
||||
if r.State != "SENDING" || r.TaskID != nil || r.SendCount != 1 {
|
||||
t.Fatal("local receipt already confirmed")
|
||||
}
|
||||
if err := first.cmd.Process.Kill(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-first.done:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("sender kill not observed")
|
||||
}
|
||||
state, ok := first.cmd.ProcessState.Sys().(syscall.WaitStatus)
|
||||
if !ok || !state.Signaled() || state.Signal() != syscall.SIGKILL {
|
||||
t.Fatal("sender was not killed", first.err)
|
||||
}
|
||||
restarted := startSender(t, s, "recover")
|
||||
result := restarted.marker(t, ctx, "accepted")
|
||||
select {
|
||||
case <-restarted.done:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("replacement sender not exited")
|
||||
}
|
||||
if restarted.err != nil || result.TaskID != committed.TaskID || result.Sends != 2 || !result.Duplicate {
|
||||
t.Fatal("replacement did not recover original receipt", restarted.err)
|
||||
}
|
||||
r = status(t, s, id)
|
||||
if r.State != "ACCEPTED" || r.TaskID == nil || *r.TaskID != committed.TaskID {
|
||||
t.Fatal("durable local acceptance missing")
|
||||
}
|
||||
page, err := api.ListTasks(ctx, orbit.ListTasksRequest{QueueID: s.queue})
|
||||
if err != nil || len(page.Items) != 1 || countRows(t, s, "sender_jobs") != 1 {
|
||||
t.Fatal("restart duplicated remote task or business record", err)
|
||||
}
|
||||
cancelAccepted(t, api, committed.TaskID)
|
||||
t.Log("sender SIGKILL observed after actual Orbit commit, before local confirmation; new process waited for natural lease expiry and recovered one task using original key")
|
||||
}
|
||||
42
examples/task-outbox/internal/outbox/schema.sql
Normal file
42
examples/task-outbox/internal/outbox/schema.sql
Normal file
@@ -0,0 +1,42 @@
|
||||
-- 业务方独立数据库的示例表,不属于平台迁移,不创建外键或级联删除。
|
||||
CREATE TABLE sender_outbox (
|
||||
namespace_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定目标命名空间,重启及换凭据不得改变',
|
||||
queue_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定目标队列,发送前核对当前授权及命名空间',
|
||||
operation_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '业务操作标识,同命名空间内唯一,调用方持久化后复用',
|
||||
idempotency_key VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '冻结平台提交键,由业务操作标识派生,禁止换键补发',
|
||||
request_json MEDIUMTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '冻结任务提交正文,保留精确数字,不存认证凭据',
|
||||
request_digest BINARY(32) NOT NULL COMMENT '冻结正文SHA256,防止同身份不同内容,非平台摘要算法',
|
||||
state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT 'PENDING待发,SENDING领取中,ACCEPTED已受理,REVIEW需核查',
|
||||
send_count INT UNSIGNED NOT NULL COMMENT '领取次数,包含领取后崩溃的次数,最多100',
|
||||
lease_epoch BIGINT UNSIGNED NOT NULL COMMENT '领取代次递增,旧发送器不能覆盖新代次结果',
|
||||
wake_at DATETIME(6) NULL COMMENT '待发下次时间或发送租约期限,终结后为空,UTC微秒',
|
||||
send_deadline_at DATETIME(6) NOT NULL COMMENT '首次入库24小时后的发送截止,短于平台幂等保留窗口',
|
||||
last_code VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '最近稳定分类,不保存服务端错误正文或秘密',
|
||||
task_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL COMMENT '已确认的平台任务标识,只在ACCEPTED非空',
|
||||
receipt_json MEDIUMTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL COMMENT '原受理回执,仅ACCEPTED非空,不代表当前执行结果',
|
||||
created_at DATETIME(6) NOT NULL COMMENT '业务事务首次创建命令UTC时间,微秒精度',
|
||||
updated_at DATETIME(6) NOT NULL COMMENT '本地状态最近变化UTC时间,微秒精度',
|
||||
PRIMARY KEY (namespace_id,operation_id),
|
||||
UNIQUE KEY uq_sender_identity(namespace_id,idempotency_key),
|
||||
KEY ix_sender_due(namespace_id,queue_id,wake_at,operation_id),
|
||||
CONSTRAINT ck_sender_document CHECK (JSON_VALID(request_json)),
|
||||
CONSTRAINT ck_sender_receipt CHECK (receipt_json IS NULL OR JSON_VALID(receipt_json)),
|
||||
CONSTRAINT ck_sender_count CHECK (send_count<=100 AND lease_epoch=send_count),
|
||||
CONSTRAINT ck_sender_state CHECK (
|
||||
(state IN ('PENDING','SENDING') AND wake_at IS NOT NULL AND task_id IS NULL AND receipt_json IS NULL)
|
||||
OR (state='REVIEW' AND wake_at IS NULL AND task_id IS NULL AND receipt_json IS NULL)
|
||||
OR (state='ACCEPTED' AND wake_at IS NULL AND task_id IS NOT NULL AND receipt_json IS NOT NULL)
|
||||
),
|
||||
CONSTRAINT ck_sender_lease CHECK (state<>'SENDING' OR lease_epoch>0),
|
||||
CONSTRAINT ck_sender_deadline CHECK (send_deadline_at>created_at AND send_deadline_at<=TIMESTAMPADD(HOUR,24,created_at))
|
||||
) ENGINE=InnoDB COMMENT='业务事务Outbox,持久化原请求与有限重发状态,不承担业务恰好一次';
|
||||
|
||||
CREATE TABLE sender_jobs (
|
||||
namespace_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '与Outbox相同的命名空间范围',
|
||||
operation_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '本地业务操作标识,与Outbox同事务创建',
|
||||
source_ref VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL COMMENT '待处理对象引用,首次入库后固定,不存对象正文或凭据',
|
||||
task_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL COMMENT '已确认的平台受理任务,为空不证明远端未受理',
|
||||
created_at DATETIME(6) NOT NULL COMMENT '业务记录创建UTC时间,微秒精度',
|
||||
updated_at DATETIME(6) NOT NULL COMMENT '关联任务最近变化UTC时间,微秒精度',
|
||||
PRIMARY KEY (namespace_id,operation_id)
|
||||
) ENGINE=InnoDB COMMENT='接入方示例业务记录,与待发请求及受理关联分别同事务保存';
|
||||
107
examples/task-outbox/internal/outbox/sender.go
Normal file
107
examples/task-outbox/internal/outbox/sender.go
Normal file
@@ -0,0 +1,107 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math/rand/v2"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
// SendOnce 有界领取并发送一项;网络交换在数据库事务外,ErrIdle表示本次无到期项。
|
||||
// 返回记录只包含本地发送状态;ACCEPTED不是业务执行成功,REVIEW必须人工核查原身份。
|
||||
func (s *Store) SendOnce(parent context.Context, api *orbit.Client) (Record, error) {
|
||||
if api == nil {
|
||||
return Record{}, ErrInvalid
|
||||
}
|
||||
// 整个网络阶段从领取前即受单调时钟15秒约束,短于30秒租约;暂停进程后不会获得新发送预算。
|
||||
work, stop := context.WithTimeout(parent, 15*time.Second)
|
||||
defer stop()
|
||||
r, err := s.claim(work)
|
||||
if err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
if r.State == "REVIEW" {
|
||||
return s.Status(parent, r.OperationID)
|
||||
}
|
||||
state, code, delay, receipt := s.deliver(work, api, r)
|
||||
// 网络超时不能阻止保存未知结果分类,收尾仍服从调用方总上下文且最多5秒。
|
||||
if err = s.finish(parent, r, state, code, delay, receipt); err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
return s.Status(parent, r.OperationID)
|
||||
}
|
||||
|
||||
func (s *Store) deliver(ctx context.Context, api *orbit.Client, r Record) (string, string, time.Duration, *orbit.SubmitTaskResult) {
|
||||
digest := sha256.Sum256(r.RequestJSON)
|
||||
if !bytes.Equal(digest[:], r.RequestDigest) || r.NamespaceID != s.namespace || r.QueueID != s.queue || r.IdempotencyKey != "outbox-task-"+r.OperationID {
|
||||
return "REVIEW", "INVALID_DOCUMENT", 0, nil
|
||||
}
|
||||
var in orbit.SubmitTaskRequest
|
||||
decoder := json.NewDecoder(bytes.NewReader(r.RequestJSON))
|
||||
decoder.DisallowUnknownFields()
|
||||
if decoder.Decode(&in) != nil || in.QueueID != s.queue {
|
||||
return "REVIEW", "INVALID_DOCUMENT", 0, nil
|
||||
}
|
||||
// 先通过当前凭据读取固定队列;不靠配置声称命名空间正确,不向错误环境盲发命令。
|
||||
queue, err := api.GetQueue(ctx, s.queue)
|
||||
if err != nil {
|
||||
state, code, delay := classify(err, r.SendCount)
|
||||
return state, code, delay, nil
|
||||
}
|
||||
if queue.ID != s.queue || queue.NamespaceID != s.namespace {
|
||||
return "REVIEW", "TARGET_MISMATCH", 0, nil
|
||||
}
|
||||
// 数据库时间决定剩余截止预算;额外查询不授予新租约,换代或临近截止时不再开始HTTP。
|
||||
var remaining int64
|
||||
err = s.db.QueryRowContext(ctx, `SELECT TIMESTAMPDIFF(MICROSECOND,UTC_TIMESTAMP(6),LEAST(wake_at,send_deadline_at)) FROM sender_outbox WHERE namespace_id=? AND queue_id=? AND operation_id=? AND state='SENDING' AND lease_epoch=?`, s.namespace, s.queue, r.OperationID, r.LeaseEpoch).Scan(&remaining)
|
||||
if err != nil || remaining <= 0 {
|
||||
return "PENDING", "UNCONFIRMED", time.Second, nil
|
||||
}
|
||||
send, stop := context.WithTimeout(ctx, min(10*time.Second, time.Duration(remaining)*time.Microsecond))
|
||||
defer stop()
|
||||
result, err := api.SubmitTask(send, r.IdempotencyKey, in)
|
||||
if err != nil {
|
||||
state, code, delay := classify(err, r.SendCount)
|
||||
return state, code, delay, nil
|
||||
}
|
||||
if !validReceipt(r, result) {
|
||||
return "REVIEW", "INVALID_RECEIPT", 0, nil
|
||||
}
|
||||
return "ACCEPTED", "ACCEPTED", 0, &result
|
||||
}
|
||||
|
||||
func validReceipt(r Record, receipt orbit.SubmitTaskResult) bool {
|
||||
t := receipt.Task
|
||||
return idPattern.MatchString(t.ID) && t.NamespaceID == r.NamespaceID && t.QueueID == r.QueueID && t.Revision > 0
|
||||
}
|
||||
|
||||
func classify(err error, count uint32) (state, code string, delay time.Duration) {
|
||||
state, code = "PENDING", "UNCONFIRMED"
|
||||
delay = min(time.Minute, time.Second*time.Duration(uint64(1)<<min(max(count, 1)-1, 6))) + time.Duration(rand.IntN(1000))*time.Millisecond
|
||||
var api *orbit.APIError
|
||||
if errors.As(err, &api) {
|
||||
if api.StatusCode == 429 {
|
||||
code = "BACKPRESSURE"
|
||||
delay = max(delay, api.RetryAfter)
|
||||
} else if api.StatusCode < 500 && api.StatusCode != 408 {
|
||||
state, code = "REVIEW", "HTTP_REJECTED"
|
||||
if api.StatusCode == 401 || api.StatusCode == 403 {
|
||||
code = "AUTHORIZATION"
|
||||
}
|
||||
if api.Code == "IDEMPOTENCY_CONFLICT" {
|
||||
code = "IDEMPOTENCY_CONFLICT"
|
||||
}
|
||||
}
|
||||
} else {
|
||||
var httpErr *orbit.HTTPError
|
||||
if errors.As(err, &httpErr) && httpErr.StatusCode < 500 && httpErr.StatusCode != 408 && httpErr.StatusCode != 429 {
|
||||
state, code = "REVIEW", "HTTP_REJECTED"
|
||||
}
|
||||
}
|
||||
return state, code, min(delay, time.Hour)
|
||||
}
|
||||
321
examples/task-outbox/internal/outbox/store.go
Normal file
321
examples/task-outbox/internal/outbox/store.go
Normal file
@@ -0,0 +1,321 @@
|
||||
// Package outbox 演示接入方本地业务事务、稳定提交身份及有限发送恢复。
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
//go:embed schema.sql
|
||||
var schema string
|
||||
|
||||
var (
|
||||
ErrInvalid = errors.New("outbox scope, operation or request is invalid")
|
||||
ErrConflict = errors.New("outbox operation already has different content; keep the original identity")
|
||||
ErrStorage = errors.New("outbox storage failed; reconcile the original operation")
|
||||
ErrCommitUnconfirmed = errors.New("outbox commit is unconfirmed; reconcile the original operation")
|
||||
ErrIdle = errors.New("no due outbox record")
|
||||
ErrNotFound = errors.New("outbox operation not found in this scope")
|
||||
ErrLeaseLost = errors.New("outbox lease was replaced; do not overwrite its result")
|
||||
idPattern = regexp.MustCompile(`^[a-f0-9]{32}$`)
|
||||
)
|
||||
|
||||
const maxSends = 100
|
||||
const recordColumns = `namespace_id,queue_id,operation_id,idempotency_key,request_json,request_digest,state,send_count,lease_epoch,wake_at,send_deadline_at,last_code,task_id,receipt_json,created_at,updated_at`
|
||||
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
namespace, queue string
|
||||
}
|
||||
|
||||
// Open 仅连接业务方数据库,不创建表,不输出原始DSN或数据库错误。
|
||||
func Open(ctx context.Context, dsn string) (*sql.DB, error) {
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil || cfg.DBName == "" {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
cfg.ParseTime = true
|
||||
cfg.Loc = time.UTC
|
||||
cfg.MultiStatements = false
|
||||
cfg.AllowAllFiles = false
|
||||
cfg.Timeout, cfg.ReadTimeout, cfg.WriteTimeout = 5*time.Second, 8*time.Second, 8*time.Second
|
||||
if cfg.Params == nil {
|
||||
cfg.Params = map[string]string{}
|
||||
}
|
||||
cfg.Params["time_zone"] = "'+00:00'"
|
||||
connector, err := mysql.NewConnector(cfg)
|
||||
if err != nil {
|
||||
return nil, ErrStorage
|
||||
}
|
||||
db := sql.OpenDB(connector)
|
||||
db.SetMaxOpenConns(8)
|
||||
db.SetMaxIdleConns(4)
|
||||
db.SetConnMaxLifetime(5 * time.Minute)
|
||||
if err = db.PingContext(ctx); err != nil {
|
||||
db.Close()
|
||||
return nil, ErrStorage
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// Init 仅用于专用空库,已有表会报错;生产接入使用业务方自己的迁移。
|
||||
func Init(ctx context.Context, db *sql.DB) error {
|
||||
for _, statement := range strings.Split(schema, ";") {
|
||||
if strings.TrimSpace(statement) == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, statement); err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func New(db *sql.DB, namespace, queue string) (*Store, error) {
|
||||
if db == nil || !idPattern.MatchString(namespace) || !idPattern.MatchString(queue) {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
return &Store{db: db, namespace: namespace, queue: queue}, nil
|
||||
}
|
||||
|
||||
// document 冻结当前v1默认值及UTC表示;Outbox按文档字节去重,不冒充平台语义摘要。
|
||||
func (s *Store) document(in orbit.SubmitTaskRequest) ([]byte, error) {
|
||||
// encoding/json会把非法UTF-8替换成U+FFFD;在冻结摘要前拒绝,不能静默改变业务标识。
|
||||
for _, value := range []string{in.QueueID, in.Handler, in.HandlerVersion, in.PayloadVersion, in.BusinessKey, in.ConcurrencyKey} {
|
||||
if !utf8.ValidString(value) {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
}
|
||||
if in.QueueID != s.queue || in.Handler == "" || in.HandlerVersion == "" || in.PayloadVersion == "" || len(in.Payload) > 256<<10 || !utf8.Valid(in.Payload) || !json.Valid(in.Payload) || bytes.Equal(bytes.TrimSpace(in.Payload), []byte("null")) {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
if in.TaskTimeoutMS == 0 {
|
||||
in.TaskTimeoutMS = 86400000
|
||||
}
|
||||
if in.RecoverySafety == "" {
|
||||
in.RecoverySafety = orbit.RecoveryReconcileOnly
|
||||
}
|
||||
if in.Retry.MaxAttempts == 0 {
|
||||
in.Retry.MaxAttempts = 1
|
||||
}
|
||||
if in.Retry.AttemptTimeoutMS == 0 {
|
||||
in.Retry.AttemptTimeoutMS = 30000
|
||||
}
|
||||
if in.Retry.MaxAttempts > 1 {
|
||||
if in.Retry.InitialDelayMS == 0 {
|
||||
in.Retry.InitialDelayMS = 1000
|
||||
}
|
||||
if in.Retry.MaxDelayMS == 0 {
|
||||
in.Retry.MaxDelayMS = 30000
|
||||
}
|
||||
}
|
||||
if in.AvailableAt != nil {
|
||||
at := in.AvailableAt.UTC()
|
||||
in.AvailableAt = &at
|
||||
}
|
||||
if in.StartDeadlineAt != nil {
|
||||
at := in.StartDeadlineAt.UTC()
|
||||
in.StartDeadlineAt = &at
|
||||
}
|
||||
raw, err := json.Marshal(in)
|
||||
if err != nil || len(raw) > 512<<10 {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// Enqueue 原子创建业务记录及Outbox。duplicate只表示该本地事务已经提交,不表示平台受理。
|
||||
// 调用方在第一次调用前持久化operationID;COMMIT回执未知时仍用原ID、原内容再次调用。
|
||||
func (s *Store) Enqueue(parent context.Context, operationID, sourceRef string, in orbit.SubmitTaskRequest) (duplicate bool, err error) {
|
||||
if !idPattern.MatchString(operationID) || !utf8.ValidString(sourceRef) || strings.TrimSpace(sourceRef) == "" || utf8.RuneCountInString(sourceRef) > 512 || strings.ContainsFunc(sourceRef, unicode.IsControl) {
|
||||
return false, ErrInvalid
|
||||
}
|
||||
raw, err := s.document(in)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
digest := sha256.Sum256(raw)
|
||||
ctx, stop := context.WithTimeout(parent, 5*time.Second)
|
||||
defer stop()
|
||||
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
return false, ErrStorage
|
||||
}
|
||||
defer tx.Rollback()
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO sender_outbox(namespace_id,queue_id,operation_id,idempotency_key,request_json,request_digest,state,send_count,lease_epoch,wake_at,send_deadline_at,last_code,created_at,updated_at) VALUES(?,?,?,?,?,?,'PENDING',0,0,UTC_TIMESTAMP(6),TIMESTAMPADD(HOUR,24,UTC_TIMESTAMP(6)),'',UTC_TIMESTAMP(6),UTC_TIMESTAMP(6))`, s.namespace, s.queue, operationID, "outbox-task-"+operationID, raw, digest[:])
|
||||
if err != nil {
|
||||
var mysqlErr *mysql.MySQLError
|
||||
if !errors.As(err, &mysqlErr) || mysqlErr.Number != 1062 {
|
||||
return false, ErrStorage
|
||||
}
|
||||
if tx.Rollback() != nil {
|
||||
return false, ErrStorage
|
||||
}
|
||||
var saved []byte
|
||||
var queue, source string
|
||||
err = s.db.QueryRowContext(ctx, `SELECT o.request_digest,o.queue_id,j.source_ref FROM sender_outbox o JOIN sender_jobs j ON j.namespace_id=o.namespace_id AND j.operation_id=o.operation_id WHERE o.namespace_id=? AND o.operation_id=?`, s.namespace, operationID).Scan(&saved, &queue, &source)
|
||||
if err != nil {
|
||||
return false, ErrStorage
|
||||
}
|
||||
if queue != s.queue || source != sourceRef || !bytes.Equal(saved, digest[:]) {
|
||||
return false, ErrConflict
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
// 锁序统一为Outbox→业务行,不能在业务行持锁时反向调用发送确认。
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO sender_jobs(namespace_id,operation_id,source_ref,created_at,updated_at) VALUES(?,?,?,UTC_TIMESTAMP(6),UTC_TIMESTAMP(6))`, s.namespace, operationID, sourceRef); err != nil {
|
||||
return false, ErrStorage
|
||||
}
|
||||
if tx.Commit() != nil {
|
||||
return false, ErrCommitUnconfirmed
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
type scanner interface{ Scan(...any) error }
|
||||
|
||||
func scanRecord(row scanner, r *Record, extra ...any) error {
|
||||
args := []any{&r.NamespaceID, &r.QueueID, &r.OperationID, &r.IdempotencyKey, &r.RequestJSON, &r.RequestDigest, &r.State, &r.SendCount, &r.LeaseEpoch, &r.WakeAt, &r.SendDeadlineAt, &r.LastCode, &r.TaskID, &r.ReceiptJSON, &r.CreatedAt, &r.UpdatedAt}
|
||||
return row.Scan(append(args, extra...)...)
|
||||
}
|
||||
func (s *Store) Status(parent context.Context, id string) (Record, error) {
|
||||
if !idPattern.MatchString(id) {
|
||||
return Record{}, ErrInvalid
|
||||
}
|
||||
ctx, stop := context.WithTimeout(parent, 5*time.Second)
|
||||
defer stop()
|
||||
var r Record
|
||||
err := scanRecord(s.db.QueryRowContext(ctx, `SELECT `+recordColumns+` FROM sender_outbox WHERE namespace_id=? AND queue_id=? AND operation_id=?`, s.namespace, s.queue, id), &r)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Record{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return Record{}, ErrStorage
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (s *Store) claim(parent context.Context) (Record, error) {
|
||||
ctx, stop := context.WithTimeout(parent, 5*time.Second)
|
||||
defer stop()
|
||||
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
return Record{}, ErrStorage
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var r Record
|
||||
var now time.Time
|
||||
err = scanRecord(tx.QueryRowContext(ctx, `SELECT `+recordColumns+`,UTC_TIMESTAMP(6) FROM sender_outbox FORCE INDEX(ix_sender_due) WHERE namespace_id=? AND queue_id=? AND wake_at<=UTC_TIMESTAMP(6) ORDER BY wake_at,operation_id LIMIT 1 FOR UPDATE SKIP LOCKED`, s.namespace, s.queue), &r, &now)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Record{}, ErrIdle
|
||||
}
|
||||
if err != nil {
|
||||
return Record{}, ErrStorage
|
||||
}
|
||||
if r.State != "PENDING" && r.State != "SENDING" {
|
||||
return Record{}, ErrStorage
|
||||
}
|
||||
if !now.Before(r.SendDeadlineAt) || r.SendCount >= maxSends {
|
||||
r.State = "REVIEW"
|
||||
r.WakeAt = nil
|
||||
r.LastCode = "SEND_BUDGET_EXHAUSTED"
|
||||
if !now.Before(r.SendDeadlineAt) {
|
||||
r.LastCode = "SEND_DEADLINE"
|
||||
}
|
||||
} else {
|
||||
r.State = "SENDING"
|
||||
r.SendCount++
|
||||
r.LeaseEpoch++
|
||||
wake := now.Add(30 * time.Second)
|
||||
if wake.After(r.SendDeadlineAt) {
|
||||
wake = r.SendDeadlineAt
|
||||
}
|
||||
r.WakeAt = &wake
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE sender_outbox SET state=?,send_count=?,lease_epoch=?,wake_at=?,last_code=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, r.State, r.SendCount, r.LeaseEpoch, r.WakeAt, r.LastCode, s.namespace, r.OperationID)
|
||||
if err != nil {
|
||||
return Record{}, ErrStorage
|
||||
}
|
||||
if tx.Commit() != nil {
|
||||
return Record{}, ErrCommitUnconfirmed
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// finish 只接受最近代次;过期但尚未被替代的成功回执可落库,重复HTTP只复用原键。
|
||||
func (s *Store) finish(parent context.Context, claim Record, state, code string, delay time.Duration, receipt *orbit.SubmitTaskResult) error {
|
||||
ctx, stop := context.WithTimeout(parent, 5*time.Second)
|
||||
defer stop()
|
||||
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var current Record
|
||||
var now time.Time
|
||||
err = scanRecord(tx.QueryRowContext(ctx, `SELECT `+recordColumns+`,UTC_TIMESTAMP(6) FROM sender_outbox WHERE namespace_id=? AND queue_id=? AND operation_id=? FOR UPDATE`, s.namespace, s.queue, claim.OperationID), ¤t, &now)
|
||||
if err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
if current.State != "SENDING" || current.LeaseEpoch != claim.LeaseEpoch {
|
||||
return ErrLeaseLost
|
||||
}
|
||||
var wake *time.Time
|
||||
var task *string
|
||||
var raw []byte
|
||||
if state == "ACCEPTED" {
|
||||
if receipt == nil || !validReceipt(current, *receipt) {
|
||||
return ErrInvalid
|
||||
}
|
||||
raw, err = json.Marshal(receipt)
|
||||
if err != nil || len(raw) > 1<<20 {
|
||||
return ErrInvalid
|
||||
}
|
||||
task = &receipt.Task.ID
|
||||
var existing *string
|
||||
if err = tx.QueryRowContext(ctx, `SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, s.namespace, current.OperationID).Scan(&existing); err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
if existing != nil && *existing != *task {
|
||||
return ErrConflict
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE sender_jobs SET task_id=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, *task, s.namespace, current.OperationID); err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
} else if state == "PENDING" {
|
||||
if delay < time.Second || delay > time.Hour {
|
||||
return ErrInvalid
|
||||
}
|
||||
if !now.Before(current.SendDeadlineAt) || current.SendCount >= maxSends {
|
||||
state = "REVIEW"
|
||||
code = "SEND_BUDGET_EXHAUSTED"
|
||||
} else {
|
||||
next := now.Add(delay)
|
||||
if next.After(current.SendDeadlineAt) {
|
||||
next = current.SendDeadlineAt
|
||||
}
|
||||
wake = &next
|
||||
}
|
||||
} else if state != "REVIEW" {
|
||||
return ErrInvalid
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE sender_outbox SET state=?,last_code=?,wake_at=?,task_id=?,receipt_json=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, state, code, wake, task, raw, s.namespace, current.OperationID); err != nil {
|
||||
return ErrStorage
|
||||
}
|
||||
if tx.Commit() != nil {
|
||||
return ErrCommitUnconfirmed
|
||||
}
|
||||
return nil
|
||||
}
|
||||
361
examples/task-outbox/internal/outbox/store_integration_test.go
Normal file
361
examples/task-outbox/internal/outbox/store_integration_test.go
Normal file
@@ -0,0 +1,361 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func integrationDB(t *testing.T) (*sql.DB, string) {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("ORBIT_TEST_SENDER_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires isolated sender database")
|
||||
}
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil || cfg.DBName != "orbit_sender_test" || !strings.HasPrefix(cfg.Addr, "127.0.0.1:") {
|
||||
t.Fatal("sender tests require disposable loopback database")
|
||||
}
|
||||
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer stop()
|
||||
db, err := Open(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
return db, dsn
|
||||
}
|
||||
func testStore(t *testing.T, db *sql.DB) *Store {
|
||||
t.Helper()
|
||||
s, err := New(db, testID(t), testID(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
func enqueue(t *testing.T, s *Store) string {
|
||||
t.Helper()
|
||||
id := testID(t)
|
||||
if _, err := s.Enqueue(context.Background(), id, "objects/test", testRequest(s.queue)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
func status(t *testing.T, s *Store, id string) Record {
|
||||
t.Helper()
|
||||
r, err := s.Status(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
func execSQL(t *testing.T, db *sql.DB, q string, args ...any) {
|
||||
t.Helper()
|
||||
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer stop()
|
||||
if _, err := db.ExecContext(ctx, q, args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
func countRows(t *testing.T, s *Store, table string) int {
|
||||
t.Helper()
|
||||
if table != "sender_jobs" && table != "sender_outbox" {
|
||||
t.Fatal("invalid test table")
|
||||
}
|
||||
var n int
|
||||
query := `SELECT COUNT(*) FROM sender_outbox WHERE namespace_id=? AND queue_id=?`
|
||||
if table == "sender_jobs" {
|
||||
query = `SELECT COUNT(*) FROM sender_jobs j JOIN sender_outbox o ON o.namespace_id=j.namespace_id AND o.operation_id=j.operation_id WHERE o.namespace_id=? AND o.queue_id=?`
|
||||
}
|
||||
if err := s.db.QueryRow(query, s.namespace, s.queue).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
func fakeReceipt(t *testing.T, s *Store) orbit.SubmitTaskResult {
|
||||
return orbit.SubmitTaskResult{Task: orbit.Task{ID: testID(t), NamespaceID: s.namespace, QueueID: s.queue, Revision: 1, State: orbit.TaskWaiting}}
|
||||
}
|
||||
|
||||
func TestIntegrationSenderSchema(t *testing.T) {
|
||||
db, _ := integrationDB(t)
|
||||
var fk int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&fk); err != nil || fk != 0 {
|
||||
t.Fatal(fk, err)
|
||||
}
|
||||
rows, err := db.Query(`SELECT COLUMN_COMMENT FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('sender_outbox','sender_jobs')`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
n := 0
|
||||
for rows.Next() {
|
||||
var c string
|
||||
if rows.Scan(&c) != nil || !strings.ContainsFunc(c, func(r rune) bool { return unicode.Is(unicode.Han, r) }) {
|
||||
t.Fatal("missing Chinese column comment")
|
||||
}
|
||||
n++
|
||||
}
|
||||
if rows.Err() != nil || n != 22 {
|
||||
t.Fatal(n, rows.Err())
|
||||
}
|
||||
rows.Close()
|
||||
s := testStore(t, db)
|
||||
id := enqueue(t, s)
|
||||
for _, assignment := range []string{"state='ACCEPTED'", "send_count=101,lease_epoch=101", "request_json='not json'", "send_deadline_at=TIMESTAMPADD(HOUR,48,created_at)"} {
|
||||
_, err = db.Exec(`UPDATE sender_outbox SET `+assignment+` WHERE namespace_id=? AND operation_id=?`, s.namespace, id)
|
||||
var rejected *mysql.MySQLError
|
||||
if !errors.As(err, &rejected) || rejected.Number != 3819 {
|
||||
t.Fatal("invalid persisted state did not violate CHECK", assignment, err)
|
||||
}
|
||||
}
|
||||
var plan string
|
||||
if err = db.QueryRow(`EXPLAIN FORMAT=JSON SELECT operation_id FROM sender_outbox FORCE INDEX(ix_sender_due) WHERE namespace_id=? AND queue_id=? AND wake_at<=UTC_TIMESTAMP(6) ORDER BY wake_at,operation_id LIMIT 1 FOR UPDATE SKIP LOCKED`, s.namespace, s.queue).Scan(&plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(plan, `"key": "ix_sender_due"`) || strings.Contains(plan, `"using_filesort": true`) {
|
||||
t.Fatal("sender due query lost ordered index", plan)
|
||||
}
|
||||
t.Log("two business tables, 22 Chinese-commented columns, zero foreign keys")
|
||||
}
|
||||
func TestIntegrationSenderConcurrentEnqueueAndConflict(t *testing.T) {
|
||||
db, _ := integrationDB(t)
|
||||
s := testStore(t, db)
|
||||
id := testID(t)
|
||||
in := testRequest(s.queue)
|
||||
invalid := in
|
||||
invalid.BusinessKey = "operation-\xff"
|
||||
if _, err := s.Enqueue(context.Background(), id, "objects/test", invalid); !errors.Is(err, ErrInvalid) || countRows(t, s, "sender_jobs") != 0 || countRows(t, s, "sender_outbox") != 0 {
|
||||
t.Fatal("invalid UTF-8 changed business state", err)
|
||||
}
|
||||
type outcome struct {
|
||||
duplicate bool
|
||||
err error
|
||||
}
|
||||
results := make(chan outcome, 8)
|
||||
var wg sync.WaitGroup
|
||||
for range 8 {
|
||||
wg.Go(func() {
|
||||
dup, err := s.Enqueue(context.Background(), id, "objects/test", in)
|
||||
results <- outcome{dup, err}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
duplicates := 0
|
||||
for r := range results {
|
||||
if r.err != nil {
|
||||
t.Fatal(r.err)
|
||||
}
|
||||
if r.duplicate {
|
||||
duplicates++
|
||||
}
|
||||
}
|
||||
if duplicates != 7 || countRows(t, s, "sender_jobs") != 1 || countRows(t, s, "sender_outbox") != 1 {
|
||||
t.Fatal("duplicate business transaction", duplicates)
|
||||
}
|
||||
if _, err := s.Enqueue(context.Background(), id, "objects/other", in); !errors.Is(err, ErrConflict) {
|
||||
t.Fatal("business identity conflict lost", err)
|
||||
}
|
||||
in.Payload = json.RawMessage(`{"sequence":9007199254740992}`)
|
||||
if _, err := s.Enqueue(context.Background(), id, "objects/test", in); !errors.Is(err, ErrConflict) {
|
||||
t.Fatal("request content conflict lost", err)
|
||||
}
|
||||
other, _ := New(db, s.namespace, testID(t))
|
||||
if _, err := other.Status(context.Background(), id); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatal("cross queue record returned", err)
|
||||
}
|
||||
if _, err := other.Enqueue(context.Background(), id, "objects/test", testRequest(other.queue)); !errors.Is(err, ErrConflict) {
|
||||
t.Fatal("operation was moved to another queue", err)
|
||||
}
|
||||
if !strings.Contains(string(status(t, s, id).RequestJSON), "9007199254740993") {
|
||||
t.Fatal("large integer rounded")
|
||||
}
|
||||
}
|
||||
func TestIntegrationSenderBusinessRollbackAndConfirmationAtomicity(t *testing.T) {
|
||||
db, _ := integrationDB(t)
|
||||
s := testStore(t, db)
|
||||
id := testID(t)
|
||||
execSQL(t, db, `CREATE TRIGGER sender_insert_fail BEFORE INSERT ON sender_jobs FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected business insert failure'`)
|
||||
t.Cleanup(func() {
|
||||
db.Exec(`DROP TRIGGER IF EXISTS sender_insert_fail`)
|
||||
db.Exec(`DROP TRIGGER IF EXISTS sender_update_fail`)
|
||||
})
|
||||
if _, err := s.Enqueue(context.Background(), id, "objects/test", testRequest(s.queue)); !errors.Is(err, ErrStorage) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if countRows(t, s, "sender_outbox") != 0 || countRows(t, s, "sender_jobs") != 0 {
|
||||
t.Fatal("business rollback left a request")
|
||||
}
|
||||
execSQL(t, db, `DROP TRIGGER sender_insert_fail`)
|
||||
if _, err := s.Enqueue(context.Background(), id, "objects/test", testRequest(s.queue)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claim, err := s.claim(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
receipt := fakeReceipt(t, s)
|
||||
execSQL(t, db, `CREATE TRIGGER sender_update_fail BEFORE UPDATE ON sender_jobs FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected business confirmation failure'`)
|
||||
if err = s.finish(context.Background(), claim, "ACCEPTED", "ACCEPTED", 0, &receipt); !errors.Is(err, ErrStorage) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r := status(t, s, id); r.State != "SENDING" || r.TaskID != nil {
|
||||
t.Fatal("partial confirmation committed")
|
||||
}
|
||||
execSQL(t, db, `DROP TRIGGER sender_update_fail`)
|
||||
if err = s.finish(context.Background(), claim, "ACCEPTED", "ACCEPTED", 0, &receipt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var task string
|
||||
if err = db.QueryRow(`SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=?`, s.namespace, id).Scan(&task); err != nil || task != receipt.Task.ID {
|
||||
t.Fatal("business confirmation missing", err)
|
||||
}
|
||||
}
|
||||
func TestIntegrationSenderConcurrentClaimsAndStaleGeneration(t *testing.T) {
|
||||
db, _ := integrationDB(t)
|
||||
s := testStore(t, db)
|
||||
for range 20 {
|
||||
enqueue(t, s)
|
||||
}
|
||||
claims := make(chan Record, 20)
|
||||
errs := make(chan error, 8)
|
||||
var wg sync.WaitGroup
|
||||
for range 8 {
|
||||
wg.Go(func() {
|
||||
for {
|
||||
r, err := s.claim(context.Background())
|
||||
if errors.Is(err, ErrIdle) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
errs <- err
|
||||
return
|
||||
}
|
||||
claims <- r
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
close(claims)
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var first Record
|
||||
for r := range claims {
|
||||
if seen[r.OperationID] || r.SendCount != 1 {
|
||||
t.Fatal("duplicate claim")
|
||||
}
|
||||
seen[r.OperationID] = true
|
||||
first = r
|
||||
}
|
||||
if len(seen) != 20 {
|
||||
t.Fatal("unclaimed records", len(seen))
|
||||
}
|
||||
execSQL(t, db, `UPDATE sender_outbox SET wake_at=TIMESTAMPADD(SECOND,-1,UTC_TIMESTAMP(6)) WHERE namespace_id=? AND operation_id=?`, s.namespace, first.OperationID)
|
||||
replacement, err := s.claim(context.Background())
|
||||
if err != nil || replacement.LeaseEpoch != 2 || replacement.IdempotencyKey != first.IdempotencyKey {
|
||||
t.Fatal(replacement, err)
|
||||
}
|
||||
receipt := fakeReceipt(t, s)
|
||||
if err = s.finish(context.Background(), first, "ACCEPTED", "ACCEPTED", 0, &receipt); !errors.Is(err, ErrLeaseLost) {
|
||||
t.Fatal("stale sender overwrote result", err)
|
||||
}
|
||||
if err = s.finish(context.Background(), replacement, "ACCEPTED", "ACCEPTED", 0, &receipt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
func TestIntegrationSenderHTTPClassificationAndBudgets(t *testing.T) {
|
||||
db, _ := integrationDB(t)
|
||||
for _, mode := range []string{"success", "backpressure", "forbidden", "conflict", "unknown", "wrong_target", "invalid_receipt", "corrupt_request", "exhausted", "last_attempt", "deadline"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
s := testStore(t, db)
|
||||
id := enqueue(t, s)
|
||||
var posts atomic.Int32
|
||||
if mode == "corrupt_request" {
|
||||
execSQL(t, db, `UPDATE sender_outbox SET request_json='{}' WHERE namespace_id=?`, s.namespace)
|
||||
}
|
||||
if mode == "exhausted" {
|
||||
execSQL(t, db, `UPDATE sender_outbox SET send_count=100,lease_epoch=100 WHERE namespace_id=?`, s.namespace)
|
||||
}
|
||||
if mode == "last_attempt" {
|
||||
execSQL(t, db, `UPDATE sender_outbox SET send_count=99,lease_epoch=99 WHERE namespace_id=?`, s.namespace)
|
||||
}
|
||||
if mode == "deadline" {
|
||||
execSQL(t, db, `UPDATE sender_outbox SET send_deadline_at=TIMESTAMPADD(MICROSECOND,1,created_at) WHERE namespace_id=?`, s.namespace)
|
||||
}
|
||||
receipt := fakeReceipt(t, s)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == "GET" {
|
||||
ns := s.namespace
|
||||
if mode == "wrong_target" {
|
||||
ns = testID(t)
|
||||
}
|
||||
json.NewEncoder(w).Encode(map[string]any{"request_id": id, "data": orbit.Queue{ID: s.queue, NamespaceID: ns}})
|
||||
return
|
||||
}
|
||||
posts.Add(1)
|
||||
if r.Header.Get("Idempotency-Key") != "outbox-task-"+id {
|
||||
t.Error("wrong submission identity")
|
||||
}
|
||||
switch mode {
|
||||
case "backpressure":
|
||||
w.Header().Set("Retry-After", "30")
|
||||
w.WriteHeader(429)
|
||||
json.NewEncoder(w).Encode(map[string]any{"request_id": id, "error": map[string]string{"code": "QUEUE_FULL", "message": "full"}})
|
||||
case "forbidden", "conflict":
|
||||
code, status := "PERMISSION_DENIED", 403
|
||||
if mode == "conflict" {
|
||||
code, status = "IDEMPOTENCY_CONFLICT", 409
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
json.NewEncoder(w).Encode(map[string]any{"request_id": id, "error": map[string]string{"code": code, "message": "rejected"}})
|
||||
case "unknown", "last_attempt":
|
||||
w.WriteHeader(502)
|
||||
default:
|
||||
reply := receipt
|
||||
if mode == "invalid_receipt" {
|
||||
reply.Task.NamespaceID = testID(t)
|
||||
}
|
||||
json.NewEncoder(w).Encode(map[string]any{"request_id": id, "data": reply})
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
api, _ := orbit.NewClient(server.URL, orbit.Options{Token: "test-token"})
|
||||
r, err := s.SendOnce(context.Background(), api)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "REVIEW"
|
||||
if mode == "success" {
|
||||
want = "ACCEPTED"
|
||||
}
|
||||
if mode == "backpressure" || mode == "unknown" {
|
||||
want = "PENDING"
|
||||
}
|
||||
if r.State != want {
|
||||
t.Fatal(mode, r.State, r.LastCode)
|
||||
}
|
||||
if (mode == "wrong_target" || mode == "corrupt_request" || mode == "exhausted" || mode == "deadline") && posts.Load() != 0 {
|
||||
t.Fatal("unsafe HTTP sent")
|
||||
}
|
||||
if mode == "backpressure" && (r.WakeAt == nil || time.Until(*r.WakeAt) < 29*time.Second) {
|
||||
t.Fatal("Retry-After was ignored")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
63
examples/task-outbox/internal/outbox/store_test.go
Normal file
63
examples/task-outbox/internal/outbox/store_test.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
func testID(t *testing.T) string {
|
||||
t.Helper()
|
||||
id, err := orbit.NewEventMessageID()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
func testRequest(queue string) orbit.SubmitTaskRequest {
|
||||
return orbit.SubmitTaskRequest{QueueID: queue, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"sequence":9007199254740993,"ref":"objects/test"}`), TaskTimeoutMS: 60000}
|
||||
}
|
||||
func TestFrozenDocumentAndFailureClassification(t *testing.T) {
|
||||
s := &Store{namespace: testID(t), queue: testID(t)}
|
||||
in := testRequest(s.queue)
|
||||
in.TaskTimeoutMS = 0
|
||||
raw, err := s.document(in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var frozen orbit.SubmitTaskRequest
|
||||
if json.Unmarshal(raw, &frozen) != nil || frozen.TaskTimeoutMS != 86400000 || frozen.Retry.MaxAttempts != 1 || frozen.Retry.AttemptTimeoutMS != 30000 || !bytes.Contains(frozen.Payload, []byte("9007199254740993")) {
|
||||
t.Fatal("frozen defaults or precision changed")
|
||||
}
|
||||
in.QueueID = testID(t)
|
||||
if _, err = s.document(in); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatal("cross queue request frozen")
|
||||
}
|
||||
in = testRequest(s.queue)
|
||||
in.BusinessKey = "operation-\xff"
|
||||
if _, err = s.document(in); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatal("invalid UTF-8 was silently rewritten")
|
||||
}
|
||||
for _, test := range []struct {
|
||||
err error
|
||||
state, code string
|
||||
}{
|
||||
{context.DeadlineExceeded, "PENDING", "UNCONFIRMED"},
|
||||
{&orbit.APIError{StatusCode: 503, Code: "UNAVAILABLE"}, "PENDING", "UNCONFIRMED"},
|
||||
{&orbit.APIError{StatusCode: 429, RetryAfter: 30 * time.Second}, "PENDING", "BACKPRESSURE"},
|
||||
{&orbit.APIError{StatusCode: 403}, "REVIEW", "AUTHORIZATION"},
|
||||
{&orbit.APIError{StatusCode: 409, Code: "IDEMPOTENCY_CONFLICT"}, "REVIEW", "IDEMPOTENCY_CONFLICT"},
|
||||
{&orbit.HTTPError{StatusCode: 302}, "REVIEW", "HTTP_REJECTED"},
|
||||
{&orbit.HTTPError{StatusCode: 502}, "PENDING", "UNCONFIRMED"},
|
||||
} {
|
||||
state, code, delay := classify(test.err, 100)
|
||||
if state != test.state || code != test.code || delay < time.Second || delay > time.Hour {
|
||||
t.Fatal(state, code, delay)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user