feat: add reusable MySQL outbox and HTTP lifecycle example

This commit is contained in:
2026-09-24 20:54:36 +08:00
parent 0140c78ea7
commit e29dec717f
52 changed files with 2488 additions and 357 deletions

View File

@@ -1,105 +0,0 @@
package outbox
import (
"context"
"io"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
)
func TestIntegrationSenderCLI(t *testing.T) {
s, api, base, _ := realStore(t)
ctx, stop := context.WithTimeout(context.Background(), 30*time.Second)
defer stop()
binary := filepath.Join(t.TempDir(), "sender")
build := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "../../cmd/sender")
if raw, err := build.CombinedOutput(); err != nil {
t.Fatalf("build sender: %v %s", err, raw)
}
values := map[string]string{"ORBIT_SENDER_DSN": os.Getenv("ORBIT_TEST_SENDER_DSN"), "ORBIT_URL": base, "ORBIT_CREDENTIALS_FILE": os.Getenv("ORBIT_TEST_SENDER_CREDENTIALS_FILE"), "ORBIT_NAMESPACE_ID": s.namespace, "ORBIT_QUEUE_ID": s.queue, "ORBIT_TOKEN": ""}
env := []string{}
for _, entry := range os.Environ() {
key, _, _ := strings.Cut(entry, "=")
if _, ok := values[key]; !ok {
env = append(env, entry)
}
}
for key, value := range values {
env = append(env, key+"="+value)
}
call := func(args ...string) string {
t.Helper()
cmd := exec.CommandContext(ctx, binary, args...)
cmd.Env = env
raw, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("sender command failed: %v %s", err, raw)
}
return string(raw)
}
id := testID(t)
if !strings.Contains(call("enqueue", id, "objects/cli.txt"), "duplicate=false") || !strings.Contains(call("enqueue", id, "objects/cli.txt"), "duplicate=true") || !strings.Contains(call("status", id), "state=PENDING") {
t.Fatal("CLI local transaction mismatch")
}
cmd := exec.CommandContext(ctx, binary, "send")
cmd.Env = env
cmd.Stdout = io.Discard
cmd.Stderr = io.Discard
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
done := make(chan struct{})
var waitErr error
go func() { waitErr = cmd.Wait(); close(done) }()
t.Cleanup(func() {
select {
case <-done:
return
default:
}
cmd.Process.Kill()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Error("owned sender CLI did not exit")
}
})
var r Record
for {
r = status(t, s, id)
if r.State == "ACCEPTED" {
break
}
if r.State == "REVIEW" {
t.Fatal("CLI needs review", r.LastCode)
}
select {
case <-done:
t.Fatal("sender exited before acceptance", waitErr)
case <-ctx.Done():
t.Fatal("CLI sender timeout")
case <-time.After(100 * time.Millisecond):
}
}
if err := cmd.Process.Signal(os.Interrupt); err != nil {
t.Fatal(err)
}
select {
case <-done:
if waitErr != nil {
t.Fatal("sender did not stop gracefully", waitErr)
}
case <-ctx.Done():
t.Fatal("sender shutdown timeout")
}
if !strings.Contains(call("status", id), "state=ACCEPTED") || countRows(t, s, "sender_jobs") != 1 || r.SendCount != 1 {
t.Fatal("CLI acceptance did not persist")
}
cancelAccepted(t, api, *r.TaskID)
t.Log("built CLI enqueue duplicate/status/send/graceful stop exercised against real business DB and Orbit")
}

View File

@@ -1,226 +0,0 @@
package outbox
import (
"context"
"encoding/binary"
"errors"
"io"
"net"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/go-sql-driver/mysql"
)
// commitLossGateway 只用于无TLS的隔离MySQL回归;捕获COMMIT的真实成功包后丢弃并断连。
// 不伪造SQL结果,也不修改数据库中的提交状态。
type commitLossGateway struct {
listener net.Listener
dropped chan struct{}
armed atomic.Bool
mu sync.Mutex
closing bool
connections map[net.Conn]struct{}
wg sync.WaitGroup
accepted chan struct{}
}
func newCommitLossGateway(t *testing.T, target string) *commitLossGateway {
t.Helper()
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
g := &commitLossGateway{listener: listener, dropped: make(chan struct{}, 1), connections: map[net.Conn]struct{}{}, accepted: make(chan struct{})}
go func() {
defer close(g.accepted)
for {
down, err := listener.Accept()
if err != nil {
return
}
g.wg.Add(1)
go func() {
defer g.wg.Done()
defer down.Close()
up, err := net.DialTimeout("tcp", target, 3*time.Second)
if err != nil {
return
}
defer up.Close()
g.mu.Lock()
if g.closing {
g.mu.Unlock()
return
}
g.connections[down] = struct{}{}
g.connections[up] = struct{}{}
g.mu.Unlock()
defer func() { g.mu.Lock(); delete(g.connections, down); delete(g.connections, up); g.mu.Unlock() }()
var committing atomic.Bool
done := make(chan struct{}, 2)
go func() {
defer func() { done <- struct{}{} }()
for {
packet, err := readMySQLPacket(down)
if err != nil {
return
}
if packet[3] == 0 && len(packet) > 4 && packet[4] == 3 {
committing.Store(strings.EqualFold(strings.TrimSpace(string(packet[5:])), "COMMIT"))
}
if writeMySQLPacket(up, packet) != nil {
return
}
}
}()
go func() {
defer func() { done <- struct{}{} }()
for {
packet, err := readMySQLPacket(up)
if err != nil {
return
}
if packet[3] == 1 && committing.Swap(false) && len(packet) > 4 && packet[4] == 0 && g.armed.CompareAndSwap(true, false) {
select {
case g.dropped <- struct{}{}:
default:
t.Error("unconsumed COMMIT fault marker")
}
return
}
if writeMySQLPacket(down, packet) != nil {
return
}
}
}()
<-done
down.Close()
up.Close()
<-done
}()
}
}()
t.Cleanup(func() {
listener.Close()
<-g.accepted
g.mu.Lock()
g.closing = true
for c := range g.connections {
c.Close()
}
g.mu.Unlock()
g.wg.Wait()
})
return g
}
func readMySQLPacket(r io.Reader) ([]byte, error) {
var header [4]byte
if _, err := io.ReadFull(r, header[:]); err != nil {
return nil, err
}
size := int(binary.LittleEndian.Uint32(header[:]) & 0x00ffffff)
packet := make([]byte, 4+size)
copy(packet, header[:])
_, err := io.ReadFull(r, packet[4:])
return packet, err
}
func writeMySQLPacket(w io.Writer, b []byte) error {
for len(b) > 0 {
n, err := w.Write(b)
if err != nil {
return err
}
if n == 0 {
return io.ErrShortWrite
}
b = b[n:]
}
return nil
}
func TestIntegrationSenderCommitResponseLoss(t *testing.T) {
direct, dsn := integrationDB(t)
cfg, err := mysql.ParseDSN(dsn)
if err != nil || cfg.TLSConfig != "" {
t.Fatal("fault requires isolated plaintext MySQL")
}
gateway := newCommitLossGateway(t, cfg.Addr)
cfg.Addr = gateway.listener.Addr().String()
ctx, stop := context.WithTimeout(context.Background(), 30*time.Second)
defer stop()
proxied, err := Open(ctx, cfg.FormatDSN())
if err != nil {
t.Fatal(err)
}
defer proxied.Close()
for _, phase := range []string{"enqueue", "claim", "confirm"} {
t.Run(phase, func(t *testing.T) {
s := testStore(t, proxied)
fresh, _ := New(direct, s.namespace, s.queue)
id := testID(t)
in := testRequest(s.queue)
if phase == "enqueue" {
gateway.armed.Store(true)
if _, err = s.Enqueue(ctx, id, "objects/test", in); !errors.Is(err, ErrCommitUnconfirmed) {
t.Fatal("enqueue reported confirmed", err)
}
dup, e := fresh.Enqueue(ctx, id, "objects/test", in)
if e != nil || !dup || countRows(t, fresh, "sender_jobs") != 1 {
t.Fatal("local commit not reconciled", e)
}
} else {
if _, err = s.Enqueue(ctx, id, "objects/test", in); err != nil {
t.Fatal(err)
}
if phase == "claim" {
gateway.armed.Store(true)
}
claim, e := s.claim(ctx)
if phase == "claim" {
if !errors.Is(e, ErrCommitUnconfirmed) {
t.Fatal("claim reported confirmed", e)
}
r := status(t, fresh, id)
if r.State != "SENDING" || r.SendCount != 1 {
t.Fatal("claim was not actually committed")
}
execSQL(t, direct, `UPDATE sender_outbox SET wake_at=TIMESTAMPADD(SECOND,-1,UTC_TIMESTAMP(6)) WHERE namespace_id=? AND operation_id=?`, s.namespace, id)
r, e = fresh.claim(ctx)
if e != nil || r.SendCount != 2 || r.IdempotencyKey != "outbox-task-"+id {
t.Fatal("unknown claim did not recover", e)
}
} else {
if e != nil {
t.Fatal(e)
}
receipt := fakeReceipt(t, s)
gateway.armed.Store(true)
if e = s.finish(ctx, claim, "ACCEPTED", "ACCEPTED", 0, &receipt); !errors.Is(e, ErrCommitUnconfirmed) {
t.Fatal("confirmation reported confirmed", e)
}
r := status(t, fresh, id)
if r.State != "ACCEPTED" || r.TaskID == nil || *r.TaskID != receipt.Task.ID {
t.Fatal("confirmed commit not visible")
}
if _, e = fresh.claim(ctx); !errors.Is(e, ErrIdle) {
t.Fatal("confirmed item eligible for resend", e)
}
var task string
if e = direct.QueryRowContext(ctx, `SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=?`, s.namespace, id).Scan(&task); e != nil || task != receipt.Task.ID {
t.Fatal("business confirmation not atomic", e)
}
}
}
select {
case <-gateway.dropped:
case <-ctx.Done():
t.Fatal("actual COMMIT OK not intercepted")
}
})
}
t.Log("real COMMIT OK discarded for business enqueue, sender claim and local acceptance; fresh stores reconciled persisted identity and state")
}

View File

@@ -1,176 +0,0 @@
package outbox
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
)
func realClient(t *testing.T) (*orbit.Client, string, string) {
t.Helper()
base, path := os.Getenv("ORBIT_TEST_SENDER_URL"), os.Getenv("ORBIT_TEST_SENDER_CREDENTIALS_FILE")
if base == "" || path == "" {
t.Skip("requires isolated Orbit process")
}
if !strings.HasPrefix(base, "http://127.0.0.1:") {
t.Fatal("sender HTTP test requires loopback")
}
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal("cannot read sender credential")
}
var cred struct {
Token string `json:"token"`
}
if json.Unmarshal(raw, &cred) != nil || cred.Token == "" {
t.Fatal("invalid sender credential")
}
api, err := orbit.NewClient(base, orbit.Options{Token: cred.Token})
if err != nil {
t.Fatal(err)
}
return api, base, cred.Token
}
func realStore(t *testing.T) (*Store, *orbit.Client, string, string) {
t.Helper()
db, _ := integrationDB(t)
api, base, token := realClient(t)
ctx, stop := context.WithTimeout(context.Background(), 10*time.Second)
defer stop()
id := testID(t)
queue, err := api.CreateQueue(ctx, "outbox-test-"+id, orbit.CreateQueueRequest{Name: "outbox-" + id, MaxConcurrency: 1, RatePerSecond: 10})
if err != nil {
t.Fatal(err)
}
s, err := New(db, queue.Queue.NamespaceID, queue.Queue.ID)
if err != nil {
t.Fatal(err)
}
return s, api, base, token
}
func cancelAccepted(t *testing.T, api *orbit.Client, id string) {
t.Helper()
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
defer stop()
task, err := api.GetTask(ctx, id)
if err != nil {
t.Fatal(err)
}
if _, err = api.CancelTask(ctx, id, "outbox-test-cancel-"+id, orbit.CancelTaskRequest{ExpectedRevision: task.Revision, Reason: "outbox regression completed"}); err != nil {
t.Fatal(err)
}
}
func TestIntegrationSenderHTTPReceiptLoss(t *testing.T) {
s, api, base, token := realStore(t)
id := enqueue(t, s)
target, _ := url.Parse(base)
var drop atomic.Bool
drop.Store(true)
var posts atomic.Int32
committed := make(chan orbit.SubmitTaskResult, 1)
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
up := r.Clone(r.Context())
up.URL.Scheme = target.Scheme
up.URL.Host = target.Host
up.Host = target.Host
up.RequestURI = ""
resp, err := http.DefaultTransport.RoundTrip(up)
if err != nil {
http.Error(w, "upstream unavailable", 502)
return
}
defer resp.Body.Close()
raw, err := io.ReadAll(io.LimitReader(resp.Body, (1<<20)+1))
if err != nil {
http.Error(w, "upstream read failed", 502)
return
}
if r.Method == "POST" && r.URL.Path == "/v1/tasks" {
posts.Add(1)
if r.Header.Get("Idempotency-Key") != "outbox-task-"+id {
t.Error("original key changed")
}
if resp.StatusCode >= 200 && resp.StatusCode < 300 && drop.CompareAndSwap(true, false) {
var envelope struct {
Data orbit.SubmitTaskResult `json:"data"`
}
if json.Unmarshal(raw, &envelope) != nil || !idPattern.MatchString(envelope.Data.Task.ID) {
t.Error("invalid real receipt")
http.Error(w, "bad receipt", 502)
return
}
committed <- envelope.Data
// 先发送成功响应头与不完整正文,避免Transport把无响应头的断线当成可透明重放。
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Content-Length", strconv.Itoa(len(raw)))
w.WriteHeader(resp.StatusCode)
w.Write(raw[:1])
w.(http.Flusher).Flush()
conn, _, err := w.(http.Hijacker).Hijack()
if err == nil {
conn.Close()
}
return
}
}
for key, values := range resp.Header {
w.Header()[key] = values
}
w.WriteHeader(resp.StatusCode)
w.Write(raw)
}))
defer proxy.Close()
through, err := orbit.NewClient(proxy.URL, orbit.Options{Token: token})
if err != nil {
t.Fatal(err)
}
ctx, stop := context.WithTimeout(context.Background(), 20*time.Second)
defer stop()
r, err := s.SendOnce(ctx, through)
if err != nil || r.State != "PENDING" || r.LastCode != "UNCONFIRMED" {
t.Fatal("unknown remote commit classified as done", r.State, err)
}
var original orbit.SubmitTaskResult
select {
case original = <-committed:
case <-ctx.Done():
t.Fatal("real accepted receipt not intercepted")
}
if _, err = api.GetTask(ctx, original.Task.ID); err != nil {
t.Fatal("remote was not actually committed", err)
}
execSQL(t, s.db, `UPDATE sender_outbox SET wake_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, s.namespace, id)
r, err = s.SendOnce(ctx, through)
if err != nil || r.State != "ACCEPTED" || r.TaskID == nil || *r.TaskID != original.Task.ID || r.SendCount != 2 {
t.Fatal("original identity not recovered", r.State, err)
}
var receipt orbit.SubmitTaskResult
if json.Unmarshal(r.ReceiptJSON, &receipt) != nil || !receipt.Deduplicated || !strings.Contains(string(receipt.Task.Payload), "9007199254740993") {
t.Fatal("replayed receipt or precision lost")
}
page, err := api.ListTasks(ctx, orbit.ListTasksRequest{QueueID: s.queue})
if err != nil || len(page.Items) != 1 || posts.Load() != 2 {
t.Fatal("remote duplicate task", len(page.Items), err)
}
// 相同凭据搭配错误的持久化目标命名空间,必须在提交前停止。
wrong, _ := New(s.db, testID(t), s.queue)
wrongID := enqueue(t, wrong)
r, err = wrong.SendOnce(ctx, through)
if err != nil || r.OperationID != wrongID || r.State != "REVIEW" || r.LastCode != "TARGET_MISMATCH" || posts.Load() != 2 {
t.Fatal("wrong namespace submitted", r.State, err)
}
cancelAccepted(t, api, original.Task.ID)
t.Log("real HTTP receipt discarded after Orbit commit; original key replay recovered exactly one task; wrong namespace never posted")
}

View File

@@ -1,26 +1,13 @@
package outbox
import "time"
import (
"time"
// Record 是业务数据库中的待发命令;只确认平台受理,不能表示业务执行成功。
type Record struct {
NamespaceID string // NamespaceID 固定目标命名空间,换凭据不能改变归属。
QueueID string // QueueID 固定目标队列,发送前通过当前凭据核对归属。
OperationID string // OperationID 调用方持久化的业务操作ID,同命名空间内唯一。
IdempotencyKey string // IdempotencyKey 由操作ID派生的固定提交键,重报禁止换键。
RequestJSON []byte // RequestJSON 冻结的任务提交正文,保留数字精度,不含凭据。
RequestDigest []byte // RequestDigest 冻结正文的SHA256,发送前重新校验,非平台语义摘要。
State string // State PENDING待发、SENDING领取中、ACCEPTED已受理、REVIEW需核查。
SendCount uint32 // SendCount 持久化领取次数,包含领取后未发HTTP即退出的次数,上限100。
LeaseEpoch uint64 // LeaseEpoch 每次领取递增;旧代次不能更新新代次的本地结果。
WakeAt *time.Time // WakeAt PENDING的下次领取时间或SENDING的租约期限;终结状态为空,UTC微秒。
SendDeadlineAt time.Time // SendDeadlineAt 首次入库后24小时停止新发送,短于平台90天幂等窗口。
LastCode string // LastCode 最近一次稳定分类,不保存HTTP错误正文、凭据或业务秘密。
TaskID *string // TaskID 仅ACCEPTED记录平台任务ID,其余状态为空。
ReceiptJSON []byte // ReceiptJSON 仅ACCEPTED保存原受理回执;不是当前执行状态。
CreatedAt time.Time // CreatedAt 业务事务首次提交命令的UTC时间,微秒精度。
UpdatedAt time.Time // UpdatedAt 最近状态变化的UTC时间,不能替代租约代次。
}
adapter "git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql"
)
// Record 复用公共适配器的中文持久化模型。
type Record = adapter.Record
// Job 演示接入方业务记录,与Record原子创建,受理确认时再原子关联任务。
type Job struct {

View File

@@ -1,239 +0,0 @@
package outbox
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"strings"
"syscall"
"testing"
"time"
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
)
type roundTripper func(*http.Request) (*http.Response, error)
func (f roundTripper) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
type senderMarker struct {
Phase string `json:"phase"`
TaskID string `json:"task_id"`
Sends uint32 `json:"sends"`
Duplicate bool `json:"duplicate"`
}
func emitMarker(m senderMarker) {
raw, _ := json.Marshal(m)
fmt.Println("ORBIT_SENDER_TEST " + string(raw))
}
// 只存在于测试二进制,使用正式Store和真实HTTP;屏障位于平台提交后、本地确认前。
func TestSenderProcessHelper(t *testing.T) {
mode := os.Getenv("ORBIT_TEST_SENDER_CHILD")
if mode == "" {
t.Skip("subprocess helper")
}
if mode != "hold_after_remote_commit" && mode != "recover" {
t.Fatal("invalid child mode")
}
db, _ := integrationDB(t)
_, base, token := realClient(t)
s, err := New(db, os.Getenv("ORBIT_TEST_SENDER_NAMESPACE"), os.Getenv("ORBIT_TEST_SENDER_QUEUE"))
if err != nil {
t.Fatal(err)
}
options := orbit.Options{Token: token}
if mode == "hold_after_remote_commit" {
options.HTTPClient = &http.Client{Timeout: 10 * time.Second, Transport: roundTripper(func(r *http.Request) (*http.Response, error) {
resp, err := http.DefaultTransport.RoundTrip(r)
if err != nil {
return nil, err
}
if r.Method != "POST" || r.URL.Path != "/v1/tasks" {
return resp, nil
}
defer resp.Body.Close()
raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if err != nil {
return nil, err
}
var envelope struct {
Data orbit.SubmitTaskResult `json:"data"`
}
if resp.StatusCode != 201 || json.Unmarshal(raw, &envelope) != nil || !idPattern.MatchString(envelope.Data.Task.ID) {
return nil, errors.New("child did not receive actual first acceptance")
}
emitMarker(senderMarker{Phase: "remote_committed", TaskID: envelope.Data.Task.ID})
<-r.Context().Done()
return nil, r.Context().Err()
})}
}
api, err := orbit.NewClient(base, options)
if err != nil {
t.Fatal(err)
}
ctx, stop := context.WithTimeout(context.Background(), 45*time.Second)
defer stop()
for ctx.Err() == nil {
r, err := s.SendOnce(ctx, api)
if err == nil {
if r.State != "ACCEPTED" {
t.Fatal("child did not accept", r.State)
}
var receipt orbit.SubmitTaskResult
if json.Unmarshal(r.ReceiptJSON, &receipt) != nil || !receipt.Deduplicated {
t.Fatal("child did not recover original receipt")
}
emitMarker(senderMarker{Phase: "accepted", TaskID: *r.TaskID, Sends: r.SendCount, Duplicate: receipt.Deduplicated})
return
}
if !errors.Is(err, ErrIdle) {
t.Fatal(err)
}
timer := time.NewTimer(100 * time.Millisecond)
select {
case <-ctx.Done():
timer.Stop()
case <-timer.C:
}
}
t.Fatal("child sender timed out")
}
type senderChild struct {
cmd *exec.Cmd
done chan struct{}
err error
markers chan senderMarker
}
func startSender(t *testing.T, s *Store, mode string) *senderChild {
t.Helper()
binary, err := os.Executable()
if err != nil {
t.Fatal(err)
}
cmd := exec.Command(binary, "-test.run=^TestSenderProcessHelper$")
values := map[string]string{"ORBIT_TEST_SENDER_CHILD": mode, "ORBIT_TEST_SENDER_NAMESPACE": s.namespace, "ORBIT_TEST_SENDER_QUEUE": s.queue}
for _, entry := range os.Environ() {
key, _, _ := strings.Cut(entry, "=")
if _, ok := values[key]; !ok {
cmd.Env = append(cmd.Env, entry)
}
}
for key, value := range values {
cmd.Env = append(cmd.Env, key+"="+value)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
t.Fatal(err)
}
cmd.Stderr = io.Discard
p := &senderChild{cmd: cmd, done: make(chan struct{}), markers: make(chan senderMarker, 4)}
if err = cmd.Start(); err != nil {
t.Fatal(err)
}
go func() {
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
if raw, ok := strings.CutPrefix(scanner.Text(), "ORBIT_SENDER_TEST "); ok {
var m senderMarker
if json.Unmarshal([]byte(raw), &m) == nil {
p.markers <- m
}
}
}
p.err = cmd.Wait()
close(p.done)
}()
t.Cleanup(func() {
select {
case <-p.done:
return
default:
}
_ = cmd.Process.Kill()
select {
case <-p.done:
case <-time.After(5 * time.Second):
t.Error("owned sender child did not exit")
}
})
return p
}
func (p *senderChild) marker(t *testing.T, ctx context.Context, phase string) senderMarker {
t.Helper()
for {
select {
case m := <-p.markers:
if m.Phase == phase {
return m
}
case <-p.done:
select {
case m := <-p.markers:
if m.Phase == phase {
return m
}
default:
}
t.Fatal("sender exited without expected marker", p.err)
case <-ctx.Done():
t.Fatal("sender marker timeout")
}
}
}
func TestIntegrationSenderProcessCommitRecovery(t *testing.T) {
s, api, _, _ := realStore(t)
id := enqueue(t, s)
ctx, stop := context.WithTimeout(context.Background(), 50*time.Second)
defer stop()
first := startSender(t, s, "hold_after_remote_commit")
committed := first.marker(t, ctx, "remote_committed")
if _, err := api.GetTask(ctx, committed.TaskID); err != nil {
t.Fatal("remote commit not confirmed", err)
}
r := status(t, s, id)
if r.State != "SENDING" || r.TaskID != nil || r.SendCount != 1 {
t.Fatal("local receipt already confirmed")
}
if err := first.cmd.Process.Kill(); err != nil {
t.Fatal(err)
}
select {
case <-first.done:
case <-ctx.Done():
t.Fatal("sender kill not observed")
}
state, ok := first.cmd.ProcessState.Sys().(syscall.WaitStatus)
if !ok || !state.Signaled() || state.Signal() != syscall.SIGKILL {
t.Fatal("sender was not killed", first.err)
}
restarted := startSender(t, s, "recover")
result := restarted.marker(t, ctx, "accepted")
select {
case <-restarted.done:
case <-ctx.Done():
t.Fatal("replacement sender not exited")
}
if restarted.err != nil || result.TaskID != committed.TaskID || result.Sends != 2 || !result.Duplicate {
t.Fatal("replacement did not recover original receipt", restarted.err)
}
r = status(t, s, id)
if r.State != "ACCEPTED" || r.TaskID == nil || *r.TaskID != committed.TaskID {
t.Fatal("durable local acceptance missing")
}
page, err := api.ListTasks(ctx, orbit.ListTasksRequest{QueueID: s.queue})
if err != nil || len(page.Items) != 1 || countRows(t, s, "sender_jobs") != 1 {
t.Fatal("restart duplicated remote task or business record", err)
}
cancelAccepted(t, api, committed.TaskID)
t.Log("sender SIGKILL observed after actual Orbit commit, before local confirmation; new process waited for natural lease expiry and recovered one task using original key")
}

View File

@@ -1,36 +1,3 @@
-- 业务方独立数据库的示例表,不属于平台迁移,不创建外键或级联删除。
CREATE TABLE sender_outbox (
namespace_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定目标命名空间,重启及换凭据不得改变',
queue_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '固定目标队列,发送前核对当前授权及命名空间',
operation_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '业务操作标识,同命名空间内唯一,调用方持久化后复用',
idempotency_key VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '冻结平台提交键,由业务操作标识派生,禁止换键补发',
request_json MEDIUMTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '冻结任务提交正文,保留精确数字,不存认证凭据',
request_digest BINARY(32) NOT NULL COMMENT '冻结正文SHA256,防止同身份不同内容,非平台摘要算法',
state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT 'PENDING待发,SENDING领取中,ACCEPTED已受理,REVIEW需核查',
send_count INT UNSIGNED NOT NULL COMMENT '领取次数,包含领取后崩溃的次数,最多100',
lease_epoch BIGINT UNSIGNED NOT NULL COMMENT '领取代次递增,旧发送器不能覆盖新代次结果',
wake_at DATETIME(6) NULL COMMENT '待发下次时间或发送租约期限,终结后为空,UTC微秒',
send_deadline_at DATETIME(6) NOT NULL COMMENT '首次入库24小时后的发送截止,短于平台幂等保留窗口',
last_code VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '最近稳定分类,不保存服务端错误正文或秘密',
task_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL COMMENT '已确认的平台任务标识,只在ACCEPTED非空',
receipt_json MEDIUMTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL COMMENT '原受理回执,仅ACCEPTED非空,不代表当前执行结果',
created_at DATETIME(6) NOT NULL COMMENT '业务事务首次创建命令UTC时间,微秒精度',
updated_at DATETIME(6) NOT NULL COMMENT '本地状态最近变化UTC时间,微秒精度',
PRIMARY KEY (namespace_id,operation_id),
UNIQUE KEY uq_sender_identity(namespace_id,idempotency_key),
KEY ix_sender_due(namespace_id,queue_id,wake_at,operation_id),
CONSTRAINT ck_sender_document CHECK (JSON_VALID(request_json)),
CONSTRAINT ck_sender_receipt CHECK (receipt_json IS NULL OR JSON_VALID(receipt_json)),
CONSTRAINT ck_sender_count CHECK (send_count<=100 AND lease_epoch=send_count),
CONSTRAINT ck_sender_state CHECK (
(state IN ('PENDING','SENDING') AND wake_at IS NOT NULL AND task_id IS NULL AND receipt_json IS NULL)
OR (state='REVIEW' AND wake_at IS NULL AND task_id IS NULL AND receipt_json IS NULL)
OR (state='ACCEPTED' AND wake_at IS NULL AND task_id IS NOT NULL AND receipt_json IS NOT NULL)
),
CONSTRAINT ck_sender_lease CHECK (state<>'SENDING' OR lease_epoch>0),
CONSTRAINT ck_sender_deadline CHECK (send_deadline_at>created_at AND send_deadline_at<=TIMESTAMPADD(HOUR,24,created_at))
) ENGINE=InnoDB COMMENT='业务事务Outbox,持久化原请求与有限重发状态,不承担业务恰好一次';
CREATE TABLE sender_jobs (
namespace_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '与Outbox相同的命名空间范围',
operation_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '本地业务操作标识,与Outbox同事务创建',

View File

@@ -1,107 +0,0 @@
package outbox
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"math/rand/v2"
"time"
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
)
// SendOnce 有界领取并发送一项;网络交换在数据库事务外,ErrIdle表示本次无到期项。
// 返回记录只包含本地发送状态;ACCEPTED不是业务执行成功,REVIEW必须人工核查原身份。
func (s *Store) SendOnce(parent context.Context, api *orbit.Client) (Record, error) {
if api == nil {
return Record{}, ErrInvalid
}
// 整个网络阶段从领取前即受单调时钟15秒约束,短于30秒租约;暂停进程后不会获得新发送预算。
work, stop := context.WithTimeout(parent, 15*time.Second)
defer stop()
r, err := s.claim(work)
if err != nil {
return Record{}, err
}
if r.State == "REVIEW" {
return s.Status(parent, r.OperationID)
}
state, code, delay, receipt := s.deliver(work, api, r)
// 网络超时不能阻止保存未知结果分类,收尾仍服从调用方总上下文且最多5秒。
if err = s.finish(parent, r, state, code, delay, receipt); err != nil {
return Record{}, err
}
return s.Status(parent, r.OperationID)
}
func (s *Store) deliver(ctx context.Context, api *orbit.Client, r Record) (string, string, time.Duration, *orbit.SubmitTaskResult) {
digest := sha256.Sum256(r.RequestJSON)
if !bytes.Equal(digest[:], r.RequestDigest) || r.NamespaceID != s.namespace || r.QueueID != s.queue || r.IdempotencyKey != "outbox-task-"+r.OperationID {
return "REVIEW", "INVALID_DOCUMENT", 0, nil
}
var in orbit.SubmitTaskRequest
decoder := json.NewDecoder(bytes.NewReader(r.RequestJSON))
decoder.DisallowUnknownFields()
if decoder.Decode(&in) != nil || in.QueueID != s.queue {
return "REVIEW", "INVALID_DOCUMENT", 0, nil
}
// 先通过当前凭据读取固定队列;不靠配置声称命名空间正确,不向错误环境盲发命令。
queue, err := api.GetQueue(ctx, s.queue)
if err != nil {
state, code, delay := classify(err, r.SendCount)
return state, code, delay, nil
}
if queue.ID != s.queue || queue.NamespaceID != s.namespace {
return "REVIEW", "TARGET_MISMATCH", 0, nil
}
// 数据库时间决定剩余截止预算;额外查询不授予新租约,换代或临近截止时不再开始HTTP。
var remaining int64
err = s.db.QueryRowContext(ctx, `SELECT TIMESTAMPDIFF(MICROSECOND,UTC_TIMESTAMP(6),LEAST(wake_at,send_deadline_at)) FROM sender_outbox WHERE namespace_id=? AND queue_id=? AND operation_id=? AND state='SENDING' AND lease_epoch=?`, s.namespace, s.queue, r.OperationID, r.LeaseEpoch).Scan(&remaining)
if err != nil || remaining <= 0 {
return "PENDING", "UNCONFIRMED", time.Second, nil
}
send, stop := context.WithTimeout(ctx, min(10*time.Second, time.Duration(remaining)*time.Microsecond))
defer stop()
result, err := api.SubmitTask(send, r.IdempotencyKey, in)
if err != nil {
state, code, delay := classify(err, r.SendCount)
return state, code, delay, nil
}
if !validReceipt(r, result) {
return "REVIEW", "INVALID_RECEIPT", 0, nil
}
return "ACCEPTED", "ACCEPTED", 0, &result
}
func validReceipt(r Record, receipt orbit.SubmitTaskResult) bool {
t := receipt.Task
return idPattern.MatchString(t.ID) && t.NamespaceID == r.NamespaceID && t.QueueID == r.QueueID && t.Revision > 0
}
func classify(err error, count uint32) (state, code string, delay time.Duration) {
state, code = "PENDING", "UNCONFIRMED"
delay = min(time.Minute, time.Second*time.Duration(uint64(1)<<min(max(count, 1)-1, 6))) + time.Duration(rand.IntN(1000))*time.Millisecond
var api *orbit.APIError
if errors.As(err, &api) {
if api.StatusCode == 429 {
code = "BACKPRESSURE"
delay = max(delay, api.RetryAfter)
} else if api.StatusCode < 500 && api.StatusCode != 408 {
state, code = "REVIEW", "HTTP_REJECTED"
if api.StatusCode == 401 || api.StatusCode == 403 {
code = "AUTHORIZATION"
}
if api.Code == "IDEMPOTENCY_CONFLICT" {
code = "IDEMPOTENCY_CONFLICT"
}
}
} else {
var httpErr *orbit.HTTPError
if errors.As(err, &httpErr) && httpErr.StatusCode < 500 && httpErr.StatusCode != 408 && httpErr.StatusCode != 429 {
state, code = "REVIEW", "HTTP_REJECTED"
}
}
return state, code, min(delay, time.Hour)
}

View File

@@ -1,321 +1,95 @@
// Package outbox 演示接入方本地业务事务、稳定提交身份及有限发送恢复。
// Package outbox 演示在业务事务中使用公共 MySQL Outbox 适配包。
package outbox
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
_ "embed"
"encoding/json"
"errors"
"regexp"
"strings"
"time"
"unicode"
"unicode/utf8"
adapter "git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql"
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
"github.com/go-sql-driver/mysql"
)
//go:embed schema.sql
var schema string
var (
ErrInvalid = errors.New("outbox scope, operation or request is invalid")
ErrConflict = errors.New("outbox operation already has different content; keep the original identity")
ErrStorage = errors.New("outbox storage failed; reconcile the original operation")
ErrCommitUnconfirmed = errors.New("outbox commit is unconfirmed; reconcile the original operation")
ErrIdle = errors.New("no due outbox record")
ErrNotFound = errors.New("outbox operation not found in this scope")
ErrLeaseLost = errors.New("outbox lease was replaced; do not overwrite its result")
idPattern = regexp.MustCompile(`^[a-f0-9]{32}$`)
ErrInvalid = adapter.ErrInvalid
ErrStorage = adapter.ErrStorage
ErrConflict = adapter.ErrConflict
ErrCommitUnconfirmed = adapter.ErrCommitUnconfirmed
ErrIdle = adapter.ErrIdle
)
const maxSends = 100
const recordColumns = `namespace_id,queue_id,operation_id,idempotency_key,request_json,request_digest,state,send_count,lease_epoch,wake_at,send_deadline_at,last_code,task_id,receipt_json,created_at,updated_at`
// Store 仅补充示例业务事务;发送、查询、租约由公共包负责。
type Store struct {
db *sql.DB
namespace, queue string
*adapter.Store
db *sql.DB
namespace string
}
// Open 仅连接业务方数据库,不创建表,不输出原始DSN或数据库错误。
func Open(ctx context.Context, dsn string) (*sql.DB, error) {
cfg, err := mysql.ParseDSN(dsn)
if err != nil || cfg.DBName == "" {
return nil, ErrInvalid
}
cfg.ParseTime = true
cfg.Loc = time.UTC
cfg.MultiStatements = false
cfg.AllowAllFiles = false
cfg.Timeout, cfg.ReadTimeout, cfg.WriteTimeout = 5*time.Second, 8*time.Second, 8*time.Second
if cfg.Params == nil {
cfg.Params = map[string]string{}
}
cfg.Params["time_zone"] = "'+00:00'"
connector, err := mysql.NewConnector(cfg)
if err != nil {
return nil, ErrStorage
}
db := sql.OpenDB(connector)
db.SetMaxOpenConns(8)
db.SetMaxIdleConns(4)
db.SetConnMaxLifetime(5 * time.Minute)
if err = db.PingContext(ctx); err != nil {
db.Close()
return nil, ErrStorage
}
return db, nil
}
var Open = adapter.Open
// Init 仅用于专用空库,已有表会报错;生产接入使用业务方自己的迁移。
func Init(ctx context.Context, db *sql.DB) error {
for _, statement := range strings.Split(schema, ";") {
if strings.TrimSpace(statement) == "" {
continue
}
if _, err := db.ExecContext(ctx, statement); err != nil {
return ErrStorage
}
if err := adapter.Init(ctx, db); err != nil {
return err
}
if _, err := db.ExecContext(ctx, schema); err != nil {
return ErrStorage
}
return nil
}
func New(db *sql.DB, namespace, queue string) (*Store, error) {
if db == nil || !idPattern.MatchString(namespace) || !idPattern.MatchString(queue) {
return nil, ErrInvalid
s, err := adapter.New(db, namespace, queue, adapter.Options{OnAccepted: accepted})
if err != nil {
return nil, err
}
return &Store{db: db, namespace: namespace, queue: queue}, nil
return &Store{s, db, namespace}, nil
}
// document 冻结当前v1默认值及UTC表示;Outbox按文档字节去重,不冒充平台语义摘要。
func (s *Store) document(in orbit.SubmitTaskRequest) ([]byte, error) {
// encoding/json会把非法UTF-8替换成U+FFFD;在冻结摘要前拒绝,不能静默改变业务标识。
for _, value := range []string{in.QueueID, in.Handler, in.HandlerVersion, in.PayloadVersion, in.BusinessKey, in.ConcurrencyKey} {
if !utf8.ValidString(value) {
return nil, ErrInvalid
}
func accepted(ctx context.Context, tx *sql.Tx, r adapter.Record, receipt orbit.SubmitTaskResult) error {
var existing *string
if err := tx.QueryRowContext(ctx, `SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, r.NamespaceID, r.OperationID).Scan(&existing); err != nil {
return ErrStorage
}
if in.QueueID != s.queue || in.Handler == "" || in.HandlerVersion == "" || in.PayloadVersion == "" || len(in.Payload) > 256<<10 || !utf8.Valid(in.Payload) || !json.Valid(in.Payload) || bytes.Equal(bytes.TrimSpace(in.Payload), []byte("null")) {
return nil, ErrInvalid
if existing != nil && *existing != receipt.Task.ID {
return ErrConflict
}
if in.TaskTimeoutMS == 0 {
in.TaskTimeoutMS = 86400000
_, err := tx.ExecContext(ctx, `UPDATE sender_jobs SET task_id=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, receipt.Task.ID, r.NamespaceID, r.OperationID)
if err != nil {
return ErrStorage
}
if in.RecoverySafety == "" {
in.RecoverySafety = orbit.RecoveryReconcileOnly
}
if in.Retry.MaxAttempts == 0 {
in.Retry.MaxAttempts = 1
}
if in.Retry.AttemptTimeoutMS == 0 {
in.Retry.AttemptTimeoutMS = 30000
}
if in.Retry.MaxAttempts > 1 {
if in.Retry.InitialDelayMS == 0 {
in.Retry.InitialDelayMS = 1000
}
if in.Retry.MaxDelayMS == 0 {
in.Retry.MaxDelayMS = 30000
}
}
if in.AvailableAt != nil {
at := in.AvailableAt.UTC()
in.AvailableAt = &at
}
if in.StartDeadlineAt != nil {
at := in.StartDeadlineAt.UTC()
in.StartDeadlineAt = &at
}
raw, err := json.Marshal(in)
if err != nil || len(raw) > 512<<10 {
return nil, ErrInvalid
}
return raw, nil
return nil
}
// Enqueue 原子创建业务记录及Outbox。duplicate只表示该本地事务已经提交,不表示平台受理。
// 调用方在第一次调用前持久化operationID;COMMIT回执未知时仍用原ID、原内容再次调用。
func (s *Store) Enqueue(parent context.Context, operationID, sourceRef string, in orbit.SubmitTaskRequest) (duplicate bool, err error) {
if !idPattern.MatchString(operationID) || !utf8.ValidString(sourceRef) || strings.TrimSpace(sourceRef) == "" || utf8.RuneCountInString(sourceRef) > 512 || strings.ContainsFunc(sourceRef, unicode.IsControl) {
func (s *Store) Enqueue(ctx context.Context, id, source string, in orbit.SubmitTaskRequest) (bool, error) {
if !utf8.ValidString(source) || strings.TrimSpace(source) == "" || utf8.RuneCountInString(source) > 512 || strings.ContainsFunc(source, unicode.IsControl) {
return false, ErrInvalid
}
raw, err := s.document(in)
if err != nil {
return false, err
}
digest := sha256.Sum256(raw)
ctx, stop := context.WithTimeout(parent, 5*time.Second)
defer stop()
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
if err != nil {
return false, ErrStorage
}
defer tx.Rollback()
_, err = tx.ExecContext(ctx, `INSERT INTO sender_outbox(namespace_id,queue_id,operation_id,idempotency_key,request_json,request_digest,state,send_count,lease_epoch,wake_at,send_deadline_at,last_code,created_at,updated_at) VALUES(?,?,?,?,?,?,'PENDING',0,0,UTC_TIMESTAMP(6),TIMESTAMPADD(HOUR,24,UTC_TIMESTAMP(6)),'',UTC_TIMESTAMP(6),UTC_TIMESTAMP(6))`, s.namespace, s.queue, operationID, "outbox-task-"+operationID, raw, digest[:])
duplicate, err := s.Store.EnqueueTx(ctx, tx, id, in)
if err != nil {
var mysqlErr *mysql.MySQLError
if !errors.As(err, &mysqlErr) || mysqlErr.Number != 1062 {
return false, err
}
if duplicate {
var saved string
if err = tx.QueryRowContext(ctx, `SELECT source_ref FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, s.namespace, id).Scan(&saved); err != nil {
return false, ErrStorage
}
if tx.Rollback() != nil {
return false, ErrStorage
}
var saved []byte
var queue, source string
err = s.db.QueryRowContext(ctx, `SELECT o.request_digest,o.queue_id,j.source_ref FROM sender_outbox o JOIN sender_jobs j ON j.namespace_id=o.namespace_id AND j.operation_id=o.operation_id WHERE o.namespace_id=? AND o.operation_id=?`, s.namespace, operationID).Scan(&saved, &queue, &source)
if err != nil {
return false, ErrStorage
}
if queue != s.queue || source != sourceRef || !bytes.Equal(saved, digest[:]) {
if saved != source {
return false, ErrConflict
}
return true, nil
}
// 锁序统一为Outbox→业务行,不能在业务行持锁时反向调用发送确认。
if _, err = tx.ExecContext(ctx, `INSERT INTO sender_jobs(namespace_id,operation_id,source_ref,created_at,updated_at) VALUES(?,?,?,UTC_TIMESTAMP(6),UTC_TIMESTAMP(6))`, s.namespace, operationID, sourceRef); err != nil {
return false, ErrStorage
} else {
if _, err = tx.ExecContext(ctx, `INSERT INTO sender_jobs(namespace_id,operation_id,source_ref,created_at,updated_at) VALUES(?,?,?,UTC_TIMESTAMP(6),UTC_TIMESTAMP(6))`, s.namespace, id, source); err != nil {
return false, ErrStorage
}
}
if tx.Commit() != nil {
return false, ErrCommitUnconfirmed
}
return false, nil
}
type scanner interface{ Scan(...any) error }
func scanRecord(row scanner, r *Record, extra ...any) error {
args := []any{&r.NamespaceID, &r.QueueID, &r.OperationID, &r.IdempotencyKey, &r.RequestJSON, &r.RequestDigest, &r.State, &r.SendCount, &r.LeaseEpoch, &r.WakeAt, &r.SendDeadlineAt, &r.LastCode, &r.TaskID, &r.ReceiptJSON, &r.CreatedAt, &r.UpdatedAt}
return row.Scan(append(args, extra...)...)
}
func (s *Store) Status(parent context.Context, id string) (Record, error) {
if !idPattern.MatchString(id) {
return Record{}, ErrInvalid
}
ctx, stop := context.WithTimeout(parent, 5*time.Second)
defer stop()
var r Record
err := scanRecord(s.db.QueryRowContext(ctx, `SELECT `+recordColumns+` FROM sender_outbox WHERE namespace_id=? AND queue_id=? AND operation_id=?`, s.namespace, s.queue, id), &r)
if errors.Is(err, sql.ErrNoRows) {
return Record{}, ErrNotFound
}
if err != nil {
return Record{}, ErrStorage
}
return r, nil
}
func (s *Store) claim(parent context.Context) (Record, error) {
ctx, stop := context.WithTimeout(parent, 5*time.Second)
defer stop()
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
if err != nil {
return Record{}, ErrStorage
}
defer tx.Rollback()
var r Record
var now time.Time
err = scanRecord(tx.QueryRowContext(ctx, `SELECT `+recordColumns+`,UTC_TIMESTAMP(6) FROM sender_outbox FORCE INDEX(ix_sender_due) WHERE namespace_id=? AND queue_id=? AND wake_at<=UTC_TIMESTAMP(6) ORDER BY wake_at,operation_id LIMIT 1 FOR UPDATE SKIP LOCKED`, s.namespace, s.queue), &r, &now)
if errors.Is(err, sql.ErrNoRows) {
return Record{}, ErrIdle
}
if err != nil {
return Record{}, ErrStorage
}
if r.State != "PENDING" && r.State != "SENDING" {
return Record{}, ErrStorage
}
if !now.Before(r.SendDeadlineAt) || r.SendCount >= maxSends {
r.State = "REVIEW"
r.WakeAt = nil
r.LastCode = "SEND_BUDGET_EXHAUSTED"
if !now.Before(r.SendDeadlineAt) {
r.LastCode = "SEND_DEADLINE"
}
} else {
r.State = "SENDING"
r.SendCount++
r.LeaseEpoch++
wake := now.Add(30 * time.Second)
if wake.After(r.SendDeadlineAt) {
wake = r.SendDeadlineAt
}
r.WakeAt = &wake
}
_, err = tx.ExecContext(ctx, `UPDATE sender_outbox SET state=?,send_count=?,lease_epoch=?,wake_at=?,last_code=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, r.State, r.SendCount, r.LeaseEpoch, r.WakeAt, r.LastCode, s.namespace, r.OperationID)
if err != nil {
return Record{}, ErrStorage
}
if tx.Commit() != nil {
return Record{}, ErrCommitUnconfirmed
}
return r, nil
}
// finish 只接受最近代次;过期但尚未被替代的成功回执可落库,重复HTTP只复用原键。
func (s *Store) finish(parent context.Context, claim Record, state, code string, delay time.Duration, receipt *orbit.SubmitTaskResult) error {
ctx, stop := context.WithTimeout(parent, 5*time.Second)
defer stop()
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
if err != nil {
return ErrStorage
}
defer tx.Rollback()
var current Record
var now time.Time
err = scanRecord(tx.QueryRowContext(ctx, `SELECT `+recordColumns+`,UTC_TIMESTAMP(6) FROM sender_outbox WHERE namespace_id=? AND queue_id=? AND operation_id=? FOR UPDATE`, s.namespace, s.queue, claim.OperationID), &current, &now)
if err != nil {
return ErrStorage
}
if current.State != "SENDING" || current.LeaseEpoch != claim.LeaseEpoch {
return ErrLeaseLost
}
var wake *time.Time
var task *string
var raw []byte
if state == "ACCEPTED" {
if receipt == nil || !validReceipt(current, *receipt) {
return ErrInvalid
}
raw, err = json.Marshal(receipt)
if err != nil || len(raw) > 1<<20 {
return ErrInvalid
}
task = &receipt.Task.ID
var existing *string
if err = tx.QueryRowContext(ctx, `SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=? FOR UPDATE`, s.namespace, current.OperationID).Scan(&existing); err != nil {
return ErrStorage
}
if existing != nil && *existing != *task {
return ErrConflict
}
if _, err = tx.ExecContext(ctx, `UPDATE sender_jobs SET task_id=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, *task, s.namespace, current.OperationID); err != nil {
return ErrStorage
}
} else if state == "PENDING" {
if delay < time.Second || delay > time.Hour {
return ErrInvalid
}
if !now.Before(current.SendDeadlineAt) || current.SendCount >= maxSends {
state = "REVIEW"
code = "SEND_BUDGET_EXHAUSTED"
} else {
next := now.Add(delay)
if next.After(current.SendDeadlineAt) {
next = current.SendDeadlineAt
}
wake = &next
}
} else if state != "REVIEW" {
return ErrInvalid
}
if _, err = tx.ExecContext(ctx, `UPDATE sender_outbox SET state=?,last_code=?,wake_at=?,task_id=?,receipt_json=?,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND operation_id=?`, state, code, wake, task, raw, s.namespace, current.OperationID); err != nil {
return ErrStorage
}
if tx.Commit() != nil {
return ErrCommitUnconfirmed
}
return nil
return duplicate, nil
}

View File

@@ -1,361 +0,0 @@
package outbox
import (
"context"
"database/sql"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"unicode"
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
"github.com/go-sql-driver/mysql"
)
func integrationDB(t *testing.T) (*sql.DB, string) {
t.Helper()
dsn := os.Getenv("ORBIT_TEST_SENDER_DSN")
if dsn == "" {
t.Skip("requires isolated sender database")
}
cfg, err := mysql.ParseDSN(dsn)
if err != nil || cfg.DBName != "orbit_sender_test" || !strings.HasPrefix(cfg.Addr, "127.0.0.1:") {
t.Fatal("sender tests require disposable loopback database")
}
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
defer stop()
db, err := Open(ctx, dsn)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { db.Close() })
return db, dsn
}
func testStore(t *testing.T, db *sql.DB) *Store {
t.Helper()
s, err := New(db, testID(t), testID(t))
if err != nil {
t.Fatal(err)
}
return s
}
func enqueue(t *testing.T, s *Store) string {
t.Helper()
id := testID(t)
if _, err := s.Enqueue(context.Background(), id, "objects/test", testRequest(s.queue)); err != nil {
t.Fatal(err)
}
return id
}
func status(t *testing.T, s *Store, id string) Record {
t.Helper()
r, err := s.Status(context.Background(), id)
if err != nil {
t.Fatal(err)
}
return r
}
func execSQL(t *testing.T, db *sql.DB, q string, args ...any) {
t.Helper()
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
defer stop()
if _, err := db.ExecContext(ctx, q, args...); err != nil {
t.Fatal(err)
}
}
func countRows(t *testing.T, s *Store, table string) int {
t.Helper()
if table != "sender_jobs" && table != "sender_outbox" {
t.Fatal("invalid test table")
}
var n int
query := `SELECT COUNT(*) FROM sender_outbox WHERE namespace_id=? AND queue_id=?`
if table == "sender_jobs" {
query = `SELECT COUNT(*) FROM sender_jobs j JOIN sender_outbox o ON o.namespace_id=j.namespace_id AND o.operation_id=j.operation_id WHERE o.namespace_id=? AND o.queue_id=?`
}
if err := s.db.QueryRow(query, s.namespace, s.queue).Scan(&n); err != nil {
t.Fatal(err)
}
return n
}
func fakeReceipt(t *testing.T, s *Store) orbit.SubmitTaskResult {
return orbit.SubmitTaskResult{Task: orbit.Task{ID: testID(t), NamespaceID: s.namespace, QueueID: s.queue, Revision: 1, State: orbit.TaskWaiting}}
}
func TestIntegrationSenderSchema(t *testing.T) {
db, _ := integrationDB(t)
var fk int
if err := db.QueryRow(`SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&fk); err != nil || fk != 0 {
t.Fatal(fk, err)
}
rows, err := db.Query(`SELECT COLUMN_COMMENT FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('sender_outbox','sender_jobs')`)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
n := 0
for rows.Next() {
var c string
if rows.Scan(&c) != nil || !strings.ContainsFunc(c, func(r rune) bool { return unicode.Is(unicode.Han, r) }) {
t.Fatal("missing Chinese column comment")
}
n++
}
if rows.Err() != nil || n != 22 {
t.Fatal(n, rows.Err())
}
rows.Close()
s := testStore(t, db)
id := enqueue(t, s)
for _, assignment := range []string{"state='ACCEPTED'", "send_count=101,lease_epoch=101", "request_json='not json'", "send_deadline_at=TIMESTAMPADD(HOUR,48,created_at)"} {
_, err = db.Exec(`UPDATE sender_outbox SET `+assignment+` WHERE namespace_id=? AND operation_id=?`, s.namespace, id)
var rejected *mysql.MySQLError
if !errors.As(err, &rejected) || rejected.Number != 3819 {
t.Fatal("invalid persisted state did not violate CHECK", assignment, err)
}
}
var plan string
if err = db.QueryRow(`EXPLAIN FORMAT=JSON SELECT operation_id FROM sender_outbox FORCE INDEX(ix_sender_due) WHERE namespace_id=? AND queue_id=? AND wake_at<=UTC_TIMESTAMP(6) ORDER BY wake_at,operation_id LIMIT 1 FOR UPDATE SKIP LOCKED`, s.namespace, s.queue).Scan(&plan); err != nil {
t.Fatal(err)
}
if !strings.Contains(plan, `"key": "ix_sender_due"`) || strings.Contains(plan, `"using_filesort": true`) {
t.Fatal("sender due query lost ordered index", plan)
}
t.Log("two business tables, 22 Chinese-commented columns, zero foreign keys")
}
func TestIntegrationSenderConcurrentEnqueueAndConflict(t *testing.T) {
db, _ := integrationDB(t)
s := testStore(t, db)
id := testID(t)
in := testRequest(s.queue)
invalid := in
invalid.BusinessKey = "operation-\xff"
if _, err := s.Enqueue(context.Background(), id, "objects/test", invalid); !errors.Is(err, ErrInvalid) || countRows(t, s, "sender_jobs") != 0 || countRows(t, s, "sender_outbox") != 0 {
t.Fatal("invalid UTF-8 changed business state", err)
}
type outcome struct {
duplicate bool
err error
}
results := make(chan outcome, 8)
var wg sync.WaitGroup
for range 8 {
wg.Go(func() {
dup, err := s.Enqueue(context.Background(), id, "objects/test", in)
results <- outcome{dup, err}
})
}
wg.Wait()
close(results)
duplicates := 0
for r := range results {
if r.err != nil {
t.Fatal(r.err)
}
if r.duplicate {
duplicates++
}
}
if duplicates != 7 || countRows(t, s, "sender_jobs") != 1 || countRows(t, s, "sender_outbox") != 1 {
t.Fatal("duplicate business transaction", duplicates)
}
if _, err := s.Enqueue(context.Background(), id, "objects/other", in); !errors.Is(err, ErrConflict) {
t.Fatal("business identity conflict lost", err)
}
in.Payload = json.RawMessage(`{"sequence":9007199254740992}`)
if _, err := s.Enqueue(context.Background(), id, "objects/test", in); !errors.Is(err, ErrConflict) {
t.Fatal("request content conflict lost", err)
}
other, _ := New(db, s.namespace, testID(t))
if _, err := other.Status(context.Background(), id); !errors.Is(err, ErrNotFound) {
t.Fatal("cross queue record returned", err)
}
if _, err := other.Enqueue(context.Background(), id, "objects/test", testRequest(other.queue)); !errors.Is(err, ErrConflict) {
t.Fatal("operation was moved to another queue", err)
}
if !strings.Contains(string(status(t, s, id).RequestJSON), "9007199254740993") {
t.Fatal("large integer rounded")
}
}
func TestIntegrationSenderBusinessRollbackAndConfirmationAtomicity(t *testing.T) {
db, _ := integrationDB(t)
s := testStore(t, db)
id := testID(t)
execSQL(t, db, `CREATE TRIGGER sender_insert_fail BEFORE INSERT ON sender_jobs FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected business insert failure'`)
t.Cleanup(func() {
db.Exec(`DROP TRIGGER IF EXISTS sender_insert_fail`)
db.Exec(`DROP TRIGGER IF EXISTS sender_update_fail`)
})
if _, err := s.Enqueue(context.Background(), id, "objects/test", testRequest(s.queue)); !errors.Is(err, ErrStorage) {
t.Fatal(err)
}
if countRows(t, s, "sender_outbox") != 0 || countRows(t, s, "sender_jobs") != 0 {
t.Fatal("business rollback left a request")
}
execSQL(t, db, `DROP TRIGGER sender_insert_fail`)
if _, err := s.Enqueue(context.Background(), id, "objects/test", testRequest(s.queue)); err != nil {
t.Fatal(err)
}
claim, err := s.claim(context.Background())
if err != nil {
t.Fatal(err)
}
receipt := fakeReceipt(t, s)
execSQL(t, db, `CREATE TRIGGER sender_update_fail BEFORE UPDATE ON sender_jobs FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected business confirmation failure'`)
if err = s.finish(context.Background(), claim, "ACCEPTED", "ACCEPTED", 0, &receipt); !errors.Is(err, ErrStorage) {
t.Fatal(err)
}
if r := status(t, s, id); r.State != "SENDING" || r.TaskID != nil {
t.Fatal("partial confirmation committed")
}
execSQL(t, db, `DROP TRIGGER sender_update_fail`)
if err = s.finish(context.Background(), claim, "ACCEPTED", "ACCEPTED", 0, &receipt); err != nil {
t.Fatal(err)
}
var task string
if err = db.QueryRow(`SELECT task_id FROM sender_jobs WHERE namespace_id=? AND operation_id=?`, s.namespace, id).Scan(&task); err != nil || task != receipt.Task.ID {
t.Fatal("business confirmation missing", err)
}
}
func TestIntegrationSenderConcurrentClaimsAndStaleGeneration(t *testing.T) {
db, _ := integrationDB(t)
s := testStore(t, db)
for range 20 {
enqueue(t, s)
}
claims := make(chan Record, 20)
errs := make(chan error, 8)
var wg sync.WaitGroup
for range 8 {
wg.Go(func() {
for {
r, err := s.claim(context.Background())
if errors.Is(err, ErrIdle) {
return
}
if err != nil {
errs <- err
return
}
claims <- r
}
})
}
wg.Wait()
close(claims)
close(errs)
for err := range errs {
t.Fatal(err)
}
seen := map[string]bool{}
var first Record
for r := range claims {
if seen[r.OperationID] || r.SendCount != 1 {
t.Fatal("duplicate claim")
}
seen[r.OperationID] = true
first = r
}
if len(seen) != 20 {
t.Fatal("unclaimed records", len(seen))
}
execSQL(t, db, `UPDATE sender_outbox SET wake_at=TIMESTAMPADD(SECOND,-1,UTC_TIMESTAMP(6)) WHERE namespace_id=? AND operation_id=?`, s.namespace, first.OperationID)
replacement, err := s.claim(context.Background())
if err != nil || replacement.LeaseEpoch != 2 || replacement.IdempotencyKey != first.IdempotencyKey {
t.Fatal(replacement, err)
}
receipt := fakeReceipt(t, s)
if err = s.finish(context.Background(), first, "ACCEPTED", "ACCEPTED", 0, &receipt); !errors.Is(err, ErrLeaseLost) {
t.Fatal("stale sender overwrote result", err)
}
if err = s.finish(context.Background(), replacement, "ACCEPTED", "ACCEPTED", 0, &receipt); err != nil {
t.Fatal(err)
}
}
func TestIntegrationSenderHTTPClassificationAndBudgets(t *testing.T) {
db, _ := integrationDB(t)
for _, mode := range []string{"success", "backpressure", "forbidden", "conflict", "unknown", "wrong_target", "invalid_receipt", "corrupt_request", "exhausted", "last_attempt", "deadline"} {
t.Run(mode, func(t *testing.T) {
s := testStore(t, db)
id := enqueue(t, s)
var posts atomic.Int32
if mode == "corrupt_request" {
execSQL(t, db, `UPDATE sender_outbox SET request_json='{}' WHERE namespace_id=?`, s.namespace)
}
if mode == "exhausted" {
execSQL(t, db, `UPDATE sender_outbox SET send_count=100,lease_epoch=100 WHERE namespace_id=?`, s.namespace)
}
if mode == "last_attempt" {
execSQL(t, db, `UPDATE sender_outbox SET send_count=99,lease_epoch=99 WHERE namespace_id=?`, s.namespace)
}
if mode == "deadline" {
execSQL(t, db, `UPDATE sender_outbox SET send_deadline_at=TIMESTAMPADD(MICROSECOND,1,created_at) WHERE namespace_id=?`, s.namespace)
}
receipt := fakeReceipt(t, s)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == "GET" {
ns := s.namespace
if mode == "wrong_target" {
ns = testID(t)
}
json.NewEncoder(w).Encode(map[string]any{"request_id": id, "data": orbit.Queue{ID: s.queue, NamespaceID: ns}})
return
}
posts.Add(1)
if r.Header.Get("Idempotency-Key") != "outbox-task-"+id {
t.Error("wrong submission identity")
}
switch mode {
case "backpressure":
w.Header().Set("Retry-After", "30")
w.WriteHeader(429)
json.NewEncoder(w).Encode(map[string]any{"request_id": id, "error": map[string]string{"code": "QUEUE_FULL", "message": "full"}})
case "forbidden", "conflict":
code, status := "PERMISSION_DENIED", 403
if mode == "conflict" {
code, status = "IDEMPOTENCY_CONFLICT", 409
}
w.WriteHeader(status)
json.NewEncoder(w).Encode(map[string]any{"request_id": id, "error": map[string]string{"code": code, "message": "rejected"}})
case "unknown", "last_attempt":
w.WriteHeader(502)
default:
reply := receipt
if mode == "invalid_receipt" {
reply.Task.NamespaceID = testID(t)
}
json.NewEncoder(w).Encode(map[string]any{"request_id": id, "data": reply})
}
}))
defer server.Close()
api, _ := orbit.NewClient(server.URL, orbit.Options{Token: "test-token"})
r, err := s.SendOnce(context.Background(), api)
if err != nil {
t.Fatal(err)
}
want := "REVIEW"
if mode == "success" {
want = "ACCEPTED"
}
if mode == "backpressure" || mode == "unknown" {
want = "PENDING"
}
if r.State != want {
t.Fatal(mode, r.State, r.LastCode)
}
if (mode == "wrong_target" || mode == "corrupt_request" || mode == "exhausted" || mode == "deadline") && posts.Load() != 0 {
t.Fatal("unsafe HTTP sent")
}
if mode == "backpressure" && (r.WakeAt == nil || time.Until(*r.WakeAt) < 29*time.Second) {
t.Fatal("Retry-After was ignored")
}
})
}
}

View File

@@ -1,64 +0,0 @@
package outbox
import (
"bytes"
"context"
"encoding/json"
"errors"
"testing"
"time"
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
)
func testID(t *testing.T) string {
t.Helper()
id, err := orbit.NewEventMessageID()
if err != nil {
t.Fatal(err)
}
return id
}
func testRequest(queue string) orbit.SubmitTaskRequest {
return orbit.SubmitTaskRequest{QueueID: queue, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"sequence":9007199254740993,"ref":"objects/test"}`), TaskTimeoutMS: 60000}
}
func TestFrozenDocumentAndFailureClassification(t *testing.T) {
s := &Store{namespace: testID(t), queue: testID(t)}
in := testRequest(s.queue)
in.TaskTimeoutMS = 0
in.DelayMS = 1500
raw, err := s.document(in)
if err != nil {
t.Fatal(err)
}
var frozen orbit.SubmitTaskRequest
if json.Unmarshal(raw, &frozen) != nil || frozen.TaskTimeoutMS != 86400000 || frozen.DelayMS != 1500 || frozen.AvailableAt != nil || frozen.Retry.MaxAttempts != 1 || frozen.Retry.AttemptTimeoutMS != 30000 || !bytes.Contains(frozen.Payload, []byte("9007199254740993")) {
t.Fatal("frozen defaults or precision changed")
}
in.QueueID = testID(t)
if _, err = s.document(in); !errors.Is(err, ErrInvalid) {
t.Fatal("cross queue request frozen")
}
in = testRequest(s.queue)
in.BusinessKey = "operation-\xff"
if _, err = s.document(in); !errors.Is(err, ErrInvalid) {
t.Fatal("invalid UTF-8 was silently rewritten")
}
for _, test := range []struct {
err error
state, code string
}{
{context.DeadlineExceeded, "PENDING", "UNCONFIRMED"},
{&orbit.APIError{StatusCode: 503, Code: "UNAVAILABLE"}, "PENDING", "UNCONFIRMED"},
{&orbit.APIError{StatusCode: 429, RetryAfter: 30 * time.Second}, "PENDING", "BACKPRESSURE"},
{&orbit.APIError{StatusCode: 403}, "REVIEW", "AUTHORIZATION"},
{&orbit.APIError{StatusCode: 409, Code: "IDEMPOTENCY_CONFLICT"}, "REVIEW", "IDEMPOTENCY_CONFLICT"},
{&orbit.HTTPError{StatusCode: 302}, "REVIEW", "HTTP_REJECTED"},
{&orbit.HTTPError{StatusCode: 502}, "PENDING", "UNCONFIRMED"},
} {
state, code, delay := classify(test.err, 100)
if state != test.state || code != test.code || delay < time.Second || delay > time.Hour {
t.Fatal(state, code, delay)
}
}
}