test: cover dispatch loss and workflow commit fault boundaries
This commit is contained in:
341
internal/storage/mysql/commit_fault_integration_test.go
Normal file
341
internal/storage/mysql/commit_fault_integration_test.go
Normal file
@@ -0,0 +1,341 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/instance"
|
||||
driver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// commitFaultProxy 只存在于测试二进制。代理原始MySQL包，不保存SQL、凭据或业务正文。
|
||||
// BEFORE在COMMIT发往MySQL前断线；AFTER收到真实MySQL成功包后、转交驱动前断线。
|
||||
type commitFaultProxy struct {
|
||||
mode atomic.Int32
|
||||
reached chan string
|
||||
mu sync.Mutex
|
||||
closed bool
|
||||
active map[net.Conn]struct{}
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
func readFaultPacket(c net.Conn) ([]byte, error) {
|
||||
var header [4]byte
|
||||
if _, err := io.ReadFull(c, header[:]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
size := int(header[0]) | int(header[1])<<8 | int(header[2])<<16
|
||||
packet := make([]byte, size+4)
|
||||
copy(packet, header[:])
|
||||
_, err := io.ReadFull(c, packet[4:])
|
||||
return packet, err
|
||||
}
|
||||
|
||||
func newCommitFaultProxy(t *testing.T, dsn string) (*commitFaultProxy, string) {
|
||||
t.Helper()
|
||||
cfg, err := driver.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
t.Fatal("invalid disposable DSN")
|
||||
}
|
||||
host, _, err := net.SplitHostPort(cfg.Addr)
|
||||
if err != nil || cfg.Net != "tcp" || host != "127.0.0.1" || cfg.TLSConfig != "" {
|
||||
t.Fatal("packet faults require disposable plaintext loopback MySQL")
|
||||
}
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := &commitFaultProxy{reached: make(chan string, 1), active: map[net.Conn]struct{}{}}
|
||||
p.wg.Add(1)
|
||||
t.Cleanup(func() {
|
||||
listener.Close()
|
||||
p.mu.Lock()
|
||||
p.closed = true
|
||||
for c := range p.active {
|
||||
c.Close()
|
||||
}
|
||||
p.mu.Unlock()
|
||||
p.wg.Wait()
|
||||
})
|
||||
target := cfg.Addr
|
||||
go func() {
|
||||
defer p.wg.Done()
|
||||
for {
|
||||
down, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
if p.closed {
|
||||
p.mu.Unlock()
|
||||
down.Close()
|
||||
return
|
||||
}
|
||||
p.active[down] = struct{}{}
|
||||
p.mu.Unlock()
|
||||
p.wg.Add(1)
|
||||
go func() {
|
||||
defer p.wg.Done()
|
||||
defer down.Close()
|
||||
defer func() { p.mu.Lock(); delete(p.active, down); p.mu.Unlock() }()
|
||||
up, err := net.DialTimeout("tcp", target, 3*time.Second)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer up.Close()
|
||||
var awaiting atomic.Bool
|
||||
clientDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(clientDone)
|
||||
defer up.Close()
|
||||
for {
|
||||
packet, err := readFaultPacket(down)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if len(packet) > 5 && packet[3] == 0 && packet[4] == 3 && strings.EqualFold(strings.TrimSpace(string(packet[5:])), "COMMIT") {
|
||||
switch p.mode.Swap(0) {
|
||||
case 1:
|
||||
p.reached <- "BEFORE"
|
||||
return
|
||||
case 2:
|
||||
awaiting.Store(true)
|
||||
}
|
||||
}
|
||||
if _, err = io.Copy(up, bytes.NewReader(packet)); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
for {
|
||||
packet, err := readFaultPacket(up)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if awaiting.Swap(false) {
|
||||
// COMMIT成功包：序号1、OK、0受影响行、0插入ID，IN_TRANS标志已清除。
|
||||
if len(packet) >= 11 && packet[3] == 1 && packet[4] == 0 && packet[5] == 0 && packet[6] == 0 && packet[7]&1 == 0 {
|
||||
p.reached <- "AFTER"
|
||||
} else {
|
||||
p.reached <- "UNEXPECTED_COMMIT_RESPONSE"
|
||||
}
|
||||
break
|
||||
}
|
||||
if _, err = io.Copy(down, bytes.NewReader(packet)); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
down.Close()
|
||||
up.Close()
|
||||
<-clientDone
|
||||
}()
|
||||
}
|
||||
}()
|
||||
cfg.Addr = listener.Addr().String()
|
||||
return p, cfg.FormatDSN()
|
||||
}
|
||||
|
||||
func (p *commitFaultProxy) expect(t *testing.T, after bool, err error) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatal("COMMIT wire cut was reported as success")
|
||||
}
|
||||
want := "BEFORE"
|
||||
if after {
|
||||
want = "AFTER"
|
||||
}
|
||||
select {
|
||||
case got := <-p.reached:
|
||||
if got != want {
|
||||
t.Fatalf("wrong COMMIT boundary: %s", got)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("COMMIT fault not reached")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationWorkflowCommitFaults(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Second)
|
||||
defer cancel()
|
||||
db, err := Open(ctx, dsn, 24, 6)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err = Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, after := range []bool{false, true} {
|
||||
mode := "before"
|
||||
if after {
|
||||
mode = "after"
|
||||
}
|
||||
for _, boundary := range []string{"start", "successor", "poll_wait", "poll_next"} {
|
||||
t.Run(boundary+"_"+mode+"_commit", func(t *testing.T) {
|
||||
f := newClockFixture(t, ctx, db)
|
||||
spec := runtimeSpec(f.resources.QueueID, 2)
|
||||
if strings.HasPrefix(boundary, "poll_") {
|
||||
spec = pollingSpec(f.resources.QueueID)
|
||||
}
|
||||
grantWorkflowInstance(t, f)
|
||||
input := publishInstanceSpec(t, f, spec)
|
||||
key := string(executionID(t))
|
||||
api := instance.New(NewWorkflowInstanceStore(db))
|
||||
var original instance.StartResult
|
||||
var c workflowCandidate
|
||||
var firstTask core.ID
|
||||
var originalNext, pollDeadline, expectedNext time.Time
|
||||
if boundary != "start" {
|
||||
original, err = api.Start(ctx, f.token, key, input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ref := retryStart(t, f)
|
||||
firstTask = ref.TaskID
|
||||
if boundary == "successor" {
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
} else {
|
||||
finishQueryOnly(t, f, ref, "PENDING")
|
||||
}
|
||||
c = runtimeInbox(t, f, ref.TaskID)
|
||||
if strings.HasPrefix(boundary, "poll_") {
|
||||
if err := db.QueryRowContext(ctx, `SELECT deadline_at FROM weave_poll_states WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, original.Instance.ID).Scan(&pollDeadline); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var occurred time.Time
|
||||
if err := db.QueryRowContext(ctx, `SELECT e.occurred_at FROM weave_inbox i JOIN orbit_events e ON e.namespace_id=i.namespace_id AND e.id=i.event_id WHERE i.namespace_id=? AND i.id=?`, f.resources.NamespaceID, c.id).Scan(&occurred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expectedNext = minimumTime(occurred.Add(time.Second), pollDeadline)
|
||||
}
|
||||
if boundary == "poll_next" {
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
c = clockCandidate(t, f, original.Instance.ID, "query")
|
||||
if err := db.QueryRowContext(ctx, `SELECT next_poll_at FROM weave_poll_states WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, original.Instance.ID).Scan(&originalNext); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitClockDue(t, f, c)
|
||||
}
|
||||
}
|
||||
proxy, proxiedDSN := newCommitFaultProxy(t, dsn)
|
||||
faultDB, err := Open(ctx, proxiedDSN, 2, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer faultDB.Close()
|
||||
if after {
|
||||
proxy.mode.Store(2)
|
||||
} else {
|
||||
proxy.mode.Store(1)
|
||||
}
|
||||
if boundary == "start" {
|
||||
_, err = instance.New(NewWorkflowInstanceStore(faultDB)).Start(ctx, f.token, key, input)
|
||||
} else {
|
||||
_, err = newRuntime(t, faultDB).process(ctx, c, false)
|
||||
}
|
||||
proxy.expect(t, after, err)
|
||||
// 回读通过直接连接，与故障连接、其内存回执和连接池完全独立。
|
||||
if boundary == "start" {
|
||||
want := 0
|
||||
if after {
|
||||
want = 1
|
||||
}
|
||||
if f.count("weave_instances", "1=1") != want || f.count("pulse_tasks", "1=1") != want {
|
||||
t.Fatal("partial start transaction")
|
||||
}
|
||||
original, err = api.Start(ctx, f.token, key, input)
|
||||
if err != nil || original.Deduplicated != after {
|
||||
t.Fatal("original start identity did not reconcile unknown COMMIT", err)
|
||||
}
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
} else {
|
||||
wantTasks := 1
|
||||
if after && (boundary == "successor" || boundary == "poll_next") {
|
||||
wantTasks = 2
|
||||
}
|
||||
if f.count("pulse_tasks", "1=1") != wantTasks {
|
||||
t.Fatal("partial or duplicated successor transaction")
|
||||
}
|
||||
if boundary != "poll_next" {
|
||||
want := 0
|
||||
if after {
|
||||
want = 1
|
||||
}
|
||||
if f.count("weave_inbox", "source_task_id=? AND state='PROCESSED'", firstTask) != want {
|
||||
t.Fatal("inbox ACK disagrees with COMMIT")
|
||||
}
|
||||
}
|
||||
if boundary == "poll_wait" {
|
||||
want := 0
|
||||
if after {
|
||||
want = 1
|
||||
}
|
||||
if f.count("weave_poll_states", "current_task_id IS NULL AND next_poll_at IS NOT NULL AND used_rounds=1") != want {
|
||||
t.Fatal("PENDING result and wait were not atomic")
|
||||
}
|
||||
}
|
||||
if boundary == "poll_next" {
|
||||
want := 1
|
||||
if after {
|
||||
want = 2
|
||||
}
|
||||
if f.count("weave_poll_rounds", "1=1") != want || f.count("weave_poll_states", "used_rounds=?", want) != 1 {
|
||||
t.Fatal("new round and budget were not atomic")
|
||||
}
|
||||
if !after && f.count("weave_poll_states", "next_poll_at=?", originalNext) != 1 {
|
||||
t.Fatal("rollback changed original next wake")
|
||||
}
|
||||
}
|
||||
if _, err = newRuntime(t, db).process(ctx, c, false); err != nil {
|
||||
t.Fatal("fresh runtime could not reconcile original candidate", err)
|
||||
}
|
||||
if boundary == "poll_wait" {
|
||||
if f.count("weave_poll_states", "next_poll_at=?", expectedNext) != 1 {
|
||||
t.Fatal("reconciled wait was timed from retry rather than original result")
|
||||
}
|
||||
clock := clockCandidate(t, f, original.Instance.ID, "query")
|
||||
waitClockDue(t, f, clock)
|
||||
runClock(t, f, clock)
|
||||
}
|
||||
}
|
||||
last := retryStart(t, f)
|
||||
if strings.HasPrefix(boundary, "poll_") {
|
||||
finishPoll(t, f, last, "DONE")
|
||||
} else {
|
||||
retryFinish(t, f, last, "SUCCEEDED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), last.TaskID)
|
||||
}
|
||||
v := runtimeView(t, f, original.Instance.ID, "SUCCEEDED")
|
||||
if v.DefinitionVersionID != original.Instance.DefinitionVersionID || !v.DeadlineAt.Equal(original.Instance.DeadlineAt) || f.count("pulse_tasks", "1=1") != 2 || f.count("pulse_attempts", "1=1") != 2 || f.used() != 0 || f.count("weave_inbox", "state='PENDING'") != 0 {
|
||||
t.Fatal("COMMIT recovery duplicated work or reset state")
|
||||
}
|
||||
if strings.HasPrefix(boundary, "poll_") && f.count("weave_poll_states", "used_rounds=2 AND current_task_id IS NULL AND resolved_at IS NOT NULL") != 1 {
|
||||
t.Fatal("COMMIT recovery lost final query budget")
|
||||
}
|
||||
if strings.HasPrefix(boundary, "poll_") && f.count("weave_poll_states", "deadline_at=? AND max_rounds=3 AND max_attempts_per_round=1", pollDeadline) != 1 {
|
||||
t.Fatal("COMMIT recovery reset original poll deadline or attempt budget")
|
||||
}
|
||||
replay, err := api.Start(ctx, f.token, key, input)
|
||||
if err != nil || !replay.Deduplicated || replay.Instance.ID != original.Instance.ID || !replay.Instance.DeadlineAt.Equal(original.Instance.DeadlineAt) {
|
||||
t.Fatal("start receipt changed after completion", err)
|
||||
}
|
||||
t.Logf("%s: real TCP COMMIT %s cut; same identity/candidate resumed; exactly two tasks/attempts and original deadline", boundary, mode)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user