342 lines
11 KiB
Go
342 lines
11 KiB
Go
package mysql
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"io"
|
||
"net"
|
||
"os"
|
||
"strings"
|
||
"sync"
|
||
"sync/atomic"
|
||
"testing"
|
||
"time"
|
||
|
||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||
"git.yuntuo.hk/k2safe/orbit/weave/instance"
|
||
driver "github.com/go-sql-driver/mysql"
|
||
)
|
||
|
||
// commitFaultProxy 只存在于测试二进制。代理原始MySQL包，不保存SQL、凭据或业务正文。
|
||
// BEFORE在COMMIT发往MySQL前断线；AFTER收到真实MySQL成功包后、转交驱动前断线。
|
||
type commitFaultProxy struct {
|
||
mode atomic.Int32
|
||
reached chan string
|
||
mu sync.Mutex
|
||
closed bool
|
||
active map[net.Conn]struct{}
|
||
wg sync.WaitGroup
|
||
}
|
||
|
||
func readFaultPacket(c net.Conn) ([]byte, error) {
|
||
var header [4]byte
|
||
if _, err := io.ReadFull(c, header[:]); err != nil {
|
||
return nil, err
|
||
}
|
||
size := int(header[0]) | int(header[1])<<8 | int(header[2])<<16
|
||
packet := make([]byte, size+4)
|
||
copy(packet, header[:])
|
||
_, err := io.ReadFull(c, packet[4:])
|
||
return packet, err
|
||
}
|
||
|
||
func newCommitFaultProxy(t *testing.T, dsn string) (*commitFaultProxy, string) {
|
||
t.Helper()
|
||
cfg, err := driver.ParseDSN(dsn)
|
||
if err != nil {
|
||
t.Fatal("invalid disposable DSN")
|
||
}
|
||
host, _, err := net.SplitHostPort(cfg.Addr)
|
||
if err != nil || cfg.Net != "tcp" || host != "127.0.0.1" || cfg.TLSConfig != "" {
|
||
t.Fatal("packet faults require disposable plaintext loopback MySQL")
|
||
}
|
||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
p := &commitFaultProxy{reached: make(chan string, 1), active: map[net.Conn]struct{}{}}
|
||
p.wg.Add(1)
|
||
t.Cleanup(func() {
|
||
listener.Close()
|
||
p.mu.Lock()
|
||
p.closed = true
|
||
for c := range p.active {
|
||
c.Close()
|
||
}
|
||
p.mu.Unlock()
|
||
p.wg.Wait()
|
||
})
|
||
target := cfg.Addr
|
||
go func() {
|
||
defer p.wg.Done()
|
||
for {
|
||
down, err := listener.Accept()
|
||
if err != nil {
|
||
return
|
||
}
|
||
p.mu.Lock()
|
||
if p.closed {
|
||
p.mu.Unlock()
|
||
down.Close()
|
||
return
|
||
}
|
||
p.active[down] = struct{}{}
|
||
p.mu.Unlock()
|
||
p.wg.Add(1)
|
||
go func() {
|
||
defer p.wg.Done()
|
||
defer down.Close()
|
||
defer func() { p.mu.Lock(); delete(p.active, down); p.mu.Unlock() }()
|
||
up, err := net.DialTimeout("tcp", target, 3*time.Second)
|
||
if err != nil {
|
||
return
|
||
}
|
||
defer up.Close()
|
||
var awaiting atomic.Bool
|
||
clientDone := make(chan struct{})
|
||
go func() {
|
||
defer close(clientDone)
|
||
defer up.Close()
|
||
for {
|
||
packet, err := readFaultPacket(down)
|
||
if err != nil {
|
||
return
|
||
}
|
||
if len(packet) > 5 && packet[3] == 0 && packet[4] == 3 && strings.EqualFold(strings.TrimSpace(string(packet[5:])), "COMMIT") {
|
||
switch p.mode.Swap(0) {
|
||
case 1:
|
||
p.reached <- "BEFORE"
|
||
return
|
||
case 2:
|
||
awaiting.Store(true)
|
||
}
|
||
}
|
||
if _, err = io.Copy(up, bytes.NewReader(packet)); err != nil {
|
||
return
|
||
}
|
||
}
|
||
}()
|
||
for {
|
||
packet, err := readFaultPacket(up)
|
||
if err != nil {
|
||
break
|
||
}
|
||
if awaiting.Swap(false) {
|
||
// COMMIT成功包：序号1、OK、0受影响行、0插入ID，IN_TRANS标志已清除。
|
||
if len(packet) >= 11 && packet[3] == 1 && packet[4] == 0 && packet[5] == 0 && packet[6] == 0 && packet[7]&1 == 0 {
|
||
p.reached <- "AFTER"
|
||
} else {
|
||
p.reached <- "UNEXPECTED_COMMIT_RESPONSE"
|
||
}
|
||
break
|
||
}
|
||
if _, err = io.Copy(down, bytes.NewReader(packet)); err != nil {
|
||
break
|
||
}
|
||
}
|
||
down.Close()
|
||
up.Close()
|
||
<-clientDone
|
||
}()
|
||
}
|
||
}()
|
||
cfg.Addr = listener.Addr().String()
|
||
return p, cfg.FormatDSN()
|
||
}
|
||
|
||
func (p *commitFaultProxy) expect(t *testing.T, after bool, err error) {
|
||
t.Helper()
|
||
if err == nil {
|
||
t.Fatal("COMMIT wire cut was reported as success")
|
||
}
|
||
want := "BEFORE"
|
||
if after {
|
||
want = "AFTER"
|
||
}
|
||
select {
|
||
case got := <-p.reached:
|
||
if got != want {
|
||
t.Fatalf("wrong COMMIT boundary: %s", got)
|
||
}
|
||
case <-time.After(time.Second):
|
||
t.Fatal("COMMIT fault not reached")
|
||
}
|
||
}
|
||
|
||
func TestIntegrationWorkflowCommitFaults(t *testing.T) {
|
||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||
if dsn == "" {
|
||
t.Skip("requires disposable MySQL")
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Second)
|
||
defer cancel()
|
||
db, err := Open(ctx, dsn, 24, 6)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer db.Close()
|
||
if err = Migrate(ctx, db); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, after := range []bool{false, true} {
|
||
mode := "before"
|
||
if after {
|
||
mode = "after"
|
||
}
|
||
for _, boundary := range []string{"start", "successor", "poll_wait", "poll_next"} {
|
||
t.Run(boundary+"_"+mode+"_commit", func(t *testing.T) {
|
||
f := newClockFixture(t, ctx, db)
|
||
spec := runtimeSpec(f.resources.QueueID, 2)
|
||
if strings.HasPrefix(boundary, "poll_") {
|
||
spec = pollingSpec(f.resources.QueueID)
|
||
}
|
||
grantWorkflowInstance(t, f)
|
||
input := publishInstanceSpec(t, f, spec)
|
||
key := string(executionID(t))
|
||
api := instance.New(NewWorkflowInstanceStore(db))
|
||
var original instance.StartResult
|
||
var c workflowCandidate
|
||
var firstTask core.ID
|
||
var originalNext, pollDeadline, expectedNext time.Time
|
||
if boundary != "start" {
|
||
original, err = api.Start(ctx, f.token, key, input)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
ref := retryStart(t, f)
|
||
firstTask = ref.TaskID
|
||
if boundary == "successor" {
|
||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||
} else {
|
||
finishQueryOnly(t, f, ref, "PENDING")
|
||
}
|
||
c = runtimeInbox(t, f, ref.TaskID)
|
||
if strings.HasPrefix(boundary, "poll_") {
|
||
if err := db.QueryRowContext(ctx, `SELECT deadline_at FROM weave_poll_states WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, original.Instance.ID).Scan(&pollDeadline); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var occurred time.Time
|
||
if err := db.QueryRowContext(ctx, `SELECT e.occurred_at FROM weave_inbox i JOIN orbit_events e ON e.namespace_id=i.namespace_id AND e.id=i.event_id WHERE i.namespace_id=? AND i.id=?`, f.resources.NamespaceID, c.id).Scan(&occurred); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
expectedNext = minimumTime(occurred.Add(time.Second), pollDeadline)
|
||
}
|
||
if boundary == "poll_next" {
|
||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||
c = clockCandidate(t, f, original.Instance.ID, "query")
|
||
if err := db.QueryRowContext(ctx, `SELECT next_poll_at FROM weave_poll_states WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, original.Instance.ID).Scan(&originalNext); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
waitClockDue(t, f, c)
|
||
}
|
||
}
|
||
proxy, proxiedDSN := newCommitFaultProxy(t, dsn)
|
||
faultDB, err := Open(ctx, proxiedDSN, 2, 1)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer faultDB.Close()
|
||
if after {
|
||
proxy.mode.Store(2)
|
||
} else {
|
||
proxy.mode.Store(1)
|
||
}
|
||
if boundary == "start" {
|
||
_, err = instance.New(NewWorkflowInstanceStore(faultDB)).Start(ctx, f.token, key, input)
|
||
} else {
|
||
_, err = newRuntime(t, faultDB).process(ctx, c, false)
|
||
}
|
||
proxy.expect(t, after, err)
|
||
// 回读通过直接连接，与故障连接、其内存回执和连接池完全独立。
|
||
if boundary == "start" {
|
||
want := 0
|
||
if after {
|
||
want = 1
|
||
}
|
||
if f.count("weave_instances", "1=1") != want || f.count("pulse_tasks", "1=1") != want {
|
||
t.Fatal("partial start transaction")
|
||
}
|
||
original, err = api.Start(ctx, f.token, key, input)
|
||
if err != nil || original.Deduplicated != after {
|
||
t.Fatal("original start identity did not reconcile unknown COMMIT", err)
|
||
}
|
||
ref := retryStart(t, f)
|
||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||
} else {
|
||
wantTasks := 1
|
||
if after && (boundary == "successor" || boundary == "poll_next") {
|
||
wantTasks = 2
|
||
}
|
||
if f.count("pulse_tasks", "1=1") != wantTasks {
|
||
t.Fatal("partial or duplicated successor transaction")
|
||
}
|
||
if boundary != "poll_next" {
|
||
want := 0
|
||
if after {
|
||
want = 1
|
||
}
|
||
if f.count("weave_inbox", "source_task_id=? AND state='PROCESSED'", firstTask) != want {
|
||
t.Fatal("inbox ACK disagrees with COMMIT")
|
||
}
|
||
}
|
||
if boundary == "poll_wait" {
|
||
want := 0
|
||
if after {
|
||
want = 1
|
||
}
|
||
if f.count("weave_poll_states", "current_task_id IS NULL AND next_poll_at IS NOT NULL AND used_rounds=1") != want {
|
||
t.Fatal("PENDING result and wait were not atomic")
|
||
}
|
||
}
|
||
if boundary == "poll_next" {
|
||
want := 1
|
||
if after {
|
||
want = 2
|
||
}
|
||
if f.count("weave_poll_rounds", "1=1") != want || f.count("weave_poll_states", "used_rounds=?", want) != 1 {
|
||
t.Fatal("new round and budget were not atomic")
|
||
}
|
||
if !after && f.count("weave_poll_states", "next_poll_at=?", originalNext) != 1 {
|
||
t.Fatal("rollback changed original next wake")
|
||
}
|
||
}
|
||
if _, err = newRuntime(t, db).process(ctx, c, false); err != nil {
|
||
t.Fatal("fresh runtime could not reconcile original candidate", err)
|
||
}
|
||
if boundary == "poll_wait" {
|
||
if f.count("weave_poll_states", "next_poll_at=?", expectedNext) != 1 {
|
||
t.Fatal("reconciled wait was timed from retry rather than original result")
|
||
}
|
||
clock := clockCandidate(t, f, original.Instance.ID, "query")
|
||
waitClockDue(t, f, clock)
|
||
runClock(t, f, clock)
|
||
}
|
||
}
|
||
last := retryStart(t, f)
|
||
if strings.HasPrefix(boundary, "poll_") {
|
||
finishPoll(t, f, last, "DONE")
|
||
} else {
|
||
retryFinish(t, f, last, "SUCCEEDED", false)
|
||
runtimeConsume(t, f, newRuntime(t, db), last.TaskID)
|
||
}
|
||
v := runtimeView(t, f, original.Instance.ID, "SUCCEEDED")
|
||
if v.DefinitionVersionID != original.Instance.DefinitionVersionID || !v.DeadlineAt.Equal(original.Instance.DeadlineAt) || f.count("pulse_tasks", "1=1") != 2 || f.count("pulse_attempts", "1=1") != 2 || f.used() != 0 || f.count("weave_inbox", "state='PENDING'") != 0 {
|
||
t.Fatal("COMMIT recovery duplicated work or reset state")
|
||
}
|
||
if strings.HasPrefix(boundary, "poll_") && f.count("weave_poll_states", "used_rounds=2 AND current_task_id IS NULL AND resolved_at IS NOT NULL") != 1 {
|
||
t.Fatal("COMMIT recovery lost final query budget")
|
||
}
|
||
if strings.HasPrefix(boundary, "poll_") && f.count("weave_poll_states", "deadline_at=? AND max_rounds=3 AND max_attempts_per_round=1", pollDeadline) != 1 {
|
||
t.Fatal("COMMIT recovery reset original poll deadline or attempt budget")
|
||
}
|
||
replay, err := api.Start(ctx, f.token, key, input)
|
||
if err != nil || !replay.Deduplicated || replay.Instance.ID != original.Instance.ID || !replay.Instance.DeadlineAt.Equal(original.Instance.DeadlineAt) {
|
||
t.Fatal("start receipt changed after completion", err)
|
||
}
|
||
t.Logf("%s: real TCP COMMIT %s cut; same identity/candidate resumed; exactly two tasks/attempts and original deadline", boundary, mode)
|
||
})
|
||
}
|
||
}
|
||
}
|