package mysql import ( "bytes" "context" "io" "net" "os" "strings" "sync" "sync/atomic" "testing" "time" core "git.yuntuo.hk/k2safe/orbit/model" "git.yuntuo.hk/k2safe/orbit/weave/instance" driver "github.com/go-sql-driver/mysql" ) // commitFaultProxy 只存在于测试二进制。代理原始MySQL包,不保存SQL、凭据或业务正文。 // BEFORE在COMMIT发往MySQL前断线;AFTER收到真实MySQL成功包后、转交驱动前断线。 type commitFaultProxy struct { mode atomic.Int32 reached chan string mu sync.Mutex closed bool active map[net.Conn]struct{} wg sync.WaitGroup } func readFaultPacket(c net.Conn) ([]byte, error) { var header [4]byte if _, err := io.ReadFull(c, header[:]); err != nil { return nil, err } size := int(header[0]) | int(header[1])<<8 | int(header[2])<<16 packet := make([]byte, size+4) copy(packet, header[:]) _, err := io.ReadFull(c, packet[4:]) return packet, err } func newCommitFaultProxy(t *testing.T, dsn string) (*commitFaultProxy, string) { t.Helper() cfg, err := driver.ParseDSN(dsn) if err != nil { t.Fatal("invalid disposable DSN") } host, _, err := net.SplitHostPort(cfg.Addr) if err != nil || cfg.Net != "tcp" || host != "127.0.0.1" || cfg.TLSConfig != "" { t.Fatal("packet faults require disposable plaintext loopback MySQL") } listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } p := &commitFaultProxy{reached: make(chan string, 1), active: map[net.Conn]struct{}{}} p.wg.Add(1) t.Cleanup(func() { listener.Close() p.mu.Lock() p.closed = true for c := range p.active { c.Close() } p.mu.Unlock() p.wg.Wait() }) target := cfg.Addr go func() { defer p.wg.Done() for { down, err := listener.Accept() if err != nil { return } p.mu.Lock() if p.closed { p.mu.Unlock() down.Close() return } p.active[down] = struct{}{} p.mu.Unlock() p.wg.Add(1) go func() { defer p.wg.Done() defer down.Close() defer func() { p.mu.Lock(); delete(p.active, down); p.mu.Unlock() }() up, err := net.DialTimeout("tcp", target, 3*time.Second) if err != nil { return } defer up.Close() var awaiting atomic.Bool clientDone := make(chan struct{}) go func() { defer close(clientDone) defer up.Close() for { packet, err := readFaultPacket(down) if err != nil { return } if len(packet) > 5 && packet[3] == 0 && packet[4] == 3 && strings.EqualFold(strings.TrimSpace(string(packet[5:])), "COMMIT") { switch p.mode.Swap(0) { case 1: p.reached <- "BEFORE" return case 2: awaiting.Store(true) } } if _, err = io.Copy(up, bytes.NewReader(packet)); err != nil { return } } }() for { packet, err := readFaultPacket(up) if err != nil { break } if awaiting.Swap(false) { // COMMIT成功包:序号1、OK、0受影响行、0插入ID,IN_TRANS标志已清除。 if len(packet) >= 11 && packet[3] == 1 && packet[4] == 0 && packet[5] == 0 && packet[6] == 0 && packet[7]&1 == 0 { p.reached <- "AFTER" } else { p.reached <- "UNEXPECTED_COMMIT_RESPONSE" } break } if _, err = io.Copy(down, bytes.NewReader(packet)); err != nil { break } } down.Close() up.Close() <-clientDone }() } }() cfg.Addr = listener.Addr().String() return p, cfg.FormatDSN() } func (p *commitFaultProxy) expect(t *testing.T, after bool, err error) { t.Helper() if err == nil { t.Fatal("COMMIT wire cut was reported as success") } want := "BEFORE" if after { want = "AFTER" } select { case got := <-p.reached: if got != want { t.Fatalf("wrong COMMIT boundary: %s", got) } case <-time.After(time.Second): t.Fatal("COMMIT fault not reached") } } func TestIntegrationWorkflowCommitFaults(t *testing.T) { dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN") if dsn == "" { t.Skip("requires disposable MySQL") } ctx, cancel := context.WithTimeout(context.Background(), 100*time.Second) defer cancel() db, err := Open(ctx, dsn, 24, 6) if err != nil { t.Fatal(err) } defer db.Close() if err = Migrate(ctx, db); err != nil { t.Fatal(err) } for _, after := range []bool{false, true} { mode := "before" if after { mode = "after" } for _, boundary := range []string{"start", "successor", "poll_wait", "poll_next"} { t.Run(boundary+"_"+mode+"_commit", func(t *testing.T) { f := newClockFixture(t, ctx, db) spec := runtimeSpec(f.resources.QueueID, 2) if strings.HasPrefix(boundary, "poll_") { spec = pollingSpec(f.resources.QueueID) } grantWorkflowInstance(t, f) input := publishInstanceSpec(t, f, spec) key := string(executionID(t)) api := instance.New(NewWorkflowInstanceStore(db)) var original instance.StartResult var c workflowCandidate var firstTask core.ID var originalNext, pollDeadline, expectedNext time.Time if boundary != "start" { original, err = api.Start(ctx, f.token, key, input) if err != nil { t.Fatal(err) } ref := retryStart(t, f) firstTask = ref.TaskID if boundary == "successor" { retryFinish(t, f, ref, "SUCCEEDED", false) } else { finishQueryOnly(t, f, ref, "PENDING") } c = runtimeInbox(t, f, ref.TaskID) if strings.HasPrefix(boundary, "poll_") { if err := db.QueryRowContext(ctx, `SELECT deadline_at FROM weave_poll_states WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, original.Instance.ID).Scan(&pollDeadline); err != nil { t.Fatal(err) } var occurred time.Time if err := db.QueryRowContext(ctx, `SELECT e.occurred_at FROM weave_inbox i JOIN orbit_events e ON e.namespace_id=i.namespace_id AND e.id=i.event_id WHERE i.namespace_id=? AND i.id=?`, f.resources.NamespaceID, c.id).Scan(&occurred); err != nil { t.Fatal(err) } expectedNext = minimumTime(occurred.Add(time.Second), pollDeadline) } if boundary == "poll_next" { runtimeConsume(t, f, newRuntime(t, db), ref.TaskID) c = clockCandidate(t, f, original.Instance.ID, "query") if err := db.QueryRowContext(ctx, `SELECT next_poll_at FROM weave_poll_states WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, original.Instance.ID).Scan(&originalNext); err != nil { t.Fatal(err) } waitClockDue(t, f, c) } } proxy, proxiedDSN := newCommitFaultProxy(t, dsn) faultDB, err := Open(ctx, proxiedDSN, 2, 1) if err != nil { t.Fatal(err) } defer faultDB.Close() if after { proxy.mode.Store(2) } else { proxy.mode.Store(1) } if boundary == "start" { _, err = instance.New(NewWorkflowInstanceStore(faultDB)).Start(ctx, f.token, key, input) } else { _, err = newRuntime(t, faultDB).process(ctx, c, false) } proxy.expect(t, after, err) // 回读通过直接连接,与故障连接、其内存回执和连接池完全独立。 if boundary == "start" { want := 0 if after { want = 1 } if f.count("weave_instances", "1=1") != want || f.count("pulse_tasks", "1=1") != want { t.Fatal("partial start transaction") } original, err = api.Start(ctx, f.token, key, input) if err != nil || original.Deduplicated != after { t.Fatal("original start identity did not reconcile unknown COMMIT", err) } ref := retryStart(t, f) retryFinish(t, f, ref, "SUCCEEDED", false) runtimeConsume(t, f, newRuntime(t, db), ref.TaskID) } else { wantTasks := 1 if after && (boundary == "successor" || boundary == "poll_next") { wantTasks = 2 } if f.count("pulse_tasks", "1=1") != wantTasks { t.Fatal("partial or duplicated successor transaction") } if boundary != "poll_next" { want := 0 if after { want = 1 } if f.count("weave_inbox", "source_task_id=? AND state='PROCESSED'", firstTask) != want { t.Fatal("inbox ACK disagrees with COMMIT") } } if boundary == "poll_wait" { want := 0 if after { want = 1 } if f.count("weave_poll_states", "current_task_id IS NULL AND next_poll_at IS NOT NULL AND used_rounds=1") != want { t.Fatal("PENDING result and wait were not atomic") } } if boundary == "poll_next" { want := 1 if after { want = 2 } if f.count("weave_poll_rounds", "1=1") != want || f.count("weave_poll_states", "used_rounds=?", want) != 1 { t.Fatal("new round and budget were not atomic") } if !after && f.count("weave_poll_states", "next_poll_at=?", originalNext) != 1 { t.Fatal("rollback changed original next wake") } } if _, err = newRuntime(t, db).process(ctx, c, false); err != nil { t.Fatal("fresh runtime could not reconcile original candidate", err) } if boundary == "poll_wait" { if f.count("weave_poll_states", "next_poll_at=?", expectedNext) != 1 { t.Fatal("reconciled wait was timed from retry rather than original result") } clock := clockCandidate(t, f, original.Instance.ID, "query") waitClockDue(t, f, clock) runClock(t, f, clock) } } last := retryStart(t, f) if strings.HasPrefix(boundary, "poll_") { finishPoll(t, f, last, "DONE") } else { retryFinish(t, f, last, "SUCCEEDED", false) runtimeConsume(t, f, newRuntime(t, db), last.TaskID) } v := runtimeView(t, f, original.Instance.ID, "SUCCEEDED") if v.DefinitionVersionID != original.Instance.DefinitionVersionID || !v.DeadlineAt.Equal(original.Instance.DeadlineAt) || f.count("pulse_tasks", "1=1") != 2 || f.count("pulse_attempts", "1=1") != 2 || f.used() != 0 || f.count("weave_inbox", "state='PENDING'") != 0 { t.Fatal("COMMIT recovery duplicated work or reset state") } if strings.HasPrefix(boundary, "poll_") && f.count("weave_poll_states", "used_rounds=2 AND current_task_id IS NULL AND resolved_at IS NOT NULL") != 1 { t.Fatal("COMMIT recovery lost final query budget") } if strings.HasPrefix(boundary, "poll_") && f.count("weave_poll_states", "deadline_at=? AND max_rounds=3 AND max_attempts_per_round=1", pollDeadline) != 1 { t.Fatal("COMMIT recovery reset original poll deadline or attempt budget") } replay, err := api.Start(ctx, f.token, key, input) if err != nil || !replay.Deduplicated || replay.Instance.ID != original.Instance.ID || !replay.Instance.DeadlineAt.Equal(original.Instance.DeadlineAt) { t.Fatal("start receipt changed after completion", err) } t.Logf("%s: real TCP COMMIT %s cut; same identity/candidate resumed; exactly two tasks/attempts and original deadline", boundary, mode) }) } } }