fix: retry precommit mysql lock conflicts

internal/storage/mysql/db.go

@@ -51,16 +51,56 @@ func Open(ctx context.Context, dsn string, maxOpen, maxIdle int) (*sql.DB, error
 	return db, nil
 }
 
-// WithTx does not replay callbacks: a failed commit may have succeeded remotely.
-// Callers reconcile by stable operation identity before retrying an ambiguous write.
+const transactionRetryLimit = 2
+
+// WithTx retries only a callback failure caused by a MySQL deadlock or lock wait
+// timeout before COMMIT. It never retries Begin/Commit errors because a commit
+// response may be lost after the database has already applied the transaction;
+// callers must reconcile such an ambiguous write by stable operation identity.
 func WithTx(ctx context.Context, db *sql.DB, fn func(*sql.Tx) error) error {
+	for attempt := 0; attempt <= transactionRetryLimit; attempt++ {
+		err, retryable := withTxAttempt(ctx, db, fn)
+		if err == nil || !retryable || attempt == transactionRetryLimit {
+			return err
+		}
+		if err := waitTransactionRetry(ctx, attempt); err != nil {
+			return err
+		}
+	}
+	return errors.New("transaction retry loop exhausted")
+}
+
+func withTxAttempt(ctx context.Context, db *sql.DB, fn func(*sql.Tx) error) (error, bool) {
 	tx, err := db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
 	if err != nil {
-		return err
+		return err, false
 	}
 	defer tx.Rollback()
 	if err := fn(tx); err != nil {
-		return err
+		return err, retryableTransactionError(err)
+	}
+	if err := tx.Commit(); err != nil {
+		return err, false
+	}
+	return nil, false
+}
+
+func retryableTransactionError(err error) bool {
+	var mysqlErr *driver.MySQLError
+	if !errors.As(err, &mysqlErr) {
+		return false
+	}
+	return mysqlErr.Number == 1205 || mysqlErr.Number == 1213
+}
+
+func waitTransactionRetry(ctx context.Context, attempt int) error {
+	delay := time.Duration(attempt+1) * 25 * time.Millisecond
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
 	}
-	return tx.Commit()
 }

internal/storage/mysql/db_retry_integration_test.go

@@ -0,0 +1,82 @@
+package mysql
+
+import (
+	"context"
+	"database/sql"
+	"os"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestIntegrationWithTxRetriesPreCommitDeadlock(t *testing.T) {
+	dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
+	if dsn == "" {
+		t.Skip("requires disposable MySQL")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	db, err := Open(ctx, dsn, 8, 4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	if _, err := db.ExecContext(ctx, `CREATE TABLE orbit_test_tx_retry (
+        id INT NOT NULL COMMENT '测试锁顺序行标识',
+        value INT NOT NULL COMMENT '测试事务累加值',
+        PRIMARY KEY (id)
+    ) ENGINE=InnoDB COMMENT='测试提交前死锁重试表'`); err != nil {
+		t.Fatal(err)
+	}
+	defer db.ExecContext(context.Background(), "DROP TABLE orbit_test_tx_retry")
+	if _, err := db.ExecContext(ctx, "INSERT INTO orbit_test_tx_retry(id,value) VALUES (1,0),(2,0)"); err != nil {
+		t.Fatal(err)
+	}
+
+	var barrier sync.WaitGroup
+	barrier.Add(2)
+	var callbackCalls atomic.Int32
+	run := func(first, second int) error {
+		attempt := 0
+		return WithTx(ctx, db, func(tx *sql.Tx) error {
+			attempt++
+			callbackCalls.Add(1)
+			order := []int{first, second}
+			if attempt > 1 {
+				order = []int{1, 2}
+			}
+			for index, id := range order {
+				if _, err := tx.ExecContext(ctx, "SELECT id FROM orbit_test_tx_retry WHERE id=? FOR UPDATE", id); err != nil {
+					return err
+				}
+				if attempt == 1 && index == 0 {
+					barrier.Done()
+					barrier.Wait()
+				}
+			}
+			_, err := tx.ExecContext(ctx, "UPDATE orbit_test_tx_retry SET value=value+1 WHERE id IN (1,2)")
+			return err
+		})
+	}
+
+	results := make(chan error, 2)
+	go func() { results <- run(1, 2) }()
+	go func() { results <- run(2, 1) }()
+	for i := 0; i < 2; i++ {
+		if err := <-results; err != nil {
+			t.Fatal(err)
+		}
+	}
+	if callbackCalls.Load() < 3 {
+		t.Fatalf("deadlock did not cause a bounded pre-commit retry; callback calls=%d", callbackCalls.Load())
+	}
+	var value int
+	if err := db.QueryRowContext(ctx, "SELECT COALESCE(MIN(value),0) FROM orbit_test_tx_retry").Scan(&value); err != nil {
+		t.Fatal(err)
+	}
+	if value != 2 {
+		t.Fatalf("committed value=%d want 2", value)
+	}
+	t.Logf("pre-commit deadlock recovered with %d callback attempts and two committed transactions", callbackCalls.Load())
+}

internal/storage/mysql/db_test.go

@@ -0,0 +1,38 @@
+package mysql
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+
+	driver "github.com/go-sql-driver/mysql"
+)
+
+func TestRetryableTransactionErrorOnlyMatchesPreCommitLockFailures(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{name: "deadlock", err: &driver.MySQLError{Number: 1213}, want: true},
+		{name: "lock wait timeout", err: fmt.Errorf("wrapped: %w", &driver.MySQLError{Number: 1205}), want: true},
+		{name: "duplicate key", err: &driver.MySQLError{Number: 1062}, want: false},
+		{name: "application error", err: errors.New("validation failed"), want: false},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := retryableTransactionError(test.err); got != test.want {
+				t.Fatalf("retryableTransactionError()=%v want %v", got, test.want)
+			}
+		})
+	}
+}
+
+func TestWaitTransactionRetryHonorsCancellation(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if err := waitTransactionRetry(ctx, 0); !errors.Is(err, context.Canceled) {
+		t.Fatalf("waitTransactionRetry()=%v want context canceled", err)
+	}
+}