test: verify workflow version isolation and pause recovery races
This commit is contained in:
390
internal/storage/mysql/workflow_races_integration_test.go
Normal file
390
internal/storage/mysql/workflow_races_integration_test.go
Normal file
@@ -0,0 +1,390 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/control"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/definition"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/instance"
|
||||
"git.yuntuo.hk/k2safe/orbit/weave/signal"
|
||||
)
|
||||
|
||||
func workflowRaceNumber(raw json.RawMessage, want string) bool {
|
||||
var data struct {
|
||||
N json.Number `json:"n"`
|
||||
}
|
||||
return json.Unmarshal(raw, &data) == nil && data.N.String() == want
|
||||
}
|
||||
|
||||
// 先锁住真实应用额度行，再确认所有竞争事务进入InnoDB锁等待。
|
||||
// 不用goroutine启动次数或sleep推断事务已重叠；不修改业务时间和状态。
|
||||
func competeWorkflow(t *testing.T, f *executionFixture, beforeRelease func(), operations ...func() error) []error {
|
||||
t.Helper()
|
||||
guard, err := f.db.BeginTx(f.ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer guard.Rollback()
|
||||
var connection uint64
|
||||
if err = guard.QueryRowContext(f.ctx, `SELECT CONNECTION_ID()`).Scan(&connection); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = lockApplicationResources(f.ctx, guard, f.resources.NamespaceID, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
results := make([]chan error, len(operations))
|
||||
for i, operation := range operations {
|
||||
results[i] = make(chan error, 1)
|
||||
go func(ch chan error, run func() error) { ch <- run() }(results[i], operation)
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case early := <-results[i]:
|
||||
t.Fatalf("operation %d returned before entering the database race: %v", i, early)
|
||||
default:
|
||||
}
|
||||
var waiting int
|
||||
err = f.db.QueryRowContext(f.ctx, `SELECT COUNT(DISTINCT w.REQUESTING_THREAD_ID) FROM performance_schema.data_lock_waits w JOIN performance_schema.threads b ON b.THREAD_ID=w.BLOCKING_THREAD_ID WHERE b.PROCESSLIST_ID=?`, connection).Scan(&waiting)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if waiting == i+1 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("only %d/%d competing transactions reached the held database lock", waiting, i+1)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
if beforeRelease != nil {
|
||||
beforeRelease()
|
||||
}
|
||||
if err = guard.Commit(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
errs := make([]error, len(results))
|
||||
for i, ch := range results {
|
||||
select {
|
||||
case errs[i] = <-ch:
|
||||
case <-f.ctx.Done():
|
||||
t.Fatal(f.ctx.Err())
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func TestIntegrationWorkflowVersionAndRaces(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
db, err := Open(ctx, dsn, 32, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err = Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Run("v1_waiting_then_v2_published_old_successor_executes", func(t *testing.T) {
|
||||
f, v1, _ := signalStart(t, ctx, db, false)
|
||||
before := clockCandidate(t, f, v1.ID, "signal")
|
||||
if runClock(t, f, before) || f.count("pulse_tasks", "1=1") != 0 {
|
||||
t.Fatal("v1 did not wait")
|
||||
}
|
||||
changed := signalSpec(f.resources.QueueID, false)
|
||||
changed.TimeoutMS = 60000
|
||||
changed.Body.Steps[1].Task.HandlerVersion = "v2"
|
||||
changed.Body.Steps[1].Task.Input = definition.Value{Kind: "LITERAL", Literal: json.RawMessage(`{"n":7}`)}
|
||||
if _, err := definition.New(NewWorkflowDefinitionStore(db)).Publish(ctx, f.token, v1.DefinitionID, "publish-v2", definition.PublishInput{Version: "v2", Specification: changed, Reason: "new successor semantics while v1 waits"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sendSignal(t, f, v1, signalInput(v1))
|
||||
runClock(t, f, clockCandidate(t, f, v1.ID, "signal"))
|
||||
assignment, ref := f.claim()
|
||||
if assignment.Task.HandlerVersion != "v1" || !workflowRaceNumber(assignment.Task.Payload, "9007199254740993") {
|
||||
t.Fatal("v1 successor used v2 handler or input")
|
||||
}
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
finished := runtimeView(t, f, v1.ID, "SUCCEEDED")
|
||||
if finished.DefinitionVersionID != v1.DefinitionVersionID || !finished.DeadlineAt.Equal(v1.DeadlineAt) || !workflowRaceNumber(finished.Output, "9007199254740993") {
|
||||
t.Fatal("v1 fixed contract changed")
|
||||
}
|
||||
second, err := instance.New(NewWorkflowInstanceStore(db)).Start(ctx, f.token, "start-v2", instance.StartInput{DefinitionID: v1.DefinitionID, Version: "v2", Input: v1.Input, BusinessKey: v1.BusinessKey})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v2 := second.Instance
|
||||
sendSignal(t, f, v2, signalInput(v2))
|
||||
runClock(t, f, clockCandidate(t, f, v2.ID, "signal"))
|
||||
f.hello.Capabilities = append(f.hello.Capabilities, worker.Capability{Handler: "object.prepare", HandlerVersion: "v2", PayloadVersion: "v1"})
|
||||
// 能力集合属于Worker进程身份，换版本需注册新进程。
|
||||
f.hello.InstanceID = executionID(t)
|
||||
f.binding, _, err = f.a.Connect(ctx, f.token, f.hello)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assignment, ref = f.claim()
|
||||
if assignment.Task.HandlerVersion != "v2" || !workflowRaceNumber(assignment.Task.Payload, "7") || v2.DefinitionVersionID == v1.DefinitionVersionID {
|
||||
t.Fatal("v2 semantics did not take effect for new instance")
|
||||
}
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{"n":7}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
newFinished := runtimeView(t, f, v2.ID, "SUCCEEDED")
|
||||
if !workflowRaceNumber(newFinished.Output, "7") || !newFinished.DeadlineAt.Equal(v2.DeadlineAt) || f.count("pulse_tasks", "1=1") != 2 || f.used() != 0 {
|
||||
t.Fatal("version executions duplicated or lost output/budget")
|
||||
}
|
||||
})
|
||||
|
||||
for _, kind := range []string{"start", "claim", "successor", "retry"} {
|
||||
for _, pauseFirst := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("pause_%s_pause_queued_first_%t", kind, pauseFirst), func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
spec := runtimeSpec(f.resources.QueueID, 2)
|
||||
spec.Body.Steps[0].Task.Retry.MaxAttempts = 2
|
||||
spec.Body.Steps[0].Task.Retry.InitialDelayMS = 100
|
||||
spec.Body.Steps[0].Task.Retry.MaxDelayMS = 100
|
||||
v := runtimeStart(t, f, spec)
|
||||
grantWorkflowControl(f)
|
||||
var ref worker.AttemptRef
|
||||
var claimed *worker.Assignment
|
||||
var taskDeadline time.Time
|
||||
if kind == "start" {
|
||||
_, ref = f.claim()
|
||||
}
|
||||
if kind == "successor" || kind == "retry" {
|
||||
ref = retryStart(t, f)
|
||||
}
|
||||
if ref.TaskID != "" {
|
||||
taskDeadline = f.get(ref.TaskID).DeadlineAt
|
||||
}
|
||||
if kind == "successor" {
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
}
|
||||
var candidate workflowCandidate
|
||||
if kind == "successor" {
|
||||
candidate = runtimeInbox(t, f, ref.TaskID)
|
||||
}
|
||||
runtime := newRuntime(t, db)
|
||||
command := executionID(t)
|
||||
operation := func() error {
|
||||
var err error
|
||||
switch kind {
|
||||
case "start":
|
||||
_, err = f.a.Start(ctx, f.token, f.binding, command, ref)
|
||||
case "claim":
|
||||
claimed, err = f.a.Claim(ctx, f.token, f.binding)
|
||||
case "successor":
|
||||
_, err = runtime.process(ctx, candidate, false)
|
||||
case "retry":
|
||||
_, err = f.a.Finish(ctx, f.token, f.binding, command, worker.FinishInput{Ref: ref, Outcome: "FAILED", Retryable: true, ErrorCode: "SAFE_RETRY", Result: json.RawMessage(`{"n":9007199254740993}`)})
|
||||
}
|
||||
return err
|
||||
}
|
||||
pauseKey := string(executionID(t))
|
||||
var pauseResult control.Result
|
||||
pause := func() error {
|
||||
var err error
|
||||
pauseResult, err = control.New(NewWorkflowControlStore(db)).Apply(ctx, f.token, v.ID, pauseKey, control.Input{Operation: "PAUSE", ExpectedControlRevision: v.ControlRevision, Reason: "overlapping database transactions"})
|
||||
return err
|
||||
}
|
||||
ops := []func() error{operation, pause}
|
||||
if pauseFirst {
|
||||
ops[0], ops[1] = ops[1], ops[0]
|
||||
}
|
||||
errs := competeWorkflow(t, f, nil, ops...)
|
||||
operationIndex, controlIndex := 0, 1
|
||||
if pauseFirst {
|
||||
operationIndex, controlIndex = 1, 0
|
||||
}
|
||||
if errs[controlIndex] != nil {
|
||||
t.Fatal(errs[controlIndex])
|
||||
}
|
||||
if err := errs[operationIndex]; err != nil && !(kind == "start" && fault.Code(err) == "EXECUTION_PAUSED") {
|
||||
t.Fatal(err)
|
||||
}
|
||||
paused := runtimeView(t, f, v.ID, "RUNNING")
|
||||
if !paused.Paused || paused.ControlRevision != 2 || !paused.DeadlineAt.Equal(v.DeadlineAt) {
|
||||
t.Fatal("pause/deadline not durable")
|
||||
}
|
||||
// 以控制事务冻结的数据库时间校验先后，不用goroutine完成顺序代替授权顺序。
|
||||
pauseTime := pauseResult.Instance.UpdatedAt
|
||||
if f.count("pulse_tasks", "created_at>?", pauseTime) != 0 || f.count("pulse_attempts", "(created_at>? OR authorized_at>?)", pauseTime, pauseTime) != 0 {
|
||||
t.Fatal("task creation, claim or start authorization occurred after PAUSE committed")
|
||||
}
|
||||
if got, err := f.a.Claim(ctx, f.token, f.binding); err != nil || got != nil {
|
||||
t.Fatal("paused workflow claimed work", err)
|
||||
}
|
||||
if kind == "claim" && claimed != nil {
|
||||
ref = worker.AttemptRef{TaskID: claimed.Task.ID, AttemptID: claimed.AttemptID, Epoch: claimed.Epoch, LeaseSecret: claimed.LeaseSecret}
|
||||
}
|
||||
if (kind == "claim" && claimed != nil) || (kind == "start" && f.get(ref.TaskID).State == "LEASED") {
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); fault.Code(err) != "EXECUTION_PAUSED" {
|
||||
t.Fatal("unstarted paused lease authorized", err)
|
||||
}
|
||||
}
|
||||
if kind == "retry" {
|
||||
if f.get(ref.TaskID).AttemptCount != 1 || f.used() != 0 || !f.get(ref.TaskID).DeadlineAt.Equal(taskDeadline) {
|
||||
t.Fatal("retry reset budget or retained permit")
|
||||
}
|
||||
// 等待原退避自然到期，暂停时仍不得产生下一次尝试。
|
||||
time.Sleep(120 * time.Millisecond)
|
||||
if got, err := f.a.Claim(ctx, f.token, f.binding); err != nil || got != nil {
|
||||
t.Fatal("paused retry was claimed", err)
|
||||
}
|
||||
}
|
||||
if kind == "successor" && (f.count("pulse_tasks", "1=1") < 1 || f.count("pulse_tasks", "1=1") > 2) {
|
||||
t.Fatal("successor duplication")
|
||||
}
|
||||
collectedWhilePaused := kind == "start" && f.get(ref.TaskID).State == "RUNNING"
|
||||
if collectedWhilePaused {
|
||||
if f.used() != 1 {
|
||||
t.Fatal("pause released a running permit")
|
||||
}
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
if f.count("pulse_tasks", "1=1") != 1 || f.used() != 0 {
|
||||
t.Fatal("paused result spawned successor or retained permit")
|
||||
}
|
||||
}
|
||||
t.Logf("overlapping pause/%s; pause queued first=%t; operation error=%v; tasks=%d; running result collected=%t", kind, pauseFirst, errs[operationIndex], f.count("pulse_tasks", "1=1"), collectedWhilePaused)
|
||||
applyControl(t, f, paused, "RESUME")
|
||||
if kind == "successor" || collectedWhilePaused {
|
||||
runtimeAdvance(t, f, newRuntime(t, db), v.ID)
|
||||
} else {
|
||||
if kind == "retry" || ref.TaskID == "" {
|
||||
_, ref = f.claim()
|
||||
}
|
||||
if f.get(ref.TaskID).State == "LEASED" {
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), ref.TaskID)
|
||||
}
|
||||
last := retryStart(t, f)
|
||||
retryFinish(t, f, last, "SUCCEEDED", false)
|
||||
runtimeConsume(t, f, newRuntime(t, db), last.TaskID)
|
||||
done := runtimeView(t, f, v.ID, "SUCCEEDED")
|
||||
attempts := 2
|
||||
if kind == "retry" {
|
||||
attempts++
|
||||
}
|
||||
if f.count("pulse_tasks", "1=1") != 2 || f.count("pulse_attempts", "1=1") != attempts || f.used() != 0 || !done.DeadlineAt.Equal(v.DeadlineAt) {
|
||||
t.Fatal("resume lost work, reset deadline or duplicated attempt")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 三个实际事务均已进入锁等待后，等待原截止自然到达，再一起放行。
|
||||
// 六种入队次序分别覆盖取消/截止先处理；及时信号与新到信号分开保存。
|
||||
for _, timely := range []bool{false, true} {
|
||||
for _, order := range [][3]int{{0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}} {
|
||||
t.Run(fmt.Sprintf("signal_deadline_cancel_timely_%t_order_%v", timely, order), func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
spec := signalSpec(f.resources.QueueID, false)
|
||||
spec.Body.Steps[0].Signal.TimeoutMS = 1000
|
||||
v := runtimeStart(t, f, spec)
|
||||
grantWorkflowSignal(f)
|
||||
grantWorkflowControl(f)
|
||||
var prior core.ID
|
||||
if timely {
|
||||
prior = sendSignal(t, f, v, signalInput(v)).Signal.ID
|
||||
}
|
||||
clock := clockCandidate(t, f, v.ID, "signal")
|
||||
var due time.Time
|
||||
if err := db.QueryRowContext(ctx, `SELECT deadline_at FROM weave_waits WHERE namespace_id=? AND workflow_id=?`, f.resources.NamespaceID, v.ID).Scan(&due); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
api := signal.New(NewWorkflowSignalStore(db))
|
||||
runtime := newRuntime(t, db)
|
||||
key, cancelKey := string(executionID(t)), string(executionID(t))
|
||||
var arrived signal.Result
|
||||
ops := []func() error{
|
||||
func() error {
|
||||
var err error
|
||||
arrived, err = api.Send(ctx, f.token, v.ID, key, signalInput(v))
|
||||
return err
|
||||
},
|
||||
func() error { _, err := runtime.process(ctx, clock, false); return err },
|
||||
func() error {
|
||||
_, err := control.New(NewWorkflowControlStore(db)).Apply(ctx, f.token, v.ID, cancelKey, control.Input{Operation: "CANCEL", ExpectedControlRevision: 1, Reason: "race natural deadline and signal"})
|
||||
return err
|
||||
},
|
||||
}
|
||||
errs := competeWorkflow(t, f, func() {
|
||||
for {
|
||||
var reached bool
|
||||
if err := db.QueryRowContext(ctx, `SELECT UTC_TIMESTAMP(6)>=?`, due).Scan(&reached); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reached {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}, ops[order[0]], ops[order[1]], ops[order[2]])
|
||||
for _, err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 40 && stopStep(t, f, v.ID); i++ {
|
||||
}
|
||||
current, err := instance.New(NewWorkflowInstanceStore(db)).Get(ctx, f.token, v.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if current.State != "CANCELLED" && current.State != "NEEDS_REVIEW" {
|
||||
t.Fatal("unsettled triple race", current.State)
|
||||
}
|
||||
if current.StopKind == nil || *current.StopKind != "CANCEL" || !current.DeadlineAt.Equal(v.DeadlineAt) || f.used() != 0 {
|
||||
t.Fatal("lost cancellation, deadline or permits")
|
||||
}
|
||||
if f.count("pulse_tasks", "1=1") > 1 || f.count("pulse_attempts", "1=1") != 0 || f.count("weave_signals", "state='PENDING'") != 0 {
|
||||
t.Fatal("triple race duplicated or stranded work")
|
||||
}
|
||||
late, err := api.Get(ctx, f.token, v.ID, arrived.Signal.ID)
|
||||
if err != nil || late.State == "CONSUMED" || late.State == "PENDING" || late.AcceptedAt.Before(due) {
|
||||
t.Fatal("late signal overrode winner or time", err)
|
||||
}
|
||||
if timely {
|
||||
original, err := api.Get(ctx, f.token, v.ID, prior)
|
||||
if err != nil || original.State != "CONSUMED" || !original.AcceptedAt.Before(due) || f.count("weave_waits", "winner_signal_id=?", prior) != 1 {
|
||||
t.Fatal("timely durable signal lost in cancellation/expiry race", err)
|
||||
}
|
||||
} else if f.count("weave_signals", "state='CONSUMED'") != 0 || f.count("pulse_tasks", "1=1") != 0 {
|
||||
t.Fatal("expired signal advanced workflow")
|
||||
}
|
||||
replay, err := api.Send(ctx, f.token, v.ID, key, signalInput(v))
|
||||
if err != nil || !replay.Deduplicated || replay.Signal.ID != arrived.Signal.ID || replay.Signal.State != arrived.Signal.State {
|
||||
t.Fatal("race receipt changed", err)
|
||||
}
|
||||
t.Logf("three overlapping transactions; timely=%t order=%v state=%s late=%s", timely, order, current.State, late.Disposition)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user