feat: add audited task review and linked manual retries
This commit is contained in:
345
internal/storage/mysql/task_recovery_integration_test.go
Normal file
345
internal/storage/mysql/task_recovery_integration_test.go
Normal file
@@ -0,0 +1,345 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/events"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
func grantTaskRecovery(f *executionFixture) {
|
||||
f.exec(`UPDATE orbit_credentials SET permissions=JSON_SET(permissions,'$.actions',JSON_ARRAY_APPEND(JSON_EXTRACT(permissions,'$.actions'),'$','tasks:review','$','tasks:retry','$','tasks:audit:read')) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, f.resources.CredentialID)
|
||||
}
|
||||
func unknownTask(t *testing.T, f *executionFixture) (service.TaskView, worker.AttemptRef) {
|
||||
t.Helper()
|
||||
v := f.submit(1)
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "UNKNOWN", false)
|
||||
v = f.get(v.ID)
|
||||
if v.State != pulse.TaskNeedsReview || !v.EffectUncertain {
|
||||
t.Fatal("fixture did not reach uncertainty")
|
||||
}
|
||||
return v, ref
|
||||
}
|
||||
func reviewInput(v service.TaskView, outcome pulse.TaskState) service.ReviewInput {
|
||||
return service.ReviewInput{ExpectedRevision: v.Revision, Reason: "operator verified execution and external result", EvidenceRef: "evidence:reconciliation-001", ExecutionSettled: true, Outcome: outcome, Result: json.RawMessage(`{"verified":true,"n":9007199254740993}`)}
|
||||
}
|
||||
func manualRetryInput(v service.TaskView) service.ManualRetryInput {
|
||||
return service.ManualRetryInput{ExpectedRevision: v.Revision, Reason: "operator confirmed safe same-operation retry", EvidenceRef: "evidence:safe-retry-001", SafeToRetry: true, TaskTimeoutMS: 600000}
|
||||
}
|
||||
|
||||
func TestIntegrationTaskRecovery(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, stop := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer stop()
|
||||
db, err := Open(ctx, dsn, 32, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fixture := func(t *testing.T) *executionFixture {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
grantTaskRecovery(f)
|
||||
return f
|
||||
}
|
||||
t.Run("review_preserves_attempts_and_publishes_new_authoritative_result", func(t *testing.T) {
|
||||
f := newEventFixture(t, ctx, db)
|
||||
grantTaskRecovery(f.executionFixture)
|
||||
sub := f.create(events.CreateInput{})
|
||||
v, ref := unknownTask(t, f.executionFixture)
|
||||
before, err := f.tasks.GetTaskAttempt(ctx, f.token, v.ID, ref.AttemptID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input := reviewInput(v, pulse.TaskSucceeded)
|
||||
out, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", input)
|
||||
if err != nil || out.Task.State != pulse.TaskSucceeded || out.Task.EffectUncertain || out.Task.AttemptCount != v.AttemptCount || out.Task.Revision != v.Revision+1 || string(out.Task.Result) != string(input.Result) {
|
||||
t.Fatal(out, err)
|
||||
}
|
||||
after, err := f.tasks.GetTaskAttempt(ctx, f.token, v.ID, ref.AttemptID)
|
||||
if err != nil || !reflect.DeepEqual(before, after) {
|
||||
t.Fatal("review rewrote worker history", err)
|
||||
}
|
||||
assertQueueCount(t, f.executionFixture, 0)
|
||||
assertApplicationCounts(t, f.executionFixture, 0, 0, 0)
|
||||
replay, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", input)
|
||||
if err != nil || !replay.Deduplicated || replay.AuditID != out.AuditID || replay.Task.Revision != out.Task.Revision {
|
||||
t.Fatal("review identity changed", err)
|
||||
}
|
||||
input.Outcome = pulse.TaskFailed
|
||||
_, err = f.tasks.ReviewTask(ctx, f.token, v.ID, "review", input)
|
||||
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
|
||||
audit, err := f.tasks.GetTaskOperation(ctx, f.token, v.ID, out.AuditID)
|
||||
if err != nil || audit.EvidenceRef != "evidence:reconciliation-001" || audit.ActorID != string(f.token.CredentialID) || audit.Action != "task.review" {
|
||||
t.Fatal(audit, err)
|
||||
}
|
||||
// 旧UNKNOWN结果仍作为原快照交付；核查终态成为下一条独立不可变事件。
|
||||
g := f.grant(sub.ID)
|
||||
if g.Event.Type != "task.needs_review" || !g.Event.Snapshot.EffectUncertain {
|
||||
t.Fatal("old snapshot was rewritten", g.Event)
|
||||
}
|
||||
f.ack(sub.ID, g)
|
||||
g = f.grant(sub.ID)
|
||||
if g.Event.Type != "task.succeeded" || g.Event.Snapshot.EffectUncertain || g.Event.Snapshot.Revision != out.Task.Revision {
|
||||
t.Fatal("review result not journaled", g.Event)
|
||||
}
|
||||
f.ack(sub.ID, g)
|
||||
late := retryFinish(t, f.executionFixture, ref, "SUCCEEDED", false)
|
||||
if late.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(v.ID).Revision != out.Task.Revision {
|
||||
t.Fatal("late worker result overwrote review")
|
||||
}
|
||||
if f.count("orbit_audit_events", "resource_id=? AND action='task.review'", v.ID) != 1 {
|
||||
t.Fatal("duplicate review audit")
|
||||
}
|
||||
})
|
||||
t.Run("review_authority_is_explicit_and_history_is_scoped", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
v, _ := unknownTask(t, f)
|
||||
_, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
|
||||
expectWorkerCode(t, err, "PERMISSION_DENIED")
|
||||
_, err = f.tasks.ListTaskOperations(ctx, f.token, v.ID, service.HistoryPageInput{})
|
||||
expectWorkerCode(t, err, "PERMISSION_DENIED")
|
||||
grantTaskRecovery(f)
|
||||
out, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := fixture(t)
|
||||
_, err = g.tasks.GetTaskOperation(ctx, g.token, v.ID, out.AuditID)
|
||||
expectWorkerCode(t, err, "NOT_FOUND")
|
||||
other := f.submit(1)
|
||||
_, err = f.tasks.GetTaskOperation(ctx, f.token, other.ID, out.AuditID)
|
||||
expectWorkerCode(t, err, "NOT_FOUND")
|
||||
page, err := f.tasks.ListTaskOperations(ctx, f.token, v.ID, service.HistoryPageInput{Limit: 1})
|
||||
if err != nil || len(page.Items) != 1 || page.NextCursor == "" {
|
||||
t.Fatal(page, err)
|
||||
}
|
||||
next, err := f.tasks.ListTaskOperations(ctx, f.token, v.ID, service.HistoryPageInput{Limit: 1, Cursor: page.NextCursor})
|
||||
if err != nil || len(next.Items) != 1 || next.Items[0].ID == page.Items[0].ID {
|
||||
t.Fatal(next, err)
|
||||
}
|
||||
_, err = f.tasks.ListTaskOperations(ctx, f.token, other.ID, service.HistoryPageInput{Cursor: page.NextCursor})
|
||||
expectWorkerCode(t, err, "INVALID_ARGUMENT")
|
||||
_, err = f.tasks.ListTaskAttempts(ctx, f.token, v.ID, service.HistoryPageInput{Cursor: page.NextCursor})
|
||||
expectWorkerCode(t, err, "INVALID_ARGUMENT")
|
||||
f.exec(`UPDATE orbit_credentials SET permissions=JSON_SET(permissions,'$.queues',JSON_ARRAY(?)) WHERE namespace_id=? AND id=?`, string(g.resources.QueueID), f.resources.NamespaceID, f.resources.CredentialID)
|
||||
_, err = f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
|
||||
expectWorkerCode(t, err, "NOT_FOUND")
|
||||
})
|
||||
t.Run("review_cas_competition_and_atomic_audit_failure", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
v, _ := unknownTask(t, f)
|
||||
input := reviewInput(v, pulse.TaskFailed)
|
||||
trigger := "review_audit_" + string(executionID(t))
|
||||
f.exec("CREATE TRIGGER " + trigger + " BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='controlled review audit failure'")
|
||||
_, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", input)
|
||||
f.exec("DROP TRIGGER " + trigger)
|
||||
if err == nil || !reflect.DeepEqual(v, f.get(v.ID)) {
|
||||
t.Fatal("failed audit did not roll back review", err)
|
||||
}
|
||||
assertQueueCount(t, f, 1)
|
||||
assertApplicationCounts(t, f, 1, 0, 0)
|
||||
if f.count("orbit_idempotency_keys", "operation='task.review'") != 0 || f.count("orbit_result_events", "1=1") != 1 {
|
||||
t.Fatal("review leaked identity or result event")
|
||||
}
|
||||
keys := []string{"review", "competing-review"}
|
||||
ch := make(chan error, 2)
|
||||
var wg sync.WaitGroup
|
||||
for _, key := range keys {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); _, err := f.tasks.ReviewTask(ctx, f.token, v.ID, key, input); ch <- err }()
|
||||
}
|
||||
wg.Wait()
|
||||
close(ch)
|
||||
success, conflicts := 0, 0
|
||||
for err := range ch {
|
||||
if err == nil {
|
||||
success++
|
||||
} else if fault.Code(err) == "STATE_CONFLICT" {
|
||||
conflicts++
|
||||
} else {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if success != 1 || conflicts != 1 {
|
||||
t.Fatal(success, conflicts)
|
||||
}
|
||||
assertQueueCount(t, f, 0)
|
||||
})
|
||||
t.Run("retry_new_identity_preserves_source_and_respects_backpressure", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
configureResources(t, f, 10, 4, 1, false)
|
||||
v, oldRef := unknownTask(t, f)
|
||||
_, err := f.tasks.RetryTask(ctx, f.token, v.ID, "retry", manualRetryInput(v))
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
review, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v = review.Task
|
||||
occupant := f.submit(1)
|
||||
in := manualRetryInput(v)
|
||||
_, err = f.tasks.RetryTask(ctx, f.token, v.ID, "retry", in)
|
||||
expectWorkerCode(t, err, "QUEUE_FULL")
|
||||
if f.count("pulse_tasks", "retry_of=?", v.ID) != 0 || f.count("orbit_idempotency_keys", "operation='task.retry'") != 0 {
|
||||
t.Fatal("quota rejection left a child")
|
||||
}
|
||||
if _, err := f.tasks.CancelTask(ctx, f.token, occupant.ID, "free", service.CancelInput{ExpectedRevision: occupant.Revision, Reason: "free retry quota"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := f.tasks.RetryTask(ctx, f.token, v.ID, "retry", in)
|
||||
if err != nil || out.Task.ID == v.ID || out.Task.RetryOf == nil || *out.Task.RetryOf != v.ID || out.Task.AttemptCount != 0 || out.Task.Revision != 1 || out.Task.Retry != v.Retry || string(out.Task.Payload) != string(v.Payload) || !reflect.DeepEqual(v, f.get(v.ID)) {
|
||||
t.Fatal(out, err)
|
||||
}
|
||||
if !out.Task.DeadlineAt.After(out.Task.CreatedAt) || out.Task.DeadlineAt.Sub(out.Task.CreatedAt) != 10*time.Minute {
|
||||
t.Fatal("new budget incorrect")
|
||||
}
|
||||
replay, err := f.tasks.RetryTask(ctx, f.token, v.ID, "retry", in)
|
||||
if err != nil || !replay.Deduplicated || replay.Task.ID != out.Task.ID || replay.AuditID != out.AuditID {
|
||||
t.Fatal("retry replay changed identity", err)
|
||||
}
|
||||
_, err = f.tasks.RetryTask(ctx, f.token, v.ID, "another", in)
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
late := retryFinish(t, f, oldRef, "SUCCEEDED", false)
|
||||
if late.Receipt.Disposition != "EVIDENCE_ONLY" || !reflect.DeepEqual(v, f.get(v.ID)) {
|
||||
t.Fatal("old worker changed resolved task")
|
||||
}
|
||||
assignment, ref := f.claim()
|
||||
if assignment.Task.ID != out.Task.ID || assignment.Task.RetryOf == nil || *assignment.Task.RetryOf != v.ID {
|
||||
t.Fatal("worker lost retry lineage")
|
||||
}
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
assertQueueCount(t, f, 0)
|
||||
})
|
||||
t.Run("retry_competing_keys_and_audit_rollback", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
v := f.submit(1)
|
||||
cancelled, err := f.tasks.CancelTask(ctx, f.token, v.ID, "cancel", service.CancelInput{ExpectedRevision: v.Revision, Reason: "stop original"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v = cancelled.Task
|
||||
trigger := "retry_audit_" + string(executionID(t))
|
||||
f.exec("CREATE TRIGGER " + trigger + " BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='controlled retry audit failure'")
|
||||
_, err = f.tasks.RetryTask(ctx, f.token, v.ID, "retry", manualRetryInput(v))
|
||||
f.exec("DROP TRIGGER " + trigger)
|
||||
if err == nil || f.count("pulse_tasks", "retry_of=?", v.ID) != 0 {
|
||||
t.Fatal("retry audit failure leaked child", err)
|
||||
}
|
||||
assertQueueCount(t, f, 0)
|
||||
ch := make(chan error, 8)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 8; i++ {
|
||||
key := string(executionID(t))
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, err := f.tasks.RetryTask(ctx, f.token, v.ID, key, manualRetryInput(v))
|
||||
ch <- err
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(ch)
|
||||
success := 0
|
||||
for err := range ch {
|
||||
if err == nil {
|
||||
success++
|
||||
} else if fault.Code(err) != "STATE_CONFLICT" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if success != 1 || f.count("pulse_tasks", "retry_of=?", v.ID) != 1 || f.count("orbit_audit_events", "action='task.retry'") != 1 {
|
||||
t.Fatal("multiple manual children", success)
|
||||
}
|
||||
assertApplicationCounts(t, f, 1, 0, 0)
|
||||
})
|
||||
t.Run("review_unblocks_serial_schedule_without_manual_child", func(t *testing.T) {
|
||||
f := newScheduleFixture(t, newExecutionFixture(t, ctx, db))
|
||||
grantTaskRecovery(f.executionFixture)
|
||||
in := f.definition()
|
||||
in.Policy.Overlap = "SERIAL"
|
||||
plan := f.create(in)
|
||||
f.due(plan, 2*time.Second)
|
||||
f.tick()
|
||||
items := f.occurrences(plan.ID)
|
||||
if len(items) != 1 || items[0].TaskID == nil {
|
||||
t.Fatal("missing scheduled task")
|
||||
}
|
||||
ref := retryStart(t, f.executionFixture)
|
||||
retryFinish(t, f.executionFixture, ref, "UNKNOWN", false)
|
||||
v := f.get(*items[0].TaskID)
|
||||
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, plan.ID)
|
||||
f.tick()
|
||||
if len(f.occurrences(plan.ID)) != 1 {
|
||||
t.Fatal("unknown execution did not block serial schedule")
|
||||
}
|
||||
out, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = f.tasks.RetryTask(ctx, f.token, v.ID, "retry", manualRetryInput(out.Task))
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, plan.ID)
|
||||
f.tick()
|
||||
if len(f.occurrences(plan.ID)) != 2 {
|
||||
t.Fatal("review did not release serial schedule")
|
||||
}
|
||||
})
|
||||
t.Run("recovery_rejects_gates_bad_owner_active_and_success", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
waiting := f.submit(1)
|
||||
_, err := f.tasks.ReviewTask(ctx, f.token, waiting.ID, "waiting", reviewInput(waiting, pulse.TaskFailed))
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
ref := retryStart(t, f)
|
||||
active := f.get(waiting.ID)
|
||||
_, err = f.tasks.ReviewTask(ctx, f.token, active.ID, "active", reviewInput(active, pulse.TaskFailed))
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
success := f.get(waiting.ID)
|
||||
_, err = f.tasks.RetryTask(ctx, f.token, success.ID, "retry", manualRetryInput(success))
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
v, _ := unknownTask(t, f)
|
||||
f.exec(`UPDATE pulse_tasks SET gate_id=? WHERE namespace_id=? AND id=?`, executionID(t), f.resources.NamespaceID, v.ID)
|
||||
_, err = f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
f.exec(`UPDATE pulse_tasks SET gate_id=NULL,owner_kind='schedule',owner_id=? WHERE namespace_id=? AND id=?`, executionID(t), f.resources.NamespaceID, v.ID)
|
||||
_, err = f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
if f.get(v.ID).Revision != v.Revision {
|
||||
t.Fatal("rejected recovery changed task")
|
||||
}
|
||||
})
|
||||
t.Run("managed_tasks_cannot_bypass_recovery_owner", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
v, _ := unknownTask(t, f)
|
||||
f.exec(`UPDATE pulse_tasks SET owner_kind='workflow',owner_id=? WHERE namespace_id=? AND id=?`, executionID(t), f.resources.NamespaceID, v.ID)
|
||||
_, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
f.exec(`UPDATE pulse_tasks SET owner_kind=NULL,owner_id=NULL WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
|
||||
out, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.exec(`UPDATE pulse_tasks SET owner_kind='schedule',owner_id=? WHERE namespace_id=? AND id=?`, executionID(t), f.resources.NamespaceID, v.ID)
|
||||
_, err = f.tasks.RetryTask(ctx, f.token, v.ID, "retry", manualRetryInput(out.Task))
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user