orbit/internal/storage/mysql/task_recovery_integration_test.go

package mysql

import (
	"context"
	"encoding/json"
	"os"
	"reflect"
	"sync"
	"testing"
	"time"

	"git.yuntuo.hk/k2safe/orbit/internal/fault"
	"git.yuntuo.hk/k2safe/orbit/platform/events"
	pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
	"git.yuntuo.hk/k2safe/orbit/pulse/service"
	"git.yuntuo.hk/k2safe/orbit/pulse/worker"
)

func grantTaskRecovery(f *executionFixture) {
	f.exec(`UPDATE orbit_credentials SET permissions=JSON_SET(permissions,'$.actions',JSON_ARRAY_APPEND(JSON_EXTRACT(permissions,'$.actions'),'$','tasks:review','$','tasks:retry','$','tasks:audit:read')) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, f.resources.CredentialID)
}
func unknownTask(t *testing.T, f *executionFixture) (service.TaskView, worker.AttemptRef) {
	t.Helper()
	v := f.submit(1)
	ref := retryStart(t, f)
	retryFinish(t, f, ref, "UNKNOWN", false)
	v = f.get(v.ID)
	if v.State != pulse.TaskNeedsReview || !v.EffectUncertain {
		t.Fatal("fixture did not reach uncertainty")
	}
	return v, ref
}
func reviewInput(v service.TaskView, outcome pulse.TaskState) service.ReviewInput {
	return service.ReviewInput{ExpectedRevision: v.Revision, Reason: "operator verified execution and external result", EvidenceRef: "evidence:reconciliation-001", ExecutionSettled: true, Outcome: outcome, Result: json.RawMessage(`{"verified":true,"n":9007199254740993}`)}
}
func manualRetryInput(v service.TaskView) service.ManualRetryInput {
	return service.ManualRetryInput{ExpectedRevision: v.Revision, Reason: "operator confirmed safe same-operation retry", EvidenceRef: "evidence:safe-retry-001", SafeToRetry: true, TaskTimeoutMS: 600000}
}

func TestIntegrationTaskRecovery(t *testing.T) {
	dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
	if dsn == "" {
		t.Skip("requires disposable MySQL")
	}
	ctx, stop := context.WithTimeout(context.Background(), 90*time.Second)
	defer stop()
	db, err := Open(ctx, dsn, 32, 8)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()
	if err := Migrate(ctx, db); err != nil {
		t.Fatal(err)
	}
	fixture := func(t *testing.T) *executionFixture {
		f := newExecutionFixture(t, ctx, db)
		grantTaskRecovery(f)
		return f
	}
	t.Run("review_preserves_attempts_and_publishes_new_authoritative_result", func(t *testing.T) {
		f := newEventFixture(t, ctx, db)
		grantTaskRecovery(f.executionFixture)
		sub := f.create(events.CreateInput{})
		v, ref := unknownTask(t, f.executionFixture)
		before, err := f.tasks.GetTaskAttempt(ctx, f.token, v.ID, ref.AttemptID)
		if err != nil {
			t.Fatal(err)
		}
		input := reviewInput(v, pulse.TaskSucceeded)
		out, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", input)
		if err != nil || out.Task.State != pulse.TaskSucceeded || out.Task.EffectUncertain || out.Task.AttemptCount != v.AttemptCount || out.Task.Revision != v.Revision+1 || string(out.Task.Result) != string(input.Result) {
			t.Fatal(out, err)
		}
		after, err := f.tasks.GetTaskAttempt(ctx, f.token, v.ID, ref.AttemptID)
		if err != nil || !reflect.DeepEqual(before, after) {
			t.Fatal("review rewrote worker history", err)
		}
		assertQueueCount(t, f.executionFixture, 0)
		assertApplicationCounts(t, f.executionFixture, 0, 0, 0)
		replay, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", input)
		if err != nil || !replay.Deduplicated || replay.AuditID != out.AuditID || replay.Task.Revision != out.Task.Revision {
			t.Fatal("review identity changed", err)
		}
		input.Outcome = pulse.TaskFailed
		_, err = f.tasks.ReviewTask(ctx, f.token, v.ID, "review", input)
		expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
		audit, err := f.tasks.GetTaskOperation(ctx, f.token, v.ID, out.AuditID)
		if err != nil || audit.EvidenceRef != "evidence:reconciliation-001" || audit.ActorID != string(f.token.CredentialID) || audit.Action != "task.review" {
			t.Fatal(audit, err)
		}
		// 旧UNKNOWN结果仍作为原快照交付；核查终态成为下一条独立不可变事件。
		g := f.grant(sub.ID)
		if g.Event.Type != "task.needs_review" || !g.Event.Snapshot.EffectUncertain {
			t.Fatal("old snapshot was rewritten", g.Event)
		}
		f.ack(sub.ID, g)
		g = f.grant(sub.ID)
		if g.Event.Type != "task.succeeded" || g.Event.Snapshot.EffectUncertain || g.Event.Snapshot.Revision != out.Task.Revision {
			t.Fatal("review result not journaled", g.Event)
		}
		f.ack(sub.ID, g)
		late := retryFinish(t, f.executionFixture, ref, "SUCCEEDED", false)
		if late.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(v.ID).Revision != out.Task.Revision {
			t.Fatal("late worker result overwrote review")
		}
		if f.count("orbit_audit_events", "resource_id=? AND action='task.review'", v.ID) != 1 {
			t.Fatal("duplicate review audit")
		}
	})
	t.Run("review_authority_is_explicit_and_history_is_scoped", func(t *testing.T) {
		f := newExecutionFixture(t, ctx, db)
		v, _ := unknownTask(t, f)
		_, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
		expectWorkerCode(t, err, "PERMISSION_DENIED")
		_, err = f.tasks.ListTaskOperations(ctx, f.token, v.ID, service.HistoryPageInput{})
		expectWorkerCode(t, err, "PERMISSION_DENIED")
		grantTaskRecovery(f)
		out, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
		if err != nil {
			t.Fatal(err)
		}
		g := fixture(t)
		_, err = g.tasks.GetTaskOperation(ctx, g.token, v.ID, out.AuditID)
		expectWorkerCode(t, err, "NOT_FOUND")
		other := f.submit(1)
		_, err = f.tasks.GetTaskOperation(ctx, f.token, other.ID, out.AuditID)
		expectWorkerCode(t, err, "NOT_FOUND")
		page, err := f.tasks.ListTaskOperations(ctx, f.token, v.ID, service.HistoryPageInput{Limit: 1})
		if err != nil || len(page.Items) != 1 || page.NextCursor == "" {
			t.Fatal(page, err)
		}
		next, err := f.tasks.ListTaskOperations(ctx, f.token, v.ID, service.HistoryPageInput{Limit: 1, Cursor: page.NextCursor})
		if err != nil || len(next.Items) != 1 || next.Items[0].ID == page.Items[0].ID {
			t.Fatal(next, err)
		}
		_, err = f.tasks.ListTaskOperations(ctx, f.token, other.ID, service.HistoryPageInput{Cursor: page.NextCursor})
		expectWorkerCode(t, err, "INVALID_ARGUMENT")
		_, err = f.tasks.ListTaskAttempts(ctx, f.token, v.ID, service.HistoryPageInput{Cursor: page.NextCursor})
		expectWorkerCode(t, err, "INVALID_ARGUMENT")
		f.exec(`UPDATE orbit_credentials SET permissions=JSON_SET(permissions,'$.queues',JSON_ARRAY(?)) WHERE namespace_id=? AND id=?`, string(g.resources.QueueID), f.resources.NamespaceID, f.resources.CredentialID)
		_, err = f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
		expectWorkerCode(t, err, "NOT_FOUND")
	})
	t.Run("review_cas_competition_and_atomic_audit_failure", func(t *testing.T) {
		f := fixture(t)
		v, _ := unknownTask(t, f)
		input := reviewInput(v, pulse.TaskFailed)
		trigger := "review_audit_" + string(executionID(t))
		f.exec("CREATE TRIGGER " + trigger + " BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='controlled review audit failure'")
		_, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", input)
		f.exec("DROP TRIGGER " + trigger)
		if err == nil || !reflect.DeepEqual(v, f.get(v.ID)) {
			t.Fatal("failed audit did not roll back review", err)
		}
		assertQueueCount(t, f, 1)
		assertApplicationCounts(t, f, 1, 0, 0)
		if f.count("orbit_idempotency_keys", "operation='task.review'") != 0 || f.count("orbit_result_events", "1=1") != 1 {
			t.Fatal("review leaked identity or result event")
		}
		keys := []string{"review", "competing-review"}
		ch := make(chan error, 2)
		var wg sync.WaitGroup
		for _, key := range keys {
			wg.Add(1)
			go func() { defer wg.Done(); _, err := f.tasks.ReviewTask(ctx, f.token, v.ID, key, input); ch <- err }()
		}
		wg.Wait()
		close(ch)
		success, conflicts := 0, 0
		for err := range ch {
			if err == nil {
				success++
			} else if fault.Code(err) == "STATE_CONFLICT" {
				conflicts++
			} else {
				t.Fatal(err)
			}
		}
		if success != 1 || conflicts != 1 {
			t.Fatal(success, conflicts)
		}
		assertQueueCount(t, f, 0)
	})
	t.Run("retry_new_identity_preserves_source_and_respects_backpressure", func(t *testing.T) {
		f := fixture(t)
		configureResources(t, f, 10, 4, 1, false)
		v, oldRef := unknownTask(t, f)
		_, err := f.tasks.RetryTask(ctx, f.token, v.ID, "retry", manualRetryInput(v))
		expectWorkerCode(t, err, "STATE_CONFLICT")
		review, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
		if err != nil {
			t.Fatal(err)
		}
		v = review.Task
		occupant := f.submit(1)
		in := manualRetryInput(v)
		_, err = f.tasks.RetryTask(ctx, f.token, v.ID, "retry", in)
		expectWorkerCode(t, err, "QUEUE_FULL")
		if f.count("pulse_tasks", "retry_of=?", v.ID) != 0 || f.count("orbit_idempotency_keys", "operation='task.retry'") != 0 {
			t.Fatal("quota rejection left a child")
		}
		if _, err := f.tasks.CancelTask(ctx, f.token, occupant.ID, "free", service.CancelInput{ExpectedRevision: occupant.Revision, Reason: "free retry quota"}); err != nil {
			t.Fatal(err)
		}
		out, err := f.tasks.RetryTask(ctx, f.token, v.ID, "retry", in)
		if err != nil || out.Task.ID == v.ID || out.Task.RetryOf == nil || *out.Task.RetryOf != v.ID || out.Task.AttemptCount != 0 || out.Task.Revision != 1 || out.Task.Retry != v.Retry || string(out.Task.Payload) != string(v.Payload) || !reflect.DeepEqual(v, f.get(v.ID)) {
			t.Fatal(out, err)
		}
		if !out.Task.DeadlineAt.After(out.Task.CreatedAt) || out.Task.DeadlineAt.Sub(out.Task.CreatedAt) != 10*time.Minute {
			t.Fatal("new budget incorrect")
		}
		replay, err := f.tasks.RetryTask(ctx, f.token, v.ID, "retry", in)
		if err != nil || !replay.Deduplicated || replay.Task.ID != out.Task.ID || replay.AuditID != out.AuditID {
			t.Fatal("retry replay changed identity", err)
		}
		_, err = f.tasks.RetryTask(ctx, f.token, v.ID, "another", in)
		expectWorkerCode(t, err, "STATE_CONFLICT")
		late := retryFinish(t, f, oldRef, "SUCCEEDED", false)
		if late.Receipt.Disposition != "EVIDENCE_ONLY" || !reflect.DeepEqual(v, f.get(v.ID)) {
			t.Fatal("old worker changed resolved task")
		}
		assignment, ref := f.claim()
		if assignment.Task.ID != out.Task.ID || assignment.Task.RetryOf == nil || *assignment.Task.RetryOf != v.ID {
			t.Fatal("worker lost retry lineage")
		}
		if _, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref); err != nil {
			t.Fatal(err)
		}
		retryFinish(t, f, ref, "SUCCEEDED", false)
		assertQueueCount(t, f, 0)
	})
	t.Run("retry_competing_keys_and_audit_rollback", func(t *testing.T) {
		f := fixture(t)
		v := f.submit(1)
		cancelled, err := f.tasks.CancelTask(ctx, f.token, v.ID, "cancel", service.CancelInput{ExpectedRevision: v.Revision, Reason: "stop original"})
		if err != nil {
			t.Fatal(err)
		}
		v = cancelled.Task
		trigger := "retry_audit_" + string(executionID(t))
		f.exec("CREATE TRIGGER " + trigger + " BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='controlled retry audit failure'")
		_, err = f.tasks.RetryTask(ctx, f.token, v.ID, "retry", manualRetryInput(v))
		f.exec("DROP TRIGGER " + trigger)
		if err == nil || f.count("pulse_tasks", "retry_of=?", v.ID) != 0 {
			t.Fatal("retry audit failure leaked child", err)
		}
		assertQueueCount(t, f, 0)
		ch := make(chan error, 8)
		var wg sync.WaitGroup
		for i := 0; i < 8; i++ {
			key := string(executionID(t))
			wg.Add(1)
			go func() {
				defer wg.Done()
				_, err := f.tasks.RetryTask(ctx, f.token, v.ID, key, manualRetryInput(v))
				ch <- err
			}()
		}
		wg.Wait()
		close(ch)
		success := 0
		for err := range ch {
			if err == nil {
				success++
			} else if fault.Code(err) != "STATE_CONFLICT" {
				t.Fatal(err)
			}
		}
		if success != 1 || f.count("pulse_tasks", "retry_of=?", v.ID) != 1 || f.count("orbit_audit_events", "action='task.retry'") != 1 {
			t.Fatal("multiple manual children", success)
		}
		assertApplicationCounts(t, f, 1, 0, 0)
	})
	t.Run("review_unblocks_serial_schedule_without_manual_child", func(t *testing.T) {
		f := newScheduleFixture(t, newExecutionFixture(t, ctx, db))
		grantTaskRecovery(f.executionFixture)
		in := f.definition()
		in.Policy.Overlap = "SERIAL"
		plan := f.create(in)
		f.due(plan, 2*time.Second)
		f.tick()
		items := f.occurrences(plan.ID)
		if len(items) != 1 || items[0].TaskID == nil {
			t.Fatal("missing scheduled task")
		}
		ref := retryStart(t, f.executionFixture)
		retryFinish(t, f.executionFixture, ref, "UNKNOWN", false)
		v := f.get(*items[0].TaskID)
		f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, plan.ID)
		f.tick()
		if len(f.occurrences(plan.ID)) != 1 {
			t.Fatal("unknown execution did not block serial schedule")
		}
		out, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
		if err != nil {
			t.Fatal(err)
		}
		_, err = f.tasks.RetryTask(ctx, f.token, v.ID, "retry", manualRetryInput(out.Task))
		expectWorkerCode(t, err, "STATE_CONFLICT")
		f.exec(`UPDATE pulse_schedules SET check_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, plan.ID)
		f.tick()
		if len(f.occurrences(plan.ID)) != 2 {
			t.Fatal("review did not release serial schedule")
		}
	})
	t.Run("recovery_rejects_gates_bad_owner_active_and_success", func(t *testing.T) {
		f := fixture(t)
		waiting := f.submit(1)
		_, err := f.tasks.ReviewTask(ctx, f.token, waiting.ID, "waiting", reviewInput(waiting, pulse.TaskFailed))
		expectWorkerCode(t, err, "STATE_CONFLICT")
		ref := retryStart(t, f)
		active := f.get(waiting.ID)
		_, err = f.tasks.ReviewTask(ctx, f.token, active.ID, "active", reviewInput(active, pulse.TaskFailed))
		expectWorkerCode(t, err, "STATE_CONFLICT")
		retryFinish(t, f, ref, "SUCCEEDED", false)
		success := f.get(waiting.ID)
		_, err = f.tasks.RetryTask(ctx, f.token, success.ID, "retry", manualRetryInput(success))
		expectWorkerCode(t, err, "STATE_CONFLICT")
		v, _ := unknownTask(t, f)
		f.exec(`UPDATE pulse_tasks SET gate_id=? WHERE namespace_id=? AND id=?`, executionID(t), f.resources.NamespaceID, v.ID)
		_, err = f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
		expectWorkerCode(t, err, "STATE_CONFLICT")
		f.exec(`UPDATE pulse_tasks SET gate_id=NULL,owner_kind='schedule',owner_id=? WHERE namespace_id=? AND id=?`, executionID(t), f.resources.NamespaceID, v.ID)
		_, err = f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
		expectWorkerCode(t, err, "STATE_CONFLICT")
		if f.get(v.ID).Revision != v.Revision {
			t.Fatal("rejected recovery changed task")
		}
	})
	t.Run("managed_tasks_cannot_bypass_recovery_owner", func(t *testing.T) {
		f := fixture(t)
		v, _ := unknownTask(t, f)
		f.exec(`UPDATE pulse_tasks SET owner_kind='workflow',owner_id=? WHERE namespace_id=? AND id=?`, executionID(t), f.resources.NamespaceID, v.ID)
		_, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
		expectWorkerCode(t, err, "STATE_CONFLICT")
		f.exec(`UPDATE pulse_tasks SET owner_kind=NULL,owner_id=NULL WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, v.ID)
		out, err := f.tasks.ReviewTask(ctx, f.token, v.ID, "review", reviewInput(v, pulse.TaskFailed))
		if err != nil {
			t.Fatal(err)
		}
		f.exec(`UPDATE pulse_tasks SET owner_kind='schedule',owner_id=? WHERE namespace_id=? AND id=?`, executionID(t), f.resources.NamespaceID, v.ID)
		_, err = f.tasks.RetryTask(ctx, f.token, v.ID, "retry", manualRetryInput(out.Task))
		expectWorkerCode(t, err, "STATE_CONFLICT")
	})
}