package orbit

import (
	"context"
	"encoding/json"
	"errors"
	"reflect"
	"testing"
	"time"
)

// 首次失败已提交并进入退避后，强杀唯一服务进程；两个新进程仅靠持久化状态接管。
// 所有时间自然流逝，不修改数据库的退避、截止或租约字段。
func TestIntegrationWorkerProcessRetryBackoffRecovery(t *testing.T) {
	for _, tc := range []struct {
		name    string
		unknown bool
		expire  bool
		final   TaskState
	}{
		{"known_then_success", false, false, TaskSucceeded},
		{"unknown_then_exhausted", true, false, TaskNeedsReview},
		{"known_deadline_during_outage", false, true, TaskFailed},
		{"unknown_deadline_during_outage", true, true, TaskNeedsReview},
	} {
		t.Run(tc.name, func(t *testing.T) {
			a := startOrbitProcess(t)
			_, _, token := integrationWorkerClient(t)
			api := processClient(t, a, token)
			ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
			defer cancel()
			name := "retry-crash." + workerTestID(t)
			q, err := api.CreateQueue(ctx, name, CreateQueueRequest{Name: name, MaxConcurrency: 2, RatePerSecond: 100})
			if err != nil {
				t.Fatal(err)
			}
			connect := func(client *Client) *WorkerConnection {
				t.Helper()
				r, err := NewWorkerRegistration(q.Queue.ID, 1, []WorkerCapability{{Handler: name, HandlerVersion: "v1", PayloadVersion: "v1"}})
				if err != nil {
					t.Fatal(err)
				}
				w, err := client.DialWorker(ctx, r)
				if err != nil {
					t.Fatal(err)
				}
				t.Cleanup(func() { _ = w.Close() })
				if _, err := w.Capacity(ctx, workerTestID(t), WorkerCapacity{Sequence: 1, Capacity: 1}); err != nil {
					t.Fatal(err)
				}
				return w
			}
			in := SubmitTaskRequest{QueueID: q.Queue.ID, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`), RecoverySafety: RecoveryReconcileOnly, TaskTimeoutMS: 40000, Retry: RetryPolicy{MaxAttempts: 2, AttemptTimeoutMS: 3000, InitialDelayMS: 8000, MaxDelayMS: 8000}}
			if tc.unknown {
				in.RecoverySafety = RecoveryIdempotent
			}
			if tc.expire {
				in.TaskTimeoutMS = 5000
				in.Retry.InitialDelayMS, in.Retry.MaxDelayMS = 1000, 1000
			}
			accepted, err := api.SubmitTask(ctx, workerTestID(t), in)
			if err != nil {
				t.Fatal(err)
			}
			w := connect(api)
			first, err := w.NextAssignment(ctx)
			if err != nil || first.Task.ID != accepted.Task.ID || first.Epoch != 1 {
				t.Fatal("first attempt identity", err)
			}
			if _, err := w.Start(ctx, workerTestID(t), first.Attempt()); err != nil {
				t.Fatal(err)
			}
			finish := WorkerFinish{Ref: first.Attempt(), Outcome: WorkerFailed, Retryable: true, ErrorCode: "TEMPORARY_REJECTION", Result: json.RawMessage(`{"evidence_ref":"diagnostics/first"}`)}
			if tc.unknown {
				finish.Outcome, finish.Retryable, finish.ErrorCode = WorkerUnknown, false, "RESULT_UNKNOWN"
			}
			receipt, err := w.Finish(ctx, workerTestID(t), finish)
			if err != nil || receipt.Receipt.TaskState != TaskWaiting || receipt.Lease.Active {
				t.Fatal("first result did not durably enter backoff", err)
			}
			pending, err := api.GetTask(ctx, accepted.Task.ID)
			if err != nil || pending.State != TaskWaiting || pending.AttemptCount != 1 || pending.EffectUncertain != tc.unknown || !pending.AvailableAt.After(receipt.Lease.ServerTime) {
				t.Fatal("retry facts were not persisted", err)
			}
			old, err := api.GetTaskAttempt(ctx, pending.ID, first.AttemptID)
			if err != nil || old.EndedAt == nil || old.Retryable == tc.unknown || string(old.Result) != string(finish.Result) {
				t.Fatal("first attempt evidence missing", err)
			}
			if delay := pending.AvailableAt.Sub(*old.EndedAt); delay < time.Duration(in.Retry.InitialDelayMS/2)*time.Millisecond || delay > time.Duration(in.Retry.MaxDelayMS)*time.Millisecond {
				t.Fatal("persisted backoff outside policy")
			}
			a.kill(t)
			_ = w.Close()
			if tc.expire {
				// 用已确认的数据库时间差等待，恢复后的状态/历史仍以数据库事实作断言。
				timer := time.NewTimer(pending.DeadlineAt.Sub(receipt.Lease.ServerTime) + 100*time.Millisecond)
				defer timer.Stop()
				select {
				case <-timer.C:
				case <-ctx.Done():
					t.Fatal("outage exceeded test budget")
				}
			}
			b, c := startOrbitProcess(t), startOrbitProcess(t)
			if a.nodeID == b.nodeID || a.nodeID == c.nodeID || b.nodeID == c.nodeID {
				t.Fatal("replacement reused node identity")
			}
			apiB, apiC := processClient(t, b, token), processClient(t, c, token)
			if !tc.expire {
				recovered, err := apiB.GetTask(ctx, pending.ID)
				if err != nil || !reflect.DeepEqual(recovered, pending) {
					t.Fatal("restart changed frozen backoff, budget, deadline or uncertainty", err)
				}
			}
			workers := []*WorkerConnection{connect(apiB), connect(apiC)}
			var current Task
			wantAttempts := uint32(1)
			if !tc.expire {
				type assignmentResult struct {
					index      int
					assignment WorkerAssignment
					err        error
				}
				wait, stop := context.WithCancel(ctx)
				defer stop()
				results := make(chan assignmentResult, 2)
				for i, worker := range workers {
					go func() {
						assignment, err := worker.NextAssignment(wait)
						results <- assignmentResult{i, assignment, err}
					}()
				}
				winner := <-results
				second := winner.assignment
				if winner.err != nil || second.Task.ID != pending.ID || second.AttemptID == first.AttemptID || second.Epoch != 2 {
					t.Fatal("retry did not preserve task identity and consumed budget", winner.err)
				}
				history, err := apiB.GetTaskAttempt(ctx, pending.ID, second.AttemptID)
				if err != nil || history.CreatedAt.Before(pending.AvailableAt) || !history.CreatedAt.Before(pending.DeadlineAt) {
					t.Fatal("retry ignored persisted timing", err)
				}
				owner := workers[winner.index]
				if _, err := owner.Start(ctx, workerTestID(t), second.Attempt()); err != nil {
					t.Fatal(err)
				}
				last := WorkerFinish{Ref: second.Attempt(), Outcome: WorkerSucceeded, Result: json.RawMessage(`{"n":9007199254740993}`)}
				if tc.unknown {
					last.Outcome, last.Retryable, last.ErrorCode = WorkerFailed, true, "TEMPORARY_REJECTION"
				}
				if _, err := owner.Finish(ctx, workerTestID(t), last); err != nil {
					t.Fatal(err)
				}
				stop()
				loser := <-results
				if !errors.Is(loser.err, context.Canceled) {
					t.Fatal("two replacement nodes received a retry assignment", loser.err)
				}
				wantAttempts = 2
			}
			current = waitRuntimeState(t, ctx, apiB, pending.ID, tc.final)
			if current.AttemptCount != wantAttempts || current.EffectUncertain != tc.unknown || current.Retry != pending.Retry || !current.DeadlineAt.Equal(pending.DeadlineAt) || current.RecoverySafety != pending.RecoverySafety || string(current.Payload) != string(pending.Payload) {
				t.Fatal("recovery reset policy, input, deadline, attempts or uncertainty")
			}
			// 两个节点仍在线且都有可用容量，预算/截止应阻止任何额外领取。
			for _, worker := range workers {
				probe, stop := context.WithTimeout(ctx, 250*time.Millisecond)
				_, err := worker.NextAssignment(probe)
				stop()
				if !errors.Is(err, context.DeadlineExceeded) {
					t.Fatal("unexpected assignment after final decision", err)
				}
			}
			attempts, err := apiC.ListTaskAttempts(ctx, pending.ID, HistoryPageRequest{})
			if err != nil || len(attempts.Items) != int(wantAttempts) || attempts.NextCursor != "" {
				t.Fatal("unexpected execution history after failover", err)
			}
			preserved, err := apiC.GetTaskAttempt(ctx, pending.ID, first.AttemptID)
			if err != nil || !reflect.DeepEqual(preserved, old) {
				t.Fatal("restart or subsequent execution rewrote first result", err)
			}
			queue, err := apiC.GetQueue(ctx, q.Queue.ID)
			wantUnfinished := uint64(0)
			if tc.final == TaskNeedsReview {
				wantUnfinished = 1
			}
			if err != nil || queue.UnfinishedTasks != wantUnfinished {
				t.Fatal("recovery lost unfinished-task accounting", err)
			}
			t.Logf("sole node killed; two fresh nodes recovered state=%s attempts=%d uncertainty=%t; original evidence and frozen policy preserved", current.State, wantAttempts, current.EffectUncertain)
		})
	}
}