feat: add durable relative task delays and outage recovery regression

sdk/go/examples/delayed-task/main.go

@@ -0,0 +1,129 @@
+// delayed-task 演示数据库计时的相对延迟、原键核对和实际 Worker 执行。
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	token, queue := os.Getenv("ORBIT_TOKEN"), os.Getenv("ORBIT_QUEUE_ID")
+	if path := os.Getenv("ORBIT_CREDENTIALS_FILE"); path != "" {
+		raw, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("read private credentials: %w", err)
+		}
+		var credentials struct {
+			Token   string `json:"token"`
+			QueueID string `json:"queue_id"`
+		}
+		if json.Unmarshal(raw, &credentials) != nil {
+			return errors.New("invalid credentials file")
+		}
+		if token == "" {
+			token = credentials.Token
+		}
+		if queue == "" {
+			queue = credentials.QueueID
+		}
+	}
+	client, err := orbit.NewClient(os.Getenv("ORBIT_URL"), orbit.Options{Token: token})
+	if err != nil {
+		return err
+	}
+	op, err := orbit.NewWorkerMessageID()
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	in := orbit.SubmitTaskRequest{QueueID: queue, Handler: "object.delayed." + op, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"object_ref":"examples/input","sequence":9007199254740993}`), DelayMS: 1500, TaskTimeoutMS: 15000, Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: 5000}}
+	accepted, err := client.SubmitTask(ctx, op, in)
+	if err != nil {
+		return err
+	}
+	// 先输出任务身份，后续失败可据此核对；不输出凭据或任意业务正文。
+	fmt.Printf("task=%s delay_ms=%d\n", accepted.Task.ID, in.DelayMS)
+	if !accepted.Task.AvailableAt.Equal(accepted.Task.CreatedAt.Add(1500 * time.Millisecond)) {
+		return errors.New("delay was not based on database acceptance")
+	}
+	w, err := orbit.NewWorker(client, orbit.WorkerOptions{QueueID: queue, Capacity: 1, ShutdownTimeout: 5 * time.Second, Handlers: []orbit.WorkerHandler{{Capability: orbit.WorkerCapability{Handler: in.Handler, HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(ctx context.Context, e orbit.WorkerExecution) (json.RawMessage, error) {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+		// 无外部副作用的示例处理器，保持精确数字文本。
+		return e.Task.Payload, nil
+	}}}})
+	if err != nil {
+		return err
+	}
+	work, stop := context.WithCancel(ctx)
+	done := make(chan error, 1)
+	go func() { done <- w.Run(work) }()
+	waited := false
+	defer func() {
+		stop()
+		if !waited {
+			<-done
+		}
+	}()
+	for {
+		current, err := client.GetTask(ctx, accepted.Task.ID)
+		if err != nil {
+			return err
+		}
+		if current.State == orbit.TaskSucceeded {
+			history, err := client.ListTaskAttempts(ctx, current.ID, orbit.HistoryPageRequest{})
+			if err != nil {
+				return err
+			}
+			if len(history.Items) != 1 || history.Items[0].AuthorizedAt == nil || history.Items[0].CreatedAt.Before(accepted.Task.AvailableAt) || history.Items[0].AuthorizedAt.Before(accepted.Task.AvailableAt) {
+				return errors.New("execution preceded the saved release time")
+			}
+			break
+		}
+		if current.State == orbit.TaskFailed || current.State == orbit.TaskNeedsReview || current.State == orbit.TaskCancelled {
+			return fmt.Errorf("task requires attention: %s", current.State)
+		}
+		select {
+		case err := <-done:
+			waited = true
+			return fmt.Errorf("Worker stopped before completion: %v", err)
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(100 * time.Millisecond):
+		}
+	}
+	duplicate, err := client.SubmitTask(ctx, op, in)
+	if err != nil {
+		return err
+	}
+	if !duplicate.Deduplicated || duplicate.Task.ID != accepted.Task.ID || !duplicate.Task.AvailableAt.Equal(accepted.Task.AvailableAt) {
+		return errors.New("replay changed the original delayed task")
+	}
+	stop()
+	err = <-done
+	waited = true
+	if err != nil {
+		return err
+	}
+	stats := w.Stats()
+	if stats.Started != 1 || stats.ReportsConfirmed != 1 || stats.Running != 0 {
+		return errors.New("Worker did not confirm and drain")
+	}
+	fmt.Println("state=SUCCEEDED attempts=1 no_early_start=true original_receipt=true worker_drained=true")
+	return nil
+}

sdk/go/examples/task-query/main.go

@@ -52,15 +52,32 @@ func run() error {
 	if err != nil {
 		return err
 	}
-	businessKey := "object /+% é " + op + " "
+	businessKey := "对象 /+% é " + op + " "
 	var tasks []orbit.Task
 	for i, key := range []string{businessKey, businessKey, businessKey[:len(businessKey)-1]} {
-		out, err := client.SubmitTask(ctx, fmt.Sprintf("query-%s-%d", op, i), orbit.SubmitTaskRequest{QueueID: queue, Handler: "query." + op, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"object_ref":"examples/input"}`), BusinessKey: key, TaskTimeoutMS: 60000})
+		payload := json.RawMessage(`{"object_ref":"examples/input","format":"text"}`)
+		if i == 1 {
+			payload = json.RawMessage(`{"object_ref":"examples/input","format":"json"}`)
+		}
+		out, err := client.SubmitTask(ctx, fmt.Sprintf("query-%s-%d", op, i), orbit.SubmitTaskRequest{QueueID: queue, Handler: "query." + op, HandlerVersion: "v1", PayloadVersion: "v1", Payload: payload, BusinessKey: key, TaskTimeoutMS: 60000})
 		if err != nil {
 			return err
 		}
 		tasks = append(tasks, out.Task)
 	}
+	// 改参数使用新身份与相同关联键，读取旧任务确认原输入仍保留。
+	for _, task := range tasks[:2] {
+		current, err := client.GetTask(ctx, task.ID)
+		if err != nil {
+			return err
+		}
+		if string(current.Payload) != string(task.Payload) {
+			return errors.New("new linked submission changed existing input")
+		}
+	}
+	if tasks[0].ID == tasks[1].ID || string(tasks[0].Payload) == string(tasks[1].Payload) {
+		return errors.New("parameter change did not create an independent task")
+	}
 	from, before := tasks[0].CreatedAt, tasks[2].CreatedAt.Add(time.Microsecond)
 	query := orbit.ListTasksRequest{QueueID: queue, State: orbit.TaskWaiting, BusinessKey: businessKey, CreatedFrom: &from, CreatedBefore: &before, Limit: 1}
 	matched, pages := 0, 0
@@ -99,6 +116,6 @@ func run() error {
 			return err
 		}
 	}
-	fmt.Printf("matched=%d pages=%d before_excluded=true cancelled=%d\n", matched, pages, len(tasks))
+	fmt.Printf("matched=%d pages=%d before_excluded=true original_payload_preserved=true cancelled=%d\n", matched, pages, len(tasks))
 	return nil
 }

sdk/go/retry_process_integration_test.go

@@ -0,0 +1,192 @@
+package orbit
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"reflect"
+	"testing"
+	"time"
+)
+
+// 首次失败已提交并进入退避后，强杀唯一服务进程；两个新进程仅靠持久化状态接管。
+// 所有时间自然流逝，不修改数据库的退避、截止或租约字段。
+func TestIntegrationWorkerProcessRetryBackoffRecovery(t *testing.T) {
+	for _, tc := range []struct {
+		name    string
+		unknown bool
+		expire  bool
+		final   TaskState
+	}{
+		{"known_then_success", false, false, TaskSucceeded},
+		{"unknown_then_exhausted", true, false, TaskNeedsReview},
+		{"known_deadline_during_outage", false, true, TaskFailed},
+		{"unknown_deadline_during_outage", true, true, TaskNeedsReview},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			a := startOrbitProcess(t)
+			_, _, token := integrationWorkerClient(t)
+			api := processClient(t, a, token)
+			ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
+			defer cancel()
+			name := "retry-crash." + workerTestID(t)
+			q, err := api.CreateQueue(ctx, name, CreateQueueRequest{Name: name, MaxConcurrency: 2, RatePerSecond: 100})
+			if err != nil {
+				t.Fatal(err)
+			}
+			connect := func(client *Client) *WorkerConnection {
+				t.Helper()
+				r, err := NewWorkerRegistration(q.Queue.ID, 1, []WorkerCapability{{Handler: name, HandlerVersion: "v1", PayloadVersion: "v1"}})
+				if err != nil {
+					t.Fatal(err)
+				}
+				w, err := client.DialWorker(ctx, r)
+				if err != nil {
+					t.Fatal(err)
+				}
+				t.Cleanup(func() { _ = w.Close() })
+				if _, err := w.Capacity(ctx, workerTestID(t), WorkerCapacity{Sequence: 1, Capacity: 1}); err != nil {
+					t.Fatal(err)
+				}
+				return w
+			}
+			in := SubmitTaskRequest{QueueID: q.Queue.ID, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`), RecoverySafety: RecoveryReconcileOnly, TaskTimeoutMS: 40000, Retry: RetryPolicy{MaxAttempts: 2, AttemptTimeoutMS: 3000, InitialDelayMS: 8000, MaxDelayMS: 8000}}
+			if tc.unknown {
+				in.RecoverySafety = RecoveryIdempotent
+			}
+			if tc.expire {
+				in.TaskTimeoutMS = 5000
+				in.Retry.InitialDelayMS, in.Retry.MaxDelayMS = 1000, 1000
+			}
+			accepted, err := api.SubmitTask(ctx, workerTestID(t), in)
+			if err != nil {
+				t.Fatal(err)
+			}
+			w := connect(api)
+			first, err := w.NextAssignment(ctx)
+			if err != nil || first.Task.ID != accepted.Task.ID || first.Epoch != 1 {
+				t.Fatal("first attempt identity", err)
+			}
+			if _, err := w.Start(ctx, workerTestID(t), first.Attempt()); err != nil {
+				t.Fatal(err)
+			}
+			finish := WorkerFinish{Ref: first.Attempt(), Outcome: WorkerFailed, Retryable: true, ErrorCode: "TEMPORARY_REJECTION", Result: json.RawMessage(`{"evidence_ref":"diagnostics/first"}`)}
+			if tc.unknown {
+				finish.Outcome, finish.Retryable, finish.ErrorCode = WorkerUnknown, false, "RESULT_UNKNOWN"
+			}
+			receipt, err := w.Finish(ctx, workerTestID(t), finish)
+			if err != nil || receipt.Receipt.TaskState != TaskWaiting || receipt.Lease.Active {
+				t.Fatal("first result did not durably enter backoff", err)
+			}
+			pending, err := api.GetTask(ctx, accepted.Task.ID)
+			if err != nil || pending.State != TaskWaiting || pending.AttemptCount != 1 || pending.EffectUncertain != tc.unknown || !pending.AvailableAt.After(receipt.Lease.ServerTime) {
+				t.Fatal("retry facts were not persisted", err)
+			}
+			old, err := api.GetTaskAttempt(ctx, pending.ID, first.AttemptID)
+			if err != nil || old.EndedAt == nil || old.Retryable == tc.unknown || string(old.Result) != string(finish.Result) {
+				t.Fatal("first attempt evidence missing", err)
+			}
+			if delay := pending.AvailableAt.Sub(*old.EndedAt); delay < time.Duration(in.Retry.InitialDelayMS/2)*time.Millisecond || delay > time.Duration(in.Retry.MaxDelayMS)*time.Millisecond {
+				t.Fatal("persisted backoff outside policy")
+			}
+			a.kill(t)
+			_ = w.Close()
+			if tc.expire {
+				// 用已确认的数据库时间差等待，恢复后的状态/历史仍以数据库事实作断言。
+				timer := time.NewTimer(pending.DeadlineAt.Sub(receipt.Lease.ServerTime) + 100*time.Millisecond)
+				defer timer.Stop()
+				select {
+				case <-timer.C:
+				case <-ctx.Done():
+					t.Fatal("outage exceeded test budget")
+				}
+			}
+			b, c := startOrbitProcess(t), startOrbitProcess(t)
+			if a.nodeID == b.nodeID || a.nodeID == c.nodeID || b.nodeID == c.nodeID {
+				t.Fatal("replacement reused node identity")
+			}
+			apiB, apiC := processClient(t, b, token), processClient(t, c, token)
+			if !tc.expire {
+				recovered, err := apiB.GetTask(ctx, pending.ID)
+				if err != nil || !reflect.DeepEqual(recovered, pending) {
+					t.Fatal("restart changed frozen backoff, budget, deadline or uncertainty", err)
+				}
+			}
+			workers := []*WorkerConnection{connect(apiB), connect(apiC)}
+			var current Task
+			wantAttempts := uint32(1)
+			if !tc.expire {
+				type assignmentResult struct {
+					index      int
+					assignment WorkerAssignment
+					err        error
+				}
+				wait, stop := context.WithCancel(ctx)
+				defer stop()
+				results := make(chan assignmentResult, 2)
+				for i, worker := range workers {
+					go func() {
+						assignment, err := worker.NextAssignment(wait)
+						results <- assignmentResult{i, assignment, err}
+					}()
+				}
+				winner := <-results
+				second := winner.assignment
+				if winner.err != nil || second.Task.ID != pending.ID || second.AttemptID == first.AttemptID || second.Epoch != 2 {
+					t.Fatal("retry did not preserve task identity and consumed budget", winner.err)
+				}
+				history, err := apiB.GetTaskAttempt(ctx, pending.ID, second.AttemptID)
+				if err != nil || history.CreatedAt.Before(pending.AvailableAt) || !history.CreatedAt.Before(pending.DeadlineAt) {
+					t.Fatal("retry ignored persisted timing", err)
+				}
+				owner := workers[winner.index]
+				if _, err := owner.Start(ctx, workerTestID(t), second.Attempt()); err != nil {
+					t.Fatal(err)
+				}
+				last := WorkerFinish{Ref: second.Attempt(), Outcome: WorkerSucceeded, Result: json.RawMessage(`{"n":9007199254740993}`)}
+				if tc.unknown {
+					last.Outcome, last.Retryable, last.ErrorCode = WorkerFailed, true, "TEMPORARY_REJECTION"
+				}
+				if _, err := owner.Finish(ctx, workerTestID(t), last); err != nil {
+					t.Fatal(err)
+				}
+				stop()
+				loser := <-results
+				if !errors.Is(loser.err, context.Canceled) {
+					t.Fatal("two replacement nodes received a retry assignment", loser.err)
+				}
+				wantAttempts = 2
+			}
+			current = waitRuntimeState(t, ctx, apiB, pending.ID, tc.final)
+			if current.AttemptCount != wantAttempts || current.EffectUncertain != tc.unknown || current.Retry != pending.Retry || !current.DeadlineAt.Equal(pending.DeadlineAt) || current.RecoverySafety != pending.RecoverySafety || string(current.Payload) != string(pending.Payload) {
+				t.Fatal("recovery reset policy, input, deadline, attempts or uncertainty")
+			}
+			// 两个节点仍在线且都有可用容量，预算/截止应阻止任何额外领取。
+			for _, worker := range workers {
+				probe, stop := context.WithTimeout(ctx, 250*time.Millisecond)
+				_, err := worker.NextAssignment(probe)
+				stop()
+				if !errors.Is(err, context.DeadlineExceeded) {
+					t.Fatal("unexpected assignment after final decision", err)
+				}
+			}
+			attempts, err := apiC.ListTaskAttempts(ctx, pending.ID, HistoryPageRequest{})
+			if err != nil || len(attempts.Items) != int(wantAttempts) || attempts.NextCursor != "" {
+				t.Fatal("unexpected execution history after failover", err)
+			}
+			preserved, err := apiC.GetTaskAttempt(ctx, pending.ID, first.AttemptID)
+			if err != nil || !reflect.DeepEqual(preserved, old) {
+				t.Fatal("restart or subsequent execution rewrote first result", err)
+			}
+			queue, err := apiC.GetQueue(ctx, q.Queue.ID)
+			wantUnfinished := uint64(0)
+			if tc.final == TaskNeedsReview {
+				wantUnfinished = 1
+			}
+			if err != nil || queue.UnfinishedTasks != wantUnfinished {
+				t.Fatal("recovery lost unfinished-task accounting", err)
+			}
+			t.Logf("sole node killed; two fresh nodes recovered state=%s attempts=%d uncertainty=%t; original evidence and frozen policy preserved", current.State, wantAttempts, current.EffectUncertain)
+		})
+	}
+}

sdk/go/task_delay_integration_test.go

@@ -0,0 +1,66 @@
+package orbit
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+)
+
+func TestIntegrationWorkerRuntimeDelayedSubmission(t *testing.T) {
+	api, queue, _ := integrationWorkerClient(t)
+	ctx, stop := context.WithTimeout(context.Background(), 20*time.Second)
+	defer stop()
+	name := "delay." + workerTestID(t)
+	key := workerTestID(t)
+	in := SubmitTaskRequest{QueueID: queue, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`), DelayMS: 1500, TaskTimeoutMS: 15000, Retry: RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: 5000}}
+	delayed, err := api.SubmitTask(ctx, key, in)
+	if err != nil || !delayed.Task.AvailableAt.Equal(delayed.Task.CreatedAt.Add(1500*time.Millisecond)) {
+		t.Fatal("relative delay not persisted", err)
+	}
+	at := delayed.Task.CreatedAt.Add(3 * time.Second)
+	absoluteInput := in
+	absoluteInput.DelayMS, absoluteInput.AvailableAt = 0, &at
+	absolute, err := api.SubmitTask(ctx, workerTestID(t), absoluteInput)
+	if err != nil || !absolute.Task.AvailableAt.Equal(at) {
+		t.Fatal("absolute release time changed", err)
+	}
+	r, err := NewWorkerRegistration(queue, 1, []WorkerCapability{{Handler: name, HandlerVersion: "v1", PayloadVersion: "v1"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	w, err := api.DialWorker(ctx, r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer w.Close()
+	if _, err := w.Capacity(ctx, workerTestID(t), WorkerCapacity{Sequence: 1, Capacity: 1}); err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []Task{delayed.Task, absolute.Task} {
+		a, err := w.NextAssignment(ctx)
+		if err != nil || a.Task.ID != want.ID || a.ServerTime.Before(want.AvailableAt) || a.Epoch != 1 {
+			t.Fatal("task dispatched before its persisted release time", err)
+		}
+		if _, err := w.Start(ctx, workerTestID(t), a.Attempt()); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := w.Finish(ctx, workerTestID(t), WorkerFinish{Ref: a.Attempt(), Outcome: WorkerSucceeded, Result: a.Task.Payload}); err != nil {
+			t.Fatal(err)
+		}
+		history, err := api.GetTaskAttempt(ctx, want.ID, a.AttemptID)
+		if err != nil || history.CreatedAt.Before(want.AvailableAt) || history.AuthorizedAt == nil || history.AuthorizedAt.Before(want.AvailableAt) {
+			t.Fatal("persisted history allowed an early start", err)
+		}
+	}
+	// 原等待时长已经经过，原键重报仍返回首次回执，不能建立新的计时或任务。
+	dupe, err := api.SubmitTask(ctx, key, in)
+	if err != nil || !dupe.Deduplicated || dupe.Task.ID != delayed.Task.ID || !dupe.Task.AvailableAt.Equal(delayed.Task.AvailableAt) || !dupe.Task.DeadlineAt.Equal(delayed.Task.DeadlineAt) {
+		t.Fatal("late replay reset original delay", err)
+	}
+	current, err := api.GetTask(ctx, delayed.Task.ID)
+	if err != nil || current.State != TaskSucceeded || current.AttemptCount != 1 {
+		t.Fatal("late replay reopened a completed task", err)
+	}
+	t.Log("relative and absolute release times obey database history; replay after elapsed delay returns original receipt without reopening")
+}

sdk/go/types.go

@@ -24,7 +24,9 @@ type SubmitTaskRequest struct {
 	ConcurrencyKey string `json:"concurrency_key,omitempty"`
 	// Priority 冻结优先级，0至9，每级30秒排序优势，不改变最早执行时间。
 	Priority int16 `json:"priority,omitempty"`
-	// AvailableAt 最早执行时间；为空表示受理后即可进入派发范围。
+	// DelayMS 相对首次受理数据库时间的等待毫秒数；原键重报不重新计时，非零不能与 AvailableAt 同用。
+	DelayMS uint64 `json:"delay_ms,omitempty"`
+	// AvailableAt 最早执行时间；为空且 DelayMS 为零表示受理后即可进入派发范围。
 	AvailableAt *time.Time `json:"available_at,omitempty"`
 	// StartDeadlineAt 最晚开始时间；为空表示使用任务总截止时间。
 	StartDeadlineAt *time.Time `json:"start_deadline_at,omitempty"`