feat: add durable schedule management and multi-node crash recovery

sdk/go/worker_process_integration_test.go

@@ -392,3 +392,83 @@ func TestIntegrationWorkerProcessCrashUnknown(t *testing.T) {
 		t.Fatal("external work marker lost")
 	}
 }
+
+// 在没有其他 Orbit 节点的间隙真实强杀调度进程，再由两个新进程从数据库接续。
+func TestIntegrationWorkerProcessScheduleRecovery(t *testing.T) {
+	a := startOrbitProcess(t)
+	_, queue, token := integrationWorkerClient(t)
+	apiA := processClient(t, a, token)
+	ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second)
+	defer cancel()
+	name := "schedule-crash." + workerTestID(t)
+	start := time.Now().UTC().Truncate(time.Microsecond).Add(time.Second)
+	end := start.Add(8 * time.Second)
+	created, err := apiA.CreateSchedule(ctx, workerTestID(t), CreateScheduleRequest{Name: name, Definition: ScheduleDefinition{
+		Rule:   ScheduleRule{Kind: "FIXED_RATE", Timezone: "UTC", StartAt: start, EndAt: &end, IntervalMS: 1000},
+		Policy: SchedulePolicy{Misfire: "CATCH_UP", Overlap: "PARALLEL", CatchupLimit: 2, CatchupWindowMS: 60000, GraceMS: 1000},
+		Task:   SubmitTaskRequest{QueueID: queue, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"source_ref":"objects/recovery"}`)},
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for {
+		page, err := apiA.ListScheduleOccurrences(ctx, created.Schedule.ID, SchedulePageRequest{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(page.Items) > 0 {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			t.Fatal(ctx.Err())
+		case <-time.After(50 * time.Millisecond):
+		}
+	}
+	a.kill(t)
+	// 三秒无服务形成真实漏跑，不改数据库时钟或游标。
+	select {
+	case <-ctx.Done():
+		t.Fatal(ctx.Err())
+	case <-time.After(3 * time.Second):
+	}
+	b := startOrbitProcess(t)
+	c := startOrbitProcess(t)
+	if a.nodeID == b.nodeID || a.nodeID == c.nodeID || b.nodeID == c.nodeID {
+		t.Fatal("scheduler nodes reused identity")
+	}
+	apiB := processClient(t, b, token)
+	for {
+		current, err := apiB.GetSchedule(ctx, created.Schedule.ID)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if current.CheckAt == nil && current.NextFireAt == nil {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			t.Fatal(ctx.Err())
+		case <-time.After(80 * time.Millisecond):
+		}
+	}
+	page, err := apiB.ListScheduleOccurrences(ctx, created.Schedule.ID, SchedulePageRequest{Limit: 100})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(page.Items) != 8 {
+		t.Fatalf("expected eight durable occurrences, got %d", len(page.Items))
+	}
+	seen := map[string]bool{}
+	for i, item := range page.Items {
+		want := start.Add(time.Duration(7-i) * time.Second)
+		if item.TaskID == nil || item.Decision != "FIRED" || item.ScheduleVersion != 1 || !item.ScheduledAt.Equal(want) || seen[*item.TaskID] {
+			t.Fatal("recovery duplicated, skipped, or retimed occurrence")
+		}
+		seen[*item.TaskID] = true
+		task, err := apiB.GetTask(ctx, *item.TaskID)
+		if err != nil || task.State != TaskWaiting || string(task.Payload) != `{"source_ref":"objects/recovery"}` {
+			t.Fatal("occurrence/task transaction lost", err)
+		}
+	}
+}