feat: add durable schedule management and multi-node crash recovery

cmd/orbit-admin/main.go

@@ -33,6 +33,7 @@ func run() error {
 	flags.StringVar(&in.Queue, "queue", "", "initial queue name")
 	flags.StringVar(&output, "output", "", "new private credentials JSON file")
 	flags.BoolVar(&in.WorkerAccess, "worker-access", false, "also grant Worker connect and consume actions for development")
+	flags.BoolVar(&in.ScheduleAccess, "schedule-access", false, "also grant schedule read and manage actions for development")
 	if err := flags.Parse(os.Args[2:]); err != nil {
 		return err
 	}

cmd/orbit/main.go

@@ -15,6 +15,7 @@ import (
 	"git.yuntuo.hk/k2safe/orbit/internal/storage/mysql"
 	"git.yuntuo.hk/k2safe/orbit/internal/transport/httpserver"
 	core "git.yuntuo.hk/k2safe/orbit/model"
+	"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
 	"git.yuntuo.hk/k2safe/orbit/pulse/service"
 	"git.yuntuo.hk/k2safe/orbit/pulse/worker"
 )
@@ -61,6 +62,32 @@ func run() error {
 	}
 	hub := httpserver.NewWorkerHub(root, worker.New(executions))
 	httpserver.RegisterWorkers(server, hub)
+	schedules, err := mysql.NewScheduleStore(db, node)
+	if err != nil {
+		return err
+	}
+	httpserver.RegisterSchedules(server, schedule.New(schedules))
+	scheduleCtx, stopSchedules := context.WithCancel(root)
+	scheduleDone := make(chan struct{})
+	go func() {
+		defer close(scheduleDone)
+		ticker := time.NewTicker(time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-scheduleCtx.Done():
+				return
+			case <-ticker.C:
+				call, cancel := context.WithTimeout(scheduleCtx, 5*time.Second)
+				_, err := schedules.Tick(call, 100)
+				cancel()
+				if err != nil && scheduleCtx.Err() == nil {
+					slog.Error("schedule batch failed", "node_id", node)
+				}
+			}
+		}
+	}()
+	defer func() { stopSchedules(); <-scheduleDone }()
 	defer func() {
 		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 		defer cancel()
@@ -93,7 +120,7 @@ func run() error {
 	}
 	done := make(chan error, 1)
 	go func() { done <- server.Serve(listener) }()
-	slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task, queue and Worker APIs", "node_id", node)
+	slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task, queue, schedule and Worker APIs", "node_id", node)
 	select {
 	case err := <-done:
 		if errors.Is(err, http.ErrServerClosed) {