feat: add durable Worker execution and WSS server

cmd/orbit-admin/main.go

@@ -32,6 +32,7 @@ func run() error {
 	flags.StringVar(&in.Environment, "environment", "", "environment name")
 	flags.StringVar(&in.Queue, "queue", "", "initial queue name")
 	flags.StringVar(&output, "output", "", "new private credentials JSON file")
+	flags.BoolVar(&in.WorkerAccess, "worker-access", false, "also grant Worker connect and consume actions for development")
 	if err := flags.Parse(os.Args[2:]); err != nil {
 		return err
 	}

cmd/orbit/main.go

@@ -14,7 +14,9 @@ import (
 	"git.yuntuo.hk/k2safe/orbit/internal/config"
 	"git.yuntuo.hk/k2safe/orbit/internal/storage/mysql"
 	"git.yuntuo.hk/k2safe/orbit/internal/transport/httpserver"
+	core "git.yuntuo.hk/k2safe/orbit/model"
 	"git.yuntuo.hk/k2safe/orbit/pulse/service"
+	"git.yuntuo.hk/k2safe/orbit/pulse/worker"
 )
 
 func main() {
@@ -49,13 +51,49 @@ func run() error {
 		}
 		return mysql.CheckSchema(ctx, db)
 	}, service.New(mysql.NewTaskStore(db)))
+	node, err := core.NewID()
+	if err != nil {
+		return err
+	}
+	executions, err := mysql.NewExecutionStore(db, node, mysql.ExecutionOptions{})
+	if err != nil {
+		return err
+	}
+	hub := httpserver.NewWorkerHub(root, worker.New(executions))
+	httpserver.RegisterWorkers(server, hub)
+	defer func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		_ = hub.Shutdown(ctx)
+	}()
+	recoveryDone := make(chan struct{})
+	recoveryCtx, stopRecovery := context.WithCancel(root)
+	go func() {
+		defer close(recoveryDone)
+		ticker := time.NewTicker(2 * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-recoveryCtx.Done():
+				return
+			case <-ticker.C:
+				call, cancel := context.WithTimeout(recoveryCtx, 5*time.Second)
+				_, err := executions.Recover(call, 100)
+				cancel()
+				if err != nil && recoveryCtx.Err() == nil {
+					slog.Error("execution recovery batch failed", "node_id", node)
+				}
+			}
+		}
+	}()
+	defer func() { stopRecovery(); <-recoveryDone }()
 	listener, err := net.Listen("tcp", cfg.HTTPAddr)
 	if err != nil {
 		return err
 	}
 	done := make(chan error, 1)
 	go func() { done <- server.Serve(listener) }()
-	slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task and queue management")
+	slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task, queue and Worker APIs", "node_id", node)
 	select {
 	case err := <-done:
 		if errors.Is(err, http.ErrServerClosed) {