feat: add bounded process metrics and private monitoring endpoint

cmd/orbit/main.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"git.yuntuo.hk/k2safe/orbit/internal/config"
+	"git.yuntuo.hk/k2safe/orbit/internal/observability"
 	"git.yuntuo.hk/k2safe/orbit/internal/storage/mysql"
 	"git.yuntuo.hk/k2safe/orbit/internal/transport/httpserver"
 	core "git.yuntuo.hk/k2safe/orbit/model"
@@ -53,12 +54,24 @@ func run() error {
 		return err
 	}
 	cancel()
-	server := httpserver.New(cfg.HTTPAddr, func(ctx context.Context) error {
+	var metrics *observability.Registry
+	var metricsListener net.Listener
+	if cfg.MetricsAddr != "" {
+		metricsListener, err = net.Listen("tcp", cfg.MetricsAddr)
+		if err != nil {
+			return errors.New("metrics listener could not bind")
+		}
+		defer metricsListener.Close()
+		metrics = observability.NewRegistry()
+		metrics.RecordDatabase(db.Stats())
+	}
+	ready := func(ctx context.Context) error {
 		if err := db.PingContext(ctx); err != nil {
 			return err
 		}
 		return mysql.CheckSchema(ctx, db)
-	}, service.New(mysql.NewTaskStore(db)))
+	}
+	server := httpserver.New(cfg.HTTPAddr, ready, service.New(mysql.NewTaskStore(db)))
 	httpserver.RegisterApplicationResources(server, resources.New(mysql.NewTaskStore(db)))
 	node, err := core.NewID()
 	if err != nil {
@@ -86,6 +99,7 @@ func run() error {
 	httpserver.RegisterSchedules(server, schedule.New(schedules))
 	httpserver.RegisterWorkflowDefinitions(server, definition.New(mysql.NewWorkflowDefinitionStore(db)))
 	httpserver.RegisterWorkflows(server, instance.New(mysql.NewWorkflowInstanceStore(db)), control.New(mysql.NewWorkflowControlStore(db)), weavesignal.New(mysql.NewWorkflowSignalStore(db)), recovery.New(mysql.NewWorkflowRecoveryStore(db)))
+	httpserver.WrapMetrics(server, metrics)
 	workflows, err := mysql.NewWorkflowRuntime(db, node)
 	if err != nil {
 		return err
@@ -101,9 +115,11 @@ func run() error {
 			case <-workflowCtx.Done():
 				return
 			case <-ticker.C:
+				started := time.Now()
 				call, cancel := context.WithTimeout(workflowCtx, 5*time.Second)
 				_, err := workflows.Tick(call, 100)
 				cancel()
+				metrics.RecordBatch("workflow", started, err)
 				if err != nil && workflowCtx.Err() == nil {
 					slog.Error("workflow batch failed", "node_id", node)
 				}
@@ -122,9 +138,11 @@ func run() error {
 			case <-scheduleCtx.Done():
 				return
 			case <-ticker.C:
+				started := time.Now()
 				call, cancel := context.WithTimeout(scheduleCtx, 5*time.Second)
 				_, err := schedules.Tick(call, 100)
 				cancel()
+				metrics.RecordBatch("schedule", started, err)
 				if err != nil && scheduleCtx.Err() == nil {
 					slog.Error("schedule batch failed", "node_id", node)
 				}
@@ -148,9 +166,11 @@ func run() error {
 			case <-recoveryCtx.Done():
 				return
 			case <-ticker.C:
+				started := time.Now()
 				call, cancel := context.WithTimeout(recoveryCtx, 5*time.Second)
 				_, err := executions.Recover(call, 100)
 				cancel()
+				metrics.RecordBatch("execution_recovery", started, err)
 				if err != nil && recoveryCtx.Err() == nil {
 					slog.Error("execution recovery batch failed", "node_id", node)
 				}
@@ -158,11 +178,38 @@ func run() error {
 		}
 	}()
 	defer func() { stopRecovery(); <-recoveryDone }()
+	if metrics != nil {
+		metricsCtx, stopMetrics := context.WithCancel(root)
+		metricsDone := make(chan struct{})
+		go func() {
+			defer close(metricsDone)
+			ticker := time.NewTicker(5 * time.Second)
+			defer ticker.Stop()
+			for {
+				select {
+				case <-metricsCtx.Done():
+					return
+				case <-ticker.C:
+					metrics.RecordDatabase(db.Stats())
+					metrics.RecordProcess()
+				}
+			}
+		}()
+		defer func() { stopMetrics(); <-metricsDone }()
+	}
 	listener, err := net.Listen("tcp", cfg.HTTPAddr)
 	if err != nil {
 		return err
 	}
-	done := make(chan error, 1)
+	defer server.Close()
+	var metricsServer *http.Server
+	done := make(chan error, 2)
+	if metrics != nil {
+		metricsServer = httpserver.NewMetricsServer(cfg.MetricsAddr, metrics)
+		defer metricsServer.Close()
+		go func() { done <- metricsServer.Serve(metricsListener) }()
+		slog.Info("Orbit metrics server starting", "metrics_address", metricsListener.Addr().String())
+	}
 	go func() { done <- server.Serve(listener) }()
 	slog.Info("Orbit HTTP server starting", "address", listener.Addr().String(), "scope", "health, task, queue, schedule, subscription, workflow and Worker APIs", "node_id", node)
 	select {
@@ -178,10 +225,20 @@ func run() error {
 			_ = server.Close()
 			return err
 		}
-		err := <-done
-		if errors.Is(err, http.ErrServerClosed) {
-			return nil
+		if metricsServer != nil {
+			if err := metricsServer.Shutdown(shutdown); err != nil {
+				return err
+			}
 		}
-		return err
+		count := 1
+		if metricsServer != nil {
+			count++
+		}
+		for i := 0; i < count; i++ {
+			if err := <-done; !errors.Is(err, http.ErrServerClosed) {
+				return err
+			}
+		}
+		return nil
 	}
 }