Export prometheus metrics (#286)

src/server/GameManager.ts

@@ -40,6 +40,18 @@ export class GameManager {
     return game;
   }
 
+  activeGames(): number {
+    return this.games.size;
+  }
+
+  activeClients(): number {
+    let totalClients = 0;
+    this.games.forEach((game: GameServer) => {
+      totalClients += game.activeClients.length;
+    });
+    return totalClients;
+  }
+
   tick() {
     const active = new Map<GameID, GameServer>();
     for (const [id, game] of this.games) {

src/server/Master.ts

@@ -13,6 +13,7 @@ import path from "path";
 import rateLimit from "express-rate-limit";
 import { fileURLToPath } from "url";
 import { gatekeeper, LimiterType } from "./Gatekeeper";
+import { setupMetricsServer } from "./MasterMetrics";
 
 const config = getServerConfigFromServer();
 const readyWorkers = new Set();
@@ -20,6 +21,10 @@ const readyWorkers = new Set();
 const app = express();
 const server = http.createServer(app);
 
+// Create a separate metrics server on port 9090
+const metricsApp = express();
+const metricsServer = http.createServer(metricsApp);
+
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 app.use(express.json());
@@ -135,6 +140,9 @@ export async function startMaster() {
   server.listen(PORT, () => {
     console.log(`Master HTTP server listening on port ${PORT}`);
   });
+
+  // Setup the metrics server
+  setupMetricsServer();
 }
 
 app.get(

src/server/MasterMetrics.ts

@@ -0,0 +1,79 @@
+import express from "express";
+import http from "http";
+import promClient from "prom-client";
+import { getServerConfigFromServer } from "../core/configuration/Config";
+
+const config = getServerConfigFromServer();
+
+// Create a separate metrics server on port 9090
+const metricsApp = express();
+const metricsServer = http.createServer(metricsApp);
+
+// Initialize the Prometheus registry for the master's own metrics
+const register = new promClient.Registry();
+
+// Prometheus metrics endpoint that gathers metrics from workers
+export function setupMetricsServer() {
+  metricsApp.get("/metrics", async (req, res) => {
+    console.log("Metrics requested");
+    try {
+      // Get the master's metrics
+      const masterMetrics = await register.metrics();
+
+      // Collect metrics from all workers
+      const workerMetricsPromises = [];
+
+      // For each worker, fetch their metrics
+      for (let i = 0; i < config.numWorkers(); i++) {
+        const workerPort = config.workerPortByIndex(i);
+        const workerUrl = `http://localhost:${workerPort}/metrics`;
+        console.log(`Fetching metrics from worker ${i} at ${workerUrl}`);
+        const workerMetricsPromise = fetch(workerUrl, {
+          headers: {
+            [config.adminHeader()]: config.adminToken(),
+          },
+        })
+          .then((response) => {
+            if (!response.ok) {
+              throw new Error(`Worker ${i} returned status ${response.status}`);
+            }
+            return response.text();
+          })
+          .then((metricsText) => {
+            // Add worker label to each metric line
+            return metricsText.replace(
+              /^([a-z][a-z0-9_]*(?:{[^}]*})?)\s/gm,
+              `$1{worker="worker-${i}"} `,
+            );
+          })
+          .catch((error) => {
+            console.error(`Error fetching metrics from worker ${i}:`, error);
+            return `# Error fetching metrics from worker ${i}: ${error.message}`;
+          });
+        workerMetricsPromises.push(workerMetricsPromise);
+      }
+
+      // Wait for all worker metrics to be fetched
+      const workerMetricsArray = await Promise.all(workerMetricsPromises);
+
+      // Add worker label to the master metrics
+      const masterMetricsWithLabel = masterMetrics.replace(
+        /^([a-z][a-z0-9_]*(?:{[^}]*})?)\s/gm,
+        `$1{worker="master"} `,
+      );
+
+      // Combine all metrics and send the response
+      res.set("Content-Type", register.contentType);
+      res.end(`${masterMetricsWithLabel}\n${workerMetricsArray.join("\n")}`);
+    } catch (error) {
+      console.error("Error collecting metrics:", error);
+      res.status(500).end(`# Error collecting metrics: ${error.message}`);
+    }
+  });
+
+  // Start the metrics server on port 9090
+  const METRICS_PORT = 9090;
+  metricsServer.listen(METRICS_PORT, () => {
+    console.log(`Metrics server listening on port ${METRICS_PORT}`);
+  });
+}

src/server/Worker.ts

@@ -17,6 +17,7 @@ import { slog } from "./StructuredLog";
 import { GameType } from "../core/game/Game";
 import { archive, readGameRecord } from "./Archive";
 import { gatekeeper, LimiterType } from "./Gatekeeper";
+import { metrics } from "./WorkerMetrics";
 
 const config = getServerConfigFromServer();
 
@@ -35,6 +36,11 @@ export function startWorker() {
 
   const gm = new GameManager(config);
 
+  // Set up periodic metrics updates
+  setInterval(() => {
+    metrics.updateGameMetrics(gm);
+  }, 15000); // Update every 15 seconds
+
   // Middleware to handle /wX path prefix
   app.use((req, res, next) => {
     // Extract the original path without the worker prefix
@@ -241,6 +247,24 @@ export function startWorker() {
     }),
   );
 
+  app.get(
+    "/metrics",
+    gatekeeper.httpHandler(LimiterType.Get, async (req, res) => {
+      if (req.headers[config.adminHeader()] !== config.adminToken()) {
+        return res.status(403).end("Access denied");
+      }
+      console.log(`metrics requested on worker ${workerId}`);
+
+      try {
+        const metricsData = await metrics.register.metrics();
+        res.set("Content-Type", metrics.register.contentType);
+        res.end(metricsData);
+      } catch (error) {
+        res.status(500).end(error.message);
+      }
+    }),
+  );
+
   // WebSocket handling
   wss.on("connection", (ws: WebSocket, req) => {
     ws.on(

src/server/WorkerMetrics.ts

@@ -0,0 +1,45 @@
+import promClient from "prom-client";
+import { GameManager } from "./GameManager";
+
+// Initialize the Prometheus registry
+const register = new promClient.Registry();
+
+// Enable default Node.js metrics collection
+promClient.collectDefaultMetrics({ register });
+
+// Add worker-specific metrics
+const activeGamesGauge = new promClient.Gauge({
+  name: "active_games_count",
+  help: "Number of active games on this worker",
+  registers: [register],
+});
+
+const connectedClientsGauge = new promClient.Gauge({
+  name: "connected_clients_count",
+  help: "Number of connected clients on this worker",
+  registers: [register],
+});
+
+const memoryUsageGauge = new promClient.Gauge({
+  name: "memory_usage_bytes",
+  help: "Current memory usage of the worker process in bytes",
+  registers: [register],
+});
+
+// Export the metrics for use in the worker
+export const metrics = {
+  register,
+  activeGamesGauge,
+  connectedClientsGauge,
+  memoryUsageGauge,
+
+  // Function to update game-related metrics
+  updateGameMetrics: (gameManager: GameManager) => {
+    activeGamesGauge.set(gameManager.activeGames());
+    connectedClientsGauge.set(gameManager.activeClients());
+
+    // Update memory usage metrics
+    const memoryUsage = process.memoryUsage();
+    memoryUsageGauge.set(memoryUsage.heapUsed);
+  },
+};