mirror of
https://github.com/openfrontio/OpenFrontIO.git
synced 2026-06-29 21:32:12 +00:00
Merge branch 'main' into meta3
This commit is contained in:
1brucben
@@ -1,61 +0,0 @@
|
||||
import { SecretManagerServiceClient } from "@google-cloud/secret-manager";
|
||||
import { Client, Events, GatewayIntentBits } from "discord.js";
|
||||
|
||||
export class DiscordBot {
|
||||
private client: Client;
|
||||
private secretManager: SecretManagerServiceClient;
|
||||
|
||||
constructor() {
|
||||
this.client = new Client({
|
||||
intents: [
|
||||
GatewayIntentBits.Guilds,
|
||||
GatewayIntentBits.GuildMessages,
|
||||
GatewayIntentBits.MessageContent,
|
||||
],
|
||||
});
|
||||
this.secretManager = new SecretManagerServiceClient();
|
||||
this.setupEventHandlers();
|
||||
}
|
||||
|
||||
private setupEventHandlers(): void {
|
||||
this.client.once(Events.ClientReady, (c) => {
|
||||
console.log(`Ready! Logged in as ${c.user.tag}`);
|
||||
});
|
||||
|
||||
this.client.on(Events.MessageCreate, async (message) => {
|
||||
if (message.author.bot) return;
|
||||
|
||||
if (message.content === "!ping") {
|
||||
await message.reply("Pong! 🏓");
|
||||
}
|
||||
|
||||
if (message.content === "!hello") {
|
||||
await message.reply(`Hello ${message.author.username}! 👋`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async getToken(): Promise<string | undefined> {
|
||||
const name =
|
||||
"projects/openfrontio/secrets/discord-bot-token/versions/latest";
|
||||
const [version] = await this.secretManager.accessSecretVersion({ name });
|
||||
return version.payload?.data?.toString().trim();
|
||||
}
|
||||
|
||||
public async start(): Promise<void> {
|
||||
try {
|
||||
const token = await this.getToken();
|
||||
if (!token) {
|
||||
throw new Error("Failed to retrieve Discord token");
|
||||
}
|
||||
await this.client.login(token);
|
||||
} catch (error) {
|
||||
console.error("Failed to start bot:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
this.client.destroy();
|
||||
}
|
||||
}
|
||||
@@ -95,8 +95,8 @@ export class GameServer {
|
||||
if (gameConfig.gameMode != null) {
|
||||
this.gameConfig.gameMode = gameConfig.gameMode;
|
||||
}
|
||||
if (gameConfig.numPlayerTeams != null) {
|
||||
this.gameConfig.numPlayerTeams = gameConfig.numPlayerTeams;
|
||||
if (gameConfig.playerTeams != null) {
|
||||
this.gameConfig.playerTeams = gameConfig.playerTeams;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+56
-1
@@ -1,4 +1,56 @@
|
||||
import * as logsAPI from "@opentelemetry/api-logs";
|
||||
import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
|
||||
import {
|
||||
LoggerProvider,
|
||||
SimpleLogRecordProcessor,
|
||||
} from "@opentelemetry/sdk-logs";
|
||||
import { OpenTelemetryTransportV3 } from "@opentelemetry/winston-transport";
|
||||
import * as dotenv from "dotenv";
|
||||
import winston from "winston";
|
||||
import { GameEnv } from "../core/configuration/Config";
|
||||
import { getServerConfigFromServer } from "../core/configuration/ConfigLoader";
|
||||
import { getOtelResource } from "./OtelResource";
|
||||
dotenv.config();
|
||||
|
||||
const config = getServerConfigFromServer();
|
||||
|
||||
const resource = getOtelResource();
|
||||
|
||||
// Initialize the OpenTelemetry Logger Provider
|
||||
const loggerProvider = new LoggerProvider({
|
||||
resource,
|
||||
});
|
||||
|
||||
if (config.env() == GameEnv.Prod && config.otelEnabled()) {
|
||||
console.log("OTEL enabled");
|
||||
// Configure OpenTelemetry endpoint with basic auth (if provided)
|
||||
const headers = {};
|
||||
if (config.otelUsername() && config.otelPassword()) {
|
||||
headers["Authorization"] =
|
||||
"Basic " +
|
||||
Buffer.from(`${config.otelUsername()}:${config.otelPassword()}`).toString(
|
||||
"base64",
|
||||
);
|
||||
}
|
||||
|
||||
// Add OTLP exporter for logs
|
||||
const logExporter = new OTLPLogExporter({
|
||||
url: `${config.otelEndpoint()}/v1/logs`,
|
||||
headers,
|
||||
});
|
||||
|
||||
// Add a log processor with the exporter
|
||||
loggerProvider.addLogRecordProcessor(
|
||||
new SimpleLogRecordProcessor(logExporter),
|
||||
);
|
||||
|
||||
// Set as the global logger provider
|
||||
logsAPI.logs.setGlobalLoggerProvider(loggerProvider);
|
||||
} else {
|
||||
console.log(
|
||||
"No OTLP endpoint and credentials provided, remote logging disabled",
|
||||
);
|
||||
}
|
||||
|
||||
// Custom format to add severity tag based on log level
|
||||
const addSeverityFormat = winston.format((info) => {
|
||||
@@ -20,7 +72,10 @@ const logger = winston.createLogger({
|
||||
service: "openfront",
|
||||
environment: process.env.NODE_ENV,
|
||||
},
|
||||
transports: [new winston.transports.Console()],
|
||||
transports: [
|
||||
new winston.transports.Console(),
|
||||
new OpenTelemetryTransportV3(),
|
||||
],
|
||||
});
|
||||
|
||||
// Export both the main logger and the child logger factory
|
||||
|
||||
+108
-94
@@ -1,119 +1,133 @@
|
||||
import { GameMapType, GameMode } from "../core/game/Game";
|
||||
import { getServerConfigFromServer } from "../core/configuration/ConfigLoader";
|
||||
import { Difficulty, GameMapType, GameMode, GameType } from "../core/game/Game";
|
||||
import { PseudoRandom } from "../core/PseudoRandom";
|
||||
import { GameConfig } from "../core/Schemas";
|
||||
import { logger } from "./Logger";
|
||||
|
||||
enum PlaylistType {
|
||||
BigMaps,
|
||||
SmallMaps,
|
||||
const log = logger.child({});
|
||||
|
||||
const config = getServerConfigFromServer();
|
||||
|
||||
const frequency = {
|
||||
World: 3,
|
||||
Europe: 2,
|
||||
Africa: 2,
|
||||
Australia: 1,
|
||||
NorthAmerica: 1,
|
||||
Britannia: 1,
|
||||
GatewayToTheAtlantic: 1,
|
||||
Iceland: 1,
|
||||
SouthAmerica: 1,
|
||||
KnownWorld: 1,
|
||||
DeglaciatedAntarctica: 1,
|
||||
EuropeClassic: 1,
|
||||
Mena: 1,
|
||||
Pangaea: 1,
|
||||
Asia: 1,
|
||||
Mars: 1,
|
||||
BetweenTwoSeas: 1,
|
||||
Japan: 1,
|
||||
BlackSea: 1,
|
||||
FaroeIslands: 1,
|
||||
};
|
||||
|
||||
interface MapWithMode {
|
||||
map: GameMapType;
|
||||
mode: GameMode;
|
||||
}
|
||||
|
||||
const random = new PseudoRandom(123);
|
||||
|
||||
export class MapPlaylist {
|
||||
private gameModeRotation = [GameMode.FFA, GameMode.FFA, GameMode.Team];
|
||||
private currentGameModeIndex = 0;
|
||||
private mapsPlaylist: MapWithMode[] = [];
|
||||
|
||||
private mapsPlaylistBig: GameMapType[] = [];
|
||||
private mapsPlaylistSmall: GameMapType[] = [];
|
||||
private currentPlaylistCounter = 0;
|
||||
public gameConfig(): GameConfig {
|
||||
const { map, mode } = this.getNextMap();
|
||||
|
||||
// Get the next map in rotation
|
||||
public getNextMap(): GameMapType {
|
||||
const playlistType: PlaylistType = this.getNextPlaylistType();
|
||||
const mapsPlaylist: GameMapType[] = this.getNextMapsPlayList(playlistType);
|
||||
return mapsPlaylist.shift()!;
|
||||
const numPlayerTeams =
|
||||
mode === GameMode.Team ? 2 + Math.floor(Math.random() * 5) : undefined;
|
||||
|
||||
// Create the default public game config (from your GameManager)
|
||||
return {
|
||||
gameMap: map,
|
||||
maxPlayers: config.lobbyMaxPlayers(map, mode),
|
||||
gameType: GameType.Public,
|
||||
difficulty: Difficulty.Medium,
|
||||
infiniteGold: false,
|
||||
infiniteTroops: false,
|
||||
instantBuild: false,
|
||||
disableNPCs: mode == GameMode.Team,
|
||||
disableNukes: false,
|
||||
gameMode: mode,
|
||||
playerTeams: numPlayerTeams,
|
||||
bots: 400,
|
||||
} as GameConfig;
|
||||
}
|
||||
|
||||
public getNextGameMode(): GameMode {
|
||||
const nextGameMode = this.gameModeRotation[this.currentGameModeIndex];
|
||||
this.currentGameModeIndex =
|
||||
(this.currentGameModeIndex + 1) % this.gameModeRotation.length;
|
||||
return nextGameMode;
|
||||
}
|
||||
|
||||
private getNextMapsPlayList(playlistType: PlaylistType): GameMapType[] {
|
||||
switch (playlistType) {
|
||||
case PlaylistType.BigMaps:
|
||||
if (!(this.mapsPlaylistBig.length > 0)) {
|
||||
this.fillMapsPlaylist(playlistType, this.mapsPlaylistBig);
|
||||
private getNextMap(): MapWithMode {
|
||||
if (this.mapsPlaylist.length === 0) {
|
||||
const numAttempts = 10000;
|
||||
for (let i = 0; i < numAttempts; i++) {
|
||||
if (this.shuffleMapsPlaylist()) {
|
||||
log.info(`Generated map playlist in ${i} attempts`);
|
||||
return this.mapsPlaylist.shift()!;
|
||||
}
|
||||
return this.mapsPlaylistBig;
|
||||
|
||||
case PlaylistType.SmallMaps:
|
||||
if (!(this.mapsPlaylistSmall.length > 0)) {
|
||||
this.fillMapsPlaylist(playlistType, this.mapsPlaylistSmall);
|
||||
}
|
||||
return this.mapsPlaylistSmall;
|
||||
}
|
||||
log.error("Failed to generate a valid map playlist");
|
||||
}
|
||||
// Even if it failed, playlist will be partially populated.
|
||||
return this.mapsPlaylist.shift()!;
|
||||
}
|
||||
|
||||
private fillMapsPlaylist(
|
||||
playlistType: PlaylistType,
|
||||
mapsPlaylist: GameMapType[],
|
||||
): void {
|
||||
const frequency = this.getFrequency(playlistType);
|
||||
private shuffleMapsPlaylist(): boolean {
|
||||
const maps: GameMapType[] = [];
|
||||
Object.keys(GameMapType).forEach((key) => {
|
||||
let count = parseInt(frequency[key]);
|
||||
while (count > 0) {
|
||||
mapsPlaylist.push(GameMapType[key]);
|
||||
count--;
|
||||
for (let i = 0; i < parseInt(frequency[key]); i++) {
|
||||
maps.push(GameMapType[key]);
|
||||
}
|
||||
});
|
||||
while (!this.allNonConsecutive(mapsPlaylist)) {
|
||||
random.shuffleArray(mapsPlaylist);
|
||||
}
|
||||
}
|
||||
|
||||
// Specifically controls how the playlists rotate.
|
||||
private getNextPlaylistType(): PlaylistType {
|
||||
switch (this.currentPlaylistCounter) {
|
||||
case 0:
|
||||
case 1:
|
||||
this.currentPlaylistCounter++;
|
||||
return PlaylistType.BigMaps;
|
||||
case 2:
|
||||
this.currentPlaylistCounter = 0;
|
||||
return PlaylistType.SmallMaps;
|
||||
}
|
||||
}
|
||||
const rand = new PseudoRandom(Date.now());
|
||||
|
||||
private getFrequency(playlistType: PlaylistType) {
|
||||
switch (playlistType) {
|
||||
// Big Maps are those larger than ~2.5 mil pixels
|
||||
case PlaylistType.BigMaps:
|
||||
return {
|
||||
Europe: 2,
|
||||
NorthAmerica: 1,
|
||||
Africa: 2,
|
||||
Britannia: 1,
|
||||
GatewayToTheAtlantic: 2,
|
||||
Australia: 2,
|
||||
Iceland: 2,
|
||||
SouthAmerica: 1,
|
||||
KnownWorld: 2,
|
||||
};
|
||||
case PlaylistType.SmallMaps:
|
||||
return {
|
||||
World: 4,
|
||||
EuropeClassic: 3,
|
||||
Mena: 2,
|
||||
Pangaea: 1,
|
||||
Asia: 1,
|
||||
Mars: 1,
|
||||
BetweenTwoSeas: 2,
|
||||
Japan: 2,
|
||||
BlackSea: 1,
|
||||
FaroeIslands: 2,
|
||||
};
|
||||
}
|
||||
}
|
||||
const ffa1: GameMapType[] = rand.shuffleArray([...maps]);
|
||||
const ffa2: GameMapType[] = rand.shuffleArray([...maps]);
|
||||
const ffa3: GameMapType[] = rand.shuffleArray([...maps]);
|
||||
const team: GameMapType[] = rand.shuffleArray([...maps]);
|
||||
|
||||
// Check for consecutive duplicates in the maps array
|
||||
private allNonConsecutive(maps: GameMapType[]): boolean {
|
||||
for (let i = 0; i < maps.length - 1; i++) {
|
||||
if (maps[i] === maps[i + 1]) {
|
||||
this.mapsPlaylist = [];
|
||||
for (let i = 0; i < maps.length; i++) {
|
||||
if (!this.addNextMap(this.mapsPlaylist, ffa1, GameMode.FFA)) {
|
||||
return false;
|
||||
}
|
||||
if (!this.addNextMap(this.mapsPlaylist, ffa2, GameMode.FFA)) {
|
||||
return false;
|
||||
}
|
||||
if (!this.addNextMap(this.mapsPlaylist, ffa3, GameMode.FFA)) {
|
||||
return false;
|
||||
}
|
||||
if (!this.addNextMap(this.mapsPlaylist, team, GameMode.Team)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private addNextMap(
|
||||
playlist: MapWithMode[],
|
||||
nextEls: GameMapType[],
|
||||
mode: GameMode,
|
||||
): boolean {
|
||||
const nonConsecutiveNum = 5;
|
||||
const lastEls = playlist
|
||||
.slice(playlist.length - nonConsecutiveNum)
|
||||
.map((m) => m.map);
|
||||
for (let i = 0; i < nextEls.length; i++) {
|
||||
const next = nextEls[i];
|
||||
if (lastEls.includes(next)) {
|
||||
continue;
|
||||
}
|
||||
nextEls.splice(i, 1);
|
||||
playlist.push({ map: next, mode: mode });
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
+2
-40
@@ -5,13 +5,11 @@ import http from "http";
|
||||
import path from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
import { getServerConfigFromServer } from "../core/configuration/ConfigLoader";
|
||||
import { Difficulty, GameMode, GameType } from "../core/game/Game";
|
||||
import { GameConfig, GameInfo } from "../core/Schemas";
|
||||
import { GameInfo } from "../core/Schemas";
|
||||
import { generateID } from "../core/Util";
|
||||
import { gatekeeper, LimiterType } from "./Gatekeeper";
|
||||
import { logger } from "./Logger";
|
||||
import { MapPlaylist } from "./MapPlaylist";
|
||||
import { setupMetricsServer } from "./MasterMetrics";
|
||||
|
||||
const config = getServerConfigFromServer();
|
||||
const playlist = new MapPlaylist();
|
||||
@@ -20,10 +18,6 @@ const readyWorkers = new Set();
|
||||
const app = express();
|
||||
const server = http.createServer(app);
|
||||
|
||||
// Create a separate metrics server on port 9090
|
||||
const metricsApp = express();
|
||||
const metricsServer = http.createServer(metricsApp);
|
||||
|
||||
const log = logger.child({ comp: "m" });
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
@@ -146,9 +140,6 @@ export async function startMaster() {
|
||||
server.listen(PORT, () => {
|
||||
log.info(`Master HTTP server listening on port ${PORT}`);
|
||||
});
|
||||
|
||||
// Setup the metrics server
|
||||
setupMetricsServer();
|
||||
}
|
||||
|
||||
app.get(
|
||||
@@ -222,40 +213,11 @@ async function fetchLobbies(): Promise<number> {
|
||||
return publicLobbyIDs.size;
|
||||
}
|
||||
|
||||
let lastGameMode: GameMode = GameMode.FFA;
|
||||
|
||||
// Function to schedule a new public game
|
||||
async function schedulePublicGame(playlist: MapPlaylist) {
|
||||
const gameID = generateID();
|
||||
const map = playlist.getNextMap();
|
||||
publicLobbyIDs.add(gameID);
|
||||
|
||||
if (lastGameMode == GameMode.FFA) {
|
||||
lastGameMode = GameMode.Team;
|
||||
} else {
|
||||
lastGameMode = GameMode.FFA;
|
||||
}
|
||||
|
||||
const gameMode = playlist.getNextGameMode();
|
||||
const numPlayerTeams =
|
||||
gameMode === GameMode.Team ? 2 + Math.floor(Math.random() * 5) : undefined;
|
||||
|
||||
// Create the default public game config (from your GameManager)
|
||||
const defaultGameConfig: GameConfig = {
|
||||
gameMap: map,
|
||||
maxPlayers: config.lobbyMaxPlayers(map),
|
||||
gameType: GameType.Public,
|
||||
difficulty: Difficulty.Medium,
|
||||
infiniteGold: false,
|
||||
infiniteTroops: false,
|
||||
instantBuild: false,
|
||||
disableNPCs: gameMode == GameMode.Team,
|
||||
disableNukes: false,
|
||||
gameMode,
|
||||
numPlayerTeams,
|
||||
bots: 400,
|
||||
};
|
||||
|
||||
const workerPath = config.workerPath(gameID);
|
||||
|
||||
// Send request to the worker to start the game
|
||||
@@ -269,7 +231,7 @@ async function schedulePublicGame(playlist: MapPlaylist) {
|
||||
[config.adminHeader()]: config.adminToken(),
|
||||
},
|
||||
body: JSON.stringify({
|
||||
gameConfig: defaultGameConfig,
|
||||
gameConfig: playlist.gameConfig(),
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
@@ -1,189 +0,0 @@
|
||||
import express from "express";
|
||||
import http from "http";
|
||||
import promClient from "prom-client";
|
||||
import { getServerConfigFromServer } from "../core/configuration/ConfigLoader";
|
||||
|
||||
const config = getServerConfigFromServer();
|
||||
|
||||
// Create a separate metrics server on port 9090
|
||||
const metricsApp = express();
|
||||
const metricsServer = http.createServer(metricsApp);
|
||||
|
||||
// Initialize the Prometheus registry for the master's own metrics
|
||||
const register = new promClient.Registry();
|
||||
|
||||
// Default Prometheus metrics
|
||||
promClient.collectDefaultMetrics({ register });
|
||||
|
||||
// Prometheus metrics endpoint that gathers metrics from workers
|
||||
export function setupMetricsServer() {
|
||||
metricsApp.get("/metrics", async (req, res) => {
|
||||
// Set a timeout for the request to avoid hanging
|
||||
const timeout = setTimeout(() => {
|
||||
res.status(500).end("# Error: Request timed out after 30 seconds");
|
||||
}, 30000);
|
||||
console.log("Metrics requested");
|
||||
try {
|
||||
// Get the master's metrics
|
||||
const masterMetrics = await register.metrics();
|
||||
|
||||
// Track seen metric names to avoid duplicate metadata
|
||||
const seenMetrics = new Set();
|
||||
const processedLines = [];
|
||||
const allMetricValues = [];
|
||||
|
||||
// Process all metadata information in the master metrics first
|
||||
const masterLines = masterMetrics.split("\n");
|
||||
|
||||
for (let j = 0; j < masterLines.length; j++) {
|
||||
const line = masterLines[j];
|
||||
|
||||
if (line.startsWith("# HELP ")) {
|
||||
const metricName = line.split(" ")[2];
|
||||
seenMetrics.add(metricName);
|
||||
processedLines.push(line);
|
||||
} else if (line.startsWith("# TYPE ")) {
|
||||
const metricName = line.split(" ")[2];
|
||||
if (seenMetrics.has(metricName)) {
|
||||
processedLines.push(line);
|
||||
}
|
||||
} else if (line.trim() && !line.startsWith("#")) {
|
||||
// Add worker label to each metric line and collect for later
|
||||
const processedLine = line.replace(
|
||||
/^([a-z][a-z0-9_]*)(?:{([^}]*)})?(\s+[0-9.e+-]+.*)/,
|
||||
(match, metricName, existingLabels, valueAndRest) => {
|
||||
if (existingLabels) {
|
||||
return `${metricName}{${existingLabels},worker="master"}${valueAndRest}`;
|
||||
} else {
|
||||
return `${metricName}{worker="master"}${valueAndRest}`;
|
||||
}
|
||||
},
|
||||
);
|
||||
allMetricValues.push(processedLine);
|
||||
}
|
||||
}
|
||||
|
||||
// Collect metrics from all workers
|
||||
for (let i = 0; i < config.numWorkers(); i++) {
|
||||
const workerPort = config.workerPortByIndex(i);
|
||||
const workerUrl = `http://localhost:${workerPort}/metrics`;
|
||||
console.log(`Fetching metrics from worker ${i} at ${workerUrl}`);
|
||||
|
||||
try {
|
||||
const response = await fetch(workerUrl, {
|
||||
headers: {
|
||||
[config.adminHeader()]: config.adminToken(),
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
console.error(`Worker ${i} returned status ${response.status}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const metricsText = await response.text();
|
||||
const lines = metricsText.split("\n");
|
||||
|
||||
for (let j = 0; j < lines.length; j++) {
|
||||
const line = lines[j];
|
||||
|
||||
// Collect HELP and TYPE info if we haven't seen this metric before
|
||||
if (line.startsWith("# HELP ")) {
|
||||
const metricName = line.split(" ")[2];
|
||||
if (!seenMetrics.has(metricName)) {
|
||||
seenMetrics.add(metricName);
|
||||
processedLines.push(line);
|
||||
}
|
||||
} else if (line.startsWith("# TYPE ")) {
|
||||
const metricName = line.split(" ")[2];
|
||||
if (
|
||||
seenMetrics.has(metricName) &&
|
||||
!processedLines.some((l) =>
|
||||
l.startsWith(`# TYPE ${metricName}`),
|
||||
)
|
||||
) {
|
||||
processedLines.push(line);
|
||||
}
|
||||
} else if (line.trim() && !line.startsWith("#")) {
|
||||
// Process and collect actual metric values
|
||||
try {
|
||||
const processedLine = line.replace(
|
||||
/^([a-z][a-z0-9_]*)(?:{([^}]*)})?(\s+[0-9.e+-]+.*)/,
|
||||
(match, metricName, existingLabels, valueAndRest) => {
|
||||
if (existingLabels) {
|
||||
return `${metricName}{${existingLabels},worker="worker-${i}"}${valueAndRest}`;
|
||||
} else {
|
||||
return `${metricName}{worker="worker-${i}"}${valueAndRest}`;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// Make sure the line was actually processed (regex matched)
|
||||
if (processedLine !== line) {
|
||||
allMetricValues.push(processedLine);
|
||||
} else if (
|
||||
line.match(/^[a-z][a-z0-9_]*(?:{[^}]*})?\s+[0-9.e+-]+.*/)
|
||||
) {
|
||||
// This looks like a metric line but didn't match our regex, try a more general approach
|
||||
const parts = line.split(/({|\s+)/);
|
||||
if (parts.length >= 3) {
|
||||
const metricName = parts[0];
|
||||
if (line.includes("{")) {
|
||||
// Has labels
|
||||
const labelEndIndex = line.indexOf("}");
|
||||
const valueStartIndex = labelEndIndex + 1;
|
||||
if (labelEndIndex > 0 && valueStartIndex < line.length) {
|
||||
const labels = line.substring(
|
||||
line.indexOf("{") + 1,
|
||||
labelEndIndex,
|
||||
);
|
||||
const valueAndRest = line.substring(valueStartIndex);
|
||||
allMetricValues.push(
|
||||
`${metricName}{${labels},worker="worker-${i}"}${valueAndRest}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// No labels
|
||||
const valueAndRest = line.substring(metricName.length);
|
||||
allMetricValues.push(
|
||||
`${metricName}{worker="worker-${i}"}${valueAndRest}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error processing metric line: ${line}`, error);
|
||||
// Skip this line if there's an error
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error fetching metrics from worker ${i}:`, error);
|
||||
allMetricValues.push(
|
||||
`# Error fetching metrics from worker ${i}: ${error.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Combine metadata with all metric values and ensure it ends with a newline
|
||||
const combinedMetrics = [...processedLines, ...allMetricValues].join(
|
||||
"\n",
|
||||
);
|
||||
|
||||
// Send the combined response with a final newline to prevent unexpected end of input
|
||||
clearTimeout(timeout);
|
||||
res.set("Content-Type", register.contentType);
|
||||
res.end(combinedMetrics + "\n");
|
||||
} catch (error) {
|
||||
console.error("Error collecting metrics:", error);
|
||||
clearTimeout(timeout);
|
||||
res.status(500).end(`# Error collecting metrics: ${error.message}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Start the metrics server on port 9090
|
||||
const METRICS_PORT = 9090;
|
||||
metricsServer.listen(METRICS_PORT, () => {
|
||||
console.log(`Metrics server listening on port ${METRICS_PORT}`);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { resourceFromAttributes } from "@opentelemetry/resources";
|
||||
import {
|
||||
ATTR_SERVICE_NAME,
|
||||
ATTR_SERVICE_VERSION,
|
||||
} from "@opentelemetry/semantic-conventions";
|
||||
import { getServerConfigFromServer } from "../core/configuration/ConfigLoader";
|
||||
|
||||
const config = getServerConfigFromServer();
|
||||
|
||||
export function getOtelResource() {
|
||||
return resourceFromAttributes({
|
||||
[ATTR_SERVICE_NAME]: "openfront",
|
||||
[ATTR_SERVICE_VERSION]: "1.0.0",
|
||||
"service.instance.id": process.env.HOSTNAME,
|
||||
"openfront.environment": config.env(),
|
||||
"openfront.host": process.env.HOST,
|
||||
"openfront.domain": process.env.DOMAIN,
|
||||
"openfront.subdomain": process.env.SUBDOMAIN,
|
||||
"openfront.component": process.env.WORKER_ID
|
||||
? "Worker " + process.env.WORKER_ID
|
||||
: "Master",
|
||||
// The comma-separated list tells OpenTelemetry which resource attributes
|
||||
// should be converted to Loki labels
|
||||
"loki.resource.labels":
|
||||
"service.name,service.instance.id,openfront.environment,openfront.host,openfront.domain,openfront.subdomain,openfront.component",
|
||||
});
|
||||
}
|
||||
+5
-24
@@ -13,7 +13,7 @@ import { Client } from "./Client";
|
||||
import { GameManager } from "./GameManager";
|
||||
import { gatekeeper, LimiterType } from "./Gatekeeper";
|
||||
import { logger } from "./Logger";
|
||||
import { metrics } from "./WorkerMetrics";
|
||||
import { initWorkerMetrics } from "./WorkerMetrics";
|
||||
|
||||
const config = getServerConfigFromServer();
|
||||
|
||||
@@ -33,10 +33,9 @@ export function startWorker() {
|
||||
|
||||
const gm = new GameManager(config, log);
|
||||
|
||||
// Set up periodic metrics updates
|
||||
setInterval(() => {
|
||||
metrics.updateGameMetrics(gm);
|
||||
}, 15000); // Update every 15 seconds
|
||||
if (config.env() == GameEnv.Prod && config.otelEnabled()) {
|
||||
initWorkerMetrics(gm);
|
||||
}
|
||||
|
||||
// Middleware to handle /wX path prefix
|
||||
app.use((req, res, next) => {
|
||||
@@ -165,7 +164,7 @@ export function startWorker() {
|
||||
disableNPCs: req.body.disableNPCs,
|
||||
disableNukes: req.body.disableNukes,
|
||||
gameMode: req.body.gameMode,
|
||||
numPlayerTeams: req.body.numPlayerTeams,
|
||||
playerTeams: req.body.playerTeams,
|
||||
});
|
||||
res.status(200).json({ success: true });
|
||||
}),
|
||||
@@ -251,24 +250,6 @@ export function startWorker() {
|
||||
}),
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/metrics",
|
||||
gatekeeper.httpHandler(LimiterType.Get, async (req, res) => {
|
||||
if (req.headers[config.adminHeader()] !== config.adminToken()) {
|
||||
return res.status(403).end("Access denied");
|
||||
}
|
||||
log.info(`metrics requested on worker ${workerId}`);
|
||||
|
||||
try {
|
||||
const metricsData = await metrics.register.metrics();
|
||||
res.set("Content-Type", metrics.register.contentType);
|
||||
res.end(metricsData);
|
||||
} catch (error) {
|
||||
res.status(500).end(error.message);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
// WebSocket handling
|
||||
wss.on("connection", (ws: WebSocket, req) => {
|
||||
ws.on(
|
||||
|
||||
+82
-35
@@ -1,45 +1,92 @@
|
||||
import promClient from "prom-client";
|
||||
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
|
||||
import {
|
||||
MeterProvider,
|
||||
PeriodicExportingMetricReader,
|
||||
} from "@opentelemetry/sdk-metrics";
|
||||
import * as dotenv from "dotenv";
|
||||
import { getServerConfigFromServer } from "../core/configuration/ConfigLoader";
|
||||
import { GameManager } from "./GameManager";
|
||||
import { getOtelResource } from "./OtelResource";
|
||||
|
||||
// Initialize the Prometheus registry
|
||||
const register = new promClient.Registry();
|
||||
dotenv.config();
|
||||
|
||||
// Enable default Node.js metrics collection
|
||||
promClient.collectDefaultMetrics({ register });
|
||||
export function initWorkerMetrics(gameManager: GameManager): void {
|
||||
// Get server configuration
|
||||
const config = getServerConfigFromServer();
|
||||
|
||||
// Add worker-specific metrics
|
||||
const activeGamesGauge = new promClient.Gauge({
|
||||
name: "openfront_active_games_count",
|
||||
help: "Number of active games on this worker",
|
||||
registers: [register],
|
||||
});
|
||||
// Create resource with worker information
|
||||
const resource = getOtelResource();
|
||||
|
||||
const connectedClientsGauge = new promClient.Gauge({
|
||||
name: "openfront_connected_clients_count",
|
||||
help: "Number of connected clients on this worker",
|
||||
registers: [register],
|
||||
});
|
||||
// Configure auth headers
|
||||
const headers = {};
|
||||
if (config.otelEnabled()) {
|
||||
headers["Authorization"] =
|
||||
"Basic " +
|
||||
Buffer.from(`${config.otelUsername()}:${config.otelPassword()}`).toString(
|
||||
"base64",
|
||||
);
|
||||
}
|
||||
|
||||
const memoryUsageGauge = new promClient.Gauge({
|
||||
name: "openfront_memory_usage_bytes",
|
||||
help: "Current memory usage of the worker process in bytes",
|
||||
registers: [register],
|
||||
});
|
||||
// Create metrics exporter
|
||||
const metricExporter = new OTLPMetricExporter({
|
||||
url: `${config.otelEndpoint()}/v1/metrics`,
|
||||
headers,
|
||||
});
|
||||
|
||||
// Export the metrics for use in the worker
|
||||
export const metrics = {
|
||||
register,
|
||||
activeGamesGauge,
|
||||
connectedClientsGauge,
|
||||
memoryUsageGauge,
|
||||
// Configure the metric reader
|
||||
const metricReader = new PeriodicExportingMetricReader({
|
||||
exporter: metricExporter,
|
||||
exportIntervalMillis: 15000, // Export metrics every 15 seconds
|
||||
});
|
||||
|
||||
// Function to update game-related metrics
|
||||
updateGameMetrics: (gameManager: GameManager) => {
|
||||
activeGamesGauge.set(gameManager.activeGames());
|
||||
connectedClientsGauge.set(gameManager.activeClients());
|
||||
// Create a meter provider
|
||||
const meterProvider = new MeterProvider({
|
||||
resource,
|
||||
readers: [metricReader],
|
||||
});
|
||||
|
||||
// Update memory usage metrics
|
||||
// Get meter for creating metrics
|
||||
const meter = meterProvider.getMeter("worker-metrics");
|
||||
|
||||
// Create observable gauges
|
||||
const activeGamesGauge = meter.createObservableGauge(
|
||||
"openfront.active_games.gauge",
|
||||
{
|
||||
description: "Number of active games on this worker",
|
||||
},
|
||||
);
|
||||
|
||||
const connectedClientsGauge = meter.createObservableGauge(
|
||||
"openfront.connected_clients.gauge",
|
||||
{
|
||||
description: "Number of connected clients on this worker",
|
||||
},
|
||||
);
|
||||
|
||||
const memoryUsageGauge = meter.createObservableGauge(
|
||||
"openfront.memory_usage.bytes",
|
||||
{
|
||||
description: "Current memory usage of the worker process in bytes",
|
||||
},
|
||||
);
|
||||
|
||||
// Register callback for active games metric
|
||||
activeGamesGauge.addCallback((result) => {
|
||||
const count = gameManager.activeGames();
|
||||
result.observe(count);
|
||||
});
|
||||
|
||||
// Register callback for connected clients metric
|
||||
connectedClientsGauge.addCallback((result) => {
|
||||
const count = gameManager.activeClients();
|
||||
result.observe(count);
|
||||
});
|
||||
|
||||
// Register callback for memory usage metric
|
||||
memoryUsageGauge.addCallback((result) => {
|
||||
const memoryUsage = process.memoryUsage();
|
||||
memoryUsageGauge.set(memoryUsage.heapUsed);
|
||||
},
|
||||
};
|
||||
result.observe(memoryUsage.heapUsed);
|
||||
});
|
||||
|
||||
console.log("Metrics initialized with GameManager");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user