Files
OpenFrontIO/src/server/WorkerMetrics.ts
T
Evan 5dc38d25f0 Add desync tracking metrics (#2707)
## Description:

- Added observable gauge to track total number of client desyncs
detected across game servers
- Tracks desyncs at the individual client level (counts each out-of-sync
client)
- Exposes metric as `openfront.desyncs.gauge` for monitoring

## Please complete the following:

- [x] I have added screenshots for all UI updates
- [x] I process any text displayed to the user through translateText()
and I've added it to the en.json file
- [x] I have added relevant tests to the test directory
- [ ] I confirm I have thoroughly tested these changes and take full
responsibility for any bugs introduced

## Please put your Discord username so you can be contacted if a bug or
regression is found:

evan
2025-12-26 16:10:43 -08:00

95 lines
2.7 KiB
TypeScript

import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
import {
MeterProvider,
PeriodicExportingMetricReader,
} from "@opentelemetry/sdk-metrics";
import * as dotenv from "dotenv";
import { getServerConfigFromServer } from "../core/configuration/ConfigLoader";
import { GameManager } from "./GameManager";
import { getOtelResource, getPromLabels } from "./OtelResource";
dotenv.config();
export function initWorkerMetrics(gameManager: GameManager): void {
// Get server configuration
const config = getServerConfigFromServer();
// Create resource with worker information
const resource = getOtelResource();
// Configure auth headers
const headers: Record<string, string> = {};
if (config.otelEnabled()) {
headers["Authorization"] = "Basic " + config.otelAuthHeader();
}
// Create metrics exporter
const metricExporter = new OTLPMetricExporter({
url: `${config.otelEndpoint()}/v1/metrics`,
headers,
});
// Configure the metric reader
const metricReader = new PeriodicExportingMetricReader({
exporter: metricExporter,
exportIntervalMillis: 15000, // Export metrics every 15 seconds
});
// Create a meter provider
const meterProvider = new MeterProvider({
resource,
readers: [metricReader],
});
// Get meter for creating metrics
const meter = meterProvider.getMeter("worker-metrics");
// Create observable gauges
const activeGamesGauge = meter.createObservableGauge(
"openfront.active_games.gauge",
{
description: "Number of active games on this worker",
},
);
const connectedClientsGauge = meter.createObservableGauge(
"openfront.connected_clients.gauge",
{
description: "Number of connected clients on this worker",
},
);
const desyncsGauge = meter.createObservableGauge("openfront.desyncs.gauge", {
description: "Number of detected desyncs on active games on this worker",
});
const memoryUsageGauge = meter.createObservableGauge(
"openfront.memory_usage.bytes",
{
description: "Current memory usage of the worker process in bytes",
},
);
activeGamesGauge.addCallback((result) => {
const count = gameManager.activeGames();
result.observe(count, getPromLabels());
});
connectedClientsGauge.addCallback((result) => {
const count = gameManager.activeClients();
result.observe(count, getPromLabels());
});
desyncsGauge.addCallback((result) => {
const count = gameManager.desyncCount();
result.observe(count, getPromLabels());
});
memoryUsageGauge.addCallback((result) => {
const memoryUsage = process.memoryUsage();
result.observe(memoryUsage.heapUsed, getPromLabels());
});
console.log("Metrics initialized with GameManager");
}