Add desync tracking metrics (#2707)

## Description:

- Added observable gauge to track total number of client desyncs
detected across game servers
- Tracks desyncs at the individual client level (counts each out-of-sync
client)
- Exposes metric as `openfront.desyncs.gauge` for monitoring

## Please complete the following:

- [x] I have added screenshots for all UI updates
- [x] I process any text displayed to the user through translateText()
and I've added it to the en.json file
- [x] I have added relevant tests to the test directory
- [ ] I confirm I have thoroughly tested these changes and take full
responsibility for any bugs introduced

## Please put your Discord username so you can be contacted if a bug or
regression is found:

evan

src/server/GameManager.ts

@@ -94,6 +94,14 @@ export class GameManager {
     return totalClients;
   }
 
+  desyncCount(): number {
+    let totalDesyncs = 0;
+    this.games.forEach((game: GameServer) => {
+      totalDesyncs += game.desyncCount;
+    });
+    return totalDesyncs;
+  }
+
   tick() {
     const active = new Map<GameID, GameServer>();
     for (const [id, game] of this.games) {

src/server/GameServer.ts

@@ -69,6 +69,8 @@ export class GameServer {
     { winner: ClientSendWinnerMessage; ips: Set<string> }
   > = new Map();
 
+  public desyncCount = 0;
+
   constructor(
     public readonly id: string,
     readonly log_: Logger,
@@ -778,6 +780,8 @@ export class GameServer {
     const { mostCommonHash, outOfSyncClients } =
       this.findOutOfSyncClients(lastHashTurn);
 
+    this.desyncCount += outOfSyncClients.length;
+
     if (outOfSyncClients.length === 0) {
       this.turns[lastHashTurn].hash = mostCommonHash;
       return;

src/server/WorkerMetrics.ts

@@ -59,6 +59,10 @@ export function initWorkerMetrics(gameManager: GameManager): void {
     },
   );
 
+  const desyncsGauge = meter.createObservableGauge("openfront.desyncs.gauge", {
+    description: "Number of detected desyncs on active games on this worker",
+  });
+
   const memoryUsageGauge = meter.createObservableGauge(
     "openfront.memory_usage.bytes",
     {
@@ -66,19 +70,21 @@ export function initWorkerMetrics(gameManager: GameManager): void {
     },
   );
 
-  // Register callback for active games metric
   activeGamesGauge.addCallback((result) => {
     const count = gameManager.activeGames();
     result.observe(count, getPromLabels());
   });
 
-  // Register callback for connected clients metric
   connectedClientsGauge.addCallback((result) => {
     const count = gameManager.activeClients();
     result.observe(count, getPromLabels());
   });
 
-  // Register callback for memory usage metric
+  desyncsGauge.addCallback((result) => {
+    const count = gameManager.desyncCount();
+    result.observe(count, getPromLabels());
+  });
+
   memoryUsageGauge.addCallback((result) => {
     const memoryUsage = process.memoryUsage();
     result.observe(memoryUsage.heapUsed, getPromLabels());