Reduce Docker image size by refactoring map loading (#1621)

## Description:

This PR implements a major refactoring of how map data is stored and
loaded, as described in #1242. Previously, map data (`.bin` files) was
bundled directly into the client-side JavaScript by Webpack using
`binary-loader`. This approach led to data duplication and increased
bundle/image sizes.

This refactoring changes the strategy entirely:
- `GameMapLoader` interface has been introduced to decouple the map
loading mechanism from the components that use it.
- New `FetchGameMapLoader` implementation loads map data by fetching it
from static server endpoint.
- Webpack configuration and `Dockerfile` have been updated to serve the
map files as static assets and to remove the source `resources/maps`
directory from the final image, thus eliminating data duplication.

This leads to several key improvements:
- Docker image size is reduced from ~750 MB to ~600 MB.
- Build time is decreased. On my local machine, the docker image build
time went from 48s to 43s.
Most of this speed-up comes from faster Webpack builds (reduced from 16s
to 11s), as it no longer needs to process large binary files. This
performance gain will be noticeable for all developers during local
development, not just in the CI workflow.

## Please complete the following:

- [X] I have added screenshots for all UI updates
- [X] I process any text displayed to the user through translateText()
and I've added it to the en.json file
- [X] I have added relevant tests to the test directory
- [X] I confirm I have thoroughly tested these changes and take full
responsibility for any bugs introduced
- [X] I have read and accepted the CLA agreement (only required once).

## Please put your Discord username so you can be contacted if a bug or
regression is found:

aaa4xu

Dockerfile

@@ -34,6 +34,11 @@ RUN npm run build-prod
 # https://openfront.io/commit.txt
 RUN echo "$GIT_COMMIT" > static/commit.txt
 
+# Remove maps data from final image
+FROM base AS prod-files
+COPY . .
+RUN rm -rf resources/maps
+
 FROM dependencies AS npm-dependencies
 # Disable Husky hooks
 ENV HUSKY=0
@@ -67,7 +72,7 @@ COPY --from=npm-dependencies /usr/src/app/node_modules node_modules
 COPY package.json .
 
 # Copy the rest of the application code
-COPY . .
+COPY --from=prod-files /usr/src/app/ /usr/src/app/
 
 # Copy frontend
 COPY --from=build /usr/src/app/static static

src/client/ClientGameRunner.ts

@@ -13,6 +13,7 @@ import { ServerConfig } from "../core/configuration/Config";
 import { getConfig } from "../core/configuration/ConfigLoader";
 import { PlayerActions, UnitType } from "../core/game/Game";
 import { TileRef } from "../core/game/GameMap";
+import { GameMapLoader } from "../core/game/GameMapLoader";
 import {
   ErrorUpdate,
   GameUpdateType,
@@ -33,6 +34,7 @@ import {
 } from "./InputHandler";
 import { endGame, startGame, startTime } from "./LocalPersistantStats";
 import { getPersistentID } from "./Main";
+import { terrainMapFileLoader } from "./TerrainMapFileLoader";
 import {
   SendAttackIntentEvent,
   SendBoatAttackIntentEvent,
@@ -82,7 +84,7 @@ export function joinLobby(
   const onmessage = (message: ServerMessage) => {
     if (message.type === "prestart") {
       console.log(`lobby: game prestarting: ${JSON.stringify(message)}`);
-      terrainLoad = loadTerrainMap(message.gameMap);
+      terrainLoad = loadTerrainMap(message.gameMap, terrainMapFileLoader);
       onPrestart();
     }
     if (message.type === "start") {
@@ -98,6 +100,7 @@ export function joinLobby(
         transport,
         userSettings,
         terrainLoad,
+        terrainMapFileLoader,
       ).then((r) => r.start());
     }
     if (message.type === "error") {
@@ -125,6 +128,7 @@ async function createClientGame(
   transport: Transport,
   userSettings: UserSettings,
   terrainLoad: Promise<TerrainMapData> | null,
+  mapLoader: GameMapLoader,
 ): Promise<ClientGameRunner> {
   if (lobbyConfig.gameStartInfo === undefined) {
     throw new Error("missing gameStartInfo");
@@ -139,7 +143,10 @@ async function createClientGame(
   if (terrainLoad) {
     gameMap = await terrainLoad;
   } else {
-    gameMap = await loadTerrainMap(lobbyConfig.gameStartInfo.config.gameMap);
+    gameMap = await loadTerrainMap(
+      lobbyConfig.gameStartInfo.config.gameMap,
+      mapLoader,
+    );
   }
   const worker = new WorkerClient(
     lobbyConfig.gameStartInfo,

src/client/HostLobbyModal.ts

@@ -111,7 +111,7 @@ export class HostLobbyModal extends LitElement {
             <span class="lobby-id" @click=${this.copyToClipboard} style="cursor: pointer;">
               ${this.lobbyIdVisible ? this.lobbyId : "••••••••"}
             </span>
-            
+
             <!-- Copy icon/success indicator -->
             <div @click=${this.copyToClipboard} style="margin-left: 8px; cursor: pointer;">
               ${

src/client/Main.ts

@@ -168,6 +168,7 @@ class Client {
       "single-player-modal",
     ) as SinglePlayerModal;
     spModal instanceof SinglePlayerModal;
+
     const singlePlayer = document.getElementById("single-player");
     if (singlePlayer === null) throw new Error("Missing single-player");
     singlePlayer.addEventListener("click", () => {

src/client/PublicLobby.ts

@@ -2,10 +2,10 @@ import { LitElement, html } from "lit";
 import { customElement, state } from "lit/decorators.js";
 import { translateText } from "../client/Utils";
 import { GameMapType, GameMode } from "../core/game/Game";
-import { terrainMapFileLoader } from "../core/game/TerrainMapFileLoader";
 import { GameID, GameInfo } from "../core/Schemas";
 import { generateID } from "../core/Util";
 import { JoinLobbyEvent } from "./Main";
+import { terrainMapFileLoader } from "./TerrainMapFileLoader";
 
 @customElement("public-lobby")
 export class PublicLobby extends LitElement {

src/client/TerrainMapFileLoader.ts

@@ -0,0 +1,4 @@
+import version from "../../resources/version.txt";
+import { FetchGameMapLoader } from "../core/game/FetchGameMapLoader";
+
+export const terrainMapFileLoader = new FetchGameMapLoader(`/maps`, version);

src/client/components/Maps.ts

@@ -1,7 +1,7 @@
 import { LitElement, css, html } from "lit";
 import { customElement, property, state } from "lit/decorators.js";
 import { GameMapType } from "../../core/game/Game";
-import { terrainMapFileLoader } from "../../core/game/TerrainMapFileLoader";
+import { terrainMapFileLoader } from "../TerrainMapFileLoader";
 import { translateText } from "../Utils";
 
 // Add map descriptions

src/core/GameRunner.ts

@@ -19,6 +19,7 @@ import {
 } from "./game/Game";
 import { createGame } from "./game/GameImpl";
 import { TileRef } from "./game/GameMap";
+import { GameMapLoader } from "./game/GameMapLoader";
 import {
   ErrorUpdate,
   GameUpdateType,
@@ -33,10 +34,11 @@ import { fixProfaneUsername } from "./validations/username";
 export async function createGameRunner(
   gameStart: GameStartInfo,
   clientID: ClientID,
+  mapLoader: GameMapLoader,
   callBack: (gu: GameUpdateViewData) => void,
 ): Promise<GameRunner> {
   const config = await getConfig(gameStart.config, null);
-  const gameMap = await loadGameMap(gameStart.config.gameMap);
+  const gameMap = await loadGameMap(gameStart.config.gameMap, mapLoader);
   const random = new PseudoRandom(simpleHash(gameStart.gameID));
 
   const humans = gameStart.players.map(

src/core/game/BinaryLoaderGameMapLoader.ts

@@ -1,13 +1,7 @@
 import { GameMapType } from "./Game";
+import { GameMapLoader, MapData } from "./GameMapLoader";
 import { MapManifest } from "./TerrainMapLoader";
 
-interface MapData {
-  mapBin: () => Promise<string>;
-  miniMapBin: () => Promise<string>;
-  manifest: () => Promise<MapManifest>;
-  webpPath: () => Promise<string>;
-}
-
 export interface BinModule {
   default: string;
 }
@@ -16,7 +10,7 @@ interface NationMapModule {
   default: MapManifest;
 }
 
-class GameMapLoader {
+export class BinaryLoaderGameMapLoader implements GameMapLoader {
   private maps: Map<GameMapType, MapData>;
 
   constructor() {
@@ -31,7 +25,7 @@ class GameMapLoader {
     };
   }
 
-  public getMapData(map: GameMapType): MapData {
+  getMapData(map: GameMapType): MapData {
     const cachedMap = this.maps.get(map);
     if (cachedMap) {
       return cachedMap;
@@ -46,14 +40,14 @@ class GameMapLoader {
           import(
             `!!binary-loader!../../../resources/maps/${fileName}/map.bin`
           ) as Promise<BinModule>
-        ).then((m) => m.default),
+        ).then((m) => this.toUInt8Array(m.default)),
       ),
       miniMapBin: this.createLazyLoader(() =>
         (
           import(
             `!!binary-loader!../../../resources/maps/${fileName}/mini_map.bin`
           ) as Promise<BinModule>
-        ).then((m) => m.default),
+        ).then((m) => this.toUInt8Array(m.default)),
       ),
       manifest: this.createLazyLoader(() =>
         (
@@ -74,6 +68,18 @@ class GameMapLoader {
     this.maps.set(map, mapData);
     return mapData;
   }
-}
 
-export const terrainMapFileLoader = new GameMapLoader();
+  /**
+   * Converts a given string into a UInt8Array where each character in the string
+   * is represented as an 8-bit unsigned integer.
+   */
+  private toUInt8Array(data: string) {
+    const rawData = new Uint8Array(data.length);
+
+    for (let i = 0; i < data.length; i++) {
+      rawData[i] = data.charCodeAt(i);
+    }
+
+    return rawData;
+  }
+}

src/core/game/FetchGameMapLoader.ts

@@ -0,0 +1,69 @@
+import { GameMapType } from "./Game";
+import { GameMapLoader, MapData } from "./GameMapLoader";
+
+export class FetchGameMapLoader implements GameMapLoader {
+  private maps: Map<GameMapType, MapData>;
+
+  public constructor(
+    private readonly prefix: string,
+    private readonly cacheBuster?: string,
+  ) {
+    this.maps = new Map<GameMapType, MapData>();
+  }
+
+  public getMapData(map: GameMapType): MapData {
+    const cachedMap = this.maps.get(map);
+    if (cachedMap) {
+      return cachedMap;
+    }
+
+    const key = Object.keys(GameMapType).find((k) => GameMapType[k] === map);
+    const fileName = key?.toLowerCase();
+
+    if (!fileName) {
+      throw new Error(`Unknown map: ${map}`);
+    }
+
+    const mapData = {
+      mapBin: () => this.loadBinaryFromUrl(this.url(fileName, "map.bin")),
+      miniMapBin: () =>
+        this.loadBinaryFromUrl(this.url(fileName, "mini_map.bin")),
+      manifest: () => this.loadJsonFromUrl(this.url(fileName, "manifest.json")),
+      webpPath: async () => this.url(fileName, "thumbnail.webp"),
+    } satisfies MapData;
+
+    this.maps.set(map, mapData);
+    return mapData;
+  }
+
+  private url(map: string, path: string) {
+    let url = `${this.prefix}/${map}/${path}`;
+
+    if (this.cacheBuster) {
+      url += `${url.includes("?") ? "&" : "?"}v=${this.cacheBuster}`;
+    }
+
+    return url;
+  }
+
+  private async loadBinaryFromUrl(url: string) {
+    const response = await fetch(url);
+
+    if (!response.ok) {
+      throw new Error(`Failed to load ${url}: ${response.statusText}`);
+    }
+
+    const data = await response.arrayBuffer();
+    return new Uint8Array(data);
+  }
+
+  private async loadJsonFromUrl(url: string) {
+    const response = await fetch(url);
+
+    if (!response.ok) {
+      throw new Error(`Failed to load ${url}: ${response.statusText}`);
+    }
+
+    return response.json();
+  }
+}

src/core/game/GameMapLoader.ts

@@ -0,0 +1,13 @@
+import { GameMapType } from "./Game";
+import { MapManifest } from "./TerrainMapLoader";
+
+export interface GameMapLoader {
+  getMapData(map: GameMapType): MapData;
+}
+
+export interface MapData {
+  mapBin: () => Promise<Uint8Array>;
+  miniMapBin: () => Promise<Uint8Array>;
+  manifest: () => Promise<MapManifest>;
+  webpPath: () => Promise<string>;
+}

src/core/game/TerrainMapLoader.ts

@@ -1,6 +1,6 @@
 import { GameMapType } from "./Game";
 import { GameMap, GameMapImpl } from "./GameMap";
-import { terrainMapFileLoader } from "./TerrainMapFileLoader";
+import { GameMapLoader } from "./GameMapLoader";
 
 export type TerrainMapData = {
   manifest: MapManifest;
@@ -32,6 +32,7 @@ export interface Nation {
 
 export async function loadTerrainMap(
   map: GameMapType,
+  terrainMapFileLoader: GameMapLoader,
 ): Promise<TerrainMapData> {
   const cached = loadedMaps.get(map);
   if (cached !== undefined) return cached;
@@ -57,7 +58,7 @@ export async function loadTerrainMap(
 
 export async function genTerrainFromBin(
   mapData: MapMetadata,
-  data: string,
+  data: Uint8Array,
 ): Promise<GameMap> {
   if (data.length !== mapData.width * mapData.height) {
     throw new Error(
@@ -65,19 +66,10 @@ export async function genTerrainFromBin(
     );
   }
 
-  // Store raw data in Uint8Array
-  const rawData = new Uint8Array(mapData.width * mapData.height);
-
-  // Copy data starting after the header
-  for (let i = 0; i < mapData.width * mapData.height; i++) {
-    const packedByte = data.charCodeAt(i);
-    rawData[i] = packedByte;
-  }
-
   return new GameMapImpl(
     mapData.width,
     mapData.height,
-    rawData,
+    data,
     mapData.num_land_tiles,
   );
 }

src/core/worker/Worker.worker.ts

@@ -1,4 +1,6 @@
+import version from "../../../resources/version.txt";
 import { createGameRunner, GameRunner } from "../GameRunner";
+import { FetchGameMapLoader } from "../game/FetchGameMapLoader";
 import { GameUpdateViewData } from "../game/GameUpdates";
 import {
   AttackAveragePositionResultMessage,
@@ -13,6 +15,7 @@ import {
 
 const ctx: Worker = self as any;
 let gameRunner: Promise<GameRunner> | null = null;
+const mapLoader = new FetchGameMapLoader(`/maps`, version);
 
 function gameUpdate(gu: GameUpdateViewData) {
   sendMessage({
@@ -37,6 +40,7 @@ ctx.addEventListener("message", async (e: MessageEvent<MainThreadMessage>) => {
         gameRunner = createGameRunner(
           message.gameStartInfo,
           message.clientID,
+          mapLoader,
           gameUpdate,
         ).then((gr) => {
           sendMessage({

tests/util/Setup.ts

@@ -48,14 +48,10 @@ export async function setup(
     fs.readFileSync(manifestPath, "utf8"),
   ) satisfies MapManifest;
 
-  // Convert Buffer to string (binary encoding)
-  const mapBinString = mapBinBuffer.toString("binary");
-  const miniMapBinString = miniMapBinBuffer.toString("binary");
-
-  const gameMap = await genTerrainFromBin(manifest.map, mapBinString);
+  const gameMap = await genTerrainFromBin(manifest.map, mapBinBuffer);
   const miniGameMap = await genTerrainFromBin(
     manifest.mini_map,
-    miniMapBinString,
+    miniMapBinBuffer,
   );
 
   // Configure the game

webpack.config.js

@@ -136,9 +136,6 @@ export default async (env, argv) => {
             from: path.resolve(__dirname, "resources"),
             to: path.resolve(__dirname, "static"),
             noErrorOnMissing: true,
-            globOptions: {
-              ignore: ["resources/maps/**/*"],
-            },
           },
         ],
         options: { concurrency: 100 },