From ee459b74100921e44297b65d9e58600883e9332b Mon Sep 17 00:00:00 2001 From: Aleksey Orekhovsky Date: Fri, 1 Aug 2025 11:49:07 +0700 Subject: [PATCH] Reduce Docker image size by refactoring map loading (#1621) ## Description: This PR implements a major refactoring of how map data is stored and loaded, as described in #1242. Previously, map data (`.bin` files) was bundled directly into the client-side JavaScript by Webpack using `binary-loader`. This approach led to data duplication and increased bundle/image sizes. This refactoring changes the strategy entirely: - `GameMapLoader` interface has been introduced to decouple the map loading mechanism from the components that use it. - New `FetchGameMapLoader` implementation loads map data by fetching it from static server endpoint. - Webpack configuration and `Dockerfile` have been updated to serve the map files as static assets and to remove the source `resources/maps` directory from the final image, thus eliminating data duplication. This leads to several key improvements: - Docker image size is reduced from ~750 MB to ~600 MB. - Build time is decreased. On my local machine, the docker image build time went from 48s to 43s. Most of this speed-up comes from faster Webpack builds (reduced from 16s to 11s), as it no longer needs to process large binary files. This performance gain will be noticeable for all developers during local development, not just in the CI workflow. ## Please complete the following: - [X] I have added screenshots for all UI updates - [X] I process any text displayed to the user through translateText() and I've added it to the en.json file - [X] I have added relevant tests to the test directory - [X] I confirm I have thoroughly tested these changes and take full responsibility for any bugs introduced - [X] I have read and accepted the CLA agreement (only required once). ## Please put your Discord username so you can be contacted if a bug or regression is found: aaa4xu --- Dockerfile | 7 +- src/client/ClientGameRunner.ts | 11 ++- src/client/HostLobbyModal.ts | 2 +- src/client/Main.ts | 1 + src/client/PublicLobby.ts | 2 +- src/client/TerrainMapFileLoader.ts | 4 ++ src/client/components/Maps.ts | 2 +- src/core/GameRunner.ts | 4 +- ...Loader.ts => BinaryLoaderGameMapLoader.ts} | 32 +++++---- src/core/game/FetchGameMapLoader.ts | 69 +++++++++++++++++++ src/core/game/GameMapLoader.ts | 13 ++++ src/core/game/TerrainMapLoader.ts | 16 ++--- src/core/worker/Worker.worker.ts | 4 ++ tests/util/Setup.ts | 8 +-- webpack.config.js | 3 - 15 files changed, 137 insertions(+), 41 deletions(-) create mode 100644 src/client/TerrainMapFileLoader.ts rename src/core/game/{TerrainMapFileLoader.ts => BinaryLoaderGameMapLoader.ts} (72%) create mode 100644 src/core/game/FetchGameMapLoader.ts create mode 100644 src/core/game/GameMapLoader.ts diff --git a/Dockerfile b/Dockerfile index 480ace982..364ed9b04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,11 @@ RUN npm run build-prod # https://openfront.io/commit.txt RUN echo "$GIT_COMMIT" > static/commit.txt +# Remove maps data from final image +FROM base AS prod-files +COPY . . +RUN rm -rf resources/maps + FROM dependencies AS npm-dependencies # Disable Husky hooks ENV HUSKY=0 @@ -67,7 +72,7 @@ COPY --from=npm-dependencies /usr/src/app/node_modules node_modules COPY package.json . # Copy the rest of the application code -COPY . . +COPY --from=prod-files /usr/src/app/ /usr/src/app/ # Copy frontend COPY --from=build /usr/src/app/static static diff --git a/src/client/ClientGameRunner.ts b/src/client/ClientGameRunner.ts index b09d8198b..9904822c4 100644 --- a/src/client/ClientGameRunner.ts +++ b/src/client/ClientGameRunner.ts @@ -13,6 +13,7 @@ import { ServerConfig } from "../core/configuration/Config"; import { getConfig } from "../core/configuration/ConfigLoader"; import { PlayerActions, UnitType } from "../core/game/Game"; import { TileRef } from "../core/game/GameMap"; +import { GameMapLoader } from "../core/game/GameMapLoader"; import { ErrorUpdate, GameUpdateType, @@ -33,6 +34,7 @@ import { } from "./InputHandler"; import { endGame, startGame, startTime } from "./LocalPersistantStats"; import { getPersistentID } from "./Main"; +import { terrainMapFileLoader } from "./TerrainMapFileLoader"; import { SendAttackIntentEvent, SendBoatAttackIntentEvent, @@ -82,7 +84,7 @@ export function joinLobby( const onmessage = (message: ServerMessage) => { if (message.type === "prestart") { console.log(`lobby: game prestarting: ${JSON.stringify(message)}`); - terrainLoad = loadTerrainMap(message.gameMap); + terrainLoad = loadTerrainMap(message.gameMap, terrainMapFileLoader); onPrestart(); } if (message.type === "start") { @@ -98,6 +100,7 @@ export function joinLobby( transport, userSettings, terrainLoad, + terrainMapFileLoader, ).then((r) => r.start()); } if (message.type === "error") { @@ -125,6 +128,7 @@ async function createClientGame( transport: Transport, userSettings: UserSettings, terrainLoad: Promise | null, + mapLoader: GameMapLoader, ): Promise { if (lobbyConfig.gameStartInfo === undefined) { throw new Error("missing gameStartInfo"); @@ -139,7 +143,10 @@ async function createClientGame( if (terrainLoad) { gameMap = await terrainLoad; } else { - gameMap = await loadTerrainMap(lobbyConfig.gameStartInfo.config.gameMap); + gameMap = await loadTerrainMap( + lobbyConfig.gameStartInfo.config.gameMap, + mapLoader, + ); } const worker = new WorkerClient( lobbyConfig.gameStartInfo, diff --git a/src/client/HostLobbyModal.ts b/src/client/HostLobbyModal.ts index eeba83454..0296c29dc 100644 --- a/src/client/HostLobbyModal.ts +++ b/src/client/HostLobbyModal.ts @@ -111,7 +111,7 @@ export class HostLobbyModal extends LitElement { ${this.lobbyIdVisible ? this.lobbyId : "••••••••"} - +
${ diff --git a/src/client/Main.ts b/src/client/Main.ts index 640ae8d93..c7b00c106 100644 --- a/src/client/Main.ts +++ b/src/client/Main.ts @@ -168,6 +168,7 @@ class Client { "single-player-modal", ) as SinglePlayerModal; spModal instanceof SinglePlayerModal; + const singlePlayer = document.getElementById("single-player"); if (singlePlayer === null) throw new Error("Missing single-player"); singlePlayer.addEventListener("click", () => { diff --git a/src/client/PublicLobby.ts b/src/client/PublicLobby.ts index fa6490e55..e94ed794b 100644 --- a/src/client/PublicLobby.ts +++ b/src/client/PublicLobby.ts @@ -2,10 +2,10 @@ import { LitElement, html } from "lit"; import { customElement, state } from "lit/decorators.js"; import { translateText } from "../client/Utils"; import { GameMapType, GameMode } from "../core/game/Game"; -import { terrainMapFileLoader } from "../core/game/TerrainMapFileLoader"; import { GameID, GameInfo } from "../core/Schemas"; import { generateID } from "../core/Util"; import { JoinLobbyEvent } from "./Main"; +import { terrainMapFileLoader } from "./TerrainMapFileLoader"; @customElement("public-lobby") export class PublicLobby extends LitElement { diff --git a/src/client/TerrainMapFileLoader.ts b/src/client/TerrainMapFileLoader.ts new file mode 100644 index 000000000..957a5ea1c --- /dev/null +++ b/src/client/TerrainMapFileLoader.ts @@ -0,0 +1,4 @@ +import version from "../../resources/version.txt"; +import { FetchGameMapLoader } from "../core/game/FetchGameMapLoader"; + +export const terrainMapFileLoader = new FetchGameMapLoader(`/maps`, version); diff --git a/src/client/components/Maps.ts b/src/client/components/Maps.ts index 840c866fc..bbb9c46e7 100644 --- a/src/client/components/Maps.ts +++ b/src/client/components/Maps.ts @@ -1,7 +1,7 @@ import { LitElement, css, html } from "lit"; import { customElement, property, state } from "lit/decorators.js"; import { GameMapType } from "../../core/game/Game"; -import { terrainMapFileLoader } from "../../core/game/TerrainMapFileLoader"; +import { terrainMapFileLoader } from "../TerrainMapFileLoader"; import { translateText } from "../Utils"; // Add map descriptions diff --git a/src/core/GameRunner.ts b/src/core/GameRunner.ts index c2567aeb3..d2fd60c6e 100644 --- a/src/core/GameRunner.ts +++ b/src/core/GameRunner.ts @@ -19,6 +19,7 @@ import { } from "./game/Game"; import { createGame } from "./game/GameImpl"; import { TileRef } from "./game/GameMap"; +import { GameMapLoader } from "./game/GameMapLoader"; import { ErrorUpdate, GameUpdateType, @@ -33,10 +34,11 @@ import { fixProfaneUsername } from "./validations/username"; export async function createGameRunner( gameStart: GameStartInfo, clientID: ClientID, + mapLoader: GameMapLoader, callBack: (gu: GameUpdateViewData) => void, ): Promise { const config = await getConfig(gameStart.config, null); - const gameMap = await loadGameMap(gameStart.config.gameMap); + const gameMap = await loadGameMap(gameStart.config.gameMap, mapLoader); const random = new PseudoRandom(simpleHash(gameStart.gameID)); const humans = gameStart.players.map( diff --git a/src/core/game/TerrainMapFileLoader.ts b/src/core/game/BinaryLoaderGameMapLoader.ts similarity index 72% rename from src/core/game/TerrainMapFileLoader.ts rename to src/core/game/BinaryLoaderGameMapLoader.ts index 62bca0a2c..2107b6faa 100644 --- a/src/core/game/TerrainMapFileLoader.ts +++ b/src/core/game/BinaryLoaderGameMapLoader.ts @@ -1,13 +1,7 @@ import { GameMapType } from "./Game"; +import { GameMapLoader, MapData } from "./GameMapLoader"; import { MapManifest } from "./TerrainMapLoader"; -interface MapData { - mapBin: () => Promise; - miniMapBin: () => Promise; - manifest: () => Promise; - webpPath: () => Promise; -} - export interface BinModule { default: string; } @@ -16,7 +10,7 @@ interface NationMapModule { default: MapManifest; } -class GameMapLoader { +export class BinaryLoaderGameMapLoader implements GameMapLoader { private maps: Map; constructor() { @@ -31,7 +25,7 @@ class GameMapLoader { }; } - public getMapData(map: GameMapType): MapData { + getMapData(map: GameMapType): MapData { const cachedMap = this.maps.get(map); if (cachedMap) { return cachedMap; @@ -46,14 +40,14 @@ class GameMapLoader { import( `!!binary-loader!../../../resources/maps/${fileName}/map.bin` ) as Promise - ).then((m) => m.default), + ).then((m) => this.toUInt8Array(m.default)), ), miniMapBin: this.createLazyLoader(() => ( import( `!!binary-loader!../../../resources/maps/${fileName}/mini_map.bin` ) as Promise - ).then((m) => m.default), + ).then((m) => this.toUInt8Array(m.default)), ), manifest: this.createLazyLoader(() => ( @@ -74,6 +68,18 @@ class GameMapLoader { this.maps.set(map, mapData); return mapData; } -} -export const terrainMapFileLoader = new GameMapLoader(); + /** + * Converts a given string into a UInt8Array where each character in the string + * is represented as an 8-bit unsigned integer. + */ + private toUInt8Array(data: string) { + const rawData = new Uint8Array(data.length); + + for (let i = 0; i < data.length; i++) { + rawData[i] = data.charCodeAt(i); + } + + return rawData; + } +} diff --git a/src/core/game/FetchGameMapLoader.ts b/src/core/game/FetchGameMapLoader.ts new file mode 100644 index 000000000..5d6c72367 --- /dev/null +++ b/src/core/game/FetchGameMapLoader.ts @@ -0,0 +1,69 @@ +import { GameMapType } from "./Game"; +import { GameMapLoader, MapData } from "./GameMapLoader"; + +export class FetchGameMapLoader implements GameMapLoader { + private maps: Map; + + public constructor( + private readonly prefix: string, + private readonly cacheBuster?: string, + ) { + this.maps = new Map(); + } + + public getMapData(map: GameMapType): MapData { + const cachedMap = this.maps.get(map); + if (cachedMap) { + return cachedMap; + } + + const key = Object.keys(GameMapType).find((k) => GameMapType[k] === map); + const fileName = key?.toLowerCase(); + + if (!fileName) { + throw new Error(`Unknown map: ${map}`); + } + + const mapData = { + mapBin: () => this.loadBinaryFromUrl(this.url(fileName, "map.bin")), + miniMapBin: () => + this.loadBinaryFromUrl(this.url(fileName, "mini_map.bin")), + manifest: () => this.loadJsonFromUrl(this.url(fileName, "manifest.json")), + webpPath: async () => this.url(fileName, "thumbnail.webp"), + } satisfies MapData; + + this.maps.set(map, mapData); + return mapData; + } + + private url(map: string, path: string) { + let url = `${this.prefix}/${map}/${path}`; + + if (this.cacheBuster) { + url += `${url.includes("?") ? "&" : "?"}v=${this.cacheBuster}`; + } + + return url; + } + + private async loadBinaryFromUrl(url: string) { + const response = await fetch(url); + + if (!response.ok) { + throw new Error(`Failed to load ${url}: ${response.statusText}`); + } + + const data = await response.arrayBuffer(); + return new Uint8Array(data); + } + + private async loadJsonFromUrl(url: string) { + const response = await fetch(url); + + if (!response.ok) { + throw new Error(`Failed to load ${url}: ${response.statusText}`); + } + + return response.json(); + } +} diff --git a/src/core/game/GameMapLoader.ts b/src/core/game/GameMapLoader.ts new file mode 100644 index 000000000..abf52ccf9 --- /dev/null +++ b/src/core/game/GameMapLoader.ts @@ -0,0 +1,13 @@ +import { GameMapType } from "./Game"; +import { MapManifest } from "./TerrainMapLoader"; + +export interface GameMapLoader { + getMapData(map: GameMapType): MapData; +} + +export interface MapData { + mapBin: () => Promise; + miniMapBin: () => Promise; + manifest: () => Promise; + webpPath: () => Promise; +} diff --git a/src/core/game/TerrainMapLoader.ts b/src/core/game/TerrainMapLoader.ts index fd54cd954..766007ed7 100644 --- a/src/core/game/TerrainMapLoader.ts +++ b/src/core/game/TerrainMapLoader.ts @@ -1,6 +1,6 @@ import { GameMapType } from "./Game"; import { GameMap, GameMapImpl } from "./GameMap"; -import { terrainMapFileLoader } from "./TerrainMapFileLoader"; +import { GameMapLoader } from "./GameMapLoader"; export type TerrainMapData = { manifest: MapManifest; @@ -32,6 +32,7 @@ export interface Nation { export async function loadTerrainMap( map: GameMapType, + terrainMapFileLoader: GameMapLoader, ): Promise { const cached = loadedMaps.get(map); if (cached !== undefined) return cached; @@ -57,7 +58,7 @@ export async function loadTerrainMap( export async function genTerrainFromBin( mapData: MapMetadata, - data: string, + data: Uint8Array, ): Promise { if (data.length !== mapData.width * mapData.height) { throw new Error( @@ -65,19 +66,10 @@ export async function genTerrainFromBin( ); } - // Store raw data in Uint8Array - const rawData = new Uint8Array(mapData.width * mapData.height); - - // Copy data starting after the header - for (let i = 0; i < mapData.width * mapData.height; i++) { - const packedByte = data.charCodeAt(i); - rawData[i] = packedByte; - } - return new GameMapImpl( mapData.width, mapData.height, - rawData, + data, mapData.num_land_tiles, ); } diff --git a/src/core/worker/Worker.worker.ts b/src/core/worker/Worker.worker.ts index 4a065f45b..29e67f214 100644 --- a/src/core/worker/Worker.worker.ts +++ b/src/core/worker/Worker.worker.ts @@ -1,4 +1,6 @@ +import version from "../../../resources/version.txt"; import { createGameRunner, GameRunner } from "../GameRunner"; +import { FetchGameMapLoader } from "../game/FetchGameMapLoader"; import { GameUpdateViewData } from "../game/GameUpdates"; import { AttackAveragePositionResultMessage, @@ -13,6 +15,7 @@ import { const ctx: Worker = self as any; let gameRunner: Promise | null = null; +const mapLoader = new FetchGameMapLoader(`/maps`, version); function gameUpdate(gu: GameUpdateViewData) { sendMessage({ @@ -37,6 +40,7 @@ ctx.addEventListener("message", async (e: MessageEvent) => { gameRunner = createGameRunner( message.gameStartInfo, message.clientID, + mapLoader, gameUpdate, ).then((gr) => { sendMessage({ diff --git a/tests/util/Setup.ts b/tests/util/Setup.ts index c552d7d4f..20e8d67d5 100644 --- a/tests/util/Setup.ts +++ b/tests/util/Setup.ts @@ -48,14 +48,10 @@ export async function setup( fs.readFileSync(manifestPath, "utf8"), ) satisfies MapManifest; - // Convert Buffer to string (binary encoding) - const mapBinString = mapBinBuffer.toString("binary"); - const miniMapBinString = miniMapBinBuffer.toString("binary"); - - const gameMap = await genTerrainFromBin(manifest.map, mapBinString); + const gameMap = await genTerrainFromBin(manifest.map, mapBinBuffer); const miniGameMap = await genTerrainFromBin( manifest.mini_map, - miniMapBinString, + miniMapBinBuffer, ); // Configure the game diff --git a/webpack.config.js b/webpack.config.js index df742f7b3..0820e8cd0 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -136,9 +136,6 @@ export default async (env, argv) => { from: path.resolve(__dirname, "resources"), to: path.resolve(__dirname, "static"), noErrorOnMissing: true, - globOptions: { - ignore: ["resources/maps/**/*"], - }, }, ], options: { concurrency: 100 },