From 3494c549066f6506e00b16094dff63735273b63b Mon Sep 17 00:00:00 2001 From: Evan Date: Thu, 27 Feb 2025 10:45:00 -0800 Subject: [PATCH] use node cluster to shard server --- Dockerfile | 2 +- docker-compose.yml | 22 +- nginx.conf | 108 +++++ src/client/HostLobbyModal.ts | 100 ++--- src/client/JoinPrivateLobbyModal.ts | 28 +- src/client/LocalServer.ts | 10 +- src/client/Main.ts | 12 +- src/client/PublicLobby.ts | 16 +- src/client/SinglePlayerModal.ts | 2 +- src/client/Transport.ts | 5 +- src/core/Schemas.ts | 20 +- src/core/configuration/Config.ts | 7 +- src/core/configuration/DefaultConfig.ts | 23 +- src/server/GameManager.ts | 126 +----- src/server/GameServer.ts | 35 +- src/server/Master.ts | 255 ++++++++++++ src/server/Server.ts | 498 +----------------------- src/server/Worker.ts | 374 ++++++++++++++++++ webpack.config.js | 63 ++- 19 files changed, 1006 insertions(+), 700 deletions(-) create mode 100644 nginx.conf create mode 100644 src/server/Master.ts create mode 100644 src/server/Worker.ts diff --git a/Dockerfile b/Dockerfile index fb6cad03b..bf7787be4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,6 @@ COPY . . # Build the client-side application RUN npm run build-prod # Expose the port the app runs on -EXPOSE 3000 +EXPOSE 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 # Define the command to run the app CMD ["npm", "run", "start:server"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index abec63525..4076e1794 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,17 +2,33 @@ version: "3" services: game-server: build: . - ports: - - "3000:3000" + expose: + - "3000" + - "3001" + - "3002" + - "3003" + - "3004" + - "3005" + - "3006" + - "3007" + - "3008" + - "3009" + - "3010" + - "3011" + - "3012" + - "3013" + - "3014" + - "3015" environment: - NODE_ENV=production + nginx: image: nginx:latest ports: - "80:80" - "443:443" volumes: - - ./nginx.conf:/etc/nginx/nginx.conf + - ./nginx.conf:/etc/nginx/conf.d/default.conf - /etc/letsencrypt:/etc/letsencrypt depends_on: - game-server diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 000000000..e27817df0 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,108 @@ +resolver 127.0.0.11 valid=30s; + +# Access log format with minimal information +log_format minimal '$remote_addr - [$time_local] "$request" $status'; + +map $uri $port { + ~^/w0/ 3001; + ~^/w1/ 3002; + ~^/w2/ 3003; + ~^/w3/ 3004; + ~^/w4/ 3005; + ~^/w5/ 3006; + ~^/w6/ 3007; + ~^/w7/ 3008; + ~^/w8/ 3009; + ~^/w9/ 3010; + ~^/w10/ 3011; + ~^/w11/ 3012; + ~^/w12/ 3013; + ~^/w13/ 3014; + ~^/w14/ 3015; + default 3000; +} + +# Don't strip the path for WebSocket connections +map $http_upgrade $strip_path { + default 1; + websocket 0; +} + +map $uri $uri_path { + # Only strip path if not a WebSocket request + ~^/w\d+(/.*)?$ $1; + default $uri; +} + +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +server { + listen 80; + + # Reduced logging + access_log /var/log/nginx/access.log minimal; + error_log /var/log/nginx/error.log error; + + # Disable logging for common requests + location = /favicon.ico { + access_log off; + log_not_found off; + return 204; + } + + # WebSocket timeout settings + proxy_read_timeout 300s; + proxy_connect_timeout 75s; + proxy_send_timeout 300s; + + # Special location block just for WebSocket connections + location ~* ^/w\d+$ { + set $ws_port 0; + + if ($uri ~* ^/w0) { + set $ws_port 3001; + } + if ($uri ~* ^/w1) { + set $ws_port 3002; + } + if ($uri ~* ^/w2) { + set $ws_port 3003; + } + if ($uri ~* ^/w3) { + set $ws_port 3004; + } + if ($uri ~* ^/w4) { + set $ws_port 3005; + } + # Add more conditions for other worker paths + + proxy_pass http://game-server:$ws_port; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_buffering off; + } + + # Regular location for all other requests + location / { + set $upstream_endpoint game-server:$port; + proxy_pass http://$upstream_endpoint$uri_path; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_buffering off; + } +} \ No newline at end of file diff --git a/src/client/HostLobbyModal.ts b/src/client/HostLobbyModal.ts index 2a4cd8c4f..a70d91461 100644 --- a/src/client/HostLobbyModal.ts +++ b/src/client/HostLobbyModal.ts @@ -1,11 +1,13 @@ import { LitElement, html, css } from "lit"; import { customElement, property, state } from "lit/decorators.js"; import { Difficulty, GameMapType, GameType } from "../core/game/Game"; -import { Lobby } from "../core/Schemas"; +import { GameConfig, GameInfo } from "../core/Schemas"; import { consolex } from "../core/Consolex"; import "./components/Difficulties"; import { DifficultyDescription } from "./components/Difficulties"; import "./components/Maps"; +import { generateID } from "../core/Util"; +import { getConfig, getServerConfig } from "../core/configuration/Config"; @customElement("host-lobby-modal") export class HostLobbyModal extends LitElement { @@ -412,7 +414,7 @@ export class HostLobbyModal extends LitElement { step="1" @input=${this.handleBotsChange} @change=${this.handleBotsChange} - .value="${this.bots}" + .value="${String(this.bots)}" />
Bots: ${this.bots == 0 ? "Disabled" : this.bots} @@ -508,7 +510,7 @@ export class HostLobbyModal extends LitElement { public open() { createLobby() .then((lobby) => { - this.lobbyId = lobby.id; + this.lobbyId = lobby.gameID; // join lobby }) .then(() => { @@ -517,7 +519,7 @@ export class HostLobbyModal extends LitElement { detail: { gameType: GameType.Private, lobby: { - id: this.lobbyId, + gameID: this.lobbyId, }, map: this.selectedMap, difficulty: this.selectedDifficulty, @@ -582,21 +584,24 @@ export class HostLobbyModal extends LitElement { } private async putGameConfig() { - const response = await fetch(`/private_lobby/${this.lobbyId}`, { - method: "PUT", - headers: { - "Content-Type": "application/json", + const response = await fetch( + `${getServerConfig().workerPath(this.lobbyId)}/game/${this.lobbyId}`, + { + method: "PUT", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + gameMap: this.selectedMap, + difficulty: this.selectedDifficulty, + disableNPCs: this.disableNPCs, + bots: this.bots, + infiniteGold: this.infiniteGold, + infiniteTroops: this.infiniteTroops, + instantBuild: this.instantBuild, + } as GameConfig), }, - body: JSON.stringify({ - gameMap: this.selectedMap, - difficulty: this.selectedDifficulty, - disableNPCs: this.disableNPCs, - bots: this.bots, - infiniteGold: this.infiniteGold, - infiniteTroops: this.infiniteTroops, - instantBuild: this.instantBuild, - }), - }); + ); } private async startGame() { @@ -604,12 +609,15 @@ export class HostLobbyModal extends LitElement { `Starting private game with map: ${GameMapType[this.selectedMap]}`, ); this.close(); - const response = await fetch(`/start_private_lobby/${this.lobbyId}`, { - method: "POST", - headers: { - "Content-Type": "application/json", + const response = await fetch( + `${getServerConfig().workerPath(this.lobbyId)}/start_game/${this.lobbyId}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, }, - }); + ); } private async copyToClipboard() { @@ -623,34 +631,42 @@ export class HostLobbyModal extends LitElement { this.copySuccess = false; }, 2000); } catch (err) { - consolex.error("Failed to copy text: ", err); + consolex.error(`Failed to copy text: ${err}`); } } private async pollPlayers() { - fetch(`/lobby/${this.lobbyId}`, { - method: "GET", - headers: { - "Content-Type": "application/json", + fetch( + `/${getServerConfig().workerPath(this.lobbyId)}/game/${this.lobbyId}`, + { + method: "GET", + headers: { + "Content-Type": "application/json", + }, }, - }) + ) .then((response) => response.json()) - .then((data) => { + .then((data: GameInfo) => { console.log(`got response: ${data}`); - this.players = data.players.map((p) => p.username); + this.players = data.clients.map((p) => p.username); }); } } -async function createLobby(): Promise { +async function createLobby(): Promise { + const serverConfig = getServerConfig(); try { - const response = await fetch("/private_lobby", { - method: "POST", - headers: { - "Content-Type": "application/json", + const id = generateID(); + const response = await fetch( + `/${serverConfig.workerPath(id)}/create_game/${id}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + // body: JSON.stringify(data), // Include this if you need to send data }, - // body: JSON.stringify(data), // Include this if you need to send data - }); + ); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); @@ -659,13 +675,7 @@ async function createLobby(): Promise { const data = await response.json(); consolex.log("Success:", data); - // Assuming the server returns an object with an 'id' property - const lobby: Lobby = { - id: data.id, - // Add other properties as needed - }; - - return lobby; + return data as GameInfo; } catch (error) { consolex.error("Error creating lobby:", error); throw error; // Re-throw the error so the caller can handle it diff --git a/src/client/JoinPrivateLobbyModal.ts b/src/client/JoinPrivateLobbyModal.ts index 44c60e498..1e47e757b 100644 --- a/src/client/JoinPrivateLobbyModal.ts +++ b/src/client/JoinPrivateLobbyModal.ts @@ -2,6 +2,8 @@ import { LitElement, html, css } from "lit"; import { customElement, property, state, query } from "lit/decorators.js"; import { GameMapType, GameType } from "../core/game/Game"; import { consolex } from "../core/Consolex"; +import { getConfig, getServerConfig } from "../core/configuration/Config"; +import { GameInfo } from "../core/Schemas"; @customElement("join-private-lobby-modal") export class JoinPrivateLobbyModal extends LitElement { @@ -358,13 +360,16 @@ export class JoinPrivateLobbyModal extends LitElement { consolex.log(`Joining lobby with ID: ${lobbyId}`); this.message = "Checking lobby..."; // Set initial message - fetch(`/lobby/${lobbyId}/exists`, { + const url = `${window.location.origin}/${getServerConfig().workerPath(lobbyId)}/game/${lobbyId}/exists`; + fetch(url, { method: "GET", headers: { "Content-Type": "application/json", }, }) - .then((response) => response.json()) + .then((response) => { + return response.json(); + }) .then((data) => { if (data.exists) { this.message = "Joined successfully! Waiting for game to start..."; @@ -372,7 +377,7 @@ export class JoinPrivateLobbyModal extends LitElement { this.dispatchEvent( new CustomEvent("join-lobby", { detail: { - lobby: { id: lobbyId }, + lobby: { gameID: lobbyId }, gameType: GameType.Private, map: GameMapType.World, }, @@ -394,15 +399,18 @@ export class JoinPrivateLobbyModal extends LitElement { private async pollPlayers() { if (!this.lobbyIdInput?.value) return; - fetch(`/lobby/${this.lobbyIdInput.value}`, { - method: "GET", - headers: { - "Content-Type": "application/json", + fetch( + `${getServerConfig().workerPath(this.lobbyIdInput.value)}/lobby/${this.lobbyIdInput.value}`, + { + method: "GET", + headers: { + "Content-Type": "application/json", + }, }, - }) + ) .then((response) => response.json()) - .then((data) => { - this.players = data.players.map((p) => p.username); + .then((data: GameInfo) => { + this.players = data.clients.map((p) => p.username); }) .catch((error) => { consolex.error("Error polling players:", error); diff --git a/src/client/LocalServer.ts b/src/client/LocalServer.ts index f3f51e34d..282e916af 100644 --- a/src/client/LocalServer.ts +++ b/src/client/LocalServer.ts @@ -1,4 +1,9 @@ -import { Config, GameEnv, ServerConfig } from "../core/configuration/Config"; +import { + Config, + GameEnv, + getServerConfig, + ServerConfig, +} from "../core/configuration/Config"; import { consolex } from "../core/Consolex"; import { GameEvent } from "../core/EventBus"; import { @@ -125,6 +130,7 @@ export class LocalServer { const blob = new Blob([JSON.stringify(GameRecordSchema.parse(record))], { type: "application/json", }); - navigator.sendBeacon("/archive_singleplayer_game", blob); + const workerPath = getServerConfig().workerPath(this.lobbyConfig.gameID); + navigator.sendBeacon(`/${workerPath}/archive_singleplayer_game`, blob); } } diff --git a/src/client/Main.ts b/src/client/Main.ts index ed24a6ff8..4ada098b9 100644 --- a/src/client/Main.ts +++ b/src/client/Main.ts @@ -65,10 +65,6 @@ class Client { setFavicon(); document.addEventListener("join-lobby", this.handleJoinLobby.bind(this)); document.addEventListener("leave-lobby", this.handleLeaveLobby.bind(this)); - document.addEventListener( - "single-player", - this.handleSinglePlayer.bind(this), - ); const spModal = document.querySelector( "single-player-modal", @@ -145,7 +141,7 @@ class Client { gameType: gameType, flag: (): string => this.flagInput.getCurrentFlag(), playerName: (): string => this.usernameInput.getCurrentUsername(), - gameID: lobby.id, + gameID: lobby.gameID, persistentID: getPersistentIDFromCookie(), playerID: generateID(), clientID: generateID(), @@ -161,7 +157,7 @@ class Client { this.joinModal.close(); this.publicLobby.stop(); if (gameType != GameType.Singleplayer) { - window.history.pushState({}, "", `/join/${lobby.id}`); + window.history.pushState({}, "", `/join/${lobby.gameID}`); sessionStorage.setItem("inLobby", "true"); } }, @@ -177,10 +173,6 @@ class Client { this.gameStop = null; this.publicLobby.leaveLobby(); } - - private async handleSinglePlayer(event: CustomEvent) { - alert("coming soon"); - } } // Initialize the client when the DOM is loaded diff --git a/src/client/PublicLobby.ts b/src/client/PublicLobby.ts index 5f63f5566..2e0dc1f69 100644 --- a/src/client/PublicLobby.ts +++ b/src/client/PublicLobby.ts @@ -1,16 +1,16 @@ import { LitElement, html } from "lit"; import { customElement, state } from "lit/decorators.js"; -import { Lobby } from "../core/Schemas"; import { Difficulty, GameMapType, GameType } from "../core/game/Game"; import { consolex } from "../core/Consolex"; import { getMapsImage } from "./utilities/Maps"; +import { GameInfo } from "../core/Schemas"; @customElement("public-lobby") export class PublicLobby extends LitElement { - @state() private lobbies: Lobby[] = []; + @state() private lobbies: GameInfo[] = []; @state() public isLobbyHighlighted: boolean = false; private lobbiesInterval: number | null = null; - private currLobby: Lobby = null; + private currLobby: GameInfo = null; createRenderRoot() { return this; @@ -36,15 +36,16 @@ export class PublicLobby extends LitElement { private async fetchAndUpdateLobbies(): Promise { try { const lobbies = await this.fetchLobbies(); + console.log(`got lobbies: ${JSON.stringify(lobbies)}`); this.lobbies = lobbies; } catch (error) { consolex.error("Error fetching lobbies:", error); } } - async fetchLobbies(): Promise { + async fetchLobbies(): Promise { try { - const response = await fetch("/lobbies"); + const response = await fetch("/public_lobbies"); if (!response.ok) throw new Error(`HTTP error! status: ${response.status}`); const data = await response.json(); @@ -67,6 +68,9 @@ export class PublicLobby extends LitElement { if (this.lobbies.length === 0) return html``; const lobby = this.lobbies[0]; + if (!lobby?.gameConfig) { + return; + } const timeRemaining = Math.max(0, Math.floor(lobby.msUntilStart / 1000)); // Format time to show minutes and seconds @@ -121,7 +125,7 @@ export class PublicLobby extends LitElement { this.currLobby = null; } - private lobbyClicked(lobby: Lobby) { + private lobbyClicked(lobby: GameInfo) { if (this.currLobby == null) { this.isLobbyHighlighted = true; this.currLobby = lobby; diff --git a/src/client/SinglePlayerModal.ts b/src/client/SinglePlayerModal.ts index cd3160fdd..57ba9fb05 100644 --- a/src/client/SinglePlayerModal.ts +++ b/src/client/SinglePlayerModal.ts @@ -424,7 +424,7 @@ export class SinglePlayerModal extends LitElement { detail: { gameType: GameType.Singleplayer, lobby: { - id: generateID(), + gameID: generateID(), }, map: this.selectedMap, difficulty: this.selectedDifficulty, diff --git a/src/client/Transport.ts b/src/client/Transport.ts index 2a64b4017..87b64d149 100644 --- a/src/client/Transport.ts +++ b/src/client/Transport.ts @@ -228,9 +228,10 @@ export class Transport { ) { this.startPing(); this.maybeKillSocket(); - const wsHost = process.env.WEBSOCKET_URL || window.location.host; + const wsHost = window.location.host; const wsProtocol = window.location.protocol === "https:" ? "wss:" : "ws:"; - this.socket = new WebSocket(`${wsProtocol}//${wsHost}`); + const workerPath = this.serverConfig.workerPath(this.lobbyConfig.gameID); + this.socket = new WebSocket(`${wsProtocol}//${wsHost}/${workerPath}`); this.onconnect = onconnect; this.onmessage = onmessage; this.socket.onopen = () => { diff --git a/src/core/Schemas.ts b/src/core/Schemas.ts index 74ce4f52f..a190457fa 100644 --- a/src/core/Schemas.ts +++ b/src/core/Schemas.ts @@ -75,6 +75,17 @@ export type GameRecord = z.infer; const PlayerTypeSchema = z.nativeEnum(PlayerType); +export interface GameInfo { + gameID: GameID; + clients?: ClientInfo[]; + numClients?: number; + msUntilStart?: number; + gameConfig?: GameConfig; +} +export interface ClientInfo { + clientID: ClientID; + username: string; +} export enum LogSeverity { Debug = "DEBUG", Info = "INFO", @@ -83,15 +94,6 @@ export enum LogSeverity { Fatal = "FATAL", } -// TODO: create Cell schema - -export interface Lobby { - id: string; - msUntilStart?: number; - numClients?: number; - gameConfig?: GameConfig; -} - const GameConfigSchema = z.object({ gameMap: z.nativeEnum(GameMapType), difficulty: z.nativeEnum(Difficulty), diff --git a/src/core/configuration/Config.ts b/src/core/configuration/Config.ts index a43a8663a..06aca3a35 100644 --- a/src/core/configuration/Config.ts +++ b/src/core/configuration/Config.ts @@ -15,7 +15,7 @@ import { Colord, colord } from "colord"; import { preprodConfig } from "./PreprodConfig"; import { prodConfig } from "./ProdConfig"; import { consolex } from "../Consolex"; -import { GameConfig } from "../Schemas"; +import { GameConfig, GameID } from "../Schemas"; import { DefaultConfig } from "./DefaultConfig"; import { DevConfig, DevServerConfig } from "./DevConfig"; import { GameMap, TileRef } from "../game/GameMap"; @@ -66,6 +66,11 @@ export interface ServerConfig { gameCreationRate(): number; lobbyLifetime(): number; discordRedirectURI(): string; + numWorkers(): number; + workerIndex(gameID: GameID): number; + workerPath(gameID: GameID): string; + workerPort(gameID: GameID): number; + workerPortByIndex(workerID: number): number; env(): GameEnv; } diff --git a/src/core/configuration/DefaultConfig.ts b/src/core/configuration/DefaultConfig.ts index 9f23e200d..ae40e888a 100644 --- a/src/core/configuration/DefaultConfig.ts +++ b/src/core/configuration/DefaultConfig.ts @@ -1,10 +1,8 @@ -import { renderNumber } from "../../client/Utils"; import { Difficulty, Game, GameType, Gold, - MessageType, Player, PlayerInfo, PlayerType, @@ -14,16 +12,19 @@ import { UnitInfo, UnitType, } from "../game/Game"; -import { GameMap, TileRef } from "../game/GameMap"; +import { TileRef } from "../game/GameMap"; import { PlayerView } from "../game/GameView"; import { UserSettings } from "../game/UserSettings"; -import { GameConfig } from "../Schemas"; -import { assertNever, within } from "../Util"; +import { GameConfig, GameID } from "../Schemas"; +import { assertNever, simpleHash, within } from "../Util"; import { Config, GameEnv, ServerConfig, Theme } from "./Config"; import { pastelTheme } from "./PastelTheme"; import { pastelThemeDark } from "./PastelThemeDark"; export abstract class DefaultServerConfig implements ServerConfig { + numWorkers(): number { + return 3; + } abstract env(): GameEnv; abstract discordRedirectURI(): string; turnIntervalMs(): number { @@ -35,6 +36,18 @@ export abstract class DefaultServerConfig implements ServerConfig { lobbyLifetime(): number { return 2 * 60 * 1000; } + workerIndex(gameID: GameID): number { + return simpleHash(gameID) % this.numWorkers(); + } + workerPath(gameID: GameID): string { + return `w${this.workerIndex(gameID)}`; + } + workerPort(gameID: GameID): number { + return this.workerPortByIndex(this.workerIndex(gameID)); + } + workerPortByIndex(index: number): number { + return 3001 + index; + } } export class DefaultConfig implements Config { diff --git a/src/server/GameManager.ts b/src/server/GameManager.ts index 7b9f217c5..624ee16d2 100644 --- a/src/server/GameManager.ts +++ b/src/server/GameManager.ts @@ -1,20 +1,12 @@ -import { Config, ServerConfig } from "../core/configuration/Config"; -import { ClientID, GameConfig, GameID } from "../core/Schemas"; -import { v4 as uuidv4 } from "uuid"; +import { ServerConfig } from "../core/configuration/Config"; +import { GameConfig, GameID } from "../core/Schemas"; import { Client } from "./Client"; import { GamePhase, GameServer } from "./GameServer"; import { Difficulty, GameMapType, GameType } from "../core/game/Game"; -import { generateID } from "../core/Util"; -import { PseudoRandom } from "../core/PseudoRandom"; export class GameManager { - private lastNewLobby: number = 0; - private mapsPlaylist: GameMapType[] = []; - private games: GameServer[] = []; - private random = new PseudoRandom(123); - constructor(private config: ServerConfig) {} public game(id: GameID): GameServer | null { @@ -34,34 +26,20 @@ export class GameManager { return false; } - updateGameConfig(gameID: GameID, gameConfig: GameConfig) { - const game = this.games.find((g) => g.id == gameID); - if (game == null) { - console.warn(`game ${gameID} not found`); - return; - } - if (game.isPublic) { - console.warn(`cannot update public game ${gameID}`); - return; - } - game.updateGameConfig(gameConfig); - } - - createPrivateGame(): string { - const id = generateID(); - this.games.push( - new GameServer(id, Date.now(), false, this.config, { - gameMap: GameMapType.World, - gameType: GameType.Private, - difficulty: Difficulty.Medium, - disableNPCs: false, - infiniteGold: false, - infiniteTroops: false, - instantBuild: false, - bots: 400, - }), - ); - return id; + createGame(id: GameID, gameConfig: GameConfig | undefined) { + const game = new GameServer(id, Date.now(), this.config, { + gameMap: GameMapType.World, + gameType: GameType.Private, + difficulty: Difficulty.Medium, + disableNPCs: false, + infiniteGold: false, + infiniteTroops: false, + instantBuild: false, + bots: 400, + ...gameConfig, // TODO: make sure this works + }); + this.games.push(game); + return game; } hasActiveGame(gameID: GameID): boolean { @@ -73,83 +51,11 @@ export class GameManager { return game.length > 0; } - // TODO: stop private games to prevent memory leak. - startPrivateGame(gameID: GameID) { - const game = this.games.find((g) => g.id == gameID); - console.log(`found game ${game}`); - if (game) { - game.start(); - } else { - throw new Error(`cannot start private game, game ${gameID} not found`); - } - } - - private getNextMap(): GameMapType { - if (this.mapsPlaylist.length > 0) { - return this.mapsPlaylist.shift(); - } - - const frequency = { - World: 4, - Europe: 4, - Mena: 2, - NorthAmerica: 2, - Oceania: 1, - BlackSea: 2, - Africa: 2, - Asia: 2, - Mars: 0, - }; - - Object.keys(GameMapType).map((key) => { - let count = parseInt(frequency[key]); - - while (count > 0) { - this.mapsPlaylist.push(GameMapType[key]); - count--; - } - }); - - while (true) { - this.random.shuffleArray(this.mapsPlaylist); - if (this.allNonConsecutive(this.mapsPlaylist)) { - return this.mapsPlaylist.shift(); - } - } - } - - private allNonConsecutive(maps: GameMapType[]): boolean { - // Check for consecutive duplicates in the maps array - for (let i = 0; i < maps.length - 1; i++) { - if (maps[i] === maps[i + 1]) { - return false; - } - } - return true; - } - tick() { const lobbies = this.gamesByPhase(GamePhase.Lobby); const active = this.gamesByPhase(GamePhase.Active); const finished = this.gamesByPhase(GamePhase.Finished); - const now = Date.now(); - if (now > this.lastNewLobby + this.config.gameCreationRate()) { - this.lastNewLobby = now; - lobbies.push( - new GameServer(generateID(), now, true, this.config, { - gameMap: this.getNextMap(), - gameType: GameType.Public, - difficulty: Difficulty.Medium, - infiniteGold: false, - infiniteTroops: false, - instantBuild: false, - disableNPCs: false, - bots: 400, - }), - ); - } - active .filter((g) => !g.hasStarted() && g.isPublic) .forEach((g) => { diff --git a/src/server/GameServer.ts b/src/server/GameServer.ts index 9cb1ccb8a..c724630aa 100644 --- a/src/server/GameServer.ts +++ b/src/server/GameServer.ts @@ -1,24 +1,24 @@ +import { RateLimiterMemory } from "rate-limiter-flexible"; +import WebSocket from "ws"; import { ClientID, ClientMessage, ClientMessageSchema, GameConfig, + GameInfo, Intent, PlayerRecord, ServerDesyncSchema, - ServerMessageSchema, ServerStartGameMessageSchema, ServerTurnMessageSchema, Turn, } from "../core/Schemas"; -import { Config, GameEnv, ServerConfig } from "../core/configuration/Config"; -import { Client } from "./Client"; -import WebSocket from "ws"; -import { slog } from "./StructuredLog"; import { CreateGameRecord } from "../core/Util"; -import { archive } from "./Archive"; -import { RateLimiterMemory } from "rate-limiter-flexible"; +import { ServerConfig } from "../core/configuration/Config"; import { GameType } from "../core/game/Game"; +import { archive } from "./Archive"; +import { Client } from "./Client"; +import { slog } from "./StructuredLog"; export enum GamePhase { Lobby = "LOBBY", @@ -51,7 +51,6 @@ export class GameServer { constructor( public readonly id: string, public readonly createdAt: number, - public readonly isPublic: boolean, private config: ServerConfig, public gameConfig: GameConfig, ) {} @@ -360,7 +359,7 @@ export class GameServer { const noRecentPings = now > this.lastPingUpdate + 20 * 1000; const noActive = this.activeClients.length == 0; - if (!this.isPublic) { + if (this.gameConfig.gameType != GameType.Public) { if (this._hasStarted) { if (noActive && noRecentPings) { console.log(`${this.id}: private game: ${this.id} complete`); @@ -389,6 +388,24 @@ export class GameServer { return this._hasStarted; } + public gameInfo(): GameInfo { + return { + gameID: this.id, + clients: this.activeClients.map((c) => ({ + username: c.username, + clientID: c.clientID, + })), + gameConfig: this.gameConfig, + msUntilStart: this.isPublic() + ? this.createdAt + this.config.lobbyLifetime() + : undefined, + }; + } + + public isPublic(): boolean { + return this.gameConfig.gameType == GameType.Public; + } + private maybeSendDesync() { if (this.activeClients.length <= 1) { return; diff --git a/src/server/Master.ts b/src/server/Master.ts new file mode 100644 index 000000000..16a3377f8 --- /dev/null +++ b/src/server/Master.ts @@ -0,0 +1,255 @@ +import cluster from "cluster"; +import http from "http"; +import express from "express"; +import { GameMapType, GameType, Difficulty } from "../core/game/Game"; +import { generateID } from "../core/Util"; +import { PseudoRandom } from "../core/PseudoRandom"; +import { GameEnv, getServerConfig } from "../core/configuration/Config"; +import { GameInfo } from "../core/Schemas"; +import path from "path"; +import rateLimit from "express-rate-limit"; +import { fileURLToPath } from "url"; + +const config = getServerConfig(); + +const app = express(); +const server = http.createServer(app); + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +app.use(express.json()); +// Serve static files from the 'out' directory +app.use(express.static(path.join(__dirname, "../../out"))); +app.use(express.json()); + +app.set("trust proxy", 2); +app.use( + rateLimit({ + windowMs: 1000, // 1 second + max: 20, // 20 requests per IP per second + }), +); + +let publicLobbiesJsonStr = ""; + +let publicLobbyIDs: Set = new Set(); + +// Start the master process +export async function startMaster() { + if (!cluster.isPrimary) { + throw new Error( + "startMaster() should only be called in the primary process", + ); + } + + console.log(`Primary ${process.pid} is running`); + console.log(`Setting up ${config.numWorkers()} workers...`); + + // Fork workers + for (let i = 0; i < config.numWorkers(); i++) { + const worker = cluster.fork({ + WORKER_ID: i, + }); + + console.log(`Started worker ${i} (PID: ${worker.process.pid})`); + } + + // Handle worker crashes + cluster.on("exit", (worker, code, signal) => { + const workerId = (worker as any).process?.env?.WORKER_ID; + if (!workerId) { + console.error(`worker crashed could not find id`); + return; + } + + console.warn( + `Worker ${workerId} (PID: ${worker.process.pid}) died with code: ${code} and signal: ${signal}`, + ); + console.log(`Restarting worker ${workerId}...`); + + // Restart the worker with the same ID + const newWorker = cluster.fork({ + WORKER_ID: workerId, + }); + + console.log( + `Restarted worker ${workerId} (New PID: ${newWorker.process.pid})`, + ); + }); + + const PORT = 3000; + server.listen(PORT, () => { + console.log(`Master HTTP server listening on port ${PORT}`); + }); + sleep(5000).then(() => { + // let the workers start up + const scheduleLobbies = () => { + schedulePublicGame().catch((error) => { + console.error("Error scheduling public game:", error); + }); + }; + + scheduleLobbies(); + setInterval(scheduleLobbies, config.gameCreationRate()); + setInterval(() => fetchLobbies(), 250); + }); +} + +// Add lobbies endpoint to list public games for this worker +app.get("/public_lobbies", (req, res) => { + res.send(publicLobbiesJsonStr); +}); + +async function fetchLobbies(): Promise { + const fetchPromises = []; + + for (const gameID of publicLobbyIDs) { + const port = config.workerPort(gameID); + const promise = fetch(`http://localhost:${port}/game/${gameID}`) + .then((resp) => resp.json()) + .then((json) => { + return json as GameInfo; + }) + .catch((error) => { + console.error(`Error fetching game ${gameID}:`, error); + // Return null or a placeholder if fetch fails + return null; + }); + + fetchPromises.push(promise); + } + + // Wait for all promises to resolve + const results = await Promise.all(fetchPromises); + + // Filter out any null results from failed fetches + const lobbyInfos: GameInfo[] = results + .filter((result) => result !== null) + .map((gi: GameInfo) => { + return { + gameID: gi.gameID, + numClients: gi?.clients?.length ?? 0, + gameConfig: gi.gameConfig, + msUntilStart: (gi.msUntilStart ?? Date.now()) - Date.now(), + } as GameInfo; + }); + + lobbyInfos.forEach((l) => { + if (l.msUntilStart <= 250) { + publicLobbyIDs.delete(l.gameID); + } + }); + + // Update the JSON string + publicLobbiesJsonStr = JSON.stringify({ + lobbies: lobbyInfos, + }); +} + +// Function to schedule a new public game +async function schedulePublicGame() { + const gameID = generateID(); + publicLobbyIDs.add(gameID); + + // Create the default public game config (from your GameManager) + const defaultGameConfig = { + gameMap: getNextMap(), + gameType: GameType.Public, + difficulty: Difficulty.Medium, + infiniteGold: false, + infiniteTroops: false, + instantBuild: false, + disableNPCs: false, + bots: 400, + }; + + const workerPath = config.workerPath(gameID); + + // Send request to the worker to start the game + try { + const response = await fetch( + `http://localhost:${config.workerPort(gameID)}/create_game/${gameID}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Internal-Request": "true", // Special header for internal requests + }, + body: JSON.stringify({ + gameID: gameID, + gameConfig: defaultGameConfig, + }), + }, + ); + + if (!response.ok) { + throw new Error(`Failed to schedule public game: ${response.statusText}`); + } + + const data = await response.json(); + } catch (error) { + console.error( + `Failed to schedule public game on worker ${workerPath}:`, + error, + ); + throw error; + } +} + +// Map rotation management (moved from GameManager) +let mapsPlaylist: GameMapType[] = []; +const random = new PseudoRandom(123); + +// Get the next map in rotation +function getNextMap(): GameMapType { + if (mapsPlaylist.length > 0) { + return mapsPlaylist.shift()!; + } + + const frequency = { + World: 4, + Europe: 4, + Mena: 2, + NorthAmerica: 2, + Oceania: 1, + BlackSea: 2, + Africa: 2, + Asia: 2, + Mars: 0, + }; + + Object.keys(GameMapType).forEach((key) => { + let count = parseInt(frequency[key]); + + while (count > 0) { + mapsPlaylist.push(GameMapType[key]); + count--; + } + }); + + while (true) { + random.shuffleArray(mapsPlaylist); + if (allNonConsecutive(mapsPlaylist)) { + return mapsPlaylist.shift()!; + } + } +} + +// Check for consecutive duplicates in the maps array +function allNonConsecutive(maps: GameMapType[]): boolean { + for (let i = 0; i < maps.length - 1; i++) { + if (maps[i] === maps[i + 1]) { + return false; + } + } + return true; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +// SPA fallback route +app.get("*", function (req, res) { + res.sendFile(path.join(__dirname, "../../out/index.html")); +}); diff --git a/src/server/Server.ts b/src/server/Server.ts index 08c722d3e..fc2d6c475 100644 --- a/src/server/Server.ts +++ b/src/server/Server.ts @@ -1,484 +1,22 @@ -import express, { json, Request, Response, NextFunction } from "express"; -import http from "http"; -import { WebSocketServer } from "ws"; -import path from "path"; -import { fileURLToPath } from "url"; -import { GameManager } from "./GameManager"; -import { - ClientMessage, - ClientMessageSchema, - GameRecord, - GameRecordSchema, - LogSeverity, - ServerStartGameMessageSchema, -} from "../core/Schemas"; -import { - GameEnv, - getConfig, - getServerConfig, -} from "../core/configuration/Config"; -import { slog } from "./StructuredLog"; -import { Client } from "./Client"; -import { GamePhase, GameServer } from "./GameServer"; -import { archive, gameRecordExists, readGameRecord } from "./Archive"; -import { DiscordBot } from "./DiscordBot"; -import { - sanitizeUsername, - validateUsername, -} from "../core/validations/username"; -import { SecretManagerServiceClient } from "@google-cloud/secret-manager"; -import dotenv from "dotenv"; -import crypto from "crypto"; -dotenv.config(); -import rateLimit from "express-rate-limit"; -import { RateLimiterMemory } from "rate-limiter-flexible"; +import cluster from "cluster"; +import { startMaster } from "./Master"; +import { startWorker } from "./Worker"; -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); - -const app = express(); -const server = http.createServer(app); -const wss = new WebSocketServer({ server }); - -const serverConfig = getServerConfig(); - -// Initialize Secret Manager -const secretManager = new SecretManagerServiceClient(); - -// Discord OAuth Configuration (will be populated from secrets) -let DISCORD_CLIENT_ID: string; -let DISCORD_CLIENT_SECRET: string; - -// Serve static files from the 'out' directory -app.use(express.static(path.join(__dirname, "../../out"))); -app.use(express.json()); - -app.set("trust proxy", 2); -app.use( - rateLimit({ - windowMs: 1000, // 1 second - max: 20, // 20 requests per IP per second - }), -); - -const rateLimiter = new RateLimiterMemory({ - points: 50, // 50 messages - duration: 1, // per 1 second -}); - -const updateRateLimiter = new RateLimiterMemory({ - points: 10, - duration: 240, // 4 minutes -}); - -const gm = new GameManager(getServerConfig()); - -const bot = new DiscordBot(); -try { - await bot.start(); -} catch (error) { - console.error("Failed to start bot:", error); -} - -let lobbiesString = ""; - -// Async error wrapper with rate limiting support -const asyncHandler = - (fn: Function, limiter = null) => - async (req: Request, res: Response, next: NextFunction) => { - try { - // Apply rate limiting if a limiter is provided - if (limiter) { - const clientIP = req.ip || req.socket.remoteAddress || "unknown"; - try { - await limiter.consume(clientIP); - } catch (error) { - console.warn(`Rate limited for IP ${clientIP}`); - return res.status(429).json({ error: "Too many requests" }); - } - } - - // Execute the route handler - await fn(req, res, next); - } catch (error) { - // Pass any errors to Express error handler - next(error); - } - }; - -// Discord OAuth callback endpoint -app.get( - "/auth/callback", - asyncHandler(async (req: Request, res: Response) => { - const { code } = req.query; - - if (!code) { - return res.status(400).send("No code provided"); - } - - // Exchange code for access token - const tokenResponse = await fetch("https://discord.com/api/oauth2/token", { - method: "POST", - body: new URLSearchParams({ - client_id: DISCORD_CLIENT_ID!, - client_secret: DISCORD_CLIENT_SECRET!, - code: code as string, - grant_type: "authorization_code", - redirect_uri: serverConfig.discordRedirectURI(), - }), - headers: { - "Content-Type": "application/x-www-form-urlencoded", - }, - }); - - if (!tokenResponse.ok) { - throw new Error("Failed to get access token"); - } - - const tokenData = await tokenResponse.json(); - - // Get user information - const userResponse = await fetch("https://discord.com/api/users/@me", { - headers: { - Authorization: `Bearer ${tokenData.access_token}`, - }, - }); - - if (!userResponse.ok) { - throw new Error("Failed to get user information"); - } - - const userData = await userResponse.json(); - const sessionToken = crypto.randomBytes(32).toString("hex"); - - // TODO: store userData and sessionToken in database. - - res.cookie("session", sessionToken, { - httpOnly: true, - secure: true, - sameSite: "strict", - maxAge: 30 * 24 * 60 * 60 * 1000, // 30 days in milliseconds - }); - res.redirect(`/`); - }), -); - -app.get("/auth/discord", (req: Request, res: Response) => { - console.log("Redirecting to Discord OAuth..."); - const redirectUri = serverConfig.discordRedirectURI(); - const authorizeUrl = `https://discord.com/api/oauth2/authorize?client_id=${DISCORD_CLIENT_ID}&redirect_uri=${encodeURIComponent(redirectUri)}&response_type=code&scope=identify`; - console.log("Auth URL:", authorizeUrl); - res.redirect(authorizeUrl); -}); - -// New GET endpoint to list lobbies -app.get("/lobbies", (req: Request, res: Response) => { - res.send(lobbiesString); -}); - -app.post( - "/private_lobby", - asyncHandler(async (req, res) => { - const clientIP = req.ip || req.socket.remoteAddress || "unknown"; - const id = gm.createPrivateGame(); - console.log(`ip ${clientIP} creating private lobby with id ${id}`); - res.json({ - id: id, - }); - }, updateRateLimiter), -); - -app.post( - "/archive_singleplayer_game", - asyncHandler(async (req, res) => { - const gameRecord: GameRecord = req.body; - const clientIP = req.ip || req.socket.remoteAddress || "unknown"; - - if (!gameRecord) { - console.log("game record not found in request"); - res.status(404).json({ error: "Game record not found" }); - return; - } - gameRecord.players.forEach((p) => (p.ip = clientIP)); - archive(gameRecord); - res.json({ - success: true, - }); - }, updateRateLimiter), -); - -app.post( - "/start_private_lobby/:id", - asyncHandler(async (req, res) => { - const clientIP = req.ip || req.socket.remoteAddress || "unknown"; - console.log(`starting private lobby with id ${req.params.id}`); - gm.startPrivateGame(req.params.id); - res.status(200).json({ success: true }); - }, updateRateLimiter), -); - -app.put( - "/private_lobby/:id", - asyncHandler(async (req, res) => { - const lobbyID = req.params.id; - gm.updateGameConfig(lobbyID, { - gameMap: req.body.gameMap, - difficulty: req.body.difficulty, - infiniteGold: req.body.infiniteGold, - infiniteTroops: req.body.infiniteTroops, - instantBuild: req.body.instantBuild, - bots: req.body.bots, - disableNPCs: req.body.disableNPCs, - }); - res.status(200).json({ success: true }); - }), -); - -app.get( - "/lobby/:id/exists", - asyncHandler(async (req, res) => { - const lobbyId = req.params.id; - let gameExists = gm.hasActiveGame(lobbyId); - if (!gameExists) { - gameExists = await gameRecordExists(lobbyId); - } - res.json({ - exists: gameExists, - }); - }), -); - -app.get( - "/lobby/:id", - asyncHandler(async (req, res) => { - const game = gm.game(req.params.id); - if (game == null) { - console.log(`lobby ${req.params.id} not found`); - return res.status(404).json({ error: "Game not found" }); - } - res.json({ - players: game.activeClients.map((c) => ({ - username: c.username, - clientID: c.clientID, - })), - }); - }), -); - -app.get( - "/private_lobby/:id", - asyncHandler(async (req, res) => { - res.json({ - hi: "5", - }); - }), -); - -app.get( - "/debug-ip", - asyncHandler(async (req, res) => { - res.send({ - "x-forwarded-for": req.headers["x-forwarded-for"], - "real-ip": req.ip, - "raw-headers": req.rawHeaders, - }); - }), -); - -app.get("*", function (req, res) { - // SPA routing - res.sendFile(path.join(__dirname, "../../out/index.html")); -}); - -wss.on("connection", (ws, req) => { - ws.on("message", async (message: string) => { - let ip = ""; - try { - const forwarded = req.headers["x-forwarded-for"]; - ip = Array.isArray(forwarded) - ? forwarded[0] - : forwarded || req.socket.remoteAddress; - await rateLimiter.consume(ip); - } catch (error) { - console.warn(`rate limit exceede for ${ip}`); - return; - } - try { - let clientMsg: ClientMessage = null; - try { - clientMsg = ClientMessageSchema.parse(JSON.parse(message.toString())); - } catch (error) { - throw new Error(`error parsing zod schema for ip: ${ip}`); - } - if (clientMsg.type == "join") { - const forwarded = req.headers["x-forwarded-for"]; - let ip = Array.isArray(forwarded) - ? forwarded[0] - : forwarded || req.socket.remoteAddress; - if (Array.isArray(ip)) { - ip = ip[0]; - } - const { isValid, error } = validateUsername(clientMsg.username); - if (!isValid) { - console.log( - `game ${clientMsg.gameID}, client ${clientMsg.clientID} received invalid username, ${error}`, - ); - return; - } - clientMsg.username = sanitizeUsername(clientMsg.username); - const wasFound = gm.addClient( - new Client( - clientMsg.clientID, - clientMsg.persistentID, - ip, - clientMsg.username, - ws, - ), - clientMsg.gameID, - clientMsg.lastTurn, - ); - if (!wasFound) { - console.log(`game ${clientMsg.gameID} not found, loading from gcs`); - const record = await readGameRecord(clientMsg.gameID); - - let startGame = null; - try { - startGame = ServerStartGameMessageSchema.parse({ - type: "start", - turns: record.turns, - config: record.gameConfig, - }); - } catch (error) { - console.log(`error validating schema for ip ${ip}`); - } - ws.send(JSON.stringify(startGame)); - } - } - if (clientMsg.type == "log") { - slog({ - logKey: "client_console_log", - msg: clientMsg.log, - severity: clientMsg.severity, - clientID: clientMsg.clientID, - gameID: clientMsg.gameID, - persistentID: clientMsg.persistentID, - }); - } - } catch (error) { - console.warn( - `error handling websocket message for ${ip}: ${error}`.substring( - 0, - 250, - ), - ); - } - }); - ws.on("error", (error: Error) => { - if ((error as any).code === "WS_ERR_UNEXPECTED_RSV_1") { - ws.close(1002); - } - }); -}); - -// Global error handler -app.use((err: Error, req: Request, res: Response, next: NextFunction) => { - console.error(`Error in ${req.method} ${req.path}:`, err); - slog({ - logKey: "server_error", - msg: `Unhandled exception in ${req.method} ${req.path}: ${err.message}`, - severity: LogSeverity.Error, - stack: err.stack, - }); - res.status(500).json({ error: "An unexpected error occurred" }); -}); - -function startServer() { - setInterval(() => tick(), 1000); - setInterval(() => updateLobbies(), 100); - - initializeSecrets(); - - const PORT = process.env.PORT || 3000; - console.log(`Server will try to run on http://localhost:${PORT}`); - - server.listen(PORT, () => { - console.log(`Server is running on http://localhost:${PORT}`); - }); -} - -function tick() { - gm.tick(); -} - -function updateLobbies() { - lobbiesString = JSON.stringify({ - lobbies: gm - .gamesByPhase(GamePhase.Lobby) - .filter((g) => g.isPublic) - .map((g) => ({ - id: g.id, - msUntilStart: g.startTime() - Date.now(), - numClients: g.numClients(), - gameConfig: g.gameConfig, - })) - .sort((a, b) => a.msUntilStart - b.msUntilStart), - }); -} - -// Process-level unhandled exception handlers -process.on("uncaughtException", (err) => { - console.error("Uncaught exception:", err); - slog({ - logKey: "uncaught_exception", - msg: `Uncaught exception: ${err.message}`, - severity: LogSeverity.Error, - stack: err.stack, - }); - // Note: We're not exiting the process to maintain uptime - // but be aware the app might be in an inconsistent state -}); - -process.on("unhandledRejection", (reason, promise) => { - console.error("Unhandled rejection at:", promise, "reason:", reason); - slog({ - logKey: "unhandled_rejection", - msg: `Unhandled promise rejection: ${reason}`, - severity: LogSeverity.Error, - }); -}); - -// Initialize secrets and start server -async function initializeSecrets() { - try { - DISCORD_CLIENT_ID = await getSecret( - "DISCORD_CLIENT_ID", - serverConfig.env(), - ); - DISCORD_CLIENT_SECRET = await getSecret( - "DISCORD_CLIENT_SECRET", - serverConfig.env(), - ); - - if (!DISCORD_CLIENT_ID || !DISCORD_CLIENT_SECRET) { - throw new Error("Failed to load Discord secrets"); - } - } catch (error) { - console.error("Failed to initialize secrets:", error); +// Main entry point of the application +async function main() { + // Check if this is the primary (master) process + if (cluster.isPrimary) { + console.log("Starting master process..."); + await startMaster(); + } else { + // This is a worker process + console.log("Starting worker process..."); + await startWorker(); } } -async function getSecret(secretName: string, ge: GameEnv) { - if (ge == GameEnv.Dev) { - console.log(`loading secret ${secretName} from environment variable`); - const value = process.env[secretName]; - if (!value) { - throw Error(`error loading secret ${secretName}`); - } - } - console.log(`loading secret ${secretName} from Google secrets manager`); - const name = `projects/openfrontio/secrets/${secretName}/versions/latest`; - const [version] = await secretManager.accessSecretVersion({ name }); - return version.payload?.data?.toString(); -} - -startServer(); +// Start the application +main().catch((error) => { + console.error("Failed to start server:", error); + process.exit(1); +}); diff --git a/src/server/Worker.ts b/src/server/Worker.ts new file mode 100644 index 000000000..f7fbb13df --- /dev/null +++ b/src/server/Worker.ts @@ -0,0 +1,374 @@ +import express, { Request, Response, NextFunction } from "express"; +import http from "http"; +import { WebSocketServer } from "ws"; +import path from "path"; +import { fileURLToPath } from "url"; +import { GameManager } from "./GameManager"; +import { getServerConfig } from "../core/configuration/Config"; +import { WebSocket } from "ws"; +import { Client } from "./Client"; +import rateLimit from "express-rate-limit"; +import { RateLimiterMemory } from "rate-limiter-flexible"; +import { GameConfig, GameRecord, LogSeverity } from "../core/Schemas"; +import { slog } from "./StructuredLog"; +import { GameType } from "../core/game/Game"; +import { archive } from "./Archive"; + +const config = getServerConfig(); + +// Worker setup +export function startWorker() { + // Get worker ID from environment variable + const workerId = parseInt(process.env.WORKER_ID || "0"); + console.log(`Worker ${workerId} starting...`); + + const __filename = fileURLToPath(import.meta.url); + const __dirname = path.dirname(__filename); + + const app = express(); + const server = http.createServer(app); + const wss = new WebSocketServer({ server }); + + const gm = new GameManager(config); + + // Middleware to handle /wX path prefix + app.use((req, res, next) => { + // Extract the original path without the worker prefix + const originalPath = req.url; + const match = originalPath.match(/^\/w(\d+)(.*)$/); + + if (match) { + const pathWorkerId = parseInt(match[1]); + const actualPath = match[2] || "/"; + + // Verify this request is for the correct worker + if (pathWorkerId !== workerId) { + return res.status(404).json({ + error: "Worker mismatch", + message: `This is worker ${workerId}, but you requested worker ${pathWorkerId}`, + }); + } + + // Update the URL to remove the worker prefix + req.url = actualPath; + } + + next(); + }); + + app.set("trust proxy", 2); + app.use(express.json()); + app.use(express.static(path.join(__dirname, "../../out"))); + app.use( + rateLimit({ + windowMs: 1000, // 1 second + max: 20, // 20 requests per IP per second + }), + ); + + const rateLimiter = new RateLimiterMemory({ + points: 50, // 50 messages + duration: 1, // per 1 second + }); + + const updateRateLimiter = new RateLimiterMemory({ + points: 10, + duration: 240, // 4 minutes + }); + + // Async handler with rate limiting + const asyncHandler = + (fn: Function, limiter = null) => + async (req: Request, res: Response, next: NextFunction) => { + try { + if (limiter) { + if (!isLocalhost(req)) { + const clientIP = req.ip || req.socket.remoteAddress || "unknown"; + try { + await limiter.consume(clientIP); + } catch (error) { + console.warn(`Rate limited for IP ${clientIP}`); + return res.status(429).json({ error: "Too many requests" }); + } + } + } + await fn(req, res, next); + } catch (error) { + next(error); + } + }; + + // Endpoint to create a private lobby + app.post( + "/create_game/:id", + asyncHandler(async (req, res) => { + const id = req.params.id; + if (!id) { + console.warn(`cannot create game, id not found`); + return; + } + // TODO: if game is public make sure request came from localhohst!!! + const clientIP = req.ip || req.socket.remoteAddress || "unknown"; + const gc = req.body?.gameConfig as GameConfig; + if (gc?.gameType == GameType.Public && !isLocalhost(req)) { + console.warn( + `cannot create public game ${id}, ip ${clientIP} not localhost`, + ); + return res.status(400); + } + + // Double-check this worker should host this game + const expectedWorkerId = config.workerIndex(id); + if (expectedWorkerId !== workerId) { + console.warn( + `This game ${id} should be on worker ${expectedWorkerId}, but this is worker ${workerId}`, + ); + return res.status(400); + } + + const game = gm.createGame(id, gc); + + console.log( + `Worker ${workerId}: IP ${clientIP} creating game ${game.isPublic() ? "Public" : "Private"} with id ${id}`, + ); + res.json(game.gameInfo()); + }, updateRateLimiter), + ); + + // Add other endpoints from your original server + app.post( + "/start_game/:id", + asyncHandler(async (req, res) => { + console.log(`starting private lobby with id ${req.params.id}`); + const game = gm.game(req.params.id); + if (!game) { + return; + } + if (game.isPublic()) { + const clientIP = req.ip || req.socket.remoteAddress || "unknown"; + console.log( + `cannot start public game ${game.id}, game is public, ip: ${clientIP}`, + ); + return; + } + game.start(); + res.status(200).json({ success: true }); + }, updateRateLimiter), + ); + + app.put( + "/game/:id", + asyncHandler(async (req, res) => { + // TODO: only update public game if from local host + const lobbyID = req.params.id; + if (req.body.gameType == GameType.Public) { + console.log(`cannot update game ${lobbyID} to public`); + return res.status(400); + } + const game = gm.game(lobbyID); + if (!game) { + return res.status(400); + } + if (game.isPublic()) { + const clientIP = req.ip || req.socket.remoteAddress || "unknown"; + console.warn(`cannot update public game ${game.id}, ip: ${clientIP}`); + return res.status(400); + } + game.updateGameConfig({ + gameMap: req.body.gameMap, + difficulty: req.body.difficulty, + infiniteGold: req.body.infiniteGold, + infiniteTroops: req.body.infiniteTroops, + instantBuild: req.body.instantBuild, + bots: req.body.bots, + disableNPCs: req.body.disableNPCs, + }); + res.status(200).json({ success: true }); + }), + ); + + app.get( + "/game/:id/exists", + asyncHandler(async (req, res) => { + const lobbyId = req.params.id; + console.log(`checking if game ${lobbyId} exists`); + let gameExists = gm.hasActiveGame(lobbyId); + res.json({ + exists: gameExists, + }); + }), + ); + + app.get( + "/game/:id", + asyncHandler(async (req, res) => { + const game = gm.game(req.params.id); + if (game == null) { + console.log(`lobby ${req.params.id} not found`); + return res.status(404).json({ error: "Game not found" }); + } + res.json(game.gameInfo()); + }), + ); + + app.post( + "/archive_singleplayer_game", + asyncHandler(async (req, res) => { + const gameRecord: GameRecord = req.body; + const clientIP = req.ip || req.socket.remoteAddress || "unknown"; + + if (!gameRecord) { + console.log("game record not found in request"); + res.status(404).json({ error: "Game record not found" }); + return; + } + gameRecord.players.forEach((p) => (p.ip = clientIP)); + archive(gameRecord); + res.json({ + success: true, + }); + }, updateRateLimiter), + ); + + // WebSocket handling + wss.on("connection", (ws: WebSocket, req) => { + ws.on("message", async (message: string) => { + const forwarded = req.headers["x-forwarded-for"]; + const ip = Array.isArray(forwarded) + ? forwarded[0] + : forwarded || req.socket.remoteAddress; + try { + await rateLimiter.consume(ip); + } catch (error) { + console.warn(`rate limit exceeded for ${ip}`); + return; + } + + try { + // Process WebSocket messages as in your original code + // Parse and handle client messages + const clientMsg = JSON.parse(message.toString()); + + if (clientMsg.type == "join") { + // Verify this worker should handle this game + const expectedWorkerId = config.workerIndex(clientMsg.gameID); + if (expectedWorkerId !== workerId) { + console.warn( + `Worker mismatch: Game ${clientMsg.gameID} should be on worker ${expectedWorkerId}, but this is worker ${workerId}`, + ); + return; + } + + // Create client and add to game + const client = new Client( + clientMsg.clientID, + clientMsg.persistentID, + ip, + clientMsg.username, + ws, + ); + + const wasFound = gm.addClient( + client, + clientMsg.gameID, + clientMsg.lastTurn, + ); + + if (!wasFound) { + console.log( + `game ${clientMsg.gameID} not found on worker ${workerId}`, + ); + // Handle game not found case + } + } + + // Handle other message types + } catch (error) { + console.warn( + `error handling websocket message for ${ip}: ${error}`.substring( + 0, + 250, + ), + ); + } + }); + + ws.on("error", (error: Error) => { + if ((error as any).code === "WS_ERR_UNEXPECTED_RSV_1") { + ws.close(1002); + } + }); + }); + + // Set up ticker + setInterval(() => gm.tick(), 1000); + + // The load balancer will handle routing to this server based on path + const PORT = config.workerPortByIndex(workerId); + server.listen(PORT, () => { + console.log(`Worker ${workerId} running on http://localhost:${PORT}`); + console.log(`Handling requests with path prefix /w${workerId}/`); + }); + + // Global error handler + app.use((err: Error, req: Request, res: Response, next: NextFunction) => { + console.error(`Error in ${req.method} ${req.path}:`, err); + slog({ + logKey: "server_error", + msg: `Unhandled exception in ${req.method} ${req.path}: ${err.message}`, + severity: LogSeverity.Error, + stack: err.stack, + }); + res.status(500).json({ error: "An unexpected error occurred" }); + }); + + // Process-level error handlers + process.on("uncaughtException", (err) => { + console.error(`Worker ${workerId} uncaught exception:`, err); + slog({ + logKey: "uncaught_exception", + msg: `Worker ${workerId} uncaught exception: ${err.message}`, + severity: LogSeverity.Error, + stack: err.stack, + }); + }); + + process.on("unhandledRejection", (reason, promise) => { + console.error( + `Worker ${workerId} unhandled rejection at:`, + promise, + "reason:", + reason, + ); + slog({ + logKey: "unhandled_rejection", + msg: `Worker ${workerId} unhandled promise rejection: ${reason}`, + severity: LogSeverity.Error, + }); + }); +} + +const isLocalhost = (req: Request): boolean => { + // Get client IP address from various possible sources + const clientIP = + req.ip || + req.socket.remoteAddress || + (req.headers["x-forwarded-for"] as string)?.split(",").shift() || + "unknown"; + + // Check if the request is from a loopback address + const isLoopbackIP = + // IPv4 localhost + clientIP === "127.0.0.1" || + // IPv6 localhost + clientIP === "::1" || + // Full loopback range + clientIP.startsWith("127."); + + // Check hostname + const isLocalHostname = + req.hostname === "localhost" || req.headers.host?.startsWith("localhost:"); + + // Consider request local if either IP is loopback or hostname is localhost + return isLoopbackIP || isLocalHostname; +}; diff --git a/webpack.config.js b/webpack.config.js index ba9e1c4c0..5c8e35906 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -123,21 +123,72 @@ export default (env, argv) => { compress: true, port: 9000, proxy: [ + // WebSocket proxies { context: ["/socket"], target: "ws://localhost:3000", ws: true, + changeOrigin: true, + logLevel: "debug", + }, + // Worker WebSocket proxies - using direct paths without /socket suffix + { + context: ["/w0"], + target: "ws://localhost:3001", + ws: true, + secure: false, + changeOrigin: true, + logLevel: "debug", }, + { + context: ["/w1"], + target: "ws://localhost:3002", + ws: true, + secure: false, + changeOrigin: true, + logLevel: "debug", + }, + { + context: ["/w2"], + target: "ws://localhost:3003", + ws: true, + secure: false, + changeOrigin: true, + logLevel: "debug", + }, + // Worker proxies for HTTP requests + { + context: ["/w0"], + target: "http://localhost:3001", + pathRewrite: { "^/w0": "" }, + secure: false, + changeOrigin: true, + logLevel: "debug", + }, + { + context: ["/w1"], + target: "http://localhost:3002", + pathRewrite: { "^/w1": "" }, + secure: false, + changeOrigin: true, + logLevel: "debug", + }, + { + context: ["/w2"], + target: "http://localhost:3003", + pathRewrite: { "^/w2": "" }, + secure: false, + changeOrigin: true, + logLevel: "debug", + }, + // Original API endpoints { context: [ - "/lobbies", + "/public_lobbies", "/join_game", - "/join_lobby", - "/private_lobby", - "/start_private_lobby", - "/lobby", + "/start_game", + "/create_game", "/archive_singleplayer_game", - "/validate-username", "/debug-ip", "/auth/callback", "/auth/discord",