From f540d0302ce66d5279e6be1a6a11de7dfc0954e0 Mon Sep 17 00:00:00 2001 From: Evan Date: Thu, 27 Feb 2025 19:21:18 -0800 Subject: [PATCH] configure nginx to run in container --- Dockerfile | 33 ++++++-- docker-compose.yml | 34 -------- nginx.conf | 116 +++++++++++----------------- src/client/HostLobbyModal.ts | 4 +- src/client/JoinPrivateLobbyModal.ts | 12 +-- src/client/PublicLobby.ts | 2 +- src/server/Master.ts | 38 ++++++--- src/server/Worker.ts | 8 ++ supervisord.conf | 24 ++++++ 9 files changed, 139 insertions(+), 132 deletions(-) delete mode 100644 docker-compose.yml create mode 100644 supervisord.conf diff --git a/Dockerfile b/Dockerfile index bf7787be4..05b749124 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,18 +4,39 @@ FROM node:18 # Add environment variable ARG GAME_ENV=preprod ENV GAME_ENV=$GAME_ENV +ENV NODE_ENV=production + +# Install Nginx, Supervisor and Git (for Husky) +RUN apt-get update && apt-get install -y nginx supervisor git && \ + rm -rf /var/lib/apt/lists/* # Set the working directory in the container WORKDIR /usr/src/app + # Copy package.json and package-lock.json COPY package*.json ./ -# Install dependencies -RUN npm install + +# Install dependencies while bypassing Husky hooks +ENV HUSKY=0 +ENV NPM_CONFIG_IGNORE_SCRIPTS=1 +RUN mkdir -p .git && npm install + # Copy the rest of the application code COPY . . + # Build the client-side application RUN npm run build-prod -# Expose the port the app runs on -EXPOSE 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 -# Define the command to run the app -CMD ["npm", "run", "start:server"] \ No newline at end of file + +# Copy Nginx configuration and ensure it's used instead of the default +COPY nginx.conf /etc/nginx/conf.d/default.conf +RUN rm -f /etc/nginx/sites-enabled/default + +# Setup supervisor configuration +RUN mkdir -p /var/log/supervisor +COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf + +# Expose only the Nginx port +EXPOSE 80 443 + +# Start Supervisor to manage both Node.js and Nginx +CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 4076e1794..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,34 +0,0 @@ -version: "3" -services: - game-server: - build: . - expose: - - "3000" - - "3001" - - "3002" - - "3003" - - "3004" - - "3005" - - "3006" - - "3007" - - "3008" - - "3009" - - "3010" - - "3011" - - "3012" - - "3013" - - "3014" - - "3015" - environment: - - NODE_ENV=production - - nginx: - image: nginx:latest - ports: - - "80:80" - - "443:443" - volumes: - - ./nginx.conf:/etc/nginx/conf.d/default.conf - - /etc/letsencrypt:/etc/letsencrypt - depends_on: - - game-server diff --git a/nginx.conf b/nginx.conf index e27817df0..7d7ff2311 100644 --- a/nginx.conf +++ b/nginx.conf @@ -1,8 +1,4 @@ -resolver 127.0.0.11 valid=30s; - -# Access log format with minimal information -log_format minimal '$remote_addr - [$time_local] "$request" $status'; - +# Map URI to ports map $uri $port { ~^/w0/ 3001; ~^/w1/ 3002; @@ -22,80 +18,59 @@ map $uri $port { default 3000; } -# Don't strip the path for WebSocket connections -map $http_upgrade $strip_path { - default 1; - websocket 0; -} - -map $uri $uri_path { - # Only strip path if not a WebSocket request - ~^/w\d+(/.*)?$ $1; - default $uri; -} - +# WebSocket settings map $http_upgrade $connection_upgrade { default upgrade; '' close; } +# WebSocket path handling +map $uri $uri_path { + ~^/w\d+(/.*)?$ $1; + default $uri; +} + server { - listen 80; - - # Reduced logging - access_log /var/log/nginx/access.log minimal; - error_log /var/log/nginx/error.log error; + listen 80 default_server; - # Disable logging for common requests - location = /favicon.ico { - access_log off; - log_not_found off; - return 204; - } + # Logging + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; - # WebSocket timeout settings - proxy_read_timeout 300s; - proxy_connect_timeout 75s; - proxy_send_timeout 300s; - - # Special location block just for WebSocket connections - location ~* ^/w\d+$ { - set $ws_port 0; - - if ($uri ~* ^/w0) { - set $ws_port 3001; - } - if ($uri ~* ^/w1) { - set $ws_port 3002; - } - if ($uri ~* ^/w2) { - set $ws_port 3003; - } - if ($uri ~* ^/w3) { - set $ws_port 3004; - } - if ($uri ~* ^/w4) { - set $ws_port 3005; - } - # Add more conditions for other worker paths - - proxy_pass http://game-server:$ws_port; - - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_buffering off; - } - - # Regular location for all other requests + # Main location location / { - set $upstream_endpoint game-server:$port; - proxy_pass http://$upstream_endpoint$uri_path; - + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # Worker locations + location ~* ^/w(\d+)(/.*)?$ { + set $worker $1; + set $worker_port 3001; + + if ($worker = "0") { set $worker_port 3001; } + if ($worker = "1") { set $worker_port 3002; } + if ($worker = "2") { set $worker_port 3003; } + if ($worker = "3") { set $worker_port 3004; } + if ($worker = "4") { set $worker_port 3005; } + if ($worker = "5") { set $worker_port 3006; } + if ($worker = "6") { set $worker_port 3007; } + if ($worker = "7") { set $worker_port 3008; } + if ($worker = "8") { set $worker_port 3009; } + if ($worker = "9") { set $worker_port 3010; } + if ($worker = "10") { set $worker_port 3011; } + if ($worker = "11") { set $worker_port 3012; } + if ($worker = "12") { set $worker_port 3013; } + if ($worker = "13") { set $worker_port 3014; } + if ($worker = "14") { set $worker_port 3015; } + + proxy_pass http://127.0.0.1:$worker_port$2; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; @@ -103,6 +78,5 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_buffering off; } } \ No newline at end of file diff --git a/src/client/HostLobbyModal.ts b/src/client/HostLobbyModal.ts index a70d91461..d69f0cd29 100644 --- a/src/client/HostLobbyModal.ts +++ b/src/client/HostLobbyModal.ts @@ -585,7 +585,7 @@ export class HostLobbyModal extends LitElement { private async putGameConfig() { const response = await fetch( - `${getServerConfig().workerPath(this.lobbyId)}/game/${this.lobbyId}`, + `${window.location.origin}/${getServerConfig().workerPath(this.lobbyId)}/game/${this.lobbyId}`, { method: "PUT", headers: { @@ -610,7 +610,7 @@ export class HostLobbyModal extends LitElement { ); this.close(); const response = await fetch( - `${getServerConfig().workerPath(this.lobbyId)}/start_game/${this.lobbyId}`, + `${window.location.origin}/${getServerConfig().workerPath(this.lobbyId)}/start_game/${this.lobbyId}`, { method: "POST", headers: { diff --git a/src/client/JoinPrivateLobbyModal.ts b/src/client/JoinPrivateLobbyModal.ts index 1e47e757b..6beb3ae75 100644 --- a/src/client/JoinPrivateLobbyModal.ts +++ b/src/client/JoinPrivateLobbyModal.ts @@ -1,8 +1,8 @@ -import { LitElement, html, css } from "lit"; -import { customElement, property, state, query } from "lit/decorators.js"; -import { GameMapType, GameType } from "../core/game/Game"; +import { LitElement, css, html } from "lit"; +import { customElement, query, state } from "lit/decorators.js"; +import { getServerConfig } from "../core/configuration/Config"; import { consolex } from "../core/Consolex"; -import { getConfig, getServerConfig } from "../core/configuration/Config"; +import { GameMapType, GameType } from "../core/game/Game"; import { GameInfo } from "../core/Schemas"; @customElement("join-private-lobby-modal") @@ -360,7 +360,7 @@ export class JoinPrivateLobbyModal extends LitElement { consolex.log(`Joining lobby with ID: ${lobbyId}`); this.message = "Checking lobby..."; // Set initial message - const url = `${window.location.origin}/${getServerConfig().workerPath(lobbyId)}/game/${lobbyId}/exists`; + const url = `/${getServerConfig().workerPath(lobbyId)}/game/${lobbyId}/exists`; fetch(url, { method: "GET", headers: { @@ -400,7 +400,7 @@ export class JoinPrivateLobbyModal extends LitElement { if (!this.lobbyIdInput?.value) return; fetch( - `${getServerConfig().workerPath(this.lobbyIdInput.value)}/lobby/${this.lobbyIdInput.value}`, + `/${getServerConfig().workerPath(this.lobbyIdInput.value)}/game/${this.lobbyIdInput.value}`, { method: "GET", headers: { diff --git a/src/client/PublicLobby.ts b/src/client/PublicLobby.ts index 2e0dc1f69..6fb5ef019 100644 --- a/src/client/PublicLobby.ts +++ b/src/client/PublicLobby.ts @@ -45,7 +45,7 @@ export class PublicLobby extends LitElement { async fetchLobbies(): Promise { try { - const response = await fetch("/public_lobbies"); + const response = await fetch(`/public_lobbies`); if (!response.ok) throw new Error(`HTTP error! status: ${response.status}`); const data = await response.json(); diff --git a/src/server/Master.ts b/src/server/Master.ts index 16a3377f8..83a5da282 100644 --- a/src/server/Master.ts +++ b/src/server/Master.ts @@ -11,6 +11,7 @@ import rateLimit from "express-rate-limit"; import { fileURLToPath } from "url"; const config = getServerConfig(); +const readyWorkers = new Set(); const app = express(); const server = http.createServer(app); @@ -54,6 +55,31 @@ export async function startMaster() { console.log(`Started worker ${i} (PID: ${worker.process.pid})`); } + cluster.on("message", (worker, message) => { + if (message.type === "WORKER_READY") { + const workerId = message.workerId; + readyWorkers.add(workerId); + console.log( + `Worker ${workerId} is ready. (${readyWorkers.size}/${config.numWorkers()} ready)`, + ); + + // Start scheduling when all workers are ready + if (readyWorkers.size === config.numWorkers()) { + console.log("All workers ready, starting game scheduling"); + // let the workers start up + const scheduleLobbies = () => { + schedulePublicGame().catch((error) => { + console.error("Error scheduling public game:", error); + }); + }; + + scheduleLobbies(); + setInterval(scheduleLobbies, config.gameCreationRate()); + setInterval(() => fetchLobbies(), 250); + } + } + }); + // Handle worker crashes cluster.on("exit", (worker, code, signal) => { const workerId = (worker as any).process?.env?.WORKER_ID; @@ -81,18 +107,6 @@ export async function startMaster() { server.listen(PORT, () => { console.log(`Master HTTP server listening on port ${PORT}`); }); - sleep(5000).then(() => { - // let the workers start up - const scheduleLobbies = () => { - schedulePublicGame().catch((error) => { - console.error("Error scheduling public game:", error); - }); - }; - - scheduleLobbies(); - setInterval(scheduleLobbies, config.gameCreationRate()); - setInterval(() => fetchLobbies(), 250); - }); } // Add lobbies endpoint to list public games for this worker diff --git a/src/server/Worker.ts b/src/server/Worker.ts index f7fbb13df..f9a407cd9 100644 --- a/src/server/Worker.ts +++ b/src/server/Worker.ts @@ -308,6 +308,14 @@ export function startWorker() { server.listen(PORT, () => { console.log(`Worker ${workerId} running on http://localhost:${PORT}`); console.log(`Handling requests with path prefix /w${workerId}/`); + // Signal to the master process that this worker is ready + if (process.send) { + process.send({ + type: "WORKER_READY", + workerId: workerId, + }); + console.log(`Worker ${workerId} signaled ready state to master`); + } }); // Global error handler diff --git a/supervisord.conf b/supervisord.conf new file mode 100644 index 000000000..f6cb1191a --- /dev/null +++ b/supervisord.conf @@ -0,0 +1,24 @@ +[supervisord] +nodaemon=true +user=root +logfile=/var/log/supervisor/supervisord.log +pidfile=/var/run/supervisord.pid + +[program:nginx] +command=/usr/sbin/nginx -g "daemon off;" +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:node] +command=npm run start:server +directory=/usr/src/app +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 \ No newline at end of file