From 49b01d801476c0d10a1e6f80d8424bc6ccca3a04 Mon Sep 17 00:00:00 2001 From: evanpelle Date: Fri, 6 Jun 2025 13:43:21 -0700 Subject: [PATCH] have master create tunnels for all workers #780 (#1042) ## Description: We want to move away from using nginx to cloudflare to route among workers. This will simplify the nginx config, move routing computation off the server, and make it easier to implement a multi-host architecture. The worker tunnels are not currently used. I also moved the tunnel creation from startup.sh to Server. The shell script was getting too complex. ## Please complete the following: - [x] I have added screenshots for all UI updates - [x] I process any text displayed to the user through translateText() and I've added it to the en.json file - [x] I have added relevant tests to the test directory - [x] I confirm I have thoroughly tested these changes and take full responsibility for any bugs introduced - [x] I understand that submitting code with bugs that could have been caught through manual testing blocks releases and new features for all contributors ## Please put your Discord username so you can be contacted if a bug or regression is found: evan --- package-lock.json | 8 + package.json | 1 + src/core/configuration/Config.ts | 4 + src/core/configuration/DefaultConfig.ts | 12 + src/server/Cloudflare.ts | 280 ++++++++++++++++++++++++ src/server/Server.ts | 35 +++ startup.sh | 105 --------- supervisord.conf | 9 +- tests/util/TestServerConfig.ts | 12 + 9 files changed, 353 insertions(+), 113 deletions(-) create mode 100644 src/server/Cloudflare.ts diff --git a/package-lock.json b/package-lock.json index 642cf039e..faad9e243 100644 --- a/package-lock.json +++ b/package-lock.json @@ -84,6 +84,7 @@ "@types/d3": "^7.4.3", "@types/jest": "^29.5.12", "@types/jquery": "^3.5.31", + "@types/js-yaml": "^4.0.9", "@types/node": "^22.10.2", "@types/pg": "^8.11.11", "@types/sinon": "^17.0.3", @@ -8263,6 +8264,13 @@ "@types/sizzle": "*" } }, + "node_modules/@types/js-yaml": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", + "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", diff --git a/package.json b/package.json index eedac982d..6eb96b8c3 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "@types/d3": "^7.4.3", "@types/jest": "^29.5.12", "@types/jquery": "^3.5.31", + "@types/js-yaml": "^4.0.9", "@types/node": "^22.10.2", "@types/pg": "^8.11.11", "@types/sinon": "^17.0.3", diff --git a/src/core/configuration/Config.ts b/src/core/configuration/Config.ts index e6f4af76f..07ba46e80 100644 --- a/src/core/configuration/Config.ts +++ b/src/core/configuration/Config.ts @@ -56,6 +56,10 @@ export interface ServerConfig { jwtAudience(): string; jwtIssuer(): string; jwkPublicKey(): Promise; + domain(): string; + subdomain(): string; + cloudflareAccountId(): string; + cloudflareApiToken(): string; } export interface NukeMagnitude { diff --git a/src/core/configuration/DefaultConfig.ts b/src/core/configuration/DefaultConfig.ts index 49c3b5730..e8bad94ed 100644 --- a/src/core/configuration/DefaultConfig.ts +++ b/src/core/configuration/DefaultConfig.ts @@ -66,6 +66,18 @@ const numPlayersConfig = { } as const satisfies Record; export abstract class DefaultServerConfig implements ServerConfig { + domain(): string { + return process.env.DOMAIN ?? ""; + } + subdomain(): string { + return process.env.SUBDOMAIN ?? ""; + } + cloudflareAccountId(): string { + return process.env.CF_ACCOUNT_ID ?? ""; + } + cloudflareApiToken(): string { + return process.env.CF_API_TOKEN ?? ""; + } private publicKey: JWK; abstract jwtAudience(): string; jwtIssuer(): string { diff --git a/src/server/Cloudflare.ts b/src/server/Cloudflare.ts new file mode 100644 index 000000000..aef377fc3 --- /dev/null +++ b/src/server/Cloudflare.ts @@ -0,0 +1,280 @@ +import { spawn } from "child_process"; +import { promises as fs } from "fs"; +import yaml from "js-yaml"; +import { homedir } from "os"; +import { join } from "path"; +import { logger } from "./Logger"; + +const log = logger.child({ + module: "cloudflare", +}); + +export interface TunnelConfig { + domain: string; + subdomain: string; + subdomainToService: Map; +} + +interface TunnelResponse { + result: { + id: string; + token: string; + }; +} + +interface ZoneResponse { + result: Array<{ + id: string; + }>; +} + +interface DNSRecordResponse { + result: Array<{ + id: string; + }>; +} + +interface CloudflaredConfig { + tunnel: string; + credentials_file: string; + ingress: Array<{ + hostname?: string; + service: string; + }>; +} + +export class Cloudflare { + private baseUrl = "https://api.cloudflare.com/client/v4"; + private configDir: string; + + constructor( + private accountId: string, + private apiToken: string, + configDir: string = "~/.cloudflared", + ) { + this.configDir = configDir.startsWith("~") + ? join(homedir(), configDir.slice(1)) + : configDir; + + log.info(`Using config directory: ${this.configDir}`); + } + + private async makeRequest( + url: string, + method: string = "GET", + data?: any, + ): Promise { + const response = await fetch(url, { + method, + headers: { + Authorization: `Bearer ${this.apiToken}`, + "Content-Type": "application/json", + }, + body: data ? JSON.stringify(data) : undefined, + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `Cloudflare API error: url ${url} ${response.status} - ${errorText}`, + ); + } + + return response.json() as Promise; + } + + public async createTunnel(config: TunnelConfig): Promise<{ + tunnelId: string; + tunnelToken: string; + tunnelUrl: string; + configPath: string; + }> { + const { domain, subdomain, subdomainToService } = config; + + // Generate unique tunnel name + const timestamp = new Date().toISOString().replace(/[-:.]/g, ""); + const tunnelName = `${subdomain}-tunnel-${timestamp}`; + + log.info(`Creating tunnel with name: ${tunnelName}`); + + // Create tunnel via API to get official tunnel ID and token + const tunnelResponse = await this.makeRequest( + `${this.baseUrl}/accounts/${this.accountId}/cfd_tunnel`, + "POST", + { name: tunnelName }, + ); + + const tunnelId = tunnelResponse.result.id; + const tunnelToken = tunnelResponse.result.token; + + if (!tunnelId) { + throw new Error("Failed to create tunnel"); + } + + log.info(`Tunnel created with ID: ${tunnelId}`); + + // Create local config file instead of using API configuration + const configPath = await this.writeTunnelConfig( + tunnelId, + tunnelToken, + subdomain, + domain, + subdomainToService, + tunnelName, + ); + + // Get zone ID + const zoneResponse = await this.makeRequest( + `${this.baseUrl}/zones?name=${domain}`, + ); + + const zoneId = zoneResponse.result[0]?.id; + if (!zoneId) { + throw new Error(`Could not find zone ID for domain ${domain}`); + } + + await Promise.all( + Array.from(subdomainToService.entries()).map(([subdomain, _]) => + this.updateDNSRecord(zoneId, tunnelId, subdomain, domain), + ), + ); + + const tunnelUrl = `https://${subdomain}.${domain}`; + log.info(`Tunnel is set up! Site will be available at: ${tunnelUrl}`); + + return { tunnelId, tunnelToken, tunnelUrl, configPath }; + } + + private async writeTunnelConfig( + tunnelId: string, + tunnelToken: string, + subdomain: string, + domain: string, + subdomainToService: Map, + tunnelName: string, + ): Promise { + log.info(`Creating local config for tunnel ${subdomain}.${domain}...`); + + // Ensure config directory exists + await fs.mkdir(this.configDir, { recursive: true }); + + const configPath = join(this.configDir, `${tunnelName}.yml`); + const credentialsFile = join(this.configDir, `${tunnelId}.json`); + + const tokenData = JSON.parse( + Buffer.from(tunnelToken, "base64").toString("utf8"), + ); + + const credentials = { + AccountTag: tokenData.a || this.accountId, + TunnelID: tokenData.t || tunnelId, + TunnelName: tunnelName, + TunnelSecret: tokenData.s, + }; + + await fs.writeFile( + credentialsFile, + JSON.stringify(credentials, null, 2), + "utf8", + ); + log.info(`Created credentials file at: ${credentialsFile}`); + + const tunnelConfig: CloudflaredConfig = { + tunnel: tunnelId, + credentials_file: credentialsFile, + ingress: [ + ...Array.from(subdomainToService.entries()).map( + ([subdomain, service]) => ({ + hostname: `${subdomain}.${domain}`, + service: service, + }), + ), + { + service: "http_status:404", + }, + ], + }; + + // Write config file + await fs.writeFile(configPath, yaml.dump(tunnelConfig), "utf8"); + log.info(`Created config file at: ${configPath}`); + + return configPath; + } + + private async updateDNSRecord( + zoneId: string, + tunnelId: string, + subdomain: string, + domain: string, + ): Promise { + const existingRecords = await this.makeRequest( + `${this.baseUrl}/zones/${zoneId}/dns_records?name=${subdomain}.${domain}`, + ); + + const recordId = existingRecords.result[0]?.id; + const dnsData = { + type: "CNAME", + name: subdomain, + content: `${tunnelId}.cfargotunnel.com`, + ttl: 1, + proxied: true, + }; + + if (recordId) { + log.info(`Updating existing DNS record for ${subdomain}.${domain}...`); + await this.makeRequest( + `${this.baseUrl}/zones/${zoneId}/dns_records/${recordId}`, + "PUT", + dnsData, + ); + } else { + log.info(`Creating new DNS record for ${subdomain}.${domain}...`); + await this.makeRequest( + `${this.baseUrl}/zones/${zoneId}/dns_records`, + "POST", + dnsData, + ); + } + } + + public async startCloudflared(configPath: string) { + const cloudflared = spawn( + "cloudflared", + ["tunnel", "--config", configPath, "--loglevel", "error", "run"], + { + detached: true, + stdio: ["ignore", "pipe", "pipe"], + env: { + ...process.env, + // Set this to bypass origin cert requirement for named tunnels + TUNNEL_ORIGIN_CERT: "/dev/null", + }, + }, + ); + + cloudflared.stdout?.on("data", (data) => { + log.info(data.toString().trim()); + }); + cloudflared.stderr?.on("data", (data) => { + log.error(data.toString().trim()); + }); + + cloudflared.on("error", (error) => { + log.error("Failed to start cloudflared", { + error: error.message, + }); + }); + + cloudflared.on("exit", (code, signal) => { + if (code !== null) { + log.error(`Cloudflared exited with code ${code}`, { + exitCode: code, + signal, + }); + } + }); + + cloudflared.unref(); + } +} diff --git a/src/server/Server.ts b/src/server/Server.ts index 59468c10b..4b98e32aa 100644 --- a/src/server/Server.ts +++ b/src/server/Server.ts @@ -1,14 +1,22 @@ import cluster from "cluster"; import * as dotenv from "dotenv"; +import { GameEnv } from "../core/configuration/Config"; +import { getServerConfigFromServer } from "../core/configuration/ConfigLoader"; +import { Cloudflare, TunnelConfig } from "./Cloudflare"; import { startMaster } from "./Master"; import { startWorker } from "./Worker"; +const config = getServerConfigFromServer(); + dotenv.config(); // Main entry point of the application async function main() { // Check if this is the primary (master) process if (cluster.isPrimary) { + if (config.env() !== GameEnv.Dev) { + await setupTunnels(); + } console.log("Starting master process..."); await startMaster(); } else { @@ -23,3 +31,30 @@ main().catch((error) => { console.error("Failed to start server:", error); process.exit(1); }); + +async function setupTunnels() { + const cloudflare = new Cloudflare( + config.cloudflareAccountId(), + config.cloudflareApiToken(), + ); + + const domainToService = new Map().set( + config.subdomain(), + `http://localhost:3000`, + ); + + for (let i = 0; i < config.numWorkers(); i++) { + domainToService.set( + `w${i}-${config.subdomain()}`, + `http://localhost:${3000 + i + 1}`, + ); + } + + const tunnel = await cloudflare.createTunnel({ + subdomain: config.subdomain(), + domain: config.domain(), + subdomainToService: domainToService, + } as TunnelConfig); + + await cloudflare.startCloudflared(tunnel.configPath); +} diff --git a/startup.sh b/startup.sh index 052c3b200..28903d067 100644 --- a/startup.sh +++ b/startup.sh @@ -1,110 +1,5 @@ #!/bin/bash set -e - -# Check if required environment variables are set -if [ -z "$CF_API_TOKEN" ] || [ -z "$CF_ACCOUNT_ID" ] || [ -z "$SUBDOMAIN" ] || [ -z "$DOMAIN" ]; then - echo "Error: Required environment variables not set" - echo "Please set CF_API_TOKEN, CF_ACCOUNT_ID, SUBDOMAIN, and DOMAIN" - exit 1 -fi - -# Generate a unique tunnel name using timestamp -TIMESTAMP=$(date +%Y%m%d%H%M%S) -TUNNEL_NAME="${SUBDOMAIN}-tunnel-${TIMESTAMP}" -echo "Using unique tunnel name: ${TUNNEL_NAME}" - -# Create a new tunnel -echo "Creating Cloudflare tunnel for subdomain ${SUBDOMAIN}..." -TUNNEL_RESPONSE=$(curl -s -X POST "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "{\"name\":\"${TUNNEL_NAME}\"}") - -# Extract tunnel ID and token -TUNNEL_ID=$(echo $TUNNEL_RESPONSE | jq -r '.result.id') -TUNNEL_TOKEN=$(echo $TUNNEL_RESPONSE | jq -r '.result.token') - -if [ -z "$TUNNEL_ID" ] || [ "$TUNNEL_ID" == "null" ]; then - echo "Failed to create tunnel" - echo $TUNNEL_RESPONSE - exit 1 -fi - -echo "Tunnel created with ID: ${TUNNEL_ID}" - -# Configure the tunnel with hostname -echo "Configuring tunnel to point to ${SUBDOMAIN}.${DOMAIN}..." -curl -s -X PUT "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${TUNNEL_ID}/configurations" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "{\"config\":{\"ingress\":[{\"hostname\":\"${SUBDOMAIN}.${DOMAIN}\",\"service\":\"http://localhost:80\"},{\"service\":\"http_status:404\"}]}}" - -# Update DNS record to point to the new tunnel -echo "Updating DNS record to point to the new tunnel..." - -# First check if DNS record exists -DNS_RECORDS=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones?name=${DOMAIN}" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json") - -ZONE_ID=$(echo $DNS_RECORDS | jq -r '.result[0].id') - -if [ -z "$ZONE_ID" ] || [ "$ZONE_ID" == "null" ]; then - echo "Could not find zone ID for domain ${DOMAIN}" - exit 1 -fi - -# Check for existing record -EXISTING_RECORDS=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records?name=${SUBDOMAIN}.${DOMAIN}" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json") - -RECORD_ID=$(echo $EXISTING_RECORDS | jq -r '.result[0].id') - -# Create or update the DNS record -if [ -z "$RECORD_ID" ] || [ "$RECORD_ID" == "null" ]; then - # Create new record - echo "Creating new DNS record..." - DNS_RESPONSE=$(curl -s -X POST "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "{\"type\":\"CNAME\",\"name\":\"${SUBDOMAIN}\",\"content\":\"${TUNNEL_ID}.cfargotunnel.com\",\"ttl\":1,\"proxied\":true}") -else - # Update existing record - echo "Updating existing DNS record..." - DNS_RESPONSE=$(curl -s -X PUT "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${RECORD_ID}" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "{\"type\":\"CNAME\",\"name\":\"${SUBDOMAIN}\",\"content\":\"${TUNNEL_ID}.cfargotunnel.com\",\"ttl\":1,\"proxied\":true}") -fi - -# Log the tunnel information -echo "Tunnel is set up! Site will be available at: https://${SUBDOMAIN}.${DOMAIN}" - -# Export the tunnel token for supervisord -export CLOUDFLARE_TUNNEL_TOKEN=${TUNNEL_TOKEN} - -# Check if Basic Auth credentials are set -if [ -z "$BASIC_AUTH_USER" ] || [ -z "$BASIC_AUTH_PASS" ]; then - echo "HTTP Basic Authentication will be disabled" -else - # Create the htpasswd file - echo "Creating basic auth credentials for user: ${BASIC_AUTH_USER}" - # Ensure apache2-utils is installed for htpasswd - command -v htpasswd > /dev/null 2>&1 || { - echo "htpasswd not found, installing apache2-utils..." - apt-get update && apt-get install -y apache2-utils - } - # Create the password file - htpasswd -bc /etc/nginx/.htpasswd ${BASIC_AUTH_USER} ${BASIC_AUTH_PASS} - - # Update Nginx configuration to enable Basic Auth - sed -i '1i auth_basic "Restricted Access";' /etc/nginx/conf.d/default.conf - sed -i '2i auth_basic_user_file /etc/nginx/.htpasswd;' /etc/nginx/conf.d/default.conf - - echo "HTTP Basic Authentication enabled for user: ${BASIC_AUTH_USER}" -fi - # Start supervisord if [ "$DOMAIN" = openfront.dev ] && [ "$SUBDOMAIN" != main ]; then exec timeout 18h /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/supervisord.conf b/supervisord.conf index 61b2aec3a..c31d0429c 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -22,11 +22,4 @@ user=node stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 - -[program:cloudflared] -command=cloudflared tunnel run --token %(ENV_CLOUDFLARE_TUNNEL_TOKEN)s -autostart=true -autorestart=true -stdout_logfile=/var/log/cloudflared.log -stderr_logfile=/var/log/cloudflared-err.log \ No newline at end of file +stderr_logfile_maxbytes=0 \ No newline at end of file diff --git a/tests/util/TestServerConfig.ts b/tests/util/TestServerConfig.ts index 4c1f76eb4..7f6d88d30 100644 --- a/tests/util/TestServerConfig.ts +++ b/tests/util/TestServerConfig.ts @@ -4,6 +4,18 @@ import { GameMapType } from "../../src/core/game/Game"; import { GameID } from "../../src/core/Schemas"; export class TestServerConfig implements ServerConfig { + domain(): string { + throw new Error("Method not implemented."); + } + subdomain(): string { + throw new Error("Method not implemented."); + } + cloudflareAccountId(): string { + throw new Error("Method not implemented."); + } + cloudflareApiToken(): string { + throw new Error("Method not implemented."); + } jwtAudience(): string { throw new Error("Method not implemented."); }