From b249b364a0210385e308e358ce67bc7d92520108 Mon Sep 17 00:00:00 2001 From: evanpelle Date: Wed, 4 Jun 2025 11:20:07 -0700 Subject: [PATCH] have master create tunnels for all workers --- src/core/configuration/Config.ts | 4 + src/core/configuration/DefaultConfig.ts | 12 ++ src/server/Cloudflare.ts | 253 ++++++++++++++++++++++++ src/server/Server.ts | 35 ++++ src/server/Worker.ts | 1 - startup.sh | 105 ---------- supervisord.conf | 9 +- tests/util/TestServerConfig.ts | 12 ++ 8 files changed, 317 insertions(+), 114 deletions(-) create mode 100644 src/server/Cloudflare.ts diff --git a/src/core/configuration/Config.ts b/src/core/configuration/Config.ts index e6f4af76f..07ba46e80 100644 --- a/src/core/configuration/Config.ts +++ b/src/core/configuration/Config.ts @@ -56,6 +56,10 @@ export interface ServerConfig { jwtAudience(): string; jwtIssuer(): string; jwkPublicKey(): Promise; + domain(): string; + subdomain(): string; + cloudflareAccountId(): string; + cloudflareApiToken(): string; } export interface NukeMagnitude { diff --git a/src/core/configuration/DefaultConfig.ts b/src/core/configuration/DefaultConfig.ts index 49c3b5730..e8bad94ed 100644 --- a/src/core/configuration/DefaultConfig.ts +++ b/src/core/configuration/DefaultConfig.ts @@ -66,6 +66,18 @@ const numPlayersConfig = { } as const satisfies Record; export abstract class DefaultServerConfig implements ServerConfig { + domain(): string { + return process.env.DOMAIN ?? ""; + } + subdomain(): string { + return process.env.SUBDOMAIN ?? ""; + } + cloudflareAccountId(): string { + return process.env.CF_ACCOUNT_ID ?? ""; + } + cloudflareApiToken(): string { + return process.env.CF_API_TOKEN ?? ""; + } private publicKey: JWK; abstract jwtAudience(): string; jwtIssuer(): string { diff --git a/src/server/Cloudflare.ts b/src/server/Cloudflare.ts new file mode 100644 index 000000000..e771364c4 --- /dev/null +++ b/src/server/Cloudflare.ts @@ -0,0 +1,253 @@ +import { spawn } from "child_process"; +import * as fs from "fs"; +import { logger } from "./Logger"; + +const log = logger.child({ + module: "cloudflare", +}); + +export interface TunnelConfig { + domain: string; + subdomain: string; + subdomainToService: Map; +} + +interface TunnelResponse { + result: { + id: string; + token: string; + }; +} + +interface ZoneResponse { + result: Array<{ + id: string; + }>; +} + +interface DNSRecordResponse { + result: Array<{ + id: string; + }>; +} + +export class Cloudflare { + private baseUrl = "https://api.cloudflare.com/client/v4"; + + constructor( + private accountId: string, + private apiToken: string, + ) {} + + private async makeRequest( + url: string, + method: string = "GET", + data?: any, + ): Promise { + const response = await fetch(url, { + method, + headers: { + Authorization: `Bearer ${this.apiToken}`, + "Content-Type": "application/json", + }, + body: data ? JSON.stringify(data) : undefined, + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `Cloudflare API error: ${response.status} - ${errorText}`, + ); + } + + return response.json() as Promise; + } + + public async createTunnel(config: TunnelConfig): Promise<{ + tunnelId: string; + tunnelToken: string; + tunnelUrl: string; + }> { + const { domain, subdomain, subdomainToService } = config; + + // Generate unique tunnel name + const timestamp = new Date().toISOString().replace(/[-:.]/g, ""); + const tunnelName = `${subdomain}-tunnel-${timestamp}`; + + log.info(`Creating tunnel with name: ${tunnelName}`); + + const tunnelResponse = await this.makeRequest( + `${this.baseUrl}/accounts/${this.accountId}/cfd_tunnel`, + "POST", + { name: tunnelName }, + ); + + const tunnelId = tunnelResponse.result.id; + const tunnelToken = tunnelResponse.result.token; + + if (!tunnelId || tunnelId === "null") { + throw new Error("Failed to create tunnel"); + } + + log.info(`Tunnel created with ID: ${tunnelId}`); + + await this.configureTunnel(tunnelId, subdomain, domain, subdomainToService); + + // Get zone ID + const zoneResponse = await this.makeRequest( + `${this.baseUrl}/zones?name=${domain}`, + ); + + const zoneId = zoneResponse.result[0]?.id; + if (!zoneId) { + throw new Error(`Could not find zone ID for domain ${domain}`); + } + + await Promise.all( + Array.from(subdomainToService.entries()).map(([subdomain, _]) => + this.updateDNSRecord(zoneId, tunnelId, subdomain, domain), + ), + ); + + const tunnelUrl = `https://${subdomain}.${domain}`; + log.info(`Tunnel is set up! Site will be available at: ${tunnelUrl}`); + + return { tunnelId, tunnelToken, tunnelUrl }; + } + + private async configureTunnel( + tunnelId: string, + subdomain: string, + domain: string, + subdomainToService: Map, + ): Promise { + log.info(`Configuring tunnel to point to ${subdomain}.${domain}...`); + + const request = { + config: { + ingress: [ + ...Array.from(subdomainToService.entries()).map( + ([subdomain, service]) => ({ + hostname: `${subdomain}.${domain}`, + service: service, + }), + ), + { + service: "http_status:404", + }, + ], + }, + }; + await this.makeRequest( + `${this.baseUrl}/accounts/${this.accountId}/cfd_tunnel/${tunnelId}/configurations`, + "PUT", + request, + ); + } + + private async updateDNSRecord( + zoneId: string, + tunnelId: string, + subdomain: string, + domain: string, + ): Promise { + const existingRecords = await this.makeRequest( + `${this.baseUrl}/zones/${zoneId}/dns_records?name=${subdomain}.${domain}`, + ); + + const recordId = existingRecords.result[0]?.id; + const dnsData = { + type: "CNAME", + name: subdomain, + content: `${tunnelId}.cfargotunnel.com`, + ttl: 1, + proxied: true, + }; + + if (recordId) { + log.info(`Updating existing DNS record for ${subdomain}.${domain}...`); + await this.makeRequest( + `${this.baseUrl}/zones/${zoneId}/dns_records/${recordId}`, + "PUT", + dnsData, + ); + } else { + log.info(`Creating new DNS record for ${subdomain}.${domain}...`); + await this.makeRequest( + `${this.baseUrl}/zones/${zoneId}/dns_records`, + "POST", + dnsData, + ); + } + } + + public async deleteTunnel(tunnelId: string): Promise { + log.info(`Deleting tunnel with ID: ${tunnelId}`); + + await this.makeRequest( + `${this.baseUrl}/accounts/${this.accountId}/cfd_tunnel/${tunnelId}`, + "DELETE", + ); + + log.info("Tunnel deleted successfully"); + } + + public async listTunnels(): Promise { + const response = await this.makeRequest<{ result: any[] }>( + `${this.baseUrl}/accounts/${this.accountId}/cfd_tunnel`, + ); + + return response.result; + } + + public async deleteDNSRecord( + subdomain: string, + domain: string, + ): Promise { + log.info(`Deleting DNS record for ${subdomain}.${domain}...`); + + // Get zone ID + const zoneResponse = await this.makeRequest( + `${this.baseUrl}/zones?name=${domain}`, + ); + + const zoneId = zoneResponse.result[0]?.id; + if (!zoneId) { + throw new Error(`Could not find zone ID for domain ${domain}`); + } + + // Get DNS record + const existingRecords = await this.makeRequest( + `${this.baseUrl}/zones/${zoneId}/dns_records?name=${subdomain}.${domain}`, + ); + + const recordId = existingRecords.result[0]?.id; + if (!recordId) { + log.info("No DNS record found to delete"); + return; + } + + // Delete DNS record + await this.makeRequest( + `${this.baseUrl}/zones/${zoneId}/dns_records/${recordId}`, + "DELETE", + ); + + log.info("DNS record deleted successfully"); + } + + public async startCloudflared(tunnelToken: string) { + const out = fs.openSync("./cloudflared.out.log", "a"); + const err = fs.openSync("./cloudflared.err.log", "a"); + + const cloudflared = spawn( + "cloudflared", + ["tunnel", "run", "--token", tunnelToken], + { + detached: true, + stdio: ["ignore", out, err], + }, + ); + cloudflared.unref(); + } +} diff --git a/src/server/Server.ts b/src/server/Server.ts index 59468c10b..5c4972f1a 100644 --- a/src/server/Server.ts +++ b/src/server/Server.ts @@ -1,14 +1,22 @@ import cluster from "cluster"; import * as dotenv from "dotenv"; +import { GameEnv } from "../core/configuration/Config"; +import { getServerConfigFromServer } from "../core/configuration/ConfigLoader"; +import { Cloudflare, TunnelConfig } from "./Cloudflare"; import { startMaster } from "./Master"; import { startWorker } from "./Worker"; +const config = getServerConfigFromServer(); + dotenv.config(); // Main entry point of the application async function main() { // Check if this is the primary (master) process if (cluster.isPrimary) { + if (config.env() !== GameEnv.Dev) { + setupTunnels(); + } console.log("Starting master process..."); await startMaster(); } else { @@ -23,3 +31,30 @@ main().catch((error) => { console.error("Failed to start server:", error); process.exit(1); }); + +async function setupTunnels() { + const cloudflare = new Cloudflare( + config.cloudflareAccountId(), + config.cloudflareApiToken(), + ); + + const domainToService = new Map().set( + config.subdomain(), + `http://localhost:3000`, + ); + + for (let i = 0; i < config.numWorkers(); i++) { + domainToService.set( + `w${i}-${config.subdomain()}`, + `http://localhost:${3000 + i + 1}`, + ); + } + + const tunnel = await cloudflare.createTunnel({ + subdomain: config.subdomain(), + domain: config.domain(), + subdomainToService: domainToService, + } as TunnelConfig); + + await cloudflare.startCloudflared(tunnel.tunnelToken); +} diff --git a/src/server/Worker.ts b/src/server/Worker.ts index 3f072ebdf..5bee6b603 100644 --- a/src/server/Worker.ts +++ b/src/server/Worker.ts @@ -11,7 +11,6 @@ import { getServerConfigFromServer } from "../core/configuration/ConfigLoader"; import { GameType } from "../core/game/Game"; import { ClientJoinMessageSchema, - GameConfig, GameRecord, GameRecordSchema, } from "../core/Schemas"; diff --git a/startup.sh b/startup.sh index 052c3b200..28903d067 100644 --- a/startup.sh +++ b/startup.sh @@ -1,110 +1,5 @@ #!/bin/bash set -e - -# Check if required environment variables are set -if [ -z "$CF_API_TOKEN" ] || [ -z "$CF_ACCOUNT_ID" ] || [ -z "$SUBDOMAIN" ] || [ -z "$DOMAIN" ]; then - echo "Error: Required environment variables not set" - echo "Please set CF_API_TOKEN, CF_ACCOUNT_ID, SUBDOMAIN, and DOMAIN" - exit 1 -fi - -# Generate a unique tunnel name using timestamp -TIMESTAMP=$(date +%Y%m%d%H%M%S) -TUNNEL_NAME="${SUBDOMAIN}-tunnel-${TIMESTAMP}" -echo "Using unique tunnel name: ${TUNNEL_NAME}" - -# Create a new tunnel -echo "Creating Cloudflare tunnel for subdomain ${SUBDOMAIN}..." -TUNNEL_RESPONSE=$(curl -s -X POST "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "{\"name\":\"${TUNNEL_NAME}\"}") - -# Extract tunnel ID and token -TUNNEL_ID=$(echo $TUNNEL_RESPONSE | jq -r '.result.id') -TUNNEL_TOKEN=$(echo $TUNNEL_RESPONSE | jq -r '.result.token') - -if [ -z "$TUNNEL_ID" ] || [ "$TUNNEL_ID" == "null" ]; then - echo "Failed to create tunnel" - echo $TUNNEL_RESPONSE - exit 1 -fi - -echo "Tunnel created with ID: ${TUNNEL_ID}" - -# Configure the tunnel with hostname -echo "Configuring tunnel to point to ${SUBDOMAIN}.${DOMAIN}..." -curl -s -X PUT "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${TUNNEL_ID}/configurations" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "{\"config\":{\"ingress\":[{\"hostname\":\"${SUBDOMAIN}.${DOMAIN}\",\"service\":\"http://localhost:80\"},{\"service\":\"http_status:404\"}]}}" - -# Update DNS record to point to the new tunnel -echo "Updating DNS record to point to the new tunnel..." - -# First check if DNS record exists -DNS_RECORDS=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones?name=${DOMAIN}" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json") - -ZONE_ID=$(echo $DNS_RECORDS | jq -r '.result[0].id') - -if [ -z "$ZONE_ID" ] || [ "$ZONE_ID" == "null" ]; then - echo "Could not find zone ID for domain ${DOMAIN}" - exit 1 -fi - -# Check for existing record -EXISTING_RECORDS=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records?name=${SUBDOMAIN}.${DOMAIN}" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json") - -RECORD_ID=$(echo $EXISTING_RECORDS | jq -r '.result[0].id') - -# Create or update the DNS record -if [ -z "$RECORD_ID" ] || [ "$RECORD_ID" == "null" ]; then - # Create new record - echo "Creating new DNS record..." - DNS_RESPONSE=$(curl -s -X POST "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "{\"type\":\"CNAME\",\"name\":\"${SUBDOMAIN}\",\"content\":\"${TUNNEL_ID}.cfargotunnel.com\",\"ttl\":1,\"proxied\":true}") -else - # Update existing record - echo "Updating existing DNS record..." - DNS_RESPONSE=$(curl -s -X PUT "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${RECORD_ID}" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "{\"type\":\"CNAME\",\"name\":\"${SUBDOMAIN}\",\"content\":\"${TUNNEL_ID}.cfargotunnel.com\",\"ttl\":1,\"proxied\":true}") -fi - -# Log the tunnel information -echo "Tunnel is set up! Site will be available at: https://${SUBDOMAIN}.${DOMAIN}" - -# Export the tunnel token for supervisord -export CLOUDFLARE_TUNNEL_TOKEN=${TUNNEL_TOKEN} - -# Check if Basic Auth credentials are set -if [ -z "$BASIC_AUTH_USER" ] || [ -z "$BASIC_AUTH_PASS" ]; then - echo "HTTP Basic Authentication will be disabled" -else - # Create the htpasswd file - echo "Creating basic auth credentials for user: ${BASIC_AUTH_USER}" - # Ensure apache2-utils is installed for htpasswd - command -v htpasswd > /dev/null 2>&1 || { - echo "htpasswd not found, installing apache2-utils..." - apt-get update && apt-get install -y apache2-utils - } - # Create the password file - htpasswd -bc /etc/nginx/.htpasswd ${BASIC_AUTH_USER} ${BASIC_AUTH_PASS} - - # Update Nginx configuration to enable Basic Auth - sed -i '1i auth_basic "Restricted Access";' /etc/nginx/conf.d/default.conf - sed -i '2i auth_basic_user_file /etc/nginx/.htpasswd;' /etc/nginx/conf.d/default.conf - - echo "HTTP Basic Authentication enabled for user: ${BASIC_AUTH_USER}" -fi - # Start supervisord if [ "$DOMAIN" = openfront.dev ] && [ "$SUBDOMAIN" != main ]; then exec timeout 18h /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/supervisord.conf b/supervisord.conf index 61b2aec3a..c31d0429c 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -22,11 +22,4 @@ user=node stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 - -[program:cloudflared] -command=cloudflared tunnel run --token %(ENV_CLOUDFLARE_TUNNEL_TOKEN)s -autostart=true -autorestart=true -stdout_logfile=/var/log/cloudflared.log -stderr_logfile=/var/log/cloudflared-err.log \ No newline at end of file +stderr_logfile_maxbytes=0 \ No newline at end of file diff --git a/tests/util/TestServerConfig.ts b/tests/util/TestServerConfig.ts index 4c1f76eb4..7f6d88d30 100644 --- a/tests/util/TestServerConfig.ts +++ b/tests/util/TestServerConfig.ts @@ -4,6 +4,18 @@ import { GameMapType } from "../../src/core/game/Game"; import { GameID } from "../../src/core/Schemas"; export class TestServerConfig implements ServerConfig { + domain(): string { + throw new Error("Method not implemented."); + } + subdomain(): string { + throw new Error("Method not implemented."); + } + cloudflareAccountId(): string { + throw new Error("Method not implemented."); + } + cloudflareApiToken(): string { + throw new Error("Method not implemented."); + } jwtAudience(): string { throw new Error("Method not implemented."); }