have master create tunnels for all workers #780 (#1042)

## Description:
We want to move away from using nginx to cloudflare to route among
workers. This will simplify the nginx config, move routing computation
off the server, and make it easier to implement a multi-host
architecture.

The worker tunnels are not currently used.

I also moved the tunnel creation from startup.sh to Server. The shell
script was getting too complex.


## Please complete the following:

- [x] I have added screenshots for all UI updates
- [x] I process any text displayed to the user through translateText()
and I've added it to the en.json file
- [x] I have added relevant tests to the test directory
- [x] I confirm I have thoroughly tested these changes and take full
responsibility for any bugs introduced
- [x] I understand that submitting code with bugs that could have been
caught through manual testing blocks releases and new features for all
contributors

## Please put your Discord username so you can be contacted if a bug or
regression is found:

evan
This commit is contained in:
evanpelle
2025-06-06 13:43:21 -07:00
committed by GitHub
parent a4fffce7f3
commit 49b01d8014
9 changed files with 353 additions and 113 deletions
+8
View File
@@ -84,6 +84,7 @@
"@types/d3": "^7.4.3",
"@types/jest": "^29.5.12",
"@types/jquery": "^3.5.31",
"@types/js-yaml": "^4.0.9",
"@types/node": "^22.10.2",
"@types/pg": "^8.11.11",
"@types/sinon": "^17.0.3",
@@ -8263,6 +8264,13 @@
"@types/sizzle": "*"
}
},
"node_modules/@types/js-yaml": {
"version": "4.0.9",
"resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz",
"integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/json-schema": {
"version": "7.0.15",
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
+1
View File
@@ -32,6 +32,7 @@
"@types/d3": "^7.4.3",
"@types/jest": "^29.5.12",
"@types/jquery": "^3.5.31",
"@types/js-yaml": "^4.0.9",
"@types/node": "^22.10.2",
"@types/pg": "^8.11.11",
"@types/sinon": "^17.0.3",
+4
View File
@@ -56,6 +56,10 @@ export interface ServerConfig {
jwtAudience(): string;
jwtIssuer(): string;
jwkPublicKey(): Promise<JWK>;
domain(): string;
subdomain(): string;
cloudflareAccountId(): string;
cloudflareApiToken(): string;
}
export interface NukeMagnitude {
+12
View File
@@ -66,6 +66,18 @@ const numPlayersConfig = {
} as const satisfies Record<GameMapType, [number, number, number]>;
export abstract class DefaultServerConfig implements ServerConfig {
domain(): string {
return process.env.DOMAIN ?? "";
}
subdomain(): string {
return process.env.SUBDOMAIN ?? "";
}
cloudflareAccountId(): string {
return process.env.CF_ACCOUNT_ID ?? "";
}
cloudflareApiToken(): string {
return process.env.CF_API_TOKEN ?? "";
}
private publicKey: JWK;
abstract jwtAudience(): string;
jwtIssuer(): string {
+280
View File
@@ -0,0 +1,280 @@
import { spawn } from "child_process";
import { promises as fs } from "fs";
import yaml from "js-yaml";
import { homedir } from "os";
import { join } from "path";
import { logger } from "./Logger";
const log = logger.child({
module: "cloudflare",
});
export interface TunnelConfig {
domain: string;
subdomain: string;
subdomainToService: Map<string, string>;
}
interface TunnelResponse {
result: {
id: string;
token: string;
};
}
interface ZoneResponse {
result: Array<{
id: string;
}>;
}
interface DNSRecordResponse {
result: Array<{
id: string;
}>;
}
interface CloudflaredConfig {
tunnel: string;
credentials_file: string;
ingress: Array<{
hostname?: string;
service: string;
}>;
}
export class Cloudflare {
private baseUrl = "https://api.cloudflare.com/client/v4";
private configDir: string;
constructor(
private accountId: string,
private apiToken: string,
configDir: string = "~/.cloudflared",
) {
this.configDir = configDir.startsWith("~")
? join(homedir(), configDir.slice(1))
: configDir;
log.info(`Using config directory: ${this.configDir}`);
}
private async makeRequest<T>(
url: string,
method: string = "GET",
data?: any,
): Promise<T> {
const response = await fetch(url, {
method,
headers: {
Authorization: `Bearer ${this.apiToken}`,
"Content-Type": "application/json",
},
body: data ? JSON.stringify(data) : undefined,
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(
`Cloudflare API error: url ${url} ${response.status} - ${errorText}`,
);
}
return response.json() as Promise<T>;
}
public async createTunnel(config: TunnelConfig): Promise<{
tunnelId: string;
tunnelToken: string;
tunnelUrl: string;
configPath: string;
}> {
const { domain, subdomain, subdomainToService } = config;
// Generate unique tunnel name
const timestamp = new Date().toISOString().replace(/[-:.]/g, "");
const tunnelName = `${subdomain}-tunnel-${timestamp}`;
log.info(`Creating tunnel with name: ${tunnelName}`);
// Create tunnel via API to get official tunnel ID and token
const tunnelResponse = await this.makeRequest<TunnelResponse>(
`${this.baseUrl}/accounts/${this.accountId}/cfd_tunnel`,
"POST",
{ name: tunnelName },
);
const tunnelId = tunnelResponse.result.id;
const tunnelToken = tunnelResponse.result.token;
if (!tunnelId) {
throw new Error("Failed to create tunnel");
}
log.info(`Tunnel created with ID: ${tunnelId}`);
// Create local config file instead of using API configuration
const configPath = await this.writeTunnelConfig(
tunnelId,
tunnelToken,
subdomain,
domain,
subdomainToService,
tunnelName,
);
// Get zone ID
const zoneResponse = await this.makeRequest<ZoneResponse>(
`${this.baseUrl}/zones?name=${domain}`,
);
const zoneId = zoneResponse.result[0]?.id;
if (!zoneId) {
throw new Error(`Could not find zone ID for domain ${domain}`);
}
await Promise.all(
Array.from(subdomainToService.entries()).map(([subdomain, _]) =>
this.updateDNSRecord(zoneId, tunnelId, subdomain, domain),
),
);
const tunnelUrl = `https://${subdomain}.${domain}`;
log.info(`Tunnel is set up! Site will be available at: ${tunnelUrl}`);
return { tunnelId, tunnelToken, tunnelUrl, configPath };
}
private async writeTunnelConfig(
tunnelId: string,
tunnelToken: string,
subdomain: string,
domain: string,
subdomainToService: Map<string, string>,
tunnelName: string,
): Promise<string> {
log.info(`Creating local config for tunnel ${subdomain}.${domain}...`);
// Ensure config directory exists
await fs.mkdir(this.configDir, { recursive: true });
const configPath = join(this.configDir, `${tunnelName}.yml`);
const credentialsFile = join(this.configDir, `${tunnelId}.json`);
const tokenData = JSON.parse(
Buffer.from(tunnelToken, "base64").toString("utf8"),
);
const credentials = {
AccountTag: tokenData.a || this.accountId,
TunnelID: tokenData.t || tunnelId,
TunnelName: tunnelName,
TunnelSecret: tokenData.s,
};
await fs.writeFile(
credentialsFile,
JSON.stringify(credentials, null, 2),
"utf8",
);
log.info(`Created credentials file at: ${credentialsFile}`);
const tunnelConfig: CloudflaredConfig = {
tunnel: tunnelId,
credentials_file: credentialsFile,
ingress: [
...Array.from(subdomainToService.entries()).map(
([subdomain, service]) => ({
hostname: `${subdomain}.${domain}`,
service: service,
}),
),
{
service: "http_status:404",
},
],
};
// Write config file
await fs.writeFile(configPath, yaml.dump(tunnelConfig), "utf8");
log.info(`Created config file at: ${configPath}`);
return configPath;
}
private async updateDNSRecord(
zoneId: string,
tunnelId: string,
subdomain: string,
domain: string,
): Promise<void> {
const existingRecords = await this.makeRequest<DNSRecordResponse>(
`${this.baseUrl}/zones/${zoneId}/dns_records?name=${subdomain}.${domain}`,
);
const recordId = existingRecords.result[0]?.id;
const dnsData = {
type: "CNAME",
name: subdomain,
content: `${tunnelId}.cfargotunnel.com`,
ttl: 1,
proxied: true,
};
if (recordId) {
log.info(`Updating existing DNS record for ${subdomain}.${domain}...`);
await this.makeRequest(
`${this.baseUrl}/zones/${zoneId}/dns_records/${recordId}`,
"PUT",
dnsData,
);
} else {
log.info(`Creating new DNS record for ${subdomain}.${domain}...`);
await this.makeRequest(
`${this.baseUrl}/zones/${zoneId}/dns_records`,
"POST",
dnsData,
);
}
}
public async startCloudflared(configPath: string) {
const cloudflared = spawn(
"cloudflared",
["tunnel", "--config", configPath, "--loglevel", "error", "run"],
{
detached: true,
stdio: ["ignore", "pipe", "pipe"],
env: {
...process.env,
// Set this to bypass origin cert requirement for named tunnels
TUNNEL_ORIGIN_CERT: "/dev/null",
},
},
);
cloudflared.stdout?.on("data", (data) => {
log.info(data.toString().trim());
});
cloudflared.stderr?.on("data", (data) => {
log.error(data.toString().trim());
});
cloudflared.on("error", (error) => {
log.error("Failed to start cloudflared", {
error: error.message,
});
});
cloudflared.on("exit", (code, signal) => {
if (code !== null) {
log.error(`Cloudflared exited with code ${code}`, {
exitCode: code,
signal,
});
}
});
cloudflared.unref();
}
}
+35
View File
@@ -1,14 +1,22 @@
import cluster from "cluster";
import * as dotenv from "dotenv";
import { GameEnv } from "../core/configuration/Config";
import { getServerConfigFromServer } from "../core/configuration/ConfigLoader";
import { Cloudflare, TunnelConfig } from "./Cloudflare";
import { startMaster } from "./Master";
import { startWorker } from "./Worker";
const config = getServerConfigFromServer();
dotenv.config();
// Main entry point of the application
async function main() {
// Check if this is the primary (master) process
if (cluster.isPrimary) {
if (config.env() !== GameEnv.Dev) {
await setupTunnels();
}
console.log("Starting master process...");
await startMaster();
} else {
@@ -23,3 +31,30 @@ main().catch((error) => {
console.error("Failed to start server:", error);
process.exit(1);
});
async function setupTunnels() {
const cloudflare = new Cloudflare(
config.cloudflareAccountId(),
config.cloudflareApiToken(),
);
const domainToService = new Map<string, string>().set(
config.subdomain(),
`http://localhost:3000`,
);
for (let i = 0; i < config.numWorkers(); i++) {
domainToService.set(
`w${i}-${config.subdomain()}`,
`http://localhost:${3000 + i + 1}`,
);
}
const tunnel = await cloudflare.createTunnel({
subdomain: config.subdomain(),
domain: config.domain(),
subdomainToService: domainToService,
} as TunnelConfig);
await cloudflare.startCloudflared(tunnel.configPath);
}
-105
View File
@@ -1,110 +1,5 @@
#!/bin/bash
set -e
# Check if required environment variables are set
if [ -z "$CF_API_TOKEN" ] || [ -z "$CF_ACCOUNT_ID" ] || [ -z "$SUBDOMAIN" ] || [ -z "$DOMAIN" ]; then
echo "Error: Required environment variables not set"
echo "Please set CF_API_TOKEN, CF_ACCOUNT_ID, SUBDOMAIN, and DOMAIN"
exit 1
fi
# Generate a unique tunnel name using timestamp
TIMESTAMP=$(date +%Y%m%d%H%M%S)
TUNNEL_NAME="${SUBDOMAIN}-tunnel-${TIMESTAMP}"
echo "Using unique tunnel name: ${TUNNEL_NAME}"
# Create a new tunnel
echo "Creating Cloudflare tunnel for subdomain ${SUBDOMAIN}..."
TUNNEL_RESPONSE=$(curl -s -X POST "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel" \
-H "Authorization: Bearer ${CF_API_TOKEN}" \
-H "Content-Type: application/json" \
--data "{\"name\":\"${TUNNEL_NAME}\"}")
# Extract tunnel ID and token
TUNNEL_ID=$(echo $TUNNEL_RESPONSE | jq -r '.result.id')
TUNNEL_TOKEN=$(echo $TUNNEL_RESPONSE | jq -r '.result.token')
if [ -z "$TUNNEL_ID" ] || [ "$TUNNEL_ID" == "null" ]; then
echo "Failed to create tunnel"
echo $TUNNEL_RESPONSE
exit 1
fi
echo "Tunnel created with ID: ${TUNNEL_ID}"
# Configure the tunnel with hostname
echo "Configuring tunnel to point to ${SUBDOMAIN}.${DOMAIN}..."
curl -s -X PUT "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${TUNNEL_ID}/configurations" \
-H "Authorization: Bearer ${CF_API_TOKEN}" \
-H "Content-Type: application/json" \
--data "{\"config\":{\"ingress\":[{\"hostname\":\"${SUBDOMAIN}.${DOMAIN}\",\"service\":\"http://localhost:80\"},{\"service\":\"http_status:404\"}]}}"
# Update DNS record to point to the new tunnel
echo "Updating DNS record to point to the new tunnel..."
# First check if DNS record exists
DNS_RECORDS=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones?name=${DOMAIN}" \
-H "Authorization: Bearer ${CF_API_TOKEN}" \
-H "Content-Type: application/json")
ZONE_ID=$(echo $DNS_RECORDS | jq -r '.result[0].id')
if [ -z "$ZONE_ID" ] || [ "$ZONE_ID" == "null" ]; then
echo "Could not find zone ID for domain ${DOMAIN}"
exit 1
fi
# Check for existing record
EXISTING_RECORDS=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records?name=${SUBDOMAIN}.${DOMAIN}" \
-H "Authorization: Bearer ${CF_API_TOKEN}" \
-H "Content-Type: application/json")
RECORD_ID=$(echo $EXISTING_RECORDS | jq -r '.result[0].id')
# Create or update the DNS record
if [ -z "$RECORD_ID" ] || [ "$RECORD_ID" == "null" ]; then
# Create new record
echo "Creating new DNS record..."
DNS_RESPONSE=$(curl -s -X POST "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records" \
-H "Authorization: Bearer ${CF_API_TOKEN}" \
-H "Content-Type: application/json" \
--data "{\"type\":\"CNAME\",\"name\":\"${SUBDOMAIN}\",\"content\":\"${TUNNEL_ID}.cfargotunnel.com\",\"ttl\":1,\"proxied\":true}")
else
# Update existing record
echo "Updating existing DNS record..."
DNS_RESPONSE=$(curl -s -X PUT "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${RECORD_ID}" \
-H "Authorization: Bearer ${CF_API_TOKEN}" \
-H "Content-Type: application/json" \
--data "{\"type\":\"CNAME\",\"name\":\"${SUBDOMAIN}\",\"content\":\"${TUNNEL_ID}.cfargotunnel.com\",\"ttl\":1,\"proxied\":true}")
fi
# Log the tunnel information
echo "Tunnel is set up! Site will be available at: https://${SUBDOMAIN}.${DOMAIN}"
# Export the tunnel token for supervisord
export CLOUDFLARE_TUNNEL_TOKEN=${TUNNEL_TOKEN}
# Check if Basic Auth credentials are set
if [ -z "$BASIC_AUTH_USER" ] || [ -z "$BASIC_AUTH_PASS" ]; then
echo "HTTP Basic Authentication will be disabled"
else
# Create the htpasswd file
echo "Creating basic auth credentials for user: ${BASIC_AUTH_USER}"
# Ensure apache2-utils is installed for htpasswd
command -v htpasswd > /dev/null 2>&1 || {
echo "htpasswd not found, installing apache2-utils..."
apt-get update && apt-get install -y apache2-utils
}
# Create the password file
htpasswd -bc /etc/nginx/.htpasswd ${BASIC_AUTH_USER} ${BASIC_AUTH_PASS}
# Update Nginx configuration to enable Basic Auth
sed -i '1i auth_basic "Restricted Access";' /etc/nginx/conf.d/default.conf
sed -i '2i auth_basic_user_file /etc/nginx/.htpasswd;' /etc/nginx/conf.d/default.conf
echo "HTTP Basic Authentication enabled for user: ${BASIC_AUTH_USER}"
fi
# Start supervisord
if [ "$DOMAIN" = openfront.dev ] && [ "$SUBDOMAIN" != main ]; then
exec timeout 18h /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
+1 -8
View File
@@ -22,11 +22,4 @@ user=node
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:cloudflared]
command=cloudflared tunnel run --token %(ENV_CLOUDFLARE_TUNNEL_TOKEN)s
autostart=true
autorestart=true
stdout_logfile=/var/log/cloudflared.log
stderr_logfile=/var/log/cloudflared-err.log
stderr_logfile_maxbytes=0
+12
View File
@@ -4,6 +4,18 @@ import { GameMapType } from "../../src/core/game/Game";
import { GameID } from "../../src/core/Schemas";
export class TestServerConfig implements ServerConfig {
domain(): string {
throw new Error("Method not implemented.");
}
subdomain(): string {
throw new Error("Method not implemented.");
}
cloudflareAccountId(): string {
throw new Error("Method not implemented.");
}
cloudflareApiToken(): string {
throw new Error("Method not implemented.");
}
jwtAudience(): string {
throw new Error("Method not implemented.");
}