mirror of
https://github.com/openfrontio/OpenFrontIO.git
synced 2026-06-21 12:20:46 +00:00
tunnels
This commit is contained in:
Generated
+21
@@ -5211,6 +5211,18 @@
|
||||
"@lit-labs/ssr-dom-shim": "^1.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@noble/hashes": {
|
||||
"version": "1.8.0",
|
||||
"resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz",
|
||||
"integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^14.21.3 || >=16"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://paulmillr.com/funding/"
|
||||
}
|
||||
},
|
||||
"node_modules/@nodelib/fs.scandir": {
|
||||
"version": "2.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
|
||||
@@ -9558,6 +9570,15 @@
|
||||
"integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/bip39": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/bip39/-/bip39-3.1.0.tgz",
|
||||
"integrity": "sha512-c9kiwdk45Do5GL0vJMe7tS95VjCii65mYAH7DfWl3uW8AVzXKQVUm64i3hzVybBDMp9r7j9iNxR85+ul8MdN/A==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@noble/hashes": "^1.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bl": {
|
||||
"version": "6.0.16",
|
||||
"resolved": "https://registry.npmjs.org/bl/-/bl-6.0.16.tgz",
|
||||
|
||||
+3
-3
@@ -247,10 +247,10 @@ export function assertNever(x: never): never {
|
||||
throw new Error("Unexpected value: " + x);
|
||||
}
|
||||
|
||||
export function generateID(): GameID {
|
||||
export function generateID(length: number = 8): GameID {
|
||||
const nanoid = customAlphabet(
|
||||
"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
|
||||
8,
|
||||
"123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKMNPQRSTUVWXYZ",
|
||||
length,
|
||||
);
|
||||
return nanoid();
|
||||
}
|
||||
|
||||
@@ -45,6 +45,10 @@ export interface ServerConfig {
|
||||
r2Endpoint(): string;
|
||||
r2AccessKey(): string;
|
||||
r2SecretKey(): string;
|
||||
cloudflareAccountId(): string;
|
||||
cloudflareApiToken(): string;
|
||||
domain(): string;
|
||||
subdomain(): string;
|
||||
otelEndpoint(): string;
|
||||
otelUsername(): string;
|
||||
otelPassword(): string;
|
||||
|
||||
@@ -72,6 +72,18 @@ const TERRAIN_EFFECTS = {
|
||||
} as const;
|
||||
|
||||
export abstract class DefaultServerConfig implements ServerConfig {
|
||||
domain(): string {
|
||||
return process.env.DOMAIN ?? "";
|
||||
}
|
||||
subdomain(): string {
|
||||
return process.env.SUBDOMAIN ?? "";
|
||||
}
|
||||
cloudflareAccountId(): string {
|
||||
return process.env.CF_ACCOUNT_ID ?? "";
|
||||
}
|
||||
cloudflareApiToken(): string {
|
||||
return process.env.CF_API_TOKEN ?? "";
|
||||
}
|
||||
private publicKey: JWK;
|
||||
abstract jwtAudience(): string;
|
||||
jwtIssuer(): string {
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
import { spawn } from "child_process";
|
||||
import * as fs from "fs";
|
||||
|
||||
export interface TunnelConfig {
|
||||
domain: string;
|
||||
subdomain: string;
|
||||
subdomainToService: Map<string, string>;
|
||||
}
|
||||
|
||||
interface TunnelResponse {
|
||||
result: {
|
||||
id: string;
|
||||
token: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface ZoneResponse {
|
||||
result: Array<{
|
||||
id: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface DNSRecordResponse {
|
||||
result: Array<{
|
||||
id: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export class Cloudflare {
|
||||
private baseUrl = "https://api.cloudflare.com/client/v4";
|
||||
|
||||
constructor(
|
||||
private accountId: string,
|
||||
private apiToken: string,
|
||||
) {}
|
||||
|
||||
private async makeRequest<T>(
|
||||
url: string,
|
||||
method: string = "GET",
|
||||
data?: any,
|
||||
): Promise<T> {
|
||||
const response = await fetch(url, {
|
||||
method,
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.apiToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: data ? JSON.stringify(data) : undefined,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(
|
||||
`Cloudflare API error: ${response.status} - ${errorText}`,
|
||||
);
|
||||
}
|
||||
|
||||
return response.json() as Promise<T>;
|
||||
}
|
||||
|
||||
public async createTunnel(config: TunnelConfig): Promise<{
|
||||
tunnelId: string;
|
||||
tunnelToken: string;
|
||||
tunnelUrl: string;
|
||||
}> {
|
||||
const { domain, subdomain, subdomainToService } = config;
|
||||
|
||||
// Generate unique tunnel name
|
||||
const timestamp = new Date().toISOString().replace(/[-:.]/g, "");
|
||||
const tunnelName = `${subdomain}-tunnel-${timestamp}`;
|
||||
|
||||
console.log(`Creating tunnel with name: ${tunnelName}`);
|
||||
|
||||
// Create tunnel
|
||||
const tunnelResponse = await this.makeRequest<TunnelResponse>(
|
||||
`${this.baseUrl}/accounts/${this.accountId}/cfd_tunnel`,
|
||||
"POST",
|
||||
{ name: tunnelName },
|
||||
);
|
||||
|
||||
const tunnelId = tunnelResponse.result.id;
|
||||
const tunnelToken = tunnelResponse.result.token;
|
||||
|
||||
if (!tunnelId || tunnelId === "null") {
|
||||
throw new Error("Failed to create tunnel");
|
||||
}
|
||||
|
||||
console.log(`Tunnel created with ID: ${tunnelId}`);
|
||||
|
||||
// Configure tunnel
|
||||
await this.configureTunnel(tunnelId, subdomain, domain, subdomainToService);
|
||||
|
||||
await Promise.all(
|
||||
Array.from(subdomainToService.entries()).map(([subdomain, _]) =>
|
||||
this.updateDNSRecord(tunnelId, subdomain, domain),
|
||||
),
|
||||
);
|
||||
|
||||
const tunnelUrl = `https://${subdomain}.${domain}`;
|
||||
console.log(`Tunnel is set up! Site will be available at: ${tunnelUrl}`);
|
||||
|
||||
return { tunnelId, tunnelToken, tunnelUrl };
|
||||
}
|
||||
|
||||
private async configureTunnel(
|
||||
tunnelId: string,
|
||||
subdomain: string,
|
||||
domain: string,
|
||||
subdomainToService: Map<string, string>,
|
||||
): Promise<void> {
|
||||
console.log(`Configuring tunnel to point to ${subdomain}.${domain}...`);
|
||||
|
||||
const request = {
|
||||
config: {
|
||||
ingress: [
|
||||
...Array.from(subdomainToService.entries()).map(
|
||||
([subdomain, service]) => ({
|
||||
hostname: `${subdomain}.${domain}`,
|
||||
service: service,
|
||||
}),
|
||||
),
|
||||
{
|
||||
service: "http_status:404",
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
console.log(JSON.stringify(request, null, 2));
|
||||
await this.makeRequest(
|
||||
`${this.baseUrl}/accounts/${this.accountId}/cfd_tunnel/${tunnelId}/configurations`,
|
||||
"PUT",
|
||||
request,
|
||||
);
|
||||
}
|
||||
|
||||
private async updateDNSRecord(
|
||||
tunnelId: string,
|
||||
subdomain: string,
|
||||
domain: string,
|
||||
): Promise<void> {
|
||||
// Get zone ID
|
||||
const zoneResponse = await this.makeRequest<ZoneResponse>(
|
||||
`${this.baseUrl}/zones?name=${domain}`,
|
||||
);
|
||||
|
||||
const zoneId = zoneResponse.result[0]?.id;
|
||||
if (!zoneId) {
|
||||
throw new Error(`Could not find zone ID for domain ${domain}`);
|
||||
}
|
||||
|
||||
// Check for existing DNS record
|
||||
const existingRecords = await this.makeRequest<DNSRecordResponse>(
|
||||
`${this.baseUrl}/zones/${zoneId}/dns_records?name=${subdomain}.${domain}`,
|
||||
);
|
||||
|
||||
const recordId = existingRecords.result[0]?.id;
|
||||
const dnsData = {
|
||||
type: "CNAME",
|
||||
name: subdomain,
|
||||
content: `${tunnelId}.cfargotunnel.com`,
|
||||
ttl: 1,
|
||||
proxied: true,
|
||||
};
|
||||
|
||||
if (recordId) {
|
||||
// Update existing record
|
||||
console.log(`Updating existing DNS record for ${subdomain}.${domain}...`);
|
||||
await this.makeRequest(
|
||||
`${this.baseUrl}/zones/${zoneId}/dns_records/${recordId}`,
|
||||
"PUT",
|
||||
dnsData,
|
||||
);
|
||||
} else {
|
||||
// Create new record
|
||||
console.log(`Creating new DNS record for ${subdomain}.${domain}...`);
|
||||
await this.makeRequest(
|
||||
`${this.baseUrl}/zones/${zoneId}/dns_records`,
|
||||
"POST",
|
||||
dnsData,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public async deleteTunnel(tunnelId: string): Promise<void> {
|
||||
console.log(`Deleting tunnel with ID: ${tunnelId}`);
|
||||
|
||||
await this.makeRequest(
|
||||
`${this.baseUrl}/accounts/${this.accountId}/cfd_tunnel/${tunnelId}`,
|
||||
"DELETE",
|
||||
);
|
||||
|
||||
console.log("Tunnel deleted successfully");
|
||||
}
|
||||
|
||||
public async listTunnels(): Promise<any[]> {
|
||||
const response = await this.makeRequest<{ result: any[] }>(
|
||||
`${this.baseUrl}/accounts/${this.accountId}/cfd_tunnel`,
|
||||
);
|
||||
|
||||
return response.result;
|
||||
}
|
||||
|
||||
public async deleteDNSRecord(
|
||||
subdomain: string,
|
||||
domain: string,
|
||||
): Promise<void> {
|
||||
console.log(`Deleting DNS record for ${subdomain}.${domain}...`);
|
||||
|
||||
// Get zone ID
|
||||
const zoneResponse = await this.makeRequest<ZoneResponse>(
|
||||
`${this.baseUrl}/zones?name=${domain}`,
|
||||
);
|
||||
|
||||
const zoneId = zoneResponse.result[0]?.id;
|
||||
if (!zoneId) {
|
||||
throw new Error(`Could not find zone ID for domain ${domain}`);
|
||||
}
|
||||
|
||||
// Get DNS record
|
||||
const existingRecords = await this.makeRequest<DNSRecordResponse>(
|
||||
`${this.baseUrl}/zones/${zoneId}/dns_records?name=${subdomain}.${domain}`,
|
||||
);
|
||||
|
||||
const recordId = existingRecords.result[0]?.id;
|
||||
if (!recordId) {
|
||||
console.log("No DNS record found to delete");
|
||||
return;
|
||||
}
|
||||
|
||||
// Delete DNS record
|
||||
await this.makeRequest(
|
||||
`${this.baseUrl}/zones/${zoneId}/dns_records/${recordId}`,
|
||||
"DELETE",
|
||||
);
|
||||
|
||||
console.log("DNS record deleted successfully");
|
||||
}
|
||||
|
||||
public async startCloudflared(tunnelToken: string) {
|
||||
const out = fs.openSync("./cloudflared.out.log", "a");
|
||||
const err = fs.openSync("./cloudflared.err.log", "a");
|
||||
|
||||
const cloudflared = spawn(
|
||||
"cloudflared",
|
||||
["tunnel", "run", "--token", tunnelToken],
|
||||
{
|
||||
detached: true,
|
||||
stdio: ["ignore", out, err],
|
||||
},
|
||||
);
|
||||
cloudflared.unref();
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { Logger } from "winston";
|
||||
import { ServerConfig } from "../core/configuration/Config";
|
||||
import { Difficulty, GameMapType, GameMode, GameType } from "../core/game/Game";
|
||||
import { GameConfig, GameID } from "../core/Schemas";
|
||||
import { generateID } from "../core/Util";
|
||||
import { Client } from "./Client";
|
||||
import { GamePhase, GameServer } from "./GameServer";
|
||||
|
||||
@@ -11,10 +12,21 @@ export class GameManager {
|
||||
constructor(
|
||||
private config: ServerConfig,
|
||||
private log: Logger,
|
||||
private workerIndex: number,
|
||||
) {
|
||||
setInterval(() => this.tick(), 1000);
|
||||
}
|
||||
|
||||
public createGameID(): GameID {
|
||||
for (let i = 0; i < 1000; i++) {
|
||||
const id = generateID(4) + this.workerIndex;
|
||||
if (!this.games.has(id)) {
|
||||
return id;
|
||||
}
|
||||
}
|
||||
throw new Error("Failed to create game ID");
|
||||
}
|
||||
|
||||
public game(id: GameID): GameServer | null {
|
||||
return this.games.get(id) ?? null;
|
||||
}
|
||||
|
||||
+50
-73
@@ -1,4 +1,3 @@
|
||||
import cluster from "cluster";
|
||||
import express from "express";
|
||||
import rateLimit from "express-rate-limit";
|
||||
import http from "http";
|
||||
@@ -10,10 +9,11 @@ import { generateID } from "../core/Util";
|
||||
import { gatekeeper, LimiterType } from "./Gatekeeper";
|
||||
import { logger } from "./Logger";
|
||||
import { MapPlaylist } from "./MapPlaylist";
|
||||
import { WorkerDiscoveryService } from "./WorkerDiscoveryService";
|
||||
|
||||
const config = getServerConfigFromServer();
|
||||
const playlist = new MapPlaylist();
|
||||
const readyWorkers = new Set();
|
||||
const workerManager = new WorkerDiscoveryService();
|
||||
|
||||
const app = express();
|
||||
const server = http.createServer(app);
|
||||
@@ -65,81 +65,26 @@ const publicLobbyIDs: Set<string> = new Set();
|
||||
|
||||
// Start the master process
|
||||
export async function startMaster() {
|
||||
if (!cluster.isPrimary) {
|
||||
throw new Error(
|
||||
"startMaster() should only be called in the primary process",
|
||||
);
|
||||
}
|
||||
|
||||
log.info(`Primary ${process.pid} is running`);
|
||||
log.info(`Setting up ${config.numWorkers()} workers...`);
|
||||
|
||||
// Fork workers
|
||||
for (let i = 0; i < config.numWorkers(); i++) {
|
||||
const worker = cluster.fork({
|
||||
WORKER_ID: i,
|
||||
});
|
||||
|
||||
log.info(`Started worker ${i} (PID: ${worker.process.pid})`);
|
||||
}
|
||||
|
||||
cluster.on("message", (worker, message) => {
|
||||
if (message.type === "WORKER_READY") {
|
||||
const workerId = message.workerId;
|
||||
readyWorkers.add(workerId);
|
||||
log.info(
|
||||
`Worker ${workerId} is ready. (${readyWorkers.size}/${config.numWorkers()} ready)`,
|
||||
);
|
||||
// Start scheduling when all workers are ready
|
||||
if (readyWorkers.size === config.numWorkers()) {
|
||||
log.info("All workers ready, starting game scheduling");
|
||||
|
||||
const scheduleLobbies = () => {
|
||||
schedulePublicGame(playlist).catch((error) => {
|
||||
log.error("Error scheduling public game:", error);
|
||||
});
|
||||
};
|
||||
|
||||
setInterval(
|
||||
() =>
|
||||
fetchLobbies().then((lobbies) => {
|
||||
if (lobbies === 0) {
|
||||
scheduleLobbies();
|
||||
}
|
||||
}),
|
||||
100,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Handle worker crashes
|
||||
cluster.on("exit", (worker, code, signal) => {
|
||||
const workerId = (worker as any).process?.env?.WORKER_ID;
|
||||
if (!workerId) {
|
||||
log.error(`worker crashed could not find id`);
|
||||
return;
|
||||
}
|
||||
|
||||
log.warn(
|
||||
`Worker ${workerId} (PID: ${worker.process.pid}) died with code: ${code} and signal: ${signal}`,
|
||||
);
|
||||
log.info(`Restarting worker ${workerId}...`);
|
||||
|
||||
// Restart the worker with the same ID
|
||||
const newWorker = cluster.fork({
|
||||
WORKER_ID: workerId,
|
||||
});
|
||||
|
||||
log.info(
|
||||
`Restarted worker ${workerId} (New PID: ${newWorker.process.pid})`,
|
||||
);
|
||||
});
|
||||
|
||||
const PORT = 3000;
|
||||
server.listen(PORT, () => {
|
||||
log.info(`Master HTTP server listening on port ${PORT}`);
|
||||
});
|
||||
|
||||
const scheduleLobbies = () => {
|
||||
schedulePublicGame(playlist).catch((error) => {
|
||||
log.error("Error scheduling public game:", error);
|
||||
});
|
||||
};
|
||||
|
||||
setInterval(
|
||||
() =>
|
||||
fetchLobbies().then((lobbies) => {
|
||||
if (lobbies === 0) {
|
||||
scheduleLobbies();
|
||||
}
|
||||
}),
|
||||
100,
|
||||
);
|
||||
}
|
||||
|
||||
app.get(
|
||||
@@ -193,6 +138,38 @@ app.post(
|
||||
}),
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/worker_heartbeat",
|
||||
gatekeeper.httpHandler(LimiterType.Post, async (req, res) => {
|
||||
if (req.headers[config.adminHeader()] !== config.adminToken()) {
|
||||
res.status(401).send("Unauthorized");
|
||||
return;
|
||||
}
|
||||
|
||||
const { workerId, dns, activeClients } = req.body;
|
||||
|
||||
if (!workerId || !dns || typeof activeClients !== "number") {
|
||||
res.status(400).json({ error: "Missing required fields" });
|
||||
return;
|
||||
}
|
||||
|
||||
workerManager.updateWorkerHeartbeat(workerId, dns, activeClients);
|
||||
res.status(200).json({ success: true });
|
||||
}),
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/api/worker_dns",
|
||||
gatekeeper.httpHandler(LimiterType.Post, async (req, res) => {
|
||||
const worker = workerManager.getAvailableWorker();
|
||||
if (!worker) {
|
||||
res.status(500).json({ error: "No available workers" });
|
||||
return;
|
||||
}
|
||||
res.status(200).json({ dns: worker.dns });
|
||||
}),
|
||||
);
|
||||
|
||||
async function fetchLobbies(): Promise<number> {
|
||||
const fetchPromises: Promise<GameInfo | null>[] = [];
|
||||
|
||||
|
||||
+98
-2
@@ -1,20 +1,29 @@
|
||||
import cluster from "cluster";
|
||||
import * as dotenv from "dotenv";
|
||||
import { getServerConfigFromServer } from "../core/configuration/ConfigLoader";
|
||||
import { Cloudflare, TunnelConfig } from "./Cloudflare";
|
||||
import { logger } from "./Logger";
|
||||
import { startMaster } from "./Master";
|
||||
import { startWorker } from "./Worker";
|
||||
|
||||
const config = getServerConfigFromServer();
|
||||
|
||||
dotenv.config();
|
||||
|
||||
// Main entry point of the application
|
||||
async function main() {
|
||||
// Check if this is the primary (master) process
|
||||
if (cluster.isPrimary) {
|
||||
// if (config.env() != GameEnv.Dev) {
|
||||
setupTunnels();
|
||||
// }
|
||||
console.log("Starting master process...");
|
||||
await startMaster();
|
||||
await startWorkers();
|
||||
} else {
|
||||
// This is a worker process
|
||||
console.log("Starting worker process...");
|
||||
await startWorker();
|
||||
console.log(`Starting worker process ${process.env.WORKER_ID}...`);
|
||||
startWorker();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,3 +32,90 @@ main().catch((error) => {
|
||||
console.error("Failed to start server:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
async function setupTunnels() {
|
||||
const cloudflare = new Cloudflare(
|
||||
config.cloudflareAccountId(),
|
||||
config.cloudflareApiToken(),
|
||||
);
|
||||
|
||||
const domainToService = new Map<string, string>().set(
|
||||
config.subdomain(),
|
||||
`http://localhost:3000`,
|
||||
);
|
||||
|
||||
for (let i = 0; i < config.numWorkers(); i++) {
|
||||
domainToService.set(
|
||||
`w${i}-${config.subdomain()}`,
|
||||
`http://localhost:${3000 + i + 1}`,
|
||||
);
|
||||
}
|
||||
|
||||
const tunnel = await cloudflare.createTunnel({
|
||||
subdomain: config.subdomain(),
|
||||
domain: config.domain(),
|
||||
subdomainToService: domainToService,
|
||||
} as TunnelConfig);
|
||||
|
||||
await cloudflare.startCloudflared(tunnel.tunnelToken);
|
||||
}
|
||||
|
||||
// Start the master process
|
||||
export async function startWorkers() {
|
||||
const log = logger.child({ comp: "startup" });
|
||||
const readyWorkers = new Set();
|
||||
|
||||
log.info(`Primary ${process.pid} is running`);
|
||||
log.info(`Setting up ${config.numWorkers()} workers...`);
|
||||
|
||||
// Fork workers
|
||||
for (let i = 0; i < config.numWorkers(); i++) {
|
||||
const worker = cluster.fork({
|
||||
WORKER_ID: i,
|
||||
WORKER_DNS: getWorkerDns(i),
|
||||
});
|
||||
|
||||
log.info(`Started worker ${i} (PID: ${worker.process.pid})`);
|
||||
}
|
||||
|
||||
cluster.on("message", (worker, message) => {
|
||||
if (message.type === "WORKER_READY") {
|
||||
const workerId = message.workerId;
|
||||
readyWorkers.add(workerId);
|
||||
log.info(
|
||||
`Worker ${workerId} is ready. (${readyWorkers.size}/${config.numWorkers()} ready)`,
|
||||
);
|
||||
// Start scheduling when all workers are ready
|
||||
if (readyWorkers.size === config.numWorkers()) {
|
||||
log.info("All workers ready, starting game scheduling");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Handle worker crashes
|
||||
cluster.on("exit", (worker, code, signal) => {
|
||||
const workerId = (worker as any).process?.env?.WORKER_ID;
|
||||
if (!workerId) {
|
||||
log.error(`worker crashed could not find id`);
|
||||
return;
|
||||
}
|
||||
|
||||
log.warn(
|
||||
`Worker ${workerId} (PID: ${worker.process.pid}) died with code: ${code} and signal: ${signal}`,
|
||||
);
|
||||
log.info(`Restarting worker ${workerId}...`);
|
||||
|
||||
// Restart the worker with the same ID
|
||||
const newWorker = cluster.fork({
|
||||
WORKER_ID: workerId,
|
||||
});
|
||||
|
||||
log.info(
|
||||
`Restarted worker ${workerId} (New PID: ${newWorker.process.pid})`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function getWorkerDns(workerId: number) {
|
||||
return `w${workerId}-${config.subdomain()}.${config.domain()}`;
|
||||
}
|
||||
|
||||
+44
-3
@@ -26,6 +26,8 @@ import { initWorkerMetrics } from "./WorkerMetrics";
|
||||
const config = getServerConfigFromServer();
|
||||
|
||||
const workerId = parseInt(process.env.WORKER_ID || "0");
|
||||
const masterAddress = `${config.subdomain()}.${config.domain()}`;
|
||||
const workerDns = `${config.subdomain()}.${config.domain()}`;
|
||||
const log = logger.child({ comp: `w_${workerId}` });
|
||||
|
||||
// Worker setup
|
||||
@@ -39,7 +41,7 @@ export function startWorker() {
|
||||
const server = http.createServer(app);
|
||||
const wss = new WebSocketServer({ server });
|
||||
|
||||
const gm = new GameManager(config, log);
|
||||
const gm = new GameManager(config, log, workerId);
|
||||
|
||||
if (config.env() === GameEnv.Prod && config.otelEnabled()) {
|
||||
initWorkerMetrics(gm);
|
||||
@@ -80,10 +82,49 @@ export function startWorker() {
|
||||
}),
|
||||
);
|
||||
|
||||
async function start() {
|
||||
this.isRunning = true;
|
||||
console.log("Worker starting long polling...");
|
||||
|
||||
// TODO: remove the true
|
||||
while (true) {
|
||||
try {
|
||||
const response = await this.poll();
|
||||
} catch (error) {
|
||||
console.error("Poll error:", error);
|
||||
await this.sleep(5000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The long polling request
|
||||
async function sendHeartbeat(): Promise<any> {
|
||||
log.info("Sending heartbeat...");
|
||||
|
||||
const response = await fetch(`${masterAddress}/api/worker_heartbeat`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
[config.adminHeader()]: config.adminToken(),
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
workerId: workerId,
|
||||
dns: ,
|
||||
activeClients: gm.activeClients(),
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
app.post(
|
||||
"/api/create_game/:id",
|
||||
"/api/create_game",
|
||||
gatekeeper.httpHandler(LimiterType.Post, async (req, res) => {
|
||||
const id = req.params.id;
|
||||
const id = gm.createGameID();
|
||||
if (!id) {
|
||||
log.warn(`cannot create game, id not found`);
|
||||
return res.status(400).json({ error: "Game ID is required" });
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import { PseudoRandom } from "../core/PseudoRandom";
|
||||
|
||||
interface WorkerInfo {
|
||||
id: string;
|
||||
dns: string;
|
||||
activeClients: number;
|
||||
lastHeartbeat: Date;
|
||||
healthy: boolean;
|
||||
}
|
||||
|
||||
// WorkerDiscoveryService - manages worker registry and load balancing
|
||||
export class WorkerDiscoveryService {
|
||||
private workers: Map<string, WorkerInfo> = new Map();
|
||||
private readonly HEARTBEAT_TIMEOUT = 60000; // 60 seconds
|
||||
private readonly CLEANUP_INTERVAL = 5000; // Check every 5 seconds
|
||||
private readonly MAX_ACTIVE_CLIENTS = 500; // Can actually handle up to 1000, but we don't want to overload the workers
|
||||
private readonly rand = new PseudoRandom(1);
|
||||
|
||||
constructor() {
|
||||
// Periodically clean up dead workers
|
||||
setInterval(() => this.cleanupDeadWorkers(), this.CLEANUP_INTERVAL);
|
||||
}
|
||||
|
||||
// Worker sends heartbeat with current state
|
||||
updateWorkerHeartbeat(
|
||||
workerId: string,
|
||||
dns: string,
|
||||
activeClients: number,
|
||||
): void {
|
||||
const existingWorker = this.workers.get(workerId);
|
||||
|
||||
this.workers.set(workerId, {
|
||||
id: workerId,
|
||||
dns,
|
||||
activeClients,
|
||||
lastHeartbeat: new Date(),
|
||||
healthy: true,
|
||||
});
|
||||
|
||||
// Log if this is a new worker
|
||||
if (!existingWorker) {
|
||||
console.log(`New worker registered: ${workerId} at ${dns}`);
|
||||
}
|
||||
}
|
||||
|
||||
getAvailableWorker(): WorkerInfo | null {
|
||||
let healthyWorkers = Array.from(this.workers.values())
|
||||
.filter((w) => w.healthy && w.activeClients < this.MAX_ACTIVE_CLIENTS)
|
||||
.sort((a, b) => {
|
||||
// Sort by load percentage (ascending)
|
||||
const loadA = a.activeClients / this.MAX_ACTIVE_CLIENTS;
|
||||
const loadB = b.activeClients / this.MAX_ACTIVE_CLIENTS;
|
||||
return loadA - loadB;
|
||||
});
|
||||
|
||||
if (healthyWorkers.length === 0) {
|
||||
healthyWorkers = Array.from(this.workers.values());
|
||||
}
|
||||
|
||||
return this.rand.randElement(healthyWorkers);
|
||||
}
|
||||
|
||||
// Get specific worker info
|
||||
getWorker(workerId: string): WorkerInfo | null {
|
||||
const worker = this.workers.get(workerId);
|
||||
return worker?.healthy ? worker : null;
|
||||
}
|
||||
|
||||
// Remove dead workers
|
||||
private cleanupDeadWorkers() {
|
||||
const now = Date.now();
|
||||
|
||||
for (const [workerId, worker] of this.workers) {
|
||||
const timeSinceHeartbeat = now - worker.lastHeartbeat.getTime();
|
||||
|
||||
if (timeSinceHeartbeat > this.HEARTBEAT_TIMEOUT && worker.healthy) {
|
||||
// Mark as unhealthy first (soft delete)
|
||||
worker.healthy = false;
|
||||
console.log(
|
||||
`Worker ${workerId} marked unhealthy (no heartbeat for ${timeSinceHeartbeat}ms)`,
|
||||
);
|
||||
} else if (timeSinceHeartbeat > this.HEARTBEAT_TIMEOUT * 3) {
|
||||
// Hard delete after 30 seconds
|
||||
this.workers.delete(workerId);
|
||||
console.log(
|
||||
`Worker ${workerId} removed (dead for ${timeSinceHeartbeat}ms)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Manually remove a worker (for graceful shutdown)
|
||||
removeWorker(workerId: string): boolean {
|
||||
const existed = this.workers.has(workerId);
|
||||
this.workers.delete(workerId);
|
||||
if (existed) {
|
||||
console.log(`Worker ${workerId} manually removed`);
|
||||
}
|
||||
return existed;
|
||||
}
|
||||
}
|
||||
-105
@@ -1,110 +1,5 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Check if required environment variables are set
|
||||
if [ -z "$CF_API_TOKEN" ] || [ -z "$CF_ACCOUNT_ID" ] || [ -z "$SUBDOMAIN" ] || [ -z "$DOMAIN" ]; then
|
||||
echo "Error: Required environment variables not set"
|
||||
echo "Please set CF_API_TOKEN, CF_ACCOUNT_ID, SUBDOMAIN, and DOMAIN"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Generate a unique tunnel name using timestamp
|
||||
TIMESTAMP=$(date +%Y%m%d%H%M%S)
|
||||
TUNNEL_NAME="${SUBDOMAIN}-tunnel-${TIMESTAMP}"
|
||||
echo "Using unique tunnel name: ${TUNNEL_NAME}"
|
||||
|
||||
# Create a new tunnel
|
||||
echo "Creating Cloudflare tunnel for subdomain ${SUBDOMAIN}..."
|
||||
TUNNEL_RESPONSE=$(curl -s -X POST "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel" \
|
||||
-H "Authorization: Bearer ${CF_API_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data "{\"name\":\"${TUNNEL_NAME}\"}")
|
||||
|
||||
# Extract tunnel ID and token
|
||||
TUNNEL_ID=$(echo $TUNNEL_RESPONSE | jq -r '.result.id')
|
||||
TUNNEL_TOKEN=$(echo $TUNNEL_RESPONSE | jq -r '.result.token')
|
||||
|
||||
if [ -z "$TUNNEL_ID" ] || [ "$TUNNEL_ID" == "null" ]; then
|
||||
echo "Failed to create tunnel"
|
||||
echo $TUNNEL_RESPONSE
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Tunnel created with ID: ${TUNNEL_ID}"
|
||||
|
||||
# Configure the tunnel with hostname
|
||||
echo "Configuring tunnel to point to ${SUBDOMAIN}.${DOMAIN}..."
|
||||
curl -s -X PUT "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${TUNNEL_ID}/configurations" \
|
||||
-H "Authorization: Bearer ${CF_API_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data "{\"config\":{\"ingress\":[{\"hostname\":\"${SUBDOMAIN}.${DOMAIN}\",\"service\":\"http://localhost:80\"},{\"service\":\"http_status:404\"}]}}"
|
||||
|
||||
# Update DNS record to point to the new tunnel
|
||||
echo "Updating DNS record to point to the new tunnel..."
|
||||
|
||||
# First check if DNS record exists
|
||||
DNS_RECORDS=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones?name=${DOMAIN}" \
|
||||
-H "Authorization: Bearer ${CF_API_TOKEN}" \
|
||||
-H "Content-Type: application/json")
|
||||
|
||||
ZONE_ID=$(echo $DNS_RECORDS | jq -r '.result[0].id')
|
||||
|
||||
if [ -z "$ZONE_ID" ] || [ "$ZONE_ID" == "null" ]; then
|
||||
echo "Could not find zone ID for domain ${DOMAIN}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check for existing record
|
||||
EXISTING_RECORDS=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records?name=${SUBDOMAIN}.${DOMAIN}" \
|
||||
-H "Authorization: Bearer ${CF_API_TOKEN}" \
|
||||
-H "Content-Type: application/json")
|
||||
|
||||
RECORD_ID=$(echo $EXISTING_RECORDS | jq -r '.result[0].id')
|
||||
|
||||
# Create or update the DNS record
|
||||
if [ -z "$RECORD_ID" ] || [ "$RECORD_ID" == "null" ]; then
|
||||
# Create new record
|
||||
echo "Creating new DNS record..."
|
||||
DNS_RESPONSE=$(curl -s -X POST "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records" \
|
||||
-H "Authorization: Bearer ${CF_API_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data "{\"type\":\"CNAME\",\"name\":\"${SUBDOMAIN}\",\"content\":\"${TUNNEL_ID}.cfargotunnel.com\",\"ttl\":1,\"proxied\":true}")
|
||||
else
|
||||
# Update existing record
|
||||
echo "Updating existing DNS record..."
|
||||
DNS_RESPONSE=$(curl -s -X PUT "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${RECORD_ID}" \
|
||||
-H "Authorization: Bearer ${CF_API_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data "{\"type\":\"CNAME\",\"name\":\"${SUBDOMAIN}\",\"content\":\"${TUNNEL_ID}.cfargotunnel.com\",\"ttl\":1,\"proxied\":true}")
|
||||
fi
|
||||
|
||||
# Log the tunnel information
|
||||
echo "Tunnel is set up! Site will be available at: https://${SUBDOMAIN}.${DOMAIN}"
|
||||
|
||||
# Export the tunnel token for supervisord
|
||||
export CLOUDFLARE_TUNNEL_TOKEN=${TUNNEL_TOKEN}
|
||||
|
||||
# Check if Basic Auth credentials are set
|
||||
if [ -z "$BASIC_AUTH_USER" ] || [ -z "$BASIC_AUTH_PASS" ]; then
|
||||
echo "HTTP Basic Authentication will be disabled"
|
||||
else
|
||||
# Create the htpasswd file
|
||||
echo "Creating basic auth credentials for user: ${BASIC_AUTH_USER}"
|
||||
# Ensure apache2-utils is installed for htpasswd
|
||||
command -v htpasswd > /dev/null 2>&1 || {
|
||||
echo "htpasswd not found, installing apache2-utils..."
|
||||
apt-get update && apt-get install -y apache2-utils
|
||||
}
|
||||
# Create the password file
|
||||
htpasswd -bc /etc/nginx/.htpasswd ${BASIC_AUTH_USER} ${BASIC_AUTH_PASS}
|
||||
|
||||
# Update Nginx configuration to enable Basic Auth
|
||||
sed -i '1i auth_basic "Restricted Access";' /etc/nginx/conf.d/default.conf
|
||||
sed -i '2i auth_basic_user_file /etc/nginx/.htpasswd;' /etc/nginx/conf.d/default.conf
|
||||
|
||||
echo "HTTP Basic Authentication enabled for user: ${BASIC_AUTH_USER}"
|
||||
fi
|
||||
|
||||
# Start supervisord
|
||||
if [ "$DOMAIN" = openfront.dev ] && [ "$SUBDOMAIN" != main ]; then
|
||||
exec timeout 18h /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
|
||||
|
||||
+1
-8
@@ -22,11 +22,4 @@ user=node
|
||||
stdout_logfile=/dev/stdout
|
||||
stdout_logfile_maxbytes=0
|
||||
stderr_logfile=/dev/stderr
|
||||
stderr_logfile_maxbytes=0
|
||||
|
||||
[program:cloudflared]
|
||||
command=cloudflared tunnel run --token %(ENV_CLOUDFLARE_TUNNEL_TOKEN)s
|
||||
autostart=true
|
||||
autorestart=true
|
||||
stdout_logfile=/var/log/cloudflared.log
|
||||
stderr_logfile=/var/log/cloudflared-err.log
|
||||
stderr_logfile_maxbytes=0
|
||||
Reference in New Issue
Block a user