From 043462e28aa75ed494a337971ea89337e764111c Mon Sep 17 00:00:00 2001 From: evanpelle Date: Mon, 8 Sep 2025 16:36:20 -0700 Subject: [PATCH] Archive games by using the api service endpoint instead of R2 (#2030) ## Description: This removes the dependencies on R2, and allows contributors to replay games without R2 access. ## Please complete the following: - [x] I have added screenshots for all UI updates - [x] I process any text displayed to the user through translateText() and I've added it to the en.json file - [x] I have added relevant tests to the test directory - [x] I confirm I have thoroughly tested these changes and take full responsibility for any bugs introduced ## Please put your Discord username so you can be contacted if a bug or regression is found: evan --- .github/workflows/deploy.yml | 1 + .github/workflows/release.yml | 4 + deploy.sh | 1 + example.env | 3 + src/core/Schemas.ts | 5 +- src/core/configuration/Config.ts | 1 + src/core/configuration/DefaultConfig.ts | 4 + src/core/configuration/DevConfig.ts | 4 + src/server/Archive.ts | 198 +++++------------------- tests/util/TestServerConfig.ts | 3 + 10 files changed, 63 insertions(+), 161 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 164712bcc..c40facca2 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -120,6 +120,7 @@ jobs: R2_ACCESS_KEY: ${{ secrets.R2_ACCESS_KEY }} R2_BUCKET: ${{ secrets.R2_BUCKET }} R2_SECRET_KEY: ${{ secrets.R2_SECRET_KEY }} + API_KEY: ${{ secrets.API_KEY }} SERVER_HOST_MASTERS: ${{ secrets.SERVER_HOST_MASTERS }} SERVER_HOST_NBG1: ${{ secrets.SERVER_HOST_NBG1 }} SERVER_HOST_STAGING: ${{ secrets.SERVER_HOST_STAGING }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b1674e15f..ffe2664d4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -77,6 +77,7 @@ jobs: R2_ACCESS_KEY: ${{ secrets.R2_ACCESS_KEY }} R2_BUCKET: ${{ secrets.R2_BUCKET }} R2_SECRET_KEY: ${{ secrets.R2_SECRET_KEY }} + API_KEY: ${{ secrets.API_KEY }} SERVER_HOST_STAGING: ${{ secrets.SERVER_HOST_STAGING }} SSH_KEY: ~/.ssh/id_rsa run: | @@ -133,6 +134,7 @@ jobs: R2_ACCESS_KEY: ${{ secrets.R2_ACCESS_KEY }} R2_BUCKET: ${{ secrets.R2_BUCKET }} R2_SECRET_KEY: ${{ secrets.R2_SECRET_KEY }} + API_KEY: ${{ secrets.API_KEY }} SERVER_HOST_NBG1: ${{ secrets.SERVER_HOST_NBG1 }} SSH_KEY: ~/.ssh/id_rsa run: | @@ -189,6 +191,7 @@ jobs: R2_ACCESS_KEY: ${{ secrets.R2_ACCESS_KEY }} R2_BUCKET: ${{ secrets.R2_BUCKET }} R2_SECRET_KEY: ${{ secrets.R2_SECRET_KEY }} + API_KEY: ${{ secrets.API_KEY }} SERVER_HOST_NBG1: ${{ secrets.SERVER_HOST_NBG1 }} SSH_KEY: ~/.ssh/id_rsa run: | @@ -245,6 +248,7 @@ jobs: R2_ACCESS_KEY: ${{ secrets.R2_ACCESS_KEY }} R2_BUCKET: ${{ secrets.R2_BUCKET }} R2_SECRET_KEY: ${{ secrets.R2_SECRET_KEY }} + API_KEY: ${{ secrets.API_KEY }} SERVER_HOST_NBG1: ${{ secrets.SERVER_HOST_NBG1 }} SSH_KEY: ~/.ssh/id_rsa run: | diff --git a/deploy.sh b/deploy.sh index 2d9eb8645..6cef45e4d 100755 --- a/deploy.sh +++ b/deploy.sh @@ -176,6 +176,7 @@ R2_ACCESS_KEY=$R2_ACCESS_KEY R2_SECRET_KEY=$R2_SECRET_KEY R2_BUCKET=$R2_BUCKET CF_API_TOKEN=$CF_API_TOKEN +API_KEY=$API_KEY DOMAIN=$DOMAIN SUBDOMAIN=$SUBDOMAIN OTEL_USERNAME=$OTEL_USERNAME diff --git a/example.env b/example.env index d186dab02..0ba76e79a 100644 --- a/example.env +++ b/example.env @@ -19,6 +19,9 @@ R2_ACCESS_KEY=your_r2_access_key R2_SECRET_KEY=your_r2_secret_key R2_BUCKET=your-bucket-name +# API Key +API_KEY=your_api_key_here + # Server Hosts SERVER_HOST_STAGING=123.456.78.90 SERVER_HOST_EU=123.456.78.91 diff --git a/src/core/Schemas.ts b/src/core/Schemas.ts index ee06566ed..c1f1ff79f 100644 --- a/src/core/Schemas.ts +++ b/src/core/Schemas.ts @@ -507,7 +507,10 @@ export const GameEndInfoSchema = GameStartInfoSchema.extend({ }); export type GameEndInfo = z.infer; -const GitCommitSchema = z.string().regex(/^[0-9a-fA-F]{40}$/); +const GitCommitSchema = z + .string() + .regex(/^[0-9a-fA-F]{40}$/) + .or(z.literal("DEV")); export const AnalyticsRecordSchema = z.object({ info: GameEndInfoSchema, diff --git a/src/core/configuration/Config.ts b/src/core/configuration/Config.ts index a97dc2811..7a391b028 100644 --- a/src/core/configuration/Config.ts +++ b/src/core/configuration/Config.ts @@ -48,6 +48,7 @@ export interface ServerConfig { r2Endpoint(): string; r2AccessKey(): string; r2SecretKey(): string; + apiKey(): string; otelEndpoint(): string; otelAuthHeader(): string; otelEnabled(): boolean; diff --git a/src/core/configuration/DefaultConfig.ts b/src/core/configuration/DefaultConfig.ts index 07aa45a0f..742efe245 100644 --- a/src/core/configuration/DefaultConfig.ts +++ b/src/core/configuration/DefaultConfig.ts @@ -153,6 +153,10 @@ export abstract class DefaultServerConfig implements ServerConfig { return process.env.R2_BUCKET ?? ""; } + apiKey(): string { + return process.env.API_KEY ?? ""; + } + adminHeader(): string { return "x-admin-key"; } diff --git a/src/core/configuration/DevConfig.ts b/src/core/configuration/DevConfig.ts index 3a14b39bb..d530795c5 100644 --- a/src/core/configuration/DevConfig.ts +++ b/src/core/configuration/DevConfig.ts @@ -9,6 +9,10 @@ export class DevServerConfig extends DefaultServerConfig { return "WARNING_DEV_ADMIN_KEY_DO_NOT_USE_IN_PRODUCTION"; } + apiKey(): string { + return "WARNING_DEV_API_KEY_DO_NOT_USE_IN_PRODUCTION"; + } + env(): GameEnv { return GameEnv.Dev; } diff --git a/src/server/Archive.ts b/src/server/Archive.ts index 6b3675d94..7a29f0f3c 100644 --- a/src/server/Archive.ts +++ b/src/server/Archive.ts @@ -1,186 +1,64 @@ -import { S3 } from "@aws-sdk/client-s3"; import { getServerConfigFromServer } from "../core/configuration/ConfigLoader"; -import { AnalyticsRecord, GameID, GameRecord } from "../core/Schemas"; -import { replacer } from "../core/Util"; +import { GameID, GameRecord, GameRecordSchema, ID } from "../core/Schemas"; import { logger } from "./Logger"; const config = getServerConfigFromServer(); const log = logger.child({ component: "Archive" }); -// R2 client configuration -const r2 = new S3({ - region: "auto", // R2 ignores region, but it's required by the SDK - endpoint: config.r2Endpoint(), - credentials: { - accessKeyId: config.r2AccessKey(), - secretAccessKey: config.r2SecretKey(), - }, -}); - -const bucket = config.r2Bucket(); -const gameFolder = "games"; -const analyticsFolder = "analytics"; - export async function archive(gameRecord: GameRecord) { try { gameRecord.gitCommit = config.gitCommit(); - // Archive to R2 - await archiveAnalyticsToR2(gameRecord); - - // Archive full game if there are turns - if (gameRecord.turns.length > 0) { - log.info( - `${gameRecord.info.gameID}: game has more than zero turns, attempting to write to full game to R2`, - ); - await archiveFullGameToR2(gameRecord); - } - } catch (error: unknown) { - // If the error is not an instance of Error, log it as a string - if (!(error instanceof Error)) { - log.error( - `${gameRecord.info.gameID}: Final archive error. Non-Error type: ${String(error)}`, - ); + const url = `${config.jwtIssuer()}/game/${gameRecord.info.gameID}`; + const response = await fetch(url, { + method: "POST", + body: JSON.stringify(gameRecord), + headers: { + "Content-Type": "application/json", + "x-api-key": config.apiKey(), + }, + }); + if (!response.ok) { + log.error(`error archiving game record: ${response.statusText}`, { + gameID: gameRecord.info.gameID, + }); return; } - - const { message, stack, name } = error; - log.error(`${gameRecord.info.gameID}: Final archive error: ${error}`, { - message: message, - stack: stack, - name: name, - ...(error && typeof error === "object" ? error : {}), - }); - } -} - -async function archiveAnalyticsToR2(gameRecord: GameRecord) { - // Create analytics data object - const { info, version, gitCommit, subdomain, domain } = gameRecord; - const analyticsData: AnalyticsRecord = { - info, - version, - gitCommit, - subdomain, - domain, - }; - - try { - // Store analytics data using just the game ID as the key - const analyticsKey = `${info.gameID}.json`; - - await r2.putObject({ - Bucket: bucket, - Key: `${analyticsFolder}/${analyticsKey}`, - Body: JSON.stringify(analyticsData, replacer), - ContentType: "application/json", - }); - - log.info(`${info.gameID}: successfully wrote game analytics to R2`); - } catch (error: unknown) { - // If the error is not an instance of Error, log it as a string - if (!(error instanceof Error)) { - log.error( - `${gameRecord.info.gameID}: Error writing game analytics to R2. Non-Error type: ${String(error)}`, - ); - return; - } - - const { message, stack, name } = error; - log.error(`${info.gameID}: Error writing game analytics to R2: ${error}`, { - message: message, - stack: stack, - name: name, - ...(error && typeof error === "object" ? error : {}), - }); - throw error; - } -} - -async function archiveFullGameToR2(gameRecord: GameRecord) { - // Create a deep copy to avoid modifying the original - const recordCopy = structuredClone(gameRecord); - - // Players may see this so make sure to clear PII - recordCopy.info.players.forEach((p) => { - p.persistentID = "REDACTED"; - }); - - try { - await r2.putObject({ - Bucket: bucket, - Key: `${gameFolder}/${recordCopy.info.gameID}`, - Body: JSON.stringify(recordCopy, replacer), - ContentType: "application/json", - }); } catch (error) { - log.error(`error saving game ${gameRecord.info.gameID}`); - throw error; + log.error(`error archiving game record: ${error}`, { + gameID: gameRecord.info.gameID, + }); + return; } - - log.info(`${gameRecord.info.gameID}: game record successfully written to R2`); } export async function readGameRecord( gameId: GameID, ): Promise { try { - // Check if file exists and download in one operation - const response = await r2.getObject({ - Bucket: bucket, - Key: `${gameFolder}/${gameId}`, // Fixed - needed to include gameFolder - }); - // Parse the response body - if (response.Body === undefined) return null; - const bodyContents = await response.Body.transformToString(); - return JSON.parse(bodyContents) as GameRecord; - } catch (error: unknown) { - // If the error is not an instance of Error, log it as a string - if (!(error instanceof Error)) { - log.error( - `${gameId}: Error reading game record from R2. Non-Error type: ${String(error)}`, - ); + if (!ID.safeParse(gameId).success) { + log.error(`invalid game ID: ${gameId}`); return null; } - const { message, stack, name } = error; - // Log the error for monitoring purposes - log.error(`${gameId}: Error reading game record from R2: ${error}`, { - message: message, - stack: stack, - name: name, - ...(error && typeof error === "object" ? error : {}), + const url = `${config.jwtIssuer()}/game/${gameId}`; + const response = await fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + }); + const record = await response.json(); + if (!response.ok) { + log.error(`error reading game record: ${response.statusText}`, { + gameID: gameId, + }); + return null; + } + return GameRecordSchema.parse(record); + } catch (error) { + log.error(`error reading game record: ${error}`, { + gameID: gameId, }); - - // Return null instead of throwing the error return null; } } - -export async function gameRecordExists(gameId: GameID): Promise { - try { - await r2.headObject({ - Bucket: bucket, - Key: `${gameFolder}/${gameId}`, // Fixed - needed to include gameFolder - }); - return true; - } catch (error: unknown) { - // If the error is not an instance of Error, log it as a string - if (!(error instanceof Error)) { - log.error( - `${gameId}: Error checking archive existence. Non-Error type: ${String(error)}`, - ); - return false; - } - const { message, stack, name } = error; - if (name === "NotFound") { - return false; - } - log.error(`${gameId}: Error checking archive existence: ${error}`, { - message: message, - stack: stack, - name: name, - ...(error && typeof error === "object" ? error : {}), - }); - return false; - } -} diff --git a/tests/util/TestServerConfig.ts b/tests/util/TestServerConfig.ts index 86a277d65..6488dc99b 100644 --- a/tests/util/TestServerConfig.ts +++ b/tests/util/TestServerConfig.ts @@ -4,6 +4,9 @@ import { GameMapType } from "../../src/core/game/Game"; import { GameID } from "../../src/core/Schemas"; export class TestServerConfig implements ServerConfig { + apiKey(): string { + throw new Error("Method not implemented."); + } allowedFlares(): string[] | undefined { throw new Error("Method not implemented."); }