upload manifest to r2 before starting container (#3767)

## Description:

It first sends the manifest to the worker to get a list of missing
files, then for each missing file it uploads them to r2 via cf worker.

This PR also has us write out the manifest in plan json instead of an
mjs file. This makes it easier for the shell script to parse

## Please complete the following:

- [x] I have added screenshots for all UI updates
- [ ] I process any text displayed to the user through translateText()
and I've added it to the en.json file
- [ ] I have added relevant tests to the test directory
- [ ] I confirm I have thoroughly tested these changes and take full
responsibility for any bugs introduced

## Please put your Discord username so you can be contacted if a bug or
regression is found:

evan

setup.sh

@@ -39,6 +39,14 @@ fi
 echo "🔄 Updating system..."
 apt update && apt upgrade -y
 
+# Install jq (used by update.sh for asset upload)
+if command -v jq &> /dev/null; then
+    echo "jq is already installed"
+else
+    echo "📦 Installing jq..."
+    apt install -y jq
+fi
+
 # Check if Docker is already installed
 if command -v docker &> /dev/null; then
     echo "Docker is already installed"

src/server/PublicAssetManifest.ts

@@ -371,15 +371,11 @@ export function copyRootPublicFiles(
   }
 }
 
-export function writePublicAssetManifestModule(
+export function writePublicAssetManifest(
   outDir: string,
   assetManifest: AssetManifest,
 ): void {
-  const manifestPath = path.join(outDir, "_assets", "asset-manifest.mjs");
+  const manifestPath = path.join(outDir, "_assets", "asset-manifest.json");
   fs.mkdirSync(path.dirname(manifestPath), { recursive: true });
-  const serializedManifest = JSON.stringify(assetManifest, null, 2);
+  fs.writeFileSync(manifestPath, `${JSON.stringify(assetManifest, null, 2)}\n`);
-  fs.writeFileSync(
-    manifestPath,
-    `const assetManifest = ${serializedManifest};\nexport { assetManifest };\nexport default assetManifest;\n`,
-  );
 }

src/server/RuntimeAssetManifest.ts

@@ -1,40 +1,34 @@
 import fs from "fs";
 import path from "path";
-import { fileURLToPath, pathToFileURL } from "url";
+import { fileURLToPath } from "url";
 import type { AssetManifest } from "../core/AssetUrls";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 const staticDir = path.join(__dirname, "../../static");
-const manifestPath = path.join(staticDir, "_assets", "asset-manifest.mjs");
+const manifestPath = path.join(staticDir, "_assets", "asset-manifest.json");
 
-let manifestPromise: Promise<AssetManifest> | null = null;
+let cachedManifest: AssetManifest | null = null;
-let manifestVersion = 0;
-
-async function importRuntimeAssetManifest(
-  version: number,
-): Promise<AssetManifest> {
-  const manifestModule = (await import(
-    `${pathToFileURL(manifestPath).href}?v=${version}`
-  )) as {
-    assetManifest?: AssetManifest;
-    default?: AssetManifest;
-  };
-  return manifestModule.assetManifest ?? manifestModule.default ?? {};
-}
 
 export async function getRuntimeAssetManifest(): Promise<AssetManifest> {
-  if (!fs.existsSync(manifestPath)) {
+  if (cachedManifest !== null) {
-    return {};
+    return cachedManifest;
   }
-
+  if (!fs.existsSync(manifestPath)) {
-  manifestPromise ??= importRuntimeAssetManifest(manifestVersion).catch(
+    cachedManifest = {};
-    () => ({}),
+    return cachedManifest;
-  );
+  }
-  return manifestPromise;
+  try {
+    cachedManifest = JSON.parse(
+      fs.readFileSync(manifestPath, "utf8"),
+    ) as AssetManifest;
+  } catch (err) {
+    console.error(`Failed to parse asset manifest at ${manifestPath}:`, err);
+    cachedManifest = {};
+  }
+  return cachedManifest;
 }
 
 export function clearRuntimeAssetManifestCache(): void {
-  manifestVersion++;
+  cachedManifest = null;
-  manifestPromise = null;
 }

update.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 # update.sh - Script to update Docker container on Hetzner server
 # Called by deploy.sh after uploading Docker image to Docker Hub
+set -eo pipefail
 
 # Check if environment file is provided
 if [ $# -ne 1 ]; then
@@ -31,6 +32,117 @@ CONTAINER_NAME="openfront-${ENV}-${SUBDOMAIN}"
 echo "Pulling ${GHCR_IMAGE} from GitHub Container Registry..."
 docker pull "${GHCR_IMAGE}"
 
+# Upload hashed assets to R2 before swapping containers. If this fails the old
+# container keeps serving — better than a stop-then-fail outage.
+echo "======================================================"
+echo "📦 Uploading assets to R2 for ${DOMAIN}..."
+echo "======================================================"
+
+if [ -z "$DOMAIN" ] || [ -z "$API_KEY" ]; then
+    echo "❌ DOMAIN or API_KEY not set; cannot upload assets."
+    exit 1
+fi
+for cmd in jq curl xargs; do
+    if ! command -v "$cmd" > /dev/null 2>&1; then
+        echo "❌ Required tool '$cmd' not found. Install via setup.sh."
+        exit 1
+    fi
+done
+
+EXTRACT_DIR="$(mktemp -d -t openfront-assets-XXXXXX)"
+trap 'rm -rf "$EXTRACT_DIR"' EXIT
+
+TMP_CONTAINER="$(docker create "${GHCR_IMAGE}")"
+if ! docker cp "${TMP_CONTAINER}:/usr/src/app/static/_assets" "$EXTRACT_DIR/"; then
+    echo "❌ docker cp failed"
+    docker rm "${TMP_CONTAINER}" > /dev/null 2>&1 || true
+    exit 1
+fi
+docker rm "${TMP_CONTAINER}" > /dev/null
+
+echo "Extracted to $EXTRACT_DIR; top-level contents:"
+ls -la "$EXTRACT_DIR/" || true
+
+R2_ENDPOINT="https://api.${DOMAIN}"
+MANIFEST="$EXTRACT_DIR/_assets/asset-manifest.json"
+if [ ! -f "$MANIFEST" ]; then
+    echo "❌ Manifest not found at $MANIFEST"
+    exit 1
+fi
+
+# Manifest values are like "/_assets/foo/bar.<hash>.png"; strip the leading "/".
+KEYS_JSON="$(jq '[.[] | sub("^/"; "")]' "$MANIFEST")"
+TOTAL="$(echo "$KEYS_JSON" | jq 'length')"
+echo "Checking $TOTAL asset keys against $R2_ENDPOINT..."
+
+CHECK_BODY="$(mktemp)"
+HTTP_CODE="$(curl -sS --connect-timeout 10 --max-time 120 \
+    -o "$CHECK_BODY" -w "%{http_code}" -X POST "$R2_ENDPOINT/game_assets/check" \
+    -H "X-API-Key: $API_KEY" \
+    -H "Content-Type: application/json" \
+    -d "{\"keys\": $KEYS_JSON}")"
+if [ "$HTTP_CODE" != "200" ]; then
+    echo "❌ /check returned HTTP $HTTP_CODE:"
+    cat "$CHECK_BODY"
+    rm -f "$CHECK_BODY"
+    exit 1
+fi
+if ! jq -e '.missing | type == "array"' "$CHECK_BODY" > /dev/null; then
+    echo "❌ /check response missing '.missing' array:"
+    cat "$CHECK_BODY"
+    rm -f "$CHECK_BODY"
+    exit 1
+fi
+MISSING="$(jq -r '.missing[]' "$CHECK_BODY")"
+rm -f "$CHECK_BODY"
+
+if [ -z "$MISSING" ]; then
+    echo "✅ All $TOTAL assets already in R2; nothing to upload."
+else
+    MISSING_COUNT="$(echo "$MISSING" | wc -l | tr -d ' ')"
+    echo "Uploading $MISSING_COUNT missing asset(s)..."
+    export R2_ENDPOINT API_KEY EXTRACT_DIR
+    # KEY from the manifest is URL-encoded per segment (e.g. flags/C%C3%B4te.png).
+    # Files on disk live at the *decoded* path, so decode KEY before reading the
+    # file, then encode the whole decoded path as one URL segment for the POST.
+    if ! echo "$MISSING" | xargs -P 16 -I{} bash -euc '
+        KEY="$1"
+        # Validate KEY: only chars that encodeURIComponent leaves literal
+        # (A-Z a-z 0-9 - _ . ! ~ * ( ) plus apostrophe), "/" between segments,
+        # and well-formed %HH escapes. The %HH check makes the printf-based
+        # decoder below safe by rejecting partial escapes; excluding "\" keeps
+        # printf "%b" from interpreting unexpected backslash sequences. The
+        # regex lives in a variable so the literal apostrophe sits inside a
+        # double-quoted assignment instead of being parsed as a shell quote.
+        RE="^([A-Za-z0-9._/~!*'\''()-]|%[0-9A-Fa-f]{2})+\$"
+        [[ "$KEY" =~ $RE ]] || {
+            echo "❌ invalid key from server: $KEY" >&2; exit 1
+        }
+        DECODED="$(printf "%b" "${KEY//%/\\x}")"
+        # Defense-in-depth: refuse any decoded path that escapes the asset tree,
+        # in case the trusted /check endpoint is ever compromised.
+        case "$DECODED" in
+            /* | *..* ) echo "❌ refusing unsafe path: $DECODED" >&2; exit 1 ;;
+        esac
+        ENC="$(jq -rn --arg k "$DECODED" "\$k|@uri")"
+        if ! curl -fsS \
+            --retry 5 --retry-all-errors --retry-delay 2 \
+            --connect-timeout 10 --max-time 120 \
+            -X PUT \
+            "$R2_ENDPOINT/game_assets/upload/$ENC" \
+            -H "X-API-Key: $API_KEY" \
+            -H "Content-Type: application/octet-stream" \
+            --data-binary "@$EXTRACT_DIR/$DECODED" > /dev/null; then
+            echo "❌ Failed to upload: $DECODED" >&2
+            exit 1
+        fi
+    ' _ {}; then
+        echo "❌ One or more asset uploads failed."
+        exit 1
+    fi
+    echo "✅ Uploaded $MISSING_COUNT asset(s) to R2."
+fi
+
 echo "Checking for existing container..."
 # Use docker ps with filter for exact name match
 RUNNING_CONTAINER="$(docker ps --filter "name=^${CONTAINER_NAME}$" -q)"

vite.config.ts

@@ -13,7 +13,7 @@ import {
   createHashedPublicAssetFiles,
   getProprietaryDir,
   getResourcesDir,
-  writePublicAssetManifestModule,
+  writePublicAssetManifest,
 } from "./src/server/PublicAssetManifest";
 
 // Vite already handles these, but its good practice to define them explicitly
@@ -78,7 +78,7 @@ export default defineConfig(({ mode }) => {
       const outDir = path.join(__dirname, "static");
       copyRootPublicFiles(resourcesDir, outDir);
       createHashedPublicAssetFiles(sourceDirs, outDir, assetManifest);
-      writePublicAssetManifestModule(outDir, assetManifest);
+      writePublicAssetManifest(outDir, assetManifest);
     },
   });