diff --git a/src/client/ClientGameRunner.ts b/src/client/ClientGameRunner.ts index 5e276f382..2d1e06639 100644 --- a/src/client/ClientGameRunner.ts +++ b/src/client/ClientGameRunner.ts @@ -9,6 +9,7 @@ import { PlayerCosmeticRefs, PlayerRecord, ServerMessage, + Turn, } from "../core/Schemas"; import { createPartialGameRecord, replacer } from "../core/Util"; import { ServerConfig } from "../core/configuration/Config"; @@ -435,20 +436,43 @@ export class ClientGameRunner { goToPlayer(); } - for (const turn of message.turns) { + const normalizeTurn = (turn: Turn): Turn => + this.gameView.config().isReplay() + ? { + ...turn, + intents: turn.intents.filter((i) => i.type !== "toggle_pause"), + } + : turn; + + // Firefox in particular suffers from a storm of thousands of tiny + // postMessage() calls on reconnect. Batch turns to keep the worker + // event loop responsive for render_frame and sim scheduling. + const batchSize = 256; + let batch: Turn[] = []; + const flush = () => { + if (batch.length === 0) return; + this.worker.sendTurnBatch(batch); + batch = []; + }; + + for (const rawTurn of message.turns as Turn[]) { + const turn = normalizeTurn(rawTurn); if (turn.turnNumber < this.turnsSeen) { continue; } while (turn.turnNumber - 1 > this.turnsSeen) { - this.worker.sendTurn({ + batch.push({ turnNumber: this.turnsSeen, intents: [], }); this.turnsSeen++; + if (batch.length >= batchSize) flush(); } - this.worker.sendTurn(turn); + batch.push(turn); this.turnsSeen++; + if (batch.length >= batchSize) flush(); } + flush(); } if (message.type === "desync") { if (this.lobby.gameStartInfo === undefined) { diff --git a/src/client/graphics/layers/PerformanceOverlay.ts b/src/client/graphics/layers/PerformanceOverlay.ts index a5c0725b6..c76bd7f7c 100644 --- a/src/client/graphics/layers/PerformanceOverlay.ts +++ b/src/client/graphics/layers/PerformanceOverlay.ts @@ -538,6 +538,7 @@ export class PerformanceOverlay extends LitElement implements Layer { } private getWorkerKeyStats(metrics: WorkerMetricsMessage | null): { + intervalMs: number; loopLagAvg: number; loopLagMax: number; simDelayAvg: number; @@ -549,9 +550,18 @@ export class PerformanceOverlay extends LitElement implements Layer { rfHandlerAvg: number | null; rfHandlerMax: number | null; traceLines: string[]; + topMsgs: Array<{ + type: string; + count: number; + queueAvg: number | null; + queueMax: number | null; + handlerAvg: number | null; + handlerMax: number | null; + }>; } { if (!metrics) { return { + intervalMs: 0, loopLagAvg: 0, loopLagMax: 0, simDelayAvg: 0, @@ -563,6 +573,7 @@ export class PerformanceOverlay extends LitElement implements Layer { rfHandlerAvg: null, rfHandlerMax: null, traceLines: [], + topMsgs: [], }; } @@ -573,7 +584,32 @@ export class PerformanceOverlay extends LitElement implements Layer { const traceLines = metrics.trace && metrics.trace.length > 0 ? metrics.trace.slice(-5) : []; + const topMsgs = Object.entries(metrics.msgCounts ?? {}) + .sort((a, b) => b[1] - a[1]) + .slice(0, 6) + .map(([type, count]) => ({ + type, + count, + queueAvg: + typeof metrics.msgQueueMsAvg?.[type] === "number" + ? metrics.msgQueueMsAvg[type] + : null, + queueMax: + typeof metrics.msgQueueMsMax?.[type] === "number" + ? metrics.msgQueueMsMax[type] + : null, + handlerAvg: + typeof metrics.msgHandlerMsAvg?.[type] === "number" + ? metrics.msgHandlerMsAvg[type] + : null, + handlerMax: + typeof metrics.msgHandlerMsMax?.[type] === "number" + ? metrics.msgHandlerMsMax[type] + : null, + })); + return { + intervalMs: metrics.intervalMs, loopLagAvg: metrics.eventLoopLagMsAvg, loopLagMax: metrics.eventLoopLagMsMax, simDelayAvg: metrics.simPumpDelayMsAvg, @@ -585,6 +621,7 @@ export class PerformanceOverlay extends LitElement implements Layer { rfHandlerAvg: typeof rfHandlerAvg === "number" ? rfHandlerAvg : null, rfHandlerMax: typeof rfHandlerMax === "number" ? rfHandlerMax : null, traceLines, + topMsgs, }; } @@ -735,6 +772,12 @@ export class PerformanceOverlay extends LitElement implements Layer { >${this.formatMs(this.workerMetricsAgeMs, 0)} +
+ metrics interval (worker) + ${this.formatMs(worker.intervalMs, 0)} +
event loop lag (avg / max)
+ ${worker.topMsgs.length + ? html`
+ top msgs (count | queue avg/max | handler avg/max) +
+ ${worker.topMsgs.map( + (m) => + html`
+ ${m.type} (${m.count}) + + ${this.formatMs(m.queueAvg, 0)}/${this.formatMs( + m.queueMax, + 0, + )} + | + ${this.formatMs(m.handlerAvg, 0)}/${this.formatMs( + m.handlerMax, + 0, + )} + +
`, + )}` + : html``}
trace diff --git a/src/client/graphics/layers/TerritoryLayer.ts b/src/client/graphics/layers/TerritoryLayer.ts index a7b2971ce..ef7aaeb4e 100644 --- a/src/client/graphics/layers/TerritoryLayer.ts +++ b/src/client/graphics/layers/TerritoryLayer.ts @@ -59,6 +59,9 @@ export class TerritoryLayer implements Layer { private hoveredOwnerSmallId: number | null = null; private lastHoverUpdateMs = 0; private hoverRequestSeq = 0; + private hoverTile: TileRef | null = null; + private hoverQueryInFlight = false; + private pendingHoverTile: TileRef | null = null; constructor( private game: GameView, @@ -299,6 +302,8 @@ export class TerritoryLayer implements Layer { this.lastHoverUpdateMs = now; if (!this.lastMousePosition) { + this.hoverTile = null; + this.pendingHoverTile = null; if (this.hoveredOwnerSmallId !== null) { this.hoveredOwnerSmallId = null; this.territoryRenderer.setHighlightedOwnerId(null); @@ -311,6 +316,8 @@ export class TerritoryLayer implements Layer { this.lastMousePosition.y, ); if (!this.game.isValidCoord(cell.x, cell.y)) { + this.hoverTile = null; + this.pendingHoverTile = null; if (this.hoveredOwnerSmallId !== null) { this.hoveredOwnerSmallId = null; this.territoryRenderer.setHighlightedOwnerId(null); @@ -319,24 +326,54 @@ export class TerritoryLayer implements Layer { } const tile = this.game.ref(cell.x, cell.y); - const seq = ++this.hoverRequestSeq; - this.game.worker - .tileContext(tile) - .then((ctx) => { - if (seq !== this.hoverRequestSeq) { - return; - } - const nextOwnerSmallId = ctx.ownerSmallId; - if (nextOwnerSmallId === this.hoveredOwnerSmallId) { - return; - } - this.hoveredOwnerSmallId = nextOwnerSmallId; - this.territoryRenderer?.setHighlightedOwnerId(nextOwnerSmallId); - }) - .catch((err) => { - // Don't spam; hover is best-effort. - console.warn("tileContext hover lookup failed:", err); - }); + + // Only query on tile changes; keep at most one query in flight. + if (this.hoverTile === tile && this.pendingHoverTile === null) { + return; + } + this.hoverTile = tile; + this.pendingHoverTile = tile; + + if (this.hoverQueryInFlight) { + return; + } + + const doQuery = () => { + const nextTile = this.pendingHoverTile; + if (nextTile === null) { + this.hoverQueryInFlight = false; + return; + } + this.pendingHoverTile = null; + this.hoverQueryInFlight = true; + + const seq = ++this.hoverRequestSeq; + this.game.worker + .tileContext(nextTile) + .then((ctx) => { + if (seq !== this.hoverRequestSeq) { + return; + } + const nextOwnerSmallId = ctx.ownerSmallId; + if (nextOwnerSmallId === this.hoveredOwnerSmallId) { + return; + } + this.hoveredOwnerSmallId = nextOwnerSmallId; + this.territoryRenderer?.setHighlightedOwnerId(nextOwnerSmallId); + }) + .catch((err) => { + // Hover is best-effort; avoid spamming logs. + console.warn("tileContext hover lookup failed:", err); + }) + .finally(() => { + this.hoverQueryInFlight = false; + if (this.pendingHoverTile !== null) { + doQuery(); + } + }); + }; + + doQuery(); } private computePaletteSignature(): string { diff --git a/src/client/graphics/webgpu/core/GroundTruthData.ts b/src/client/graphics/webgpu/core/GroundTruthData.ts index 803944884..e5f9eafb0 100644 --- a/src/client/graphics/webgpu/core/GroundTruthData.ts +++ b/src/client/graphics/webgpu/core/GroundTruthData.ts @@ -58,8 +58,12 @@ export class GroundTruthData { private readonly state: Uint16Array; private readonly terrainData: Uint8Array; private needsStateUpload = true; + private stateUploadScratch: Uint32Array | null = null; + private stateUploadScratchStrideU32 = 0; private needsPaletteUpload = true; private needsTerrainDataUpload = true; + private terrainDataUploadScratch: Uint8Array | null = null; + private terrainDataUploadScratchBytesPerRow = 0; private needsTerrainParamsUpload = true; private useVisualStateTexture = false; private visualStateNeedsSync = false; @@ -514,41 +518,42 @@ export class GroundTruthData { } this.needsStateUpload = false; - // Convert 16-bit CPU state to 32-bit array - const u32State = new Uint32Array(this.state.length); - for (let i = 0; i < this.state.length; i++) { - u32State[i] = this.state[i]; - } - const bytesPerTexel = Uint32Array.BYTES_PER_ELEMENT; const fullBytesPerRow = this.mapWidth * bytesPerTexel; + const bytesPerRow = align(fullBytesPerRow, 256); + const strideU32 = bytesPerRow / 4; + const required = strideU32 * this.mapHeight; - if (fullBytesPerRow % 256 === 0) { - this.device.queue.writeTexture( - { texture: this.stateTexture }, - u32State, - { bytesPerRow: fullBytesPerRow, rowsPerImage: this.mapHeight }, - { - width: this.mapWidth, - height: this.mapHeight, - depthOrArrayLayers: 1, - }, - ); - } else { - // Fallback: upload row-by-row with padding - const paddedBytesPerRow = align(fullBytesPerRow, 256); - const scratch = new Uint32Array(paddedBytesPerRow / 4); - for (let y = 0; y < this.mapHeight; y++) { - const start = y * this.mapWidth; - scratch.set(u32State.subarray(start, start + this.mapWidth), 0); - this.device.queue.writeTexture( - { texture: this.stateTexture, origin: { x: 0, y } }, - scratch, - { bytesPerRow: paddedBytesPerRow, rowsPerImage: 1 }, - { width: this.mapWidth, height: 1, depthOrArrayLayers: 1 }, - ); + if ( + !this.stateUploadScratch || + this.stateUploadScratchStrideU32 !== strideU32 || + this.stateUploadScratch.length < required + ) { + this.stateUploadScratch = new Uint32Array(required); + this.stateUploadScratchStrideU32 = strideU32; + } + + const dst = this.stateUploadScratch; + const src = this.state; + const w = this.mapWidth; + for (let y = 0; y < this.mapHeight; y++) { + const srcBase = y * w; + const dstBase = y * strideU32; + for (let x = 0; x < w; x++) { + dst[dstBase + x] = src[srcBase + x]; } } + + this.device.queue.writeTexture( + { texture: this.stateTexture }, + dst, + { bytesPerRow, rowsPerImage: this.mapHeight }, + { + width: this.mapWidth, + height: this.mapHeight, + depthOrArrayLayers: 1, + }, + ); } /** @@ -589,36 +594,36 @@ export class GroundTruthData { } this.needsTerrainDataUpload = false; - const bytesPerRow = this.mapWidth; - const paddedBytesPerRow = align(bytesPerRow, 256); - - if (paddedBytesPerRow === bytesPerRow) { - // Direct upload if already aligned - this.device.queue.writeTexture( - { texture: this.terrainDataTexture }, - this.terrainData, - { bytesPerRow, rowsPerImage: this.mapHeight }, - { - width: this.mapWidth, - height: this.mapHeight, - depthOrArrayLayers: 1, - }, - ); - } else { - // Row-by-row upload with padding - const row = new Uint8Array(paddedBytesPerRow); - for (let y = 0; y < this.mapHeight; y++) { - row.fill(0); - const start = y * this.mapWidth; - row.set(this.terrainData.subarray(start, start + this.mapWidth), 0); - this.device.queue.writeTexture( - { texture: this.terrainDataTexture, origin: { x: 0, y } }, - row, - { bytesPerRow: paddedBytesPerRow, rowsPerImage: 1 }, - { width: this.mapWidth, height: 1, depthOrArrayLayers: 1 }, - ); - } + const bytesPerRow = align(this.mapWidth, 256); + const required = bytesPerRow * this.mapHeight; + if ( + !this.terrainDataUploadScratch || + this.terrainDataUploadScratchBytesPerRow !== bytesPerRow || + this.terrainDataUploadScratch.length < required + ) { + this.terrainDataUploadScratch = new Uint8Array(required); + this.terrainDataUploadScratchBytesPerRow = bytesPerRow; } + + const dst = this.terrainDataUploadScratch; + const src = this.terrainData; + const w = this.mapWidth; + for (let y = 0; y < this.mapHeight; y++) { + const srcStart = y * w; + const dstStart = y * bytesPerRow; + dst.set(src.subarray(srcStart, srcStart + w), dstStart); + } + + this.device.queue.writeTexture( + { texture: this.terrainDataTexture }, + dst, + { bytesPerRow, rowsPerImage: this.mapHeight }, + { + width: this.mapWidth, + height: this.mapHeight, + depthOrArrayLayers: 1, + }, + ); } uploadTerrainParams(): void { diff --git a/src/client/graphics/webgpu/core/Pipeline.ts b/src/client/graphics/webgpu/core/Pipeline.ts new file mode 100644 index 000000000..ec2b4dcc7 --- /dev/null +++ b/src/client/graphics/webgpu/core/Pipeline.ts @@ -0,0 +1,90 @@ +type AnyFn = (...args: any[]) => any; +type SlowPipelineKind = "compute" | "render"; +type SlowPipelineEvent = { + kind: SlowPipelineKind; + ms: number; + label?: string; +}; + +function getOptionalMethod( + obj: unknown, + name: string, +): T | undefined { + const anyObj = obj as any; + const value = anyObj?.[name]; + return typeof value === "function" ? (value as T) : undefined; +} + +function emitSlowPipelineEvent(event: SlowPipelineEvent): void { + const hook = (globalThis as any).__webgpuSlowPipelineHook; + if (typeof hook !== "function") { + return; + } + try { + hook(event); + } catch { + // ignore + } +} + +export async function createComputePipeline( + device: GPUDevice, + descriptor: GPUComputePipelineDescriptor, + debugLabel?: string, +): Promise { + const start = performance.now(); + + const maybeAsync = getOptionalMethod< + (desc: GPUComputePipelineDescriptor) => Promise + >(device, "createComputePipelineAsync"); + + const pipeline = maybeAsync + ? await maybeAsync.call(device, descriptor) + : device.createComputePipeline(descriptor); + + const ms = performance.now() - start; + if (ms > 250) { + console.warn("WebGPU slow compute pipeline compile", { + ms: Math.round(ms), + label: debugLabel, + }); + emitSlowPipelineEvent({ + kind: "compute", + ms, + label: debugLabel, + }); + } + + return pipeline; +} + +export async function createRenderPipeline( + device: GPUDevice, + descriptor: GPURenderPipelineDescriptor, + debugLabel?: string, +): Promise { + const start = performance.now(); + + const maybeAsync = getOptionalMethod< + (desc: GPURenderPipelineDescriptor) => Promise + >(device, "createRenderPipelineAsync"); + + const pipeline = maybeAsync + ? await maybeAsync.call(device, descriptor) + : device.createRenderPipeline(descriptor); + + const ms = performance.now() - start; + if (ms > 250) { + console.warn("WebGPU slow render pipeline compile", { + ms: Math.round(ms), + label: debugLabel, + }); + emitSlowPipelineEvent({ + kind: "render", + ms, + label: debugLabel, + }); + } + + return pipeline; +} diff --git a/src/core/worker/Worker.worker.ts b/src/core/worker/Worker.worker.ts index 93067cfb0..61f71cd52 100644 --- a/src/core/worker/Worker.worker.ts +++ b/src/core/worker/Worker.worker.ts @@ -16,7 +16,7 @@ import { } from "../game/GameUpdates"; import { createGameRunner, GameRunner } from "../GameRunner"; -import { ClientID, GameStartInfo, PlayerCosmetics } from "../Schemas"; +import { ClientID, GameStartInfo, PlayerCosmetics, Turn } from "../Schemas"; import { DirtyTileQueue } from "./DirtyTileQueue"; import { WorkerCanvas2DRenderer } from "./WorkerCanvas2DRenderer"; import { @@ -45,6 +45,7 @@ let renderer: WorkerTerritoryRenderer | WorkerCanvas2DRenderer | null = null; let dirtyTiles: DirtyTileQueue | null = null; let dirtyTilesOverflow = false; let renderTileState: Uint16Array | null = null; +const pendingTurns: Turn[] = []; type WorkerDebugConfig = { enabled: boolean; @@ -282,24 +283,36 @@ class WorkerProfiler { const profiler = new WorkerProfiler(); let simPumpScheduled = false; + function scheduleSimPump(): void { if (simPumpScheduled) { return; } simPumpScheduled = true; - const scheduledAtWallMs = Date.now(); setTimeout(async () => { simPumpScheduled = false; if (!gameRunner) { return; } + const gr = await gameRunner; profiler.recordSimDelay(Date.now() - scheduledAtWallMs); const execStart = performance.now(); + if (pendingTurns.length > 0) { + // Drain turns into GameRunner's queue in chunks so we don't block + // the worker event loop for too long (important for Firefox). + const maxDrain = 256; + for (let i = 0; i < maxDrain && pendingTurns.length > 0; i++) { + const t = pendingTurns.shift(); + if (t) { + gr.addTurn(t); + } + } + } gr.executeNextTick(); profiler.recordSimExec(performance.now() - execStart); - if (gr.hasPendingTurns()) { + if (pendingTurns.length > 0 || gr.hasPendingTurns()) { scheduleSimPump(); } }, 0); @@ -442,6 +455,9 @@ ctx.addEventListener("message", async (e: MessageEvent) => { if (!dirtyTiles) { return; } + if (dirtyTilesOverflow) { + return; + } const tile = Number(packedUpdate >> 16n) as TileRef; const state = Number(packedUpdate & 0xffffn); @@ -475,8 +491,7 @@ ctx.addEventListener("message", async (e: MessageEvent) => { } try { - const gr = await gameRunner; - gr.addTurn(message.turn); + pendingTurns.push(message.turn); scheduleSimPump(); } catch (error) { console.error("Failed to process turn:", error); @@ -484,6 +499,20 @@ ctx.addEventListener("message", async (e: MessageEvent) => { } break; + case "turn_batch": + if (!gameRunner) { + throw new Error("Game runner not initialized"); + } + + try { + pendingTurns.push(...message.turns); + scheduleSimPump(); + } catch (error) { + console.error("Failed to process turn batch:", error); + throw error; + } + break; + case "tile_context": if (!gameRunner) { throw new Error("Game runner not initialized"); diff --git a/src/core/worker/WorkerClient.ts b/src/core/worker/WorkerClient.ts index f0ba0d86a..efc641dfd 100644 --- a/src/core/worker/WorkerClient.ts +++ b/src/core/worker/WorkerClient.ts @@ -25,6 +25,10 @@ export class WorkerClient { ) => void; private workerMetricsCallback?: (metrics: WorkerMetricsMessage) => void; + private pendingTurns: Turn[] = []; + private turnFlushScheduled = false; + private readonly maxTurnsPerBatch = 256; + constructor( private gameStartInfo: GameStartInfo, private clientID: ClientID, @@ -154,15 +158,43 @@ export class WorkerClient { this.gameUpdateCallback = gameUpdate; } + private scheduleTurnFlush(): void { + if (this.turnFlushScheduled) return; + this.turnFlushScheduled = true; + setTimeout(() => { + this.turnFlushScheduled = false; + this.flushTurns(); + }, 0); + } + + private flushTurns(): void { + while (this.pendingTurns.length > 0) { + const batch = this.pendingTurns.splice(0, this.maxTurnsPerBatch); + this.postMessage({ + type: "turn_batch", + turns: batch, + }); + } + } + sendTurn(turn: Turn) { if (!this.isInitialized) { throw new Error("Worker not initialized"); } - this.postMessage({ - type: "turn", - turn, - }); + this.pendingTurns.push(turn); + this.scheduleTurnFlush(); + } + + sendTurnBatch(turns: Turn[]) { + if (!this.isInitialized) { + throw new Error("Worker not initialized"); + } + if (turns.length === 0) return; + + // Preserve order with any already queued turns. + this.pendingTurns.push(...turns); + this.scheduleTurnFlush(); } sendHeartbeat() { diff --git a/src/core/worker/WorkerMessages.ts b/src/core/worker/WorkerMessages.ts index 3cda691c7..69a0f2bbf 100644 --- a/src/core/worker/WorkerMessages.ts +++ b/src/core/worker/WorkerMessages.ts @@ -13,6 +13,7 @@ export type WorkerMessageType = | "init" | "initialized" | "turn" + | "turn_batch" | "game_update" | "tile_context" | "tile_context_result" @@ -73,6 +74,11 @@ export interface TurnMessage extends BaseWorkerMessage { turn: Turn; } +export interface TurnBatchMessage extends BaseWorkerMessage { + type: "turn_batch"; + turns: Turn[]; +} + // Messages from worker to main thread export interface InitializedMessage extends BaseWorkerMessage { type: "initialized"; @@ -337,6 +343,7 @@ export type MainThreadMessage = | HeartbeatMessage | InitMessage | TurnMessage + | TurnBatchMessage | TileContextMessage | PlayerActionsMessage | PlayerProfileMessage