Optimize turn processing and enhance performance metrics handling

- Introduced batching for turn messages in ClientGameRunner to reduce the frequency of worker communication
- Updated WorkerClient to manage pending turns and schedule flushes
- Enhanced Worker.worker.ts to process turn batches
- Added new TurnBatchMessage type for better organization of turn data sent between the main thread and worker.
- Improved PerformanceOverlay to display additional metrics related to worker performance and turn processing.
This commit is contained in:
scamiv
2026-02-04 18:20:08 +01:00
parent 45a8e33562
commit aa2578514a
8 changed files with 380 additions and 89 deletions
+27 -3
View File
@@ -9,6 +9,7 @@ import {
PlayerCosmeticRefs,
PlayerRecord,
ServerMessage,
Turn,
} from "../core/Schemas";
import { createPartialGameRecord, replacer } from "../core/Util";
import { ServerConfig } from "../core/configuration/Config";
@@ -435,20 +436,43 @@ export class ClientGameRunner {
goToPlayer();
}
for (const turn of message.turns) {
const normalizeTurn = (turn: Turn): Turn =>
this.gameView.config().isReplay()
? {
...turn,
intents: turn.intents.filter((i) => i.type !== "toggle_pause"),
}
: turn;
// Firefox in particular suffers from a storm of thousands of tiny
// postMessage() calls on reconnect. Batch turns to keep the worker
// event loop responsive for render_frame and sim scheduling.
const batchSize = 256;
let batch: Turn[] = [];
const flush = () => {
if (batch.length === 0) return;
this.worker.sendTurnBatch(batch);
batch = [];
};
for (const rawTurn of message.turns as Turn[]) {
const turn = normalizeTurn(rawTurn);
if (turn.turnNumber < this.turnsSeen) {
continue;
}
while (turn.turnNumber - 1 > this.turnsSeen) {
this.worker.sendTurn({
batch.push({
turnNumber: this.turnsSeen,
intents: [],
});
this.turnsSeen++;
if (batch.length >= batchSize) flush();
}
this.worker.sendTurn(turn);
batch.push(turn);
this.turnsSeen++;
if (batch.length >= batchSize) flush();
}
flush();
}
if (message.type === "desync") {
if (this.lobby.gameStartInfo === undefined) {
@@ -538,6 +538,7 @@ export class PerformanceOverlay extends LitElement implements Layer {
}
private getWorkerKeyStats(metrics: WorkerMetricsMessage | null): {
intervalMs: number;
loopLagAvg: number;
loopLagMax: number;
simDelayAvg: number;
@@ -549,9 +550,18 @@ export class PerformanceOverlay extends LitElement implements Layer {
rfHandlerAvg: number | null;
rfHandlerMax: number | null;
traceLines: string[];
topMsgs: Array<{
type: string;
count: number;
queueAvg: number | null;
queueMax: number | null;
handlerAvg: number | null;
handlerMax: number | null;
}>;
} {
if (!metrics) {
return {
intervalMs: 0,
loopLagAvg: 0,
loopLagMax: 0,
simDelayAvg: 0,
@@ -563,6 +573,7 @@ export class PerformanceOverlay extends LitElement implements Layer {
rfHandlerAvg: null,
rfHandlerMax: null,
traceLines: [],
topMsgs: [],
};
}
@@ -573,7 +584,32 @@ export class PerformanceOverlay extends LitElement implements Layer {
const traceLines =
metrics.trace && metrics.trace.length > 0 ? metrics.trace.slice(-5) : [];
const topMsgs = Object.entries(metrics.msgCounts ?? {})
.sort((a, b) => b[1] - a[1])
.slice(0, 6)
.map(([type, count]) => ({
type,
count,
queueAvg:
typeof metrics.msgQueueMsAvg?.[type] === "number"
? metrics.msgQueueMsAvg[type]
: null,
queueMax:
typeof metrics.msgQueueMsMax?.[type] === "number"
? metrics.msgQueueMsMax[type]
: null,
handlerAvg:
typeof metrics.msgHandlerMsAvg?.[type] === "number"
? metrics.msgHandlerMsAvg[type]
: null,
handlerMax:
typeof metrics.msgHandlerMsMax?.[type] === "number"
? metrics.msgHandlerMsMax[type]
: null,
}));
return {
intervalMs: metrics.intervalMs,
loopLagAvg: metrics.eventLoopLagMsAvg,
loopLagMax: metrics.eventLoopLagMsMax,
simDelayAvg: metrics.simPumpDelayMsAvg,
@@ -585,6 +621,7 @@ export class PerformanceOverlay extends LitElement implements Layer {
rfHandlerAvg: typeof rfHandlerAvg === "number" ? rfHandlerAvg : null,
rfHandlerMax: typeof rfHandlerMax === "number" ? rfHandlerMax : null,
traceLines,
topMsgs,
};
}
@@ -735,6 +772,12 @@ export class PerformanceOverlay extends LitElement implements Layer {
>${this.formatMs(this.workerMetricsAgeMs, 0)}</span
>
</div>
<div class="layer-row">
<span class="layer-name">metrics interval (worker)</span>
<span class="layer-metrics"
>${this.formatMs(worker.intervalMs, 0)}</span
>
</div>
<div class="layer-row">
<span class="layer-name">event loop lag (avg / max)</span>
<span class="layer-metrics"
@@ -770,6 +813,30 @@ export class PerformanceOverlay extends LitElement implements Layer {
${this.formatMs(worker.rfHandlerMax, 0)}</span
>
</div>
${worker.topMsgs.length
? html`<div class="performance-line" style="margin-top: 6px;">
top msgs (count | queue avg/max | handler avg/max)
</div>
${worker.topMsgs.map(
(m) =>
html`<div class="layer-row">
<span class="layer-name" title=${m.type}
>${m.type} (${m.count})
</span>
<span class="layer-metrics"
>${this.formatMs(m.queueAvg, 0)}/${this.formatMs(
m.queueMax,
0,
)}
|
${this.formatMs(m.handlerAvg, 0)}/${this.formatMs(
m.handlerMax,
0,
)}
</span>
</div>`,
)}`
: html``}
<div class="layer-row" style="margin-top: 4px;">
<span class="layer-name">trace</span>
<span class="layer-metrics">
+55 -18
View File
@@ -59,6 +59,9 @@ export class TerritoryLayer implements Layer {
private hoveredOwnerSmallId: number | null = null;
private lastHoverUpdateMs = 0;
private hoverRequestSeq = 0;
private hoverTile: TileRef | null = null;
private hoverQueryInFlight = false;
private pendingHoverTile: TileRef | null = null;
constructor(
private game: GameView,
@@ -299,6 +302,8 @@ export class TerritoryLayer implements Layer {
this.lastHoverUpdateMs = now;
if (!this.lastMousePosition) {
this.hoverTile = null;
this.pendingHoverTile = null;
if (this.hoveredOwnerSmallId !== null) {
this.hoveredOwnerSmallId = null;
this.territoryRenderer.setHighlightedOwnerId(null);
@@ -311,6 +316,8 @@ export class TerritoryLayer implements Layer {
this.lastMousePosition.y,
);
if (!this.game.isValidCoord(cell.x, cell.y)) {
this.hoverTile = null;
this.pendingHoverTile = null;
if (this.hoveredOwnerSmallId !== null) {
this.hoveredOwnerSmallId = null;
this.territoryRenderer.setHighlightedOwnerId(null);
@@ -319,24 +326,54 @@ export class TerritoryLayer implements Layer {
}
const tile = this.game.ref(cell.x, cell.y);
const seq = ++this.hoverRequestSeq;
this.game.worker
.tileContext(tile)
.then((ctx) => {
if (seq !== this.hoverRequestSeq) {
return;
}
const nextOwnerSmallId = ctx.ownerSmallId;
if (nextOwnerSmallId === this.hoveredOwnerSmallId) {
return;
}
this.hoveredOwnerSmallId = nextOwnerSmallId;
this.territoryRenderer?.setHighlightedOwnerId(nextOwnerSmallId);
})
.catch((err) => {
// Don't spam; hover is best-effort.
console.warn("tileContext hover lookup failed:", err);
});
// Only query on tile changes; keep at most one query in flight.
if (this.hoverTile === tile && this.pendingHoverTile === null) {
return;
}
this.hoverTile = tile;
this.pendingHoverTile = tile;
if (this.hoverQueryInFlight) {
return;
}
const doQuery = () => {
const nextTile = this.pendingHoverTile;
if (nextTile === null) {
this.hoverQueryInFlight = false;
return;
}
this.pendingHoverTile = null;
this.hoverQueryInFlight = true;
const seq = ++this.hoverRequestSeq;
this.game.worker
.tileContext(nextTile)
.then((ctx) => {
if (seq !== this.hoverRequestSeq) {
return;
}
const nextOwnerSmallId = ctx.ownerSmallId;
if (nextOwnerSmallId === this.hoveredOwnerSmallId) {
return;
}
this.hoveredOwnerSmallId = nextOwnerSmallId;
this.territoryRenderer?.setHighlightedOwnerId(nextOwnerSmallId);
})
.catch((err) => {
// Hover is best-effort; avoid spamming logs.
console.warn("tileContext hover lookup failed:", err);
})
.finally(() => {
this.hoverQueryInFlight = false;
if (this.pendingHoverTile !== null) {
doQuery();
}
});
};
doQuery();
}
private computePaletteSignature(): string {
@@ -58,8 +58,12 @@ export class GroundTruthData {
private readonly state: Uint16Array;
private readonly terrainData: Uint8Array;
private needsStateUpload = true;
private stateUploadScratch: Uint32Array | null = null;
private stateUploadScratchStrideU32 = 0;
private needsPaletteUpload = true;
private needsTerrainDataUpload = true;
private terrainDataUploadScratch: Uint8Array | null = null;
private terrainDataUploadScratchBytesPerRow = 0;
private needsTerrainParamsUpload = true;
private useVisualStateTexture = false;
private visualStateNeedsSync = false;
@@ -514,41 +518,42 @@ export class GroundTruthData {
}
this.needsStateUpload = false;
// Convert 16-bit CPU state to 32-bit array
const u32State = new Uint32Array(this.state.length);
for (let i = 0; i < this.state.length; i++) {
u32State[i] = this.state[i];
}
const bytesPerTexel = Uint32Array.BYTES_PER_ELEMENT;
const fullBytesPerRow = this.mapWidth * bytesPerTexel;
const bytesPerRow = align(fullBytesPerRow, 256);
const strideU32 = bytesPerRow / 4;
const required = strideU32 * this.mapHeight;
if (fullBytesPerRow % 256 === 0) {
this.device.queue.writeTexture(
{ texture: this.stateTexture },
u32State,
{ bytesPerRow: fullBytesPerRow, rowsPerImage: this.mapHeight },
{
width: this.mapWidth,
height: this.mapHeight,
depthOrArrayLayers: 1,
},
);
} else {
// Fallback: upload row-by-row with padding
const paddedBytesPerRow = align(fullBytesPerRow, 256);
const scratch = new Uint32Array(paddedBytesPerRow / 4);
for (let y = 0; y < this.mapHeight; y++) {
const start = y * this.mapWidth;
scratch.set(u32State.subarray(start, start + this.mapWidth), 0);
this.device.queue.writeTexture(
{ texture: this.stateTexture, origin: { x: 0, y } },
scratch,
{ bytesPerRow: paddedBytesPerRow, rowsPerImage: 1 },
{ width: this.mapWidth, height: 1, depthOrArrayLayers: 1 },
);
if (
!this.stateUploadScratch ||
this.stateUploadScratchStrideU32 !== strideU32 ||
this.stateUploadScratch.length < required
) {
this.stateUploadScratch = new Uint32Array(required);
this.stateUploadScratchStrideU32 = strideU32;
}
const dst = this.stateUploadScratch;
const src = this.state;
const w = this.mapWidth;
for (let y = 0; y < this.mapHeight; y++) {
const srcBase = y * w;
const dstBase = y * strideU32;
for (let x = 0; x < w; x++) {
dst[dstBase + x] = src[srcBase + x];
}
}
this.device.queue.writeTexture(
{ texture: this.stateTexture },
dst,
{ bytesPerRow, rowsPerImage: this.mapHeight },
{
width: this.mapWidth,
height: this.mapHeight,
depthOrArrayLayers: 1,
},
);
}
/**
@@ -589,36 +594,36 @@ export class GroundTruthData {
}
this.needsTerrainDataUpload = false;
const bytesPerRow = this.mapWidth;
const paddedBytesPerRow = align(bytesPerRow, 256);
if (paddedBytesPerRow === bytesPerRow) {
// Direct upload if already aligned
this.device.queue.writeTexture(
{ texture: this.terrainDataTexture },
this.terrainData,
{ bytesPerRow, rowsPerImage: this.mapHeight },
{
width: this.mapWidth,
height: this.mapHeight,
depthOrArrayLayers: 1,
},
);
} else {
// Row-by-row upload with padding
const row = new Uint8Array(paddedBytesPerRow);
for (let y = 0; y < this.mapHeight; y++) {
row.fill(0);
const start = y * this.mapWidth;
row.set(this.terrainData.subarray(start, start + this.mapWidth), 0);
this.device.queue.writeTexture(
{ texture: this.terrainDataTexture, origin: { x: 0, y } },
row,
{ bytesPerRow: paddedBytesPerRow, rowsPerImage: 1 },
{ width: this.mapWidth, height: 1, depthOrArrayLayers: 1 },
);
}
const bytesPerRow = align(this.mapWidth, 256);
const required = bytesPerRow * this.mapHeight;
if (
!this.terrainDataUploadScratch ||
this.terrainDataUploadScratchBytesPerRow !== bytesPerRow ||
this.terrainDataUploadScratch.length < required
) {
this.terrainDataUploadScratch = new Uint8Array(required);
this.terrainDataUploadScratchBytesPerRow = bytesPerRow;
}
const dst = this.terrainDataUploadScratch;
const src = this.terrainData;
const w = this.mapWidth;
for (let y = 0; y < this.mapHeight; y++) {
const srcStart = y * w;
const dstStart = y * bytesPerRow;
dst.set(src.subarray(srcStart, srcStart + w), dstStart);
}
this.device.queue.writeTexture(
{ texture: this.terrainDataTexture },
dst,
{ bytesPerRow, rowsPerImage: this.mapHeight },
{
width: this.mapWidth,
height: this.mapHeight,
depthOrArrayLayers: 1,
},
);
}
uploadTerrainParams(): void {
@@ -0,0 +1,90 @@
type AnyFn = (...args: any[]) => any;
type SlowPipelineKind = "compute" | "render";
type SlowPipelineEvent = {
kind: SlowPipelineKind;
ms: number;
label?: string;
};
function getOptionalMethod<T extends AnyFn>(
obj: unknown,
name: string,
): T | undefined {
const anyObj = obj as any;
const value = anyObj?.[name];
return typeof value === "function" ? (value as T) : undefined;
}
function emitSlowPipelineEvent(event: SlowPipelineEvent): void {
const hook = (globalThis as any).__webgpuSlowPipelineHook;
if (typeof hook !== "function") {
return;
}
try {
hook(event);
} catch {
// ignore
}
}
export async function createComputePipeline(
device: GPUDevice,
descriptor: GPUComputePipelineDescriptor,
debugLabel?: string,
): Promise<GPUComputePipeline> {
const start = performance.now();
const maybeAsync = getOptionalMethod<
(desc: GPUComputePipelineDescriptor) => Promise<GPUComputePipeline>
>(device, "createComputePipelineAsync");
const pipeline = maybeAsync
? await maybeAsync.call(device, descriptor)
: device.createComputePipeline(descriptor);
const ms = performance.now() - start;
if (ms > 250) {
console.warn("WebGPU slow compute pipeline compile", {
ms: Math.round(ms),
label: debugLabel,
});
emitSlowPipelineEvent({
kind: "compute",
ms,
label: debugLabel,
});
}
return pipeline;
}
export async function createRenderPipeline(
device: GPUDevice,
descriptor: GPURenderPipelineDescriptor,
debugLabel?: string,
): Promise<GPURenderPipeline> {
const start = performance.now();
const maybeAsync = getOptionalMethod<
(desc: GPURenderPipelineDescriptor) => Promise<GPURenderPipeline>
>(device, "createRenderPipelineAsync");
const pipeline = maybeAsync
? await maybeAsync.call(device, descriptor)
: device.createRenderPipeline(descriptor);
const ms = performance.now() - start;
if (ms > 250) {
console.warn("WebGPU slow render pipeline compile", {
ms: Math.round(ms),
label: debugLabel,
});
emitSlowPipelineEvent({
kind: "render",
ms,
label: debugLabel,
});
}
return pipeline;
}
+34 -5
View File
@@ -16,7 +16,7 @@ import {
} from "../game/GameUpdates";
import { createGameRunner, GameRunner } from "../GameRunner";
import { ClientID, GameStartInfo, PlayerCosmetics } from "../Schemas";
import { ClientID, GameStartInfo, PlayerCosmetics, Turn } from "../Schemas";
import { DirtyTileQueue } from "./DirtyTileQueue";
import { WorkerCanvas2DRenderer } from "./WorkerCanvas2DRenderer";
import {
@@ -45,6 +45,7 @@ let renderer: WorkerTerritoryRenderer | WorkerCanvas2DRenderer | null = null;
let dirtyTiles: DirtyTileQueue | null = null;
let dirtyTilesOverflow = false;
let renderTileState: Uint16Array | null = null;
const pendingTurns: Turn[] = [];
type WorkerDebugConfig = {
enabled: boolean;
@@ -282,24 +283,36 @@ class WorkerProfiler {
const profiler = new WorkerProfiler();
let simPumpScheduled = false;
function scheduleSimPump(): void {
if (simPumpScheduled) {
return;
}
simPumpScheduled = true;
const scheduledAtWallMs = Date.now();
setTimeout(async () => {
simPumpScheduled = false;
if (!gameRunner) {
return;
}
const gr = await gameRunner;
profiler.recordSimDelay(Date.now() - scheduledAtWallMs);
const execStart = performance.now();
if (pendingTurns.length > 0) {
// Drain turns into GameRunner's queue in chunks so we don't block
// the worker event loop for too long (important for Firefox).
const maxDrain = 256;
for (let i = 0; i < maxDrain && pendingTurns.length > 0; i++) {
const t = pendingTurns.shift();
if (t) {
gr.addTurn(t);
}
}
}
gr.executeNextTick();
profiler.recordSimExec(performance.now() - execStart);
if (gr.hasPendingTurns()) {
if (pendingTurns.length > 0 || gr.hasPendingTurns()) {
scheduleSimPump();
}
}, 0);
@@ -442,6 +455,9 @@ ctx.addEventListener("message", async (e: MessageEvent<MainThreadMessage>) => {
if (!dirtyTiles) {
return;
}
if (dirtyTilesOverflow) {
return;
}
const tile = Number(packedUpdate >> 16n) as TileRef;
const state = Number(packedUpdate & 0xffffn);
@@ -475,8 +491,7 @@ ctx.addEventListener("message", async (e: MessageEvent<MainThreadMessage>) => {
}
try {
const gr = await gameRunner;
gr.addTurn(message.turn);
pendingTurns.push(message.turn);
scheduleSimPump();
} catch (error) {
console.error("Failed to process turn:", error);
@@ -484,6 +499,20 @@ ctx.addEventListener("message", async (e: MessageEvent<MainThreadMessage>) => {
}
break;
case "turn_batch":
if (!gameRunner) {
throw new Error("Game runner not initialized");
}
try {
pendingTurns.push(...message.turns);
scheduleSimPump();
} catch (error) {
console.error("Failed to process turn batch:", error);
throw error;
}
break;
case "tile_context":
if (!gameRunner) {
throw new Error("Game runner not initialized");
+36 -4
View File
@@ -25,6 +25,10 @@ export class WorkerClient {
) => void;
private workerMetricsCallback?: (metrics: WorkerMetricsMessage) => void;
private pendingTurns: Turn[] = [];
private turnFlushScheduled = false;
private readonly maxTurnsPerBatch = 256;
constructor(
private gameStartInfo: GameStartInfo,
private clientID: ClientID,
@@ -154,15 +158,43 @@ export class WorkerClient {
this.gameUpdateCallback = gameUpdate;
}
private scheduleTurnFlush(): void {
if (this.turnFlushScheduled) return;
this.turnFlushScheduled = true;
setTimeout(() => {
this.turnFlushScheduled = false;
this.flushTurns();
}, 0);
}
private flushTurns(): void {
while (this.pendingTurns.length > 0) {
const batch = this.pendingTurns.splice(0, this.maxTurnsPerBatch);
this.postMessage({
type: "turn_batch",
turns: batch,
});
}
}
sendTurn(turn: Turn) {
if (!this.isInitialized) {
throw new Error("Worker not initialized");
}
this.postMessage({
type: "turn",
turn,
});
this.pendingTurns.push(turn);
this.scheduleTurnFlush();
}
sendTurnBatch(turns: Turn[]) {
if (!this.isInitialized) {
throw new Error("Worker not initialized");
}
if (turns.length === 0) return;
// Preserve order with any already queued turns.
this.pendingTurns.push(...turns);
this.scheduleTurnFlush();
}
sendHeartbeat() {
+7
View File
@@ -13,6 +13,7 @@ export type WorkerMessageType =
| "init"
| "initialized"
| "turn"
| "turn_batch"
| "game_update"
| "tile_context"
| "tile_context_result"
@@ -73,6 +74,11 @@ export interface TurnMessage extends BaseWorkerMessage {
turn: Turn;
}
export interface TurnBatchMessage extends BaseWorkerMessage {
type: "turn_batch";
turns: Turn[];
}
// Messages from worker to main thread
export interface InitializedMessage extends BaseWorkerMessage {
type: "initialized";
@@ -337,6 +343,7 @@ export type MainThreadMessage =
| HeartbeatMessage
| InitMessage
| TurnMessage
| TurnBatchMessage
| TileContextMessage
| PlayerActionsMessage
| PlayerProfileMessage