mirror of
https://github.com/openfrontio/OpenFrontIO.git
synced 2026-06-21 11:40:42 +00:00
Implement worker metrics and debugging events
- Introduced WorkerMetricsEvent and SetWorkerDebugEvent to facilitate communication between the main thread and worker for performance monitoring. - Enhanced ClientGameRunner to emit worker metrics and handle debug configuration updates. - Updated PerformanceOverlay to display worker metrics and allow toggling of debug settings. - Refactored Canvas2DRendererProxy and TerritoryRendererProxy to improve rendering performance and manage render cooldowns. - Added profiling capabilities in Worker.worker.ts to track event loop lag, simulation delays, and message handling metrics.
This commit is contained in:
@@ -34,7 +34,9 @@ import {
|
||||
InputHandler,
|
||||
MouseMoveEvent,
|
||||
MouseUpEvent,
|
||||
SetWorkerDebugEvent,
|
||||
TickMetricsEvent,
|
||||
WorkerMetricsEvent,
|
||||
} from "./InputHandler";
|
||||
import { endGame, startGame, startTime } from "./LocalPersistantStats";
|
||||
import { terrainMapFileLoader } from "./TerrainMapFileLoader";
|
||||
@@ -221,6 +223,12 @@ async function createClientGame(
|
||||
lobbyConfig.clientID,
|
||||
);
|
||||
await worker.initialize();
|
||||
worker.onWorkerMetrics((metrics) => {
|
||||
eventBus.emit(new WorkerMetricsEvent(metrics));
|
||||
});
|
||||
eventBus.on(SetWorkerDebugEvent, (event: SetWorkerDebugEvent) => {
|
||||
worker.setWorkerDebug(event.config);
|
||||
});
|
||||
const gameView = new GameView(
|
||||
worker,
|
||||
config,
|
||||
|
||||
@@ -2,6 +2,7 @@ import { EventBus, GameEvent } from "../core/EventBus";
|
||||
import { UnitType } from "../core/game/Game";
|
||||
import { UnitView } from "../core/game/GameView";
|
||||
import { UserSettings } from "../core/game/UserSettings";
|
||||
import type { WorkerMetricsMessage } from "../core/worker/WorkerMessages";
|
||||
import { UIState } from "./graphics/UIState";
|
||||
import { ReplaySpeedMultiplier } from "./utilities/ReplaySpeedMultiplier";
|
||||
|
||||
@@ -81,6 +82,20 @@ export class RefreshGraphicsEvent implements GameEvent {}
|
||||
|
||||
export class TogglePerformanceOverlayEvent implements GameEvent {}
|
||||
|
||||
export class SetWorkerDebugEvent implements GameEvent {
|
||||
constructor(
|
||||
public readonly config: {
|
||||
enabled: boolean;
|
||||
intervalMs?: number;
|
||||
includeTrace?: boolean;
|
||||
},
|
||||
) {}
|
||||
}
|
||||
|
||||
export class WorkerMetricsEvent implements GameEvent {
|
||||
constructor(public readonly metrics: WorkerMetricsMessage) {}
|
||||
}
|
||||
|
||||
export class ToggleStructureEvent implements GameEvent {
|
||||
constructor(public readonly structureTypes: UnitType[] | null) {}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ import { createCanvas } from "src/client/Utils";
|
||||
import { Theme } from "../../../core/configuration/Config";
|
||||
import { TileRef } from "../../../core/game/GameMap";
|
||||
import { GameView } from "../../../core/game/GameView";
|
||||
import { generateID } from "../../../core/Util";
|
||||
import { WorkerClient } from "../../../core/worker/WorkerClient";
|
||||
import {
|
||||
InitRendererMessage,
|
||||
@@ -16,7 +15,6 @@ import {
|
||||
SetPaletteMessage,
|
||||
SetPatternsEnabledMessage,
|
||||
SetShaderSettingsMessage,
|
||||
TickRendererMessage,
|
||||
ViewSize,
|
||||
ViewTransform,
|
||||
} from "../../../core/worker/WorkerMessages";
|
||||
@@ -40,6 +38,8 @@ export class Canvas2DRendererProxy {
|
||||
private lastSentViewSize: ViewSize | null = null;
|
||||
private lastSentViewTransform: ViewTransform | null = null;
|
||||
private renderInFlight = false;
|
||||
private renderSeq = 0;
|
||||
private renderCooldownUntilMs = 0;
|
||||
|
||||
private constructor(
|
||||
private readonly game: GameView,
|
||||
@@ -299,23 +299,27 @@ export class Canvas2DRendererProxy {
|
||||
}
|
||||
|
||||
tick(): void {
|
||||
const message: TickRendererMessage = { type: "tick_renderer" };
|
||||
this.sendToWorker(message);
|
||||
// No-op: worker renderer ticks from worker-side game_update.
|
||||
}
|
||||
|
||||
render(): void {
|
||||
if (this.failed) {
|
||||
return;
|
||||
}
|
||||
if (performance.now() < this.renderCooldownUntilMs) {
|
||||
return;
|
||||
}
|
||||
if (this.renderInFlight) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.renderInFlight = true;
|
||||
const renderId = `render_${generateID()}`;
|
||||
const renderId = `render_${++this.renderSeq}`;
|
||||
const sentAtWallMs = Date.now();
|
||||
|
||||
const message: RenderFrameMessage = { type: "render_frame" };
|
||||
message.id = renderId;
|
||||
message.sentAtWallMs = sentAtWallMs;
|
||||
|
||||
if (
|
||||
!this.lastSentViewSize ||
|
||||
@@ -343,15 +347,81 @@ export class Canvas2DRendererProxy {
|
||||
worker.removeMessageHandler(renderId);
|
||||
return;
|
||||
}
|
||||
this.renderInFlight = false;
|
||||
|
||||
console.warn(`render_done timeout (${renderId})`);
|
||||
worker.removeMessageHandler(renderId);
|
||||
}, 2000);
|
||||
|
||||
this.renderInFlight = false;
|
||||
this.renderCooldownUntilMs = performance.now() + 250;
|
||||
this.lastSentViewSize = null;
|
||||
this.lastSentViewTransform = null;
|
||||
}, 15000);
|
||||
|
||||
worker.addMessageHandler(renderId, (m: any) => {
|
||||
if (m?.type !== "render_done") {
|
||||
return;
|
||||
}
|
||||
clearTimeout(timeout);
|
||||
const startedAt = typeof m.startedAt === "number" ? m.startedAt : NaN;
|
||||
const endedAt = typeof m.endedAt === "number" ? m.endedAt : NaN;
|
||||
const startedAtWallMs =
|
||||
typeof m.startedAtWallMs === "number" ? m.startedAtWallMs : NaN;
|
||||
const endedAtWallMs =
|
||||
typeof m.endedAtWallMs === "number" ? m.endedAtWallMs : NaN;
|
||||
const echoedSentAtWallMs =
|
||||
typeof m.sentAtWallMs === "number" ? m.sentAtWallMs : sentAtWallMs;
|
||||
if (
|
||||
Number.isFinite(startedAt) &&
|
||||
Number.isFinite(endedAt) &&
|
||||
Number.isFinite(startedAtWallMs) &&
|
||||
Number.isFinite(endedAtWallMs) &&
|
||||
Number.isFinite(echoedSentAtWallMs)
|
||||
) {
|
||||
const queueMs = startedAtWallMs - echoedSentAtWallMs;
|
||||
const renderMs = endedAt - startedAt;
|
||||
const totalMs = endedAtWallMs - echoedSentAtWallMs;
|
||||
const breakdown =
|
||||
typeof m.renderCpuMs === "number" ||
|
||||
typeof m.renderGpuWaitMs === "number" ||
|
||||
typeof m.renderWaitPrevGpuMs === "number" ||
|
||||
typeof m.renderGetTextureMs === "number"
|
||||
? {
|
||||
waitPrevGpuMs:
|
||||
typeof m.renderWaitPrevGpuMs === "number"
|
||||
? Math.round(m.renderWaitPrevGpuMs)
|
||||
: undefined,
|
||||
waitPrevGpuTimedOut:
|
||||
typeof m.renderWaitPrevGpuTimedOut === "boolean"
|
||||
? m.renderWaitPrevGpuTimedOut
|
||||
: undefined,
|
||||
cpuMs:
|
||||
typeof m.renderCpuMs === "number"
|
||||
? Math.round(m.renderCpuMs)
|
||||
: undefined,
|
||||
getTextureMs:
|
||||
typeof m.renderGetTextureMs === "number"
|
||||
? Math.round(m.renderGetTextureMs)
|
||||
: undefined,
|
||||
gpuWaitMs:
|
||||
typeof m.renderGpuWaitMs === "number"
|
||||
? Math.round(m.renderGpuWaitMs)
|
||||
: undefined,
|
||||
gpuWaitTimedOut:
|
||||
typeof m.renderGpuWaitTimedOut === "boolean"
|
||||
? m.renderGpuWaitTimedOut
|
||||
: undefined,
|
||||
}
|
||||
: undefined;
|
||||
if (totalMs > 1000 || queueMs > 1000 || renderMs > 1000) {
|
||||
console.warn("worker render timing", {
|
||||
id: renderId,
|
||||
queueMs: Math.round(queueMs),
|
||||
renderMs: Math.round(renderMs),
|
||||
totalMs: Math.round(totalMs),
|
||||
breakdown,
|
||||
});
|
||||
}
|
||||
}
|
||||
this.renderInFlight = false;
|
||||
});
|
||||
} else {
|
||||
|
||||
@@ -2,9 +2,12 @@ import { LitElement, css, html } from "lit";
|
||||
import { customElement, property, state } from "lit/decorators.js";
|
||||
import { EventBus } from "../../../core/EventBus";
|
||||
import { UserSettings } from "../../../core/game/UserSettings";
|
||||
import type { WorkerMetricsMessage } from "../../../core/worker/WorkerMessages";
|
||||
import {
|
||||
SetWorkerDebugEvent,
|
||||
TickMetricsEvent,
|
||||
TogglePerformanceOverlayEvent,
|
||||
WorkerMetricsEvent,
|
||||
} from "../../InputHandler";
|
||||
import { translateText } from "../../Utils";
|
||||
import { FrameProfiler } from "../FrameProfiler";
|
||||
@@ -42,6 +45,18 @@ export class PerformanceOverlay extends LitElement implements Layer {
|
||||
@state()
|
||||
private isVisible: boolean = false;
|
||||
|
||||
@state()
|
||||
private workerMetrics: WorkerMetricsMessage | null = null;
|
||||
|
||||
@state()
|
||||
private workerMetricsAgeMs: number = 0;
|
||||
|
||||
@state()
|
||||
private workerIncludeTrace: boolean = false;
|
||||
|
||||
@state()
|
||||
private workerIntervalMs: number = 1000;
|
||||
|
||||
@state()
|
||||
private isDragging: boolean = false;
|
||||
|
||||
@@ -60,6 +75,7 @@ export class PerformanceOverlay extends LitElement implements Layer {
|
||||
private dragStart: { x: number; y: number } = { x: 0, y: 0 };
|
||||
private tickExecutionTimes: number[] = [];
|
||||
private tickDelayTimes: number[] = [];
|
||||
private lastWorkerMetricsWallMs: number = 0;
|
||||
|
||||
private copyStatusTimeoutId: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
@@ -232,11 +248,24 @@ export class PerformanceOverlay extends LitElement implements Layer {
|
||||
this.eventBus.on(TickMetricsEvent, (event: TickMetricsEvent) => {
|
||||
this.updateTickMetrics(event.tickExecutionDuration, event.tickDelay);
|
||||
});
|
||||
this.eventBus.on(WorkerMetricsEvent, (event: WorkerMetricsEvent) => {
|
||||
this.workerMetrics = event.metrics;
|
||||
this.lastWorkerMetricsWallMs = Date.now();
|
||||
this.workerMetricsAgeMs = 0;
|
||||
this.requestUpdate();
|
||||
});
|
||||
}
|
||||
|
||||
setVisible(visible: boolean) {
|
||||
this.isVisible = visible;
|
||||
FrameProfiler.setEnabled(visible);
|
||||
this.eventBus.emit(
|
||||
new SetWorkerDebugEvent({
|
||||
enabled: visible,
|
||||
intervalMs: this.workerIntervalMs,
|
||||
includeTrace: this.workerIncludeTrace,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
private handleClose() {
|
||||
@@ -326,10 +355,21 @@ export class PerformanceOverlay extends LitElement implements Layer {
|
||||
// Update FrameProfiler enabled state when visibility changes
|
||||
if (wasVisible !== this.isVisible) {
|
||||
FrameProfiler.setEnabled(this.isVisible);
|
||||
this.eventBus.emit(
|
||||
new SetWorkerDebugEvent({
|
||||
enabled: this.isVisible,
|
||||
intervalMs: this.workerIntervalMs,
|
||||
includeTrace: this.workerIncludeTrace,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
if (!this.isVisible) return;
|
||||
|
||||
if (this.lastWorkerMetricsWallMs > 0) {
|
||||
this.workerMetricsAgeMs = Date.now() - this.lastWorkerMetricsWallMs;
|
||||
}
|
||||
|
||||
const now = performance.now();
|
||||
|
||||
// Initialize timing on first call
|
||||
@@ -486,10 +526,99 @@ export class PerformanceOverlay extends LitElement implements Layer {
|
||||
executionSamples: [...this.tickExecutionTimes],
|
||||
delaySamples: [...this.tickDelayTimes],
|
||||
},
|
||||
worker: {
|
||||
enabled: this.isVisible,
|
||||
includeTrace: this.workerIncludeTrace,
|
||||
intervalMs: this.workerIntervalMs,
|
||||
lastMetricsAgeMs: this.workerMetricsAgeMs,
|
||||
metrics: this.workerMetrics,
|
||||
},
|
||||
layers: this.layerBreakdown.map((layer) => ({ ...layer })),
|
||||
};
|
||||
}
|
||||
|
||||
private getWorkerKeyStats(metrics: WorkerMetricsMessage | null): {
|
||||
loopLagAvg: number;
|
||||
loopLagMax: number;
|
||||
simDelayAvg: number;
|
||||
simDelayMax: number;
|
||||
simExecAvg: number;
|
||||
simExecMax: number;
|
||||
rfQueueAvg: number | null;
|
||||
rfQueueMax: number | null;
|
||||
rfHandlerAvg: number | null;
|
||||
rfHandlerMax: number | null;
|
||||
traceLines: string[];
|
||||
} {
|
||||
if (!metrics) {
|
||||
return {
|
||||
loopLagAvg: 0,
|
||||
loopLagMax: 0,
|
||||
simDelayAvg: 0,
|
||||
simDelayMax: 0,
|
||||
simExecAvg: 0,
|
||||
simExecMax: 0,
|
||||
rfQueueAvg: null,
|
||||
rfQueueMax: null,
|
||||
rfHandlerAvg: null,
|
||||
rfHandlerMax: null,
|
||||
traceLines: [],
|
||||
};
|
||||
}
|
||||
|
||||
const rfQueueAvg = metrics.msgQueueMsAvg?.["render_frame"];
|
||||
const rfQueueMax = metrics.msgQueueMsMax?.["render_frame"];
|
||||
const rfHandlerAvg = metrics.msgHandlerMsAvg?.["render_frame"];
|
||||
const rfHandlerMax = metrics.msgHandlerMsMax?.["render_frame"];
|
||||
const traceLines =
|
||||
metrics.trace && metrics.trace.length > 0 ? metrics.trace.slice(-5) : [];
|
||||
|
||||
return {
|
||||
loopLagAvg: metrics.eventLoopLagMsAvg,
|
||||
loopLagMax: metrics.eventLoopLagMsMax,
|
||||
simDelayAvg: metrics.simPumpDelayMsAvg,
|
||||
simDelayMax: metrics.simPumpDelayMsMax,
|
||||
simExecAvg: metrics.simPumpExecMsAvg,
|
||||
simExecMax: metrics.simPumpExecMsMax,
|
||||
rfQueueAvg: typeof rfQueueAvg === "number" ? rfQueueAvg : null,
|
||||
rfQueueMax: typeof rfQueueMax === "number" ? rfQueueMax : null,
|
||||
rfHandlerAvg: typeof rfHandlerAvg === "number" ? rfHandlerAvg : null,
|
||||
rfHandlerMax: typeof rfHandlerMax === "number" ? rfHandlerMax : null,
|
||||
traceLines,
|
||||
};
|
||||
}
|
||||
|
||||
private formatMs(v: number | null | undefined, digits: number = 1): string {
|
||||
if (v === null || v === undefined || !Number.isFinite(v)) return "—";
|
||||
return `${v.toFixed(digits)}ms`;
|
||||
}
|
||||
|
||||
private onWorkerTraceToggle(e: Event) {
|
||||
const target = e.target as HTMLInputElement;
|
||||
this.workerIncludeTrace = !!target.checked;
|
||||
this.eventBus.emit(
|
||||
new SetWorkerDebugEvent({
|
||||
enabled: this.isVisible,
|
||||
intervalMs: this.workerIntervalMs,
|
||||
includeTrace: this.workerIncludeTrace,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
private onWorkerIntervalChange(e: Event) {
|
||||
const target = e.target as HTMLSelectElement;
|
||||
const ms = Number.parseInt(target.value, 10);
|
||||
if (!Number.isFinite(ms) || ms <= 0) return;
|
||||
this.workerIntervalMs = ms;
|
||||
this.eventBus.emit(
|
||||
new SetWorkerDebugEvent({
|
||||
enabled: this.isVisible,
|
||||
intervalMs: this.workerIntervalMs,
|
||||
includeTrace: this.workerIncludeTrace,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
private clearCopyStatusTimeout() {
|
||||
if (this.copyStatusTimeoutId !== null) {
|
||||
clearTimeout(this.copyStatusTimeoutId);
|
||||
@@ -550,6 +679,8 @@ export class PerformanceOverlay extends LitElement implements Layer {
|
||||
? Math.max(...this.layerBreakdown.map((l) => l.avg))
|
||||
: 1;
|
||||
|
||||
const worker = this.getWorkerKeyStats(this.workerMetrics);
|
||||
|
||||
return html`
|
||||
<div
|
||||
class="performance-overlay ${this.isDragging ? "dragging" : ""}"
|
||||
@@ -596,6 +727,85 @@ export class PerformanceOverlay extends LitElement implements Layer {
|
||||
<span>${this.tickDelayAvg.toFixed(2)}ms</span>
|
||||
(max: <span>${this.tickDelayMax}ms</span>)
|
||||
</div>
|
||||
<div class="layers-section">
|
||||
<div class="performance-line">Worker</div>
|
||||
<div class="layer-row" style="margin-top: 4px;">
|
||||
<span class="layer-name">metrics age</span>
|
||||
<span class="layer-metrics"
|
||||
>${this.formatMs(this.workerMetricsAgeMs, 0)}</span
|
||||
>
|
||||
</div>
|
||||
<div class="layer-row">
|
||||
<span class="layer-name">event loop lag (avg / max)</span>
|
||||
<span class="layer-metrics"
|
||||
>${this.formatMs(worker.loopLagAvg)} /
|
||||
${this.formatMs(worker.loopLagMax, 0)}</span
|
||||
>
|
||||
</div>
|
||||
<div class="layer-row">
|
||||
<span class="layer-name">sim pump delay (avg / max)</span>
|
||||
<span class="layer-metrics"
|
||||
>${this.formatMs(worker.simDelayAvg)} /
|
||||
${this.formatMs(worker.simDelayMax, 0)}</span
|
||||
>
|
||||
</div>
|
||||
<div class="layer-row">
|
||||
<span class="layer-name">sim pump exec (avg / max)</span>
|
||||
<span class="layer-metrics"
|
||||
>${this.formatMs(worker.simExecAvg)} /
|
||||
${this.formatMs(worker.simExecMax, 0)}</span
|
||||
>
|
||||
</div>
|
||||
<div class="layer-row">
|
||||
<span class="layer-name">render_frame queue (avg / max)</span>
|
||||
<span class="layer-metrics"
|
||||
>${this.formatMs(worker.rfQueueAvg, 0)} /
|
||||
${this.formatMs(worker.rfQueueMax, 0)}</span
|
||||
>
|
||||
</div>
|
||||
<div class="layer-row">
|
||||
<span class="layer-name">render_frame handler (avg / max)</span>
|
||||
<span class="layer-metrics"
|
||||
>${this.formatMs(worker.rfHandlerAvg, 0)} /
|
||||
${this.formatMs(worker.rfHandlerMax, 0)}</span
|
||||
>
|
||||
</div>
|
||||
<div class="layer-row" style="margin-top: 4px;">
|
||||
<span class="layer-name">trace</span>
|
||||
<span class="layer-metrics">
|
||||
<label style="cursor: pointer;">
|
||||
<input
|
||||
type="checkbox"
|
||||
.checked=${this.workerIncludeTrace}
|
||||
@change=${this.onWorkerTraceToggle}
|
||||
/>
|
||||
include
|
||||
</label>
|
||||
<select
|
||||
style="margin-left: 8px;"
|
||||
.value=${String(this.workerIntervalMs)}
|
||||
@change=${this.onWorkerIntervalChange}
|
||||
>
|
||||
<option value="250">250ms</option>
|
||||
<option value="500">500ms</option>
|
||||
<option value="1000">1000ms</option>
|
||||
<option value="2000">2000ms</option>
|
||||
</select>
|
||||
</span>
|
||||
</div>
|
||||
${worker.traceLines.length
|
||||
? html`<div
|
||||
class="performance-line"
|
||||
style="margin-top: 4px; opacity: 0.85;"
|
||||
>
|
||||
<div
|
||||
style="white-space: pre-wrap; font-size: 10px; line-height: 1.2;"
|
||||
>
|
||||
${worker.traceLines.join("\n")}
|
||||
</div>
|
||||
</div>`
|
||||
: html``}
|
||||
</div>
|
||||
${this.layerBreakdown.length
|
||||
? html`<div class="layers-section">
|
||||
<div class="performance-line">
|
||||
|
||||
@@ -2,7 +2,6 @@ import { createCanvas } from "src/client/Utils";
|
||||
import { Theme } from "../../../core/configuration/Config";
|
||||
import { TileRef } from "../../../core/game/GameMap";
|
||||
import { GameView } from "../../../core/game/GameView";
|
||||
import { generateID } from "../../../core/Util";
|
||||
import { WorkerClient } from "../../../core/worker/WorkerClient";
|
||||
import {
|
||||
InitRendererMessage,
|
||||
@@ -16,7 +15,6 @@ import {
|
||||
SetPaletteMessage,
|
||||
SetPatternsEnabledMessage,
|
||||
SetShaderSettingsMessage,
|
||||
TickRendererMessage,
|
||||
ViewSize,
|
||||
ViewTransform,
|
||||
} from "../../../core/worker/WorkerMessages";
|
||||
@@ -44,6 +42,8 @@ export class TerritoryRendererProxy {
|
||||
private lastSentViewSize: ViewSize | null = null;
|
||||
private lastSentViewTransform: ViewTransform | null = null;
|
||||
private renderInFlight = false;
|
||||
private renderSeq = 0;
|
||||
private renderCooldownUntilMs = 0;
|
||||
|
||||
private constructor(
|
||||
private readonly game: GameView,
|
||||
@@ -386,25 +386,29 @@ export class TerritoryRendererProxy {
|
||||
}
|
||||
|
||||
tick(): void {
|
||||
const message: TickRendererMessage = {
|
||||
type: "tick_renderer",
|
||||
};
|
||||
this.sendToWorker(message);
|
||||
// No-op: worker renderer ticks from worker-side game_update.
|
||||
// Sending tick messages from the main thread duplicates GPU work and
|
||||
// can stall Firefox badly under load.
|
||||
}
|
||||
|
||||
render(): void {
|
||||
if (this.failed) {
|
||||
return;
|
||||
}
|
||||
if (performance.now() < this.renderCooldownUntilMs) {
|
||||
return;
|
||||
}
|
||||
if (this.renderInFlight) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.renderInFlight = true;
|
||||
const renderId = `render_${generateID()}`;
|
||||
const renderId = `render_${++this.renderSeq}`;
|
||||
const sentAtWallMs = Date.now();
|
||||
|
||||
const message: RenderFrameMessage = { type: "render_frame" };
|
||||
message.id = renderId;
|
||||
message.sentAtWallMs = sentAtWallMs;
|
||||
|
||||
if (
|
||||
!this.lastSentViewSize ||
|
||||
@@ -432,15 +436,84 @@ export class TerritoryRendererProxy {
|
||||
worker.removeMessageHandler(renderId);
|
||||
return;
|
||||
}
|
||||
this.renderInFlight = false;
|
||||
|
||||
console.warn(`render_done timeout (${renderId})`);
|
||||
worker.removeMessageHandler(renderId);
|
||||
}, 2000);
|
||||
|
||||
// Recover from lost/blocked frames without flooding the worker.
|
||||
this.renderInFlight = false;
|
||||
this.renderCooldownUntilMs = performance.now() + 250;
|
||||
|
||||
// Force a view resync on the next successful render.
|
||||
this.lastSentViewSize = null;
|
||||
this.lastSentViewTransform = null;
|
||||
}, 15000);
|
||||
|
||||
worker.addMessageHandler(renderId, (m: any) => {
|
||||
if (m?.type !== "render_done") {
|
||||
return;
|
||||
}
|
||||
clearTimeout(timeout);
|
||||
const startedAt = typeof m.startedAt === "number" ? m.startedAt : NaN;
|
||||
const endedAt = typeof m.endedAt === "number" ? m.endedAt : NaN;
|
||||
const startedAtWallMs =
|
||||
typeof m.startedAtWallMs === "number" ? m.startedAtWallMs : NaN;
|
||||
const endedAtWallMs =
|
||||
typeof m.endedAtWallMs === "number" ? m.endedAtWallMs : NaN;
|
||||
const echoedSentAtWallMs =
|
||||
typeof m.sentAtWallMs === "number" ? m.sentAtWallMs : sentAtWallMs;
|
||||
if (
|
||||
Number.isFinite(startedAt) &&
|
||||
Number.isFinite(endedAt) &&
|
||||
Number.isFinite(startedAtWallMs) &&
|
||||
Number.isFinite(endedAtWallMs) &&
|
||||
Number.isFinite(echoedSentAtWallMs)
|
||||
) {
|
||||
const queueMs = startedAtWallMs - echoedSentAtWallMs;
|
||||
const renderMs = endedAt - startedAt;
|
||||
const totalMs = endedAtWallMs - echoedSentAtWallMs;
|
||||
const breakdown =
|
||||
typeof m.renderCpuMs === "number" ||
|
||||
typeof m.renderGpuWaitMs === "number" ||
|
||||
typeof m.renderWaitPrevGpuMs === "number" ||
|
||||
typeof m.renderGetTextureMs === "number"
|
||||
? {
|
||||
waitPrevGpuMs:
|
||||
typeof m.renderWaitPrevGpuMs === "number"
|
||||
? Math.round(m.renderWaitPrevGpuMs)
|
||||
: undefined,
|
||||
waitPrevGpuTimedOut:
|
||||
typeof m.renderWaitPrevGpuTimedOut === "boolean"
|
||||
? m.renderWaitPrevGpuTimedOut
|
||||
: undefined,
|
||||
cpuMs:
|
||||
typeof m.renderCpuMs === "number"
|
||||
? Math.round(m.renderCpuMs)
|
||||
: undefined,
|
||||
getTextureMs:
|
||||
typeof m.renderGetTextureMs === "number"
|
||||
? Math.round(m.renderGetTextureMs)
|
||||
: undefined,
|
||||
gpuWaitMs:
|
||||
typeof m.renderGpuWaitMs === "number"
|
||||
? Math.round(m.renderGpuWaitMs)
|
||||
: undefined,
|
||||
gpuWaitTimedOut:
|
||||
typeof m.renderGpuWaitTimedOut === "boolean"
|
||||
? m.renderGpuWaitTimedOut
|
||||
: undefined,
|
||||
}
|
||||
: undefined;
|
||||
if (totalMs > 1000 || queueMs > 1000 || renderMs > 1000) {
|
||||
console.warn("worker render timing", {
|
||||
id: renderId,
|
||||
queueMs: Math.round(queueMs),
|
||||
renderMs: Math.round(renderMs),
|
||||
totalMs: Math.round(totalMs),
|
||||
breakdown,
|
||||
});
|
||||
}
|
||||
}
|
||||
this.renderInFlight = false;
|
||||
});
|
||||
} else {
|
||||
|
||||
+721
-387
File diff suppressed because it is too large
Load Diff
@@ -9,7 +9,12 @@ import { TileRef } from "../game/GameMap";
|
||||
import { ErrorUpdate, GameUpdateViewData } from "../game/GameUpdates";
|
||||
import { ClientID, GameStartInfo, Turn } from "../Schemas";
|
||||
import { generateID } from "../Util";
|
||||
import { TileContext, WorkerMessage } from "./WorkerMessages";
|
||||
import {
|
||||
SetWorkerDebugMessage,
|
||||
TileContext,
|
||||
WorkerMessage,
|
||||
WorkerMetricsMessage,
|
||||
} from "./WorkerMessages";
|
||||
|
||||
export class WorkerClient {
|
||||
private worker: Worker;
|
||||
@@ -18,6 +23,7 @@ export class WorkerClient {
|
||||
private gameUpdateCallback?: (
|
||||
update: GameUpdateViewData | ErrorUpdate,
|
||||
) => void;
|
||||
private workerMetricsCallback?: (metrics: WorkerMetricsMessage) => void;
|
||||
|
||||
constructor(
|
||||
private gameStartInfo: GameStartInfo,
|
||||
@@ -45,6 +51,10 @@ export class WorkerClient {
|
||||
}
|
||||
break;
|
||||
|
||||
case "worker_metrics":
|
||||
this.workerMetricsCallback?.(message);
|
||||
break;
|
||||
|
||||
case "initialized":
|
||||
case "renderer_ready":
|
||||
default:
|
||||
@@ -78,6 +88,13 @@ export class WorkerClient {
|
||||
* Post a message to the worker with optional transferables.
|
||||
*/
|
||||
postMessage(message: any, transfer?: Transferable[]): void {
|
||||
if (
|
||||
message &&
|
||||
typeof message === "object" &&
|
||||
typeof message.sentAtWallMs !== "number"
|
||||
) {
|
||||
message.sentAtWallMs = Date.now();
|
||||
}
|
||||
if (transfer && transfer.length > 0) {
|
||||
this.worker.postMessage(message, transfer);
|
||||
return;
|
||||
@@ -85,6 +102,23 @@ export class WorkerClient {
|
||||
this.worker.postMessage(message);
|
||||
}
|
||||
|
||||
onWorkerMetrics(callback?: (metrics: WorkerMetricsMessage) => void): void {
|
||||
this.workerMetricsCallback = callback;
|
||||
}
|
||||
|
||||
setWorkerDebug(config: {
|
||||
enabled: boolean;
|
||||
intervalMs?: number;
|
||||
includeTrace?: boolean;
|
||||
}): void {
|
||||
this.postMessage({
|
||||
type: "set_worker_debug",
|
||||
enabled: config.enabled,
|
||||
intervalMs: config.intervalMs,
|
||||
includeTrace: config.includeTrace,
|
||||
} satisfies SetWorkerDebugMessage);
|
||||
}
|
||||
|
||||
initialize(): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const messageId = generateID();
|
||||
@@ -96,7 +130,7 @@ export class WorkerClient {
|
||||
}
|
||||
});
|
||||
|
||||
this.worker.postMessage({
|
||||
this.postMessage({
|
||||
type: "init",
|
||||
id: messageId,
|
||||
gameStartInfo: this.gameStartInfo,
|
||||
@@ -125,14 +159,14 @@ export class WorkerClient {
|
||||
throw new Error("Worker not initialized");
|
||||
}
|
||||
|
||||
this.worker.postMessage({
|
||||
this.postMessage({
|
||||
type: "turn",
|
||||
turn,
|
||||
});
|
||||
}
|
||||
|
||||
sendHeartbeat() {
|
||||
this.worker.postMessage({
|
||||
this.postMessage({
|
||||
type: "heartbeat",
|
||||
});
|
||||
}
|
||||
@@ -155,7 +189,7 @@ export class WorkerClient {
|
||||
}
|
||||
});
|
||||
|
||||
this.worker.postMessage({
|
||||
this.postMessage({
|
||||
type: "player_profile",
|
||||
id: messageId,
|
||||
playerID: playerID,
|
||||
@@ -181,7 +215,7 @@ export class WorkerClient {
|
||||
}
|
||||
});
|
||||
|
||||
this.worker.postMessage({
|
||||
this.postMessage({
|
||||
type: "player_border_tiles",
|
||||
id: messageId,
|
||||
playerID: playerID,
|
||||
@@ -211,7 +245,7 @@ export class WorkerClient {
|
||||
}
|
||||
});
|
||||
|
||||
this.worker.postMessage({
|
||||
this.postMessage({
|
||||
type: "player_actions",
|
||||
id: messageId,
|
||||
playerID: playerID,
|
||||
@@ -247,7 +281,7 @@ export class WorkerClient {
|
||||
}
|
||||
});
|
||||
|
||||
this.worker.postMessage({
|
||||
this.postMessage({
|
||||
type: "attack_average_position",
|
||||
id: messageId,
|
||||
playerID: playerID,
|
||||
@@ -277,7 +311,7 @@ export class WorkerClient {
|
||||
}
|
||||
});
|
||||
|
||||
this.worker.postMessage({
|
||||
this.postMessage({
|
||||
type: "transport_ship_spawn",
|
||||
id: messageId,
|
||||
playerID: playerID,
|
||||
@@ -301,7 +335,7 @@ export class WorkerClient {
|
||||
}
|
||||
});
|
||||
|
||||
this.worker.postMessage({
|
||||
this.postMessage({
|
||||
type: "tile_context",
|
||||
id: messageId,
|
||||
tile,
|
||||
|
||||
@@ -42,12 +42,19 @@ export type WorkerMessageType =
|
||||
| "tick_renderer"
|
||||
| "render_frame"
|
||||
| "render_done"
|
||||
| "set_worker_debug"
|
||||
| "worker_metrics"
|
||||
| "renderer_metrics";
|
||||
|
||||
// Base interface for all messages
|
||||
interface BaseWorkerMessage {
|
||||
type: WorkerMessageType;
|
||||
id?: string;
|
||||
/**
|
||||
* Cross-thread timestamp (Date.now()) set by the sender when enqueuing the
|
||||
* message. Used for queue latency debugging.
|
||||
*/
|
||||
sentAtWallMs?: number;
|
||||
}
|
||||
|
||||
export interface HeartbeatMessage extends BaseWorkerMessage {
|
||||
@@ -258,6 +265,36 @@ export interface RenderFrameMessage extends BaseWorkerMessage {
|
||||
// Renderer messages from worker to main thread
|
||||
export interface RenderDoneMessage extends BaseWorkerMessage {
|
||||
type: "render_done";
|
||||
/**
|
||||
* Timestamp (performance.now()) in the worker right before starting work.
|
||||
*/
|
||||
startedAt?: number;
|
||||
/**
|
||||
* Timestamp (performance.now()) in the worker right after finishing work.
|
||||
*/
|
||||
endedAt?: number;
|
||||
/**
|
||||
* Echo of RenderFrameMessage.sentAtWallMs (if provided) so callers can
|
||||
* compute queue/processing latency without storing state.
|
||||
*/
|
||||
sentAtWallMs?: number;
|
||||
/**
|
||||
* Timestamps (Date.now()) in the worker. Use these for cross-thread latency
|
||||
* (Firefox may use a different time origin for performance.now()).
|
||||
*/
|
||||
startedAtWallMs?: number;
|
||||
endedAtWallMs?: number;
|
||||
|
||||
/**
|
||||
* Optional breakdown from the worker's renderAsync implementation.
|
||||
* All values are milliseconds.
|
||||
*/
|
||||
renderWaitPrevGpuMs?: number;
|
||||
renderCpuMs?: number;
|
||||
renderGetTextureMs?: number;
|
||||
renderGpuWaitMs?: number;
|
||||
renderWaitPrevGpuTimedOut?: boolean;
|
||||
renderGpuWaitTimedOut?: boolean;
|
||||
}
|
||||
|
||||
export interface RendererReadyMessage extends BaseWorkerMessage {
|
||||
@@ -271,6 +308,30 @@ export interface RendererMetricsMessage extends BaseWorkerMessage {
|
||||
computeMs: number;
|
||||
}
|
||||
|
||||
export interface SetWorkerDebugMessage extends BaseWorkerMessage {
|
||||
type: "set_worker_debug";
|
||||
enabled: boolean;
|
||||
intervalMs?: number;
|
||||
includeTrace?: boolean;
|
||||
}
|
||||
|
||||
export interface WorkerMetricsMessage extends BaseWorkerMessage {
|
||||
type: "worker_metrics";
|
||||
intervalMs: number;
|
||||
eventLoopLagMsAvg: number;
|
||||
eventLoopLagMsMax: number;
|
||||
simPumpDelayMsAvg: number;
|
||||
simPumpDelayMsMax: number;
|
||||
simPumpExecMsAvg: number;
|
||||
simPumpExecMsMax: number;
|
||||
msgCounts: Record<string, number>;
|
||||
msgHandlerMsAvg: Record<string, number>;
|
||||
msgHandlerMsMax: Record<string, number>;
|
||||
msgQueueMsAvg: Record<string, number>;
|
||||
msgQueueMsMax: Record<string, number>;
|
||||
trace?: string[];
|
||||
}
|
||||
|
||||
// Union types for type safety
|
||||
export type MainThreadMessage =
|
||||
| HeartbeatMessage
|
||||
@@ -295,6 +356,7 @@ export type MainThreadMessage =
|
||||
| RefreshPaletteMessage
|
||||
| RefreshTerrainMessage
|
||||
| TickRendererMessage
|
||||
| SetWorkerDebugMessage
|
||||
| RenderFrameMessage;
|
||||
|
||||
// Message send from worker
|
||||
@@ -309,4 +371,5 @@ export type WorkerMessage =
|
||||
| TransportShipSpawnResultMessage
|
||||
| RenderDoneMessage
|
||||
| RendererReadyMessage
|
||||
| RendererMetricsMessage;
|
||||
| RendererMetricsMessage
|
||||
| WorkerMetricsMessage;
|
||||
|
||||
@@ -28,6 +28,7 @@ export class WorkerTerritoryRenderer {
|
||||
private resources: GroundTruthData | null = null;
|
||||
private gameViewAdapter: GameViewAdapter | null = null;
|
||||
private ready = false;
|
||||
private lastGpuWork: Promise<void> | null = null;
|
||||
|
||||
// Compute passes
|
||||
private computePasses: ComputePass[] = [];
|
||||
@@ -62,6 +63,10 @@ export class WorkerTerritoryRenderer {
|
||||
private postSmoothingEnabled = false;
|
||||
private defensePostRange: number;
|
||||
private patternsEnabled = false;
|
||||
private tickPending = false;
|
||||
private tickRunning = false;
|
||||
private gpuWaitEnabled = true;
|
||||
private readonly gpuWaitTimeoutMs = 250;
|
||||
|
||||
/**
|
||||
* Initialize renderer with offscreen canvas and game data.
|
||||
@@ -548,9 +553,9 @@ export class WorkerTerritoryRenderer {
|
||||
* Perform one simulation tick.
|
||||
* Runs compute passes to update ground truth data.
|
||||
*/
|
||||
tick(): void {
|
||||
tick(): boolean {
|
||||
if (!this.ready || !this.device || !this.resources) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
this.resources.updateTickTiming(performance.now() / 1000);
|
||||
@@ -579,7 +584,7 @@ export class WorkerTerritoryRenderer {
|
||||
(this.defendedStrengthPass?.needsUpdate() ?? false);
|
||||
|
||||
if (!needsCompute) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
const encoder = this.device.device.createCommandEncoder();
|
||||
@@ -610,13 +615,61 @@ export class WorkerTerritoryRenderer {
|
||||
}
|
||||
|
||||
this.device.device.queue.submit([encoder.finish()]);
|
||||
return true;
|
||||
}
|
||||
|
||||
requestTick(): void {
|
||||
this.tickPending = true;
|
||||
if (this.tickRunning) {
|
||||
return;
|
||||
}
|
||||
this.tickRunning = true;
|
||||
void this.runTickLoop();
|
||||
}
|
||||
|
||||
private async runTickLoop(): Promise<void> {
|
||||
try {
|
||||
while (this.tickPending) {
|
||||
this.tickPending = false;
|
||||
|
||||
if (!this.ready || !this.device) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.gpuWaitEnabled && this.lastGpuWork) {
|
||||
const r = await this.awaitGpuWork(this.lastGpuWork);
|
||||
if (r.timedOut) {
|
||||
this.gpuWaitEnabled = false;
|
||||
}
|
||||
this.lastGpuWork = null;
|
||||
}
|
||||
|
||||
const submitted = this.tick();
|
||||
const q: any = this.device.device.queue as any;
|
||||
if (submitted && typeof q?.onSubmittedWorkDone === "function") {
|
||||
const p = q.onSubmittedWorkDone() as Promise<void>;
|
||||
this.lastGpuWork = p.catch(() => {});
|
||||
if (this.gpuWaitEnabled) {
|
||||
const r = await this.awaitGpuWork(this.lastGpuWork);
|
||||
if (r.timedOut) {
|
||||
this.gpuWaitEnabled = false;
|
||||
this.lastGpuWork = null;
|
||||
} else {
|
||||
this.lastGpuWork = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
this.tickRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Render one frame.
|
||||
* Runs render passes to draw to the canvas.
|
||||
*/
|
||||
render(): void {
|
||||
render(onGetTextureMs?: (ms: number) => void): void {
|
||||
if (
|
||||
!this.ready ||
|
||||
!this.device ||
|
||||
@@ -638,7 +691,11 @@ export class WorkerTerritoryRenderer {
|
||||
}
|
||||
|
||||
const encoder = this.device.device.createCommandEncoder();
|
||||
const getTexStart = performance.now();
|
||||
const swapchainView = this.device.context.getCurrentTexture().createView();
|
||||
if (onGetTextureMs) {
|
||||
onGetTextureMs(performance.now() - getTexStart);
|
||||
}
|
||||
|
||||
if (
|
||||
this.preSmoothingEnabled &&
|
||||
@@ -692,4 +749,95 @@ export class WorkerTerritoryRenderer {
|
||||
|
||||
this.device.device.queue.submit([encoder.finish()]);
|
||||
}
|
||||
|
||||
async renderAsync(): Promise<{
|
||||
waitPrevGpuMs: number;
|
||||
cpuMs: number;
|
||||
getTextureMs: number;
|
||||
gpuWaitMs: number;
|
||||
waitPrevGpuTimedOut: boolean;
|
||||
gpuWaitTimedOut: boolean;
|
||||
} | null> {
|
||||
if (!this.ready || !this.device) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let waitPrevGpuMs = 0;
|
||||
let cpuMs = 0;
|
||||
let getTextureMs = 0;
|
||||
let gpuWaitMs = 0;
|
||||
let waitPrevGpuTimedOut = false;
|
||||
let gpuWaitTimedOut = false;
|
||||
|
||||
if (this.gpuWaitEnabled && this.lastGpuWork) {
|
||||
const t0 = performance.now();
|
||||
const r = await this.awaitGpuWork(this.lastGpuWork);
|
||||
waitPrevGpuTimedOut = r.timedOut;
|
||||
if (r.timedOut) {
|
||||
this.gpuWaitEnabled = false;
|
||||
}
|
||||
waitPrevGpuMs = performance.now() - t0;
|
||||
this.lastGpuWork = null;
|
||||
}
|
||||
|
||||
const cpuStart = performance.now();
|
||||
this.render((ms) => {
|
||||
getTextureMs = ms;
|
||||
});
|
||||
cpuMs = performance.now() - cpuStart;
|
||||
|
||||
const q: any = this.device.device.queue as any;
|
||||
if (typeof q?.onSubmittedWorkDone !== "function") {
|
||||
this.lastGpuWork = null;
|
||||
return {
|
||||
waitPrevGpuMs,
|
||||
cpuMs,
|
||||
getTextureMs,
|
||||
gpuWaitMs,
|
||||
waitPrevGpuTimedOut,
|
||||
gpuWaitTimedOut,
|
||||
};
|
||||
}
|
||||
|
||||
const gpuStart = performance.now();
|
||||
const p = q.onSubmittedWorkDone() as Promise<void>;
|
||||
this.lastGpuWork = p.catch(() => {});
|
||||
if (this.gpuWaitEnabled) {
|
||||
const r = await this.awaitGpuWork(this.lastGpuWork);
|
||||
gpuWaitTimedOut = r.timedOut;
|
||||
if (r.timedOut) {
|
||||
this.gpuWaitEnabled = false;
|
||||
this.lastGpuWork = null;
|
||||
} else {
|
||||
this.lastGpuWork = null;
|
||||
}
|
||||
gpuWaitMs = performance.now() - gpuStart;
|
||||
}
|
||||
|
||||
return {
|
||||
waitPrevGpuMs,
|
||||
cpuMs,
|
||||
getTextureMs,
|
||||
gpuWaitMs,
|
||||
waitPrevGpuTimedOut,
|
||||
gpuWaitTimedOut,
|
||||
};
|
||||
}
|
||||
|
||||
private async awaitGpuWork(
|
||||
work: Promise<void>,
|
||||
): Promise<{ timedOut: boolean }> {
|
||||
let timeoutId: any = null;
|
||||
const timeout = new Promise<"timeout">((resolve) => {
|
||||
timeoutId = setTimeout(() => resolve("timeout"), this.gpuWaitTimeoutMs);
|
||||
});
|
||||
const result = await Promise.race([
|
||||
work.then(() => "done" as const),
|
||||
timeout,
|
||||
]);
|
||||
if (timeoutId !== null) {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
return { timedOut: result === "timeout" };
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user