perf: reduce core live-memory footprint by 45% on large maps (#4507)

## Summary

Reduces the simulation's steady-state memory footprint. On Giant World
Map at 20 game-minutes (12 000 ticks, 400 bots, seed `perf-default`),
live memory after a full GC drops **293 MB → 161 MB (−45%)**; unforced
peak heap drops **326 MB → 165 MB**. The simulation also runs ~10%
faster (85 → 94 ticks/s). The final game-state hash is **bit-identical**
(`57830793797434300`) — no behavior change.

## Measurement (first commit)

The full-game perf harness gains a footprint mode:

- `--footprint` — forces a full GC at every `--window` boundary and
records the live heap / ArrayBuffer / RSS curve across the game
(requires `NODE_OPTIONS=--expose-gc`).
- `--snapshot-at 0,2000,12000` — writes V8 `.heapsnapshot` files at
chosen ticks.
- `HeapSnapshotRetainers.ts` — attributes every heap node to its nearest
meaningfully-named retainer (e.g. `PlayerImpl._tiles`), plus prints
retainer chains for all nodes ≥128 KB. `HeapSnapshotSummary.ts` is a
streaming fallback for snapshots too large to `JSON.parse`.

Baseline attribution at tick 12 000: player `_tiles`/`_borderTiles` Sets
**83 MB**, GameMap `refToX`/`refToY` lookup tables **38 MB**, two
duplicate 30.5 MB visited-scratch arrays, trade-ship stepper paths **15
MB**, a construction-only flood-fill queue **9.5 MB**.

## Optimizations

**Map-sized buffers (second commit):**
- `GameMap.x()/y()` compute `ref % width` / `(ref / width) | 0` instead
of reading two per-tile Uint16 tables (−38 MB). The arithmetic is
cheaper than the tables' random-access cache misses — this is where the
speedup comes from.
- `PlayerExecution` and `SpatialQuery` each kept their own per-game
generation-stamped visited `Uint32Array`; both now share one via
`TileTraversalScratch` (−30 MB).
- `PathFinderStepper` stores numeric paths as `Uint32Array` (half the
bytes; steppers hold their full path for a unit's whole journey).
- `ConnectedComponents` frees its flood-fill queue after `initialize()`.

**Player tile sets (third commit):**
- New `TileSet`: insertion-ordered set of tile refs backed by a dense
`Uint32Array` plus an open-addressing hash index — ~12 bytes/element vs
~34 for a native `Set<number>`. Deletes tombstone; compaction is
deferred while iteration is in progress so positions never shift under
an iterator.
- Iteration semantics match `Set` exactly (insertion order, entries
added mid-iteration visited, deleted ones skipped, delete+re-add moves
to end) — the simulation relies on this order for determinism, and the
unchanged hash confirms it.
- `Player.borderTiles()` now returns `ReadonlyTileSet` (a native `Set`
still satisfies it structurally); `GameRunner.playerBorderTiles` copies
into a real `Set` since that result crosses the worker boundary via
structured clone.

## Footprint curve (giant world map, live MB after forced GC)

| checkpoint | before | after |
|---|---|---|
| spawn end | 20 + 100 buf | 20 + 55 buf |
| tick 6301 | 119 + 161 buf | 29 + 127 buf |
| tick 12301 | 130 + 161 buf | 32 + 129 buf |

## Validation

- Final hash `57830793797434300` identical across baseline / round 1 /
round 2 runs (12 000 ticks).
- Full suite passes (1798 + 126 tests), including new `TileSet` tests:
order semantics, mutation-during-iteration parity with `Set`, tombstone
compaction, and a 20 000-op randomized differential test against native
`Set`.
- Runs recorded in
`tests/perf/output/footprint-{baseline,round1,round2}-giant.txt`.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Evan
2026-07-04 15:25:29 -07:00
committed by GitHub
parent b0f85c5739
commit 7fa81c6bb9
16 changed files with 1022 additions and 105 deletions
+141
View File
@@ -0,0 +1,141 @@
import { describe, expect, it } from "vitest";
import { PseudoRandom } from "../src/core/PseudoRandom";
import { TileSet } from "../src/core/game/TileSet";
describe("TileSet", () => {
it("adds, reports membership and size", () => {
const s = new TileSet();
expect(s.size).toBe(0);
expect(s.has(5)).toBe(false);
s.add(5);
s.add(9);
s.add(5); // duplicate
expect(s.size).toBe(2);
expect(s.has(5)).toBe(true);
expect(s.has(9)).toBe(true);
expect(s.has(6)).toBe(false);
});
it("deletes and reports whether the value was present", () => {
const s = new TileSet([1, 2, 3]);
expect(s.delete(2)).toBe(true);
expect(s.delete(2)).toBe(false);
expect(s.delete(99)).toBe(false);
expect(s.size).toBe(2);
expect(s.has(2)).toBe(false);
expect([...s]).toEqual([1, 3]);
});
it("iterates in insertion order across all iteration surfaces", () => {
const values = [42, 7, 100000, 0, 13];
const s = new TileSet(values);
expect([...s]).toEqual(values);
expect(Array.from(s.values())).toEqual(values);
const seen: number[] = [];
s.forEach((t) => seen.push(t));
expect(seen).toEqual(values);
});
it("moves a value to the end on delete + re-add, matching Set", () => {
const s = new TileSet([1, 2, 3]);
s.delete(1);
s.add(1);
expect([...s]).toEqual([2, 3, 1]);
});
it("re-adding an existing value does not change its position", () => {
const s = new TileSet([1, 2, 3]);
s.add(1);
expect([...s]).toEqual([1, 2, 3]);
});
it("visits entries added during forEach, matching Set", () => {
const s = new TileSet([1, 2]);
const seen: number[] = [];
s.forEach((t) => {
seen.push(t);
if (t === 1) s.add(3);
});
expect(seen).toEqual([1, 2, 3]);
});
it("skips entries deleted during forEach, matching Set", () => {
const s = new TileSet([1, 2, 3]);
const seen: number[] = [];
s.forEach((t) => {
seen.push(t);
if (t === 1) s.delete(3);
});
expect(seen).toEqual([1, 2]);
});
it("supports deleting the current entry during iteration", () => {
const s = new TileSet([1, 2, 3]);
const seen: number[] = [];
for (const t of s) {
seen.push(t);
s.delete(t);
}
expect(seen).toEqual([1, 2, 3]);
expect(s.size).toBe(0);
});
it("preserves order through tombstone compaction", () => {
const s = new TileSet();
// Interleave adds and deletes well past the compaction thresholds.
for (let i = 0; i < 1000; i++) s.add(i);
for (let i = 0; i < 1000; i++) {
if (i % 3 !== 0) s.delete(i);
}
for (let i = 2000; i < 2100; i++) s.add(i);
const expected: number[] = [];
for (let i = 0; i < 1000; i++) {
if (i % 3 === 0) expected.push(i);
}
for (let i = 2000; i < 2100; i++) expected.push(i);
expect([...s]).toEqual(expected);
expect(s.size).toBe(expected.length);
for (const v of expected) expect(s.has(v)).toBe(true);
expect(s.has(1)).toBe(false);
});
it("clear empties the set", () => {
const s = new TileSet([1, 2, 3]);
s.clear();
expect(s.size).toBe(0);
expect(s.has(1)).toBe(false);
expect([...s]).toEqual([]);
s.add(7);
expect([...s]).toEqual([7]);
});
it("handles large tile refs (up to the 65535x65535 map bound)", () => {
const big = 65535 * 65535 - 1;
const s = new TileSet([big, 0, big - 1]);
expect(s.has(big)).toBe(true);
expect([...s]).toEqual([big, 0, big - 1]);
});
it("matches native Set behavior on a randomized operation sequence", () => {
const random = new PseudoRandom(12345);
const tileSet = new TileSet();
const reference = new Set<number>();
for (let op = 0; op < 20000; op++) {
const value = random.nextInt(0, 500);
if (random.chance(3)) {
expect(tileSet.delete(value)).toBe(reference.delete(value));
} else {
tileSet.add(value);
reference.add(value);
}
if (op % 500 === 0) {
expect(tileSet.size).toBe(reference.size);
expect([...tileSet]).toEqual([...reference]);
}
}
expect([...tileSet]).toEqual([...reference]);
for (let v = 0; v < 500; v++) {
expect(tileSet.has(v)).toBe(reference.has(v));
}
});
});
+75
View File
@@ -24,8 +24,16 @@
* [--seed perf-default] [--top 30] [--window 1000]
* [--no-cpu-profile] [--no-exec-profile]
* [--no-gc-profile] [--no-alloc-profile]
* [--footprint] [--snapshot-at 0,2000,12000]
*
* --footprint records the live heap (used heap after a forced full GC) at
* every --window boundary; it requires NODE_OPTIONS=--expose-gc.
* --snapshot-at writes .heapsnapshot files at the given game-phase ticks
* (0 = right after the spawn phase) for offline attribution; summarize them
* with tests/perf/fullgame/HeapSnapshotSummary.ts.
*/
import fs from "fs";
import v8 from "node:v8";
import path from "path";
import { fileURLToPath } from "url";
import { Config } from "../../../src/core/configuration/Config";
@@ -47,11 +55,13 @@ import { GameConfig, GameStartInfo } from "../../../src/core/Schemas";
import { simpleHash } from "../../../src/core/Util";
import {
AllocationSampler,
FootprintCheckpoint,
GcTracker,
HeapSampler,
HeapWindow,
summarizeAllocationProfile,
summarizeGcEvents,
takeFootprintCheckpoint,
} from "./GcProfiler";
import { NodeGameMapLoader } from "./NodeGameMapLoader";
import {
@@ -81,6 +91,9 @@ interface Options {
execProfile: boolean;
gcProfile: boolean;
allocProfile: boolean;
footprint: boolean;
snapshotAt: number[];
waterNukes: boolean;
}
function resolveMap(name: string): GameMapType {
@@ -109,6 +122,9 @@ function parseArgs(argv: string[]): Options {
execProfile: true,
gcProfile: true,
allocProfile: true,
footprint: false,
snapshotAt: [],
waterNukes: false,
};
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
@@ -154,6 +170,17 @@ function parseArgs(argv: string[]): Options {
case "--no-alloc-profile":
opts.allocProfile = false;
break;
case "--footprint":
opts.footprint = true;
break;
case "--snapshot-at":
opts.snapshotAt = next()
.split(",")
.map((v) => parseInt(v, 10));
break;
case "--water-nukes":
opts.waterNukes = true;
break;
default:
throw new Error(`unknown argument: ${arg}`);
}
@@ -203,6 +230,7 @@ async function main(): Promise<void> {
infiniteTroops: false,
instantBuild: false,
randomSpawn: false,
waterNukes: opts.waterNukes ? true : undefined,
};
const gameStart: GameStartInfo = {
gameID: opts.seed,
@@ -272,6 +300,28 @@ async function main(): Promise<void> {
gcTracker?.start();
const heapSampler = opts.gcProfile ? new HeapSampler() : null;
const footprints: FootprintCheckpoint[] = [];
const recordFootprint = (label: string): void => {
if (!opts.footprint) return;
const cp = takeFootprintCheckpoint(label);
if (cp === null) {
throw new Error(
"--footprint requires the gc() global; run with NODE_OPTIONS=--expose-gc",
);
}
footprints.push(cp);
};
const snapshotDir = path.join(PROJECT_ROOT, "tests/perf/output");
const writeSnapshot = (label: string): void => {
fs.mkdirSync(snapshotDir, { recursive: true });
const file = path.join(
snapshotDir,
`fullgame-${opts.map.replace(/\W+/g, "_")}-${opts.seed}-${label}.heapsnapshot`,
);
console.log(`Writing heap snapshot ${path.relative(PROJECT_ROOT, file)}`);
v8.writeHeapSnapshot(file);
};
let turnNumber = 0;
const runTick = (stats: TickStats): boolean => {
runner.addTurn({ turnNumber: turnNumber++, intents: [] });
@@ -302,6 +352,10 @@ async function main(): Promise<void> {
);
heapSampler?.closeWindow("spawn");
recordFootprint(`spawn (tick ${game.ticks() - 1})`);
if (opts.snapshotAt.includes(0)) {
writeSnapshot("tick0");
}
// Main game phase, under the CPU profiler and allocation sampler.
const cpuProfiler = opts.cpuProfile ? new CpuProfiler() : null;
@@ -328,6 +382,10 @@ async function main(): Promise<void> {
if ((i + 1) % opts.window === 0 || i === opts.ticks - 1) {
heapSampler?.closeWindow(`${windowStartTick}-${game.ticks() - 1}`);
windowStartTick = game.ticks();
recordFootprint(`tick ${game.ticks() - 1}`);
}
if (opts.snapshotAt.includes(i + 1)) {
writeSnapshot(`tick${i + 1}`);
}
}
const gamePhaseMs = performance.now() - gameStart_;
@@ -372,6 +430,23 @@ async function main(): Promise<void> {
summary.slowest.map((s) => `#${s.tick} (${fmtMs(s.ms)}ms)`).join(", "),
);
if (footprints.length > 0) {
console.log(`\n--- Live-heap footprint (after forced full GC) ---`);
console.log(
table(
["checkpoint", "live MB", "total MB", "ext MB", "arrbuf MB", "rss MB"],
footprints.map((cp) => [
cp.label,
fmtMB(cp.liveHeapBytes),
fmtMB(cp.totalHeapBytes),
fmtMB(cp.externalBytes),
fmtMB(cp.arrayBuffersBytes),
fmtMB(cp.rssBytes),
]),
),
);
}
if (opts.execProfile) {
console.log(`\n--- Time by Execution class ---`);
const rows = execProfiler.report();
+38
View File
@@ -274,3 +274,41 @@ export function summarizeAllocationProfile(
sites.sort((a, b) => b.selfBytes - a.selfBytes);
return { sites, totalBytes };
}
// ── Live-heap footprint checkpoints ──
export interface FootprintCheckpoint {
label: string;
/** used_heap_size after a forced full GC — the live set. */
liveHeapBytes: number;
totalHeapBytes: number;
externalBytes: number;
arrayBuffersBytes: number;
rssBytes: number;
}
/**
* Forces a full GC (twice, so objects freed by finalizers in the first pass
* are also collected) and returns the resulting heap statistics. Requires the
* process to run with --expose-gc; returns null otherwise.
*/
export function takeFootprintCheckpoint(
label: string,
): FootprintCheckpoint | null {
const gc = (globalThis as { gc?: () => void }).gc;
if (gc === undefined) {
return null;
}
gc();
gc();
const heap = v8.getHeapStatistics();
const mem = process.memoryUsage();
return {
label,
liveHeapBytes: heap.used_heap_size,
totalHeapBytes: heap.total_heap_size,
externalBytes: mem.external,
arrayBuffersBytes: mem.arrayBuffers,
rssBytes: mem.rss,
};
}
@@ -0,0 +1,189 @@
/**
* Retainer attribution for a V8 .heapsnapshot: aggregates every node's self
* size under a label derived from its retainer chain — the nearest ancestor
* with a project-meaningful constructor name plus the property path from it
* (e.g. "PlayerImpl._tiles.table"). Also lists the largest individual nodes
* with their full retainer chains.
*
* Loads the whole snapshot with JSON.parse, so only suitable for snapshots
* under V8's max string length (~500 MB); use HeapSnapshotSummary.ts for a
* flat by-type summary of bigger files.
*
* Usage:
* npx tsx tests/perf/fullgame/HeapSnapshotRetainers.ts <file.heapsnapshot> [top]
*/
import fs from "fs";
// Constructor names that identify a container, not an owner — the walk
// continues past these to find whose field the container is.
const GENERIC_NAMES = new Set([
"",
"Object",
"Array",
"Set",
"Map",
"WeakMap",
"WeakSet",
"ArrayBuffer",
"SharedArrayBuffer",
"DataView",
"Int8Array",
"Uint8Array",
"Uint8ClampedArray",
"Int16Array",
"Uint16Array",
"Int32Array",
"Uint32Array",
"Float32Array",
"Float64Array",
"BigInt64Array",
"BigUint64Array",
]);
function main(): void {
const file = process.argv[2];
const top = parseInt(process.argv[3] ?? "40", 10);
const snap = JSON.parse(fs.readFileSync(file, "utf8")) as {
snapshot: {
meta: {
node_fields: string[];
node_types: (string[] | string)[];
edge_fields: string[];
edge_types: (string[] | string)[];
};
node_count: number;
edge_count: number;
};
nodes: number[];
edges: number[];
strings: string[];
};
const { meta } = snap.snapshot;
const NF = meta.node_fields.length;
const N_TYPE = meta.node_fields.indexOf("type");
const N_NAME = meta.node_fields.indexOf("name");
const N_SIZE = meta.node_fields.indexOf("self_size");
const N_EDGES = meta.node_fields.indexOf("edge_count");
const nodeTypes = meta.node_types[N_TYPE] as string[];
const EF = meta.edge_fields.length;
const E_TYPE = meta.edge_fields.indexOf("type");
const E_NAME = meta.edge_fields.indexOf("name_or_index");
const E_TO = meta.edge_fields.indexOf("to_node");
const edgeTypes = meta.edge_types[E_TYPE] as string[];
const WEAK_EDGE = edgeTypes.indexOf("weak");
const ELEMENT_EDGE = edgeTypes.indexOf("element");
const HIDDEN_EDGE = edgeTypes.indexOf("hidden");
const { nodes, edges, strings } = snap;
const nodeCount = snap.snapshot.node_count;
// First retainer of each node (prefer non-weak edges), plus the edge name.
const retainer = new Int32Array(nodeCount).fill(-1);
const retainerWeak = new Uint8Array(nodeCount);
const retainerEdge = new Int32Array(nodeCount).fill(-1); // string idx or -1
let edgeIdx = 0;
for (let src = 0; src < nodeCount; src++) {
const count = nodes[src * NF + N_EDGES];
for (let e = 0; e < count; e++, edgeIdx += EF) {
const to = edges[edgeIdx + E_TO] / NF;
const type = edges[edgeIdx + E_TYPE];
const weak = type === WEAK_EDGE ? 1 : 0;
if (retainer[to] === -1 || (retainerWeak[to] === 1 && weak === 0)) {
retainer[to] = src;
retainerWeak[to] = weak;
retainerEdge[to] =
type === ELEMENT_EDGE || type === HIDDEN_EDGE
? -2 // numeric index — label as []
: edges[edgeIdx + E_NAME];
}
}
}
const nodeName = (i: number): string => strings[nodes[i * NF + N_NAME]];
const nodeType = (i: number): string => nodeTypes[nodes[i * NF + N_TYPE]];
const edgeLabel = (i: number): string =>
retainerEdge[i] === -2 ? "[]" : (strings[retainerEdge[i]] ?? "?");
// Label a node by its nearest non-generic named ancestor plus the property
// path from that ancestor (capped, deepest segments dropped first).
const labelOf = (i: number): string => {
const segments: string[] = [];
let cur = i;
for (let depth = 0; depth < 12; depth++) {
const parent = retainer[cur];
if (parent === -1) return `(root) ${segments.join(".")}`;
const t = nodeType(parent);
const name = nodeName(parent);
if (
(t === "object" || t === "closure" || t === "native") &&
!GENERIC_NAMES.has(name)
) {
return `${name}.${segments.slice(0, 3).join(".")}`;
}
if (t === "synthetic") {
return `(${name}) ${segments.slice(0, 3).join(".")}`;
}
segments.unshift(edgeLabel(cur));
cur = parent;
}
return `(deep) ${segments.slice(0, 3).join(".")}`;
};
interface Bucket {
bytes: number;
count: number;
}
const buckets = new Map<string, Bucket>();
let totalBytes = 0;
const big: { i: number; size: number }[] = [];
for (let i = 0; i < nodeCount; i++) {
const size = nodes[i * NF + N_SIZE];
if (size === 0) continue;
totalBytes += size;
const t = nodeType(i);
// Group bulk data types under their retainers; everything else by type.
const key =
t === "code" || t === "string" || t === "concatenated string"
? `(all ${t})`
: labelOf(i);
const b = buckets.get(key);
if (b) {
b.bytes += size;
b.count++;
} else {
buckets.set(key, { bytes: size, count: 1 });
}
if (size >= 128 * 1024) {
big.push({ i, size });
}
}
const fmtMB = (bytes: number): string => (bytes / 1024 / 1024).toFixed(2);
console.log(`${file}\nlive: ${fmtMB(totalBytes)} MB\n`);
console.log(`--- Top ${top} retainer groups by self size ---`);
const sorted = [...buckets.entries()].sort((a, b) => b[1].bytes - a[1].bytes);
for (const [label, b] of sorted.slice(0, top)) {
console.log(
`${fmtMB(b.bytes).padStart(9)} MB ${String(b.count).padStart(8)} ${label}`,
);
}
console.log(`\n--- Nodes ≥128KB with retainer chains ---`);
big.sort((a, b) => b.size - a.size);
for (const { i, size } of big.slice(0, top)) {
const chain: string[] = [];
let cur = i;
for (let depth = 0; depth < 8 && retainer[cur] !== -1; depth++) {
const parent = retainer[cur];
chain.push(`${nodeName(parent) || nodeType(parent)}.${edgeLabel(cur)}`);
cur = parent;
}
console.log(
`${fmtMB(size).padStart(9)} MB ${nodeType(i)} ${nodeName(i)}${chain.join(" ← ")}`,
);
}
}
main();
+250
View File
@@ -0,0 +1,250 @@
/**
* Summarizes a V8 .heapsnapshot file: total live bytes and the top heap
* consumers grouped by (node type, constructor/name), by self size.
*
* Snapshot files from a large heap are multi-GB JSON — far beyond V8's max
* string length — so this streams the file and parses just the `nodes` array
* (flat integers) and the `strings` table with a byte-level scanner.
*
* Usage:
* npx tsx tests/perf/fullgame/HeapSnapshotSummary.ts <file.heapsnapshot> [top]
*/
import fs from "fs";
interface Group {
typeIdx: number;
nameIdx: number; // -1 when the type's node names are per-instance content
count: number;
bytes: number;
}
// Node types whose per-node name is instance content (string payloads,
// function source positions, ...) rather than a meaningful grouping key.
const CONTENT_NAMED_TYPES = new Set([
"string",
"concatenated string",
"sliced string",
"number",
"bigint",
"symbol",
"regexp",
"code",
]);
async function main(): Promise<void> {
const file = process.argv[2];
const top = parseInt(process.argv[3] ?? "40", 10);
if (!file) {
console.error(
"usage: npx tsx tests/perf/fullgame/HeapSnapshotSummary.ts <file.heapsnapshot> [top]",
);
process.exit(1);
}
// ── Meta: parse the small "snapshot" header object from the file prefix ──
const fd = fs.openSync(file, "r");
const prefixBuf = Buffer.alloc(1 << 20);
const prefixLen = fs.readSync(fd, prefixBuf, 0, prefixBuf.length, 0);
fs.closeSync(fd);
const prefix = prefixBuf.subarray(0, prefixLen).toString("utf8");
const nodesKey = '"nodes":[';
const nodesIdx = prefix.indexOf(nodesKey);
if (nodesIdx < 0) {
throw new Error(`"nodes" array not found in the first 1MB of ${file}`);
}
const metaJson = prefix.slice(0, prefix.lastIndexOf(",", nodesIdx)) + "}";
const meta = JSON.parse(metaJson).snapshot.meta as {
node_fields: string[];
node_types: (string[] | string)[];
};
const fieldCount = meta.node_fields.length;
const typeField = meta.node_fields.indexOf("type");
const nameField = meta.node_fields.indexOf("name");
const sizeField = meta.node_fields.indexOf("self_size");
const typeNames = meta.node_types[typeField] as string[];
const contentNamedTypeIdx = new Set(
typeNames.flatMap((t, i) => (CONTENT_NAMED_TYPES.has(t) ? [i] : [])),
);
// ── Stream pass: aggregate the nodes array, then collect needed strings ──
const groups = new Map<number, Group>();
const groupKey = (typeIdx: number, nameIdx: number) =>
typeIdx * 0x100000000 + nameIdx + 1; // +1 so nameIdx -1 maps to 0
let totalBytes = 0;
let totalNodes = 0;
// Scanner state.
const SEEK_STRINGS = 0; // between the nodes array and the strings table
const IN_NODES = 1;
const STRINGS_BETWEEN = 2; // inside strings array, between tokens
const IN_STRING = 3;
const DONE = 4;
let state = IN_NODES;
// IN_NODES state: integer accumulator + current node's fields.
let cur = 0;
let hasCur = false;
const nodeVals = new Array<number>(fieldCount).fill(0);
let fieldIdx = 0;
const finishNumber = (): void => {
if (!hasCur) return;
nodeVals[fieldIdx] = cur;
cur = 0;
hasCur = false;
if (++fieldIdx === fieldCount) {
fieldIdx = 0;
totalNodes++;
const size = nodeVals[sizeField];
totalBytes += size;
const typeIdx = nodeVals[typeField];
const nameIdx = contentNamedTypeIdx.has(typeIdx)
? -1
: nodeVals[nameField];
const key = groupKey(typeIdx, nameIdx);
const g = groups.get(key);
if (g) {
g.count++;
g.bytes += size;
} else {
groups.set(key, { typeIdx, nameIdx, count: 1, bytes: size });
}
}
};
// SEEK_STRINGS state: match the `"strings":[` marker across chunk borders.
const stringsKey = Buffer.from('"strings":[');
let matchPos = 0;
// IN_STRING state: raw token bytes (with quotes) for JSON.parse.
let stringIdx = 0;
let escape = false;
let tokenChunks: Buffer[] = [];
let tokenStart = -1; // start of current token in current chunk, if wanted
let wantToken = false;
const names = new Map<number, string>();
const neededNames = new Set<number>();
const stream = fs.createReadStream(file, {
start: nodesIdx + nodesKey.length,
highWaterMark: 8 << 20,
});
for await (const chunk of stream as AsyncIterable<Buffer>) {
for (let i = 0; i < chunk.length; i++) {
const b = chunk[i];
switch (state) {
case IN_NODES:
if (b >= 0x30 && b <= 0x39) {
cur = cur * 10 + (b - 0x30);
hasCur = true;
} else {
finishNumber();
if (b === 0x5d) {
// "]" — end of nodes; now that groups are final, we know which
// string-table entries we need.
for (const g of groups.values()) {
if (g.nameIdx >= 0) neededNames.add(g.nameIdx);
}
state = SEEK_STRINGS;
}
}
break;
case SEEK_STRINGS:
if (b === stringsKey[matchPos]) {
if (++matchPos === stringsKey.length) {
state = STRINGS_BETWEEN;
}
} else {
matchPos = b === stringsKey[0] ? 1 : 0;
}
break;
case STRINGS_BETWEEN:
if (b === 0x22) {
state = IN_STRING;
escape = false;
wantToken = neededNames.has(stringIdx);
tokenChunks = [];
tokenStart = wantToken ? i : -1;
} else if (b === 0x5d) {
state = DONE;
}
break;
case IN_STRING:
if (escape) {
escape = false;
} else if (b === 0x5c) {
escape = true;
} else if (b === 0x22) {
if (wantToken) {
tokenChunks.push(chunk.subarray(tokenStart, i + 1));
names.set(
stringIdx,
JSON.parse(Buffer.concat(tokenChunks).toString("utf8")),
);
tokenChunks = [];
}
stringIdx++;
state = STRINGS_BETWEEN;
}
break;
case DONE:
break;
}
}
// Carry an in-progress wanted token across the chunk border.
if (state === IN_STRING && wantToken) {
tokenChunks.push(chunk.subarray(Math.max(tokenStart, 0)));
tokenStart = 0;
}
if (state === DONE) break;
}
// ── Report ──
const fmtMB = (bytes: number): string => (bytes / 1024 / 1024).toFixed(2);
const all = [...groups.values()].sort((a, b) => b.bytes - a.bytes);
console.log(
`${file}\nlive: ${fmtMB(totalBytes)} MB across ${totalNodes} nodes\n`,
);
const byType = new Map<number, { count: number; bytes: number }>();
for (const g of all) {
const t = byType.get(g.typeIdx) ?? { count: 0, bytes: 0 };
t.count += g.count;
t.bytes += g.bytes;
byType.set(g.typeIdx, t);
}
console.log("--- By node type ---");
for (const [typeIdx, t] of [...byType.entries()].sort(
(a, b) => b[1].bytes - a[1].bytes,
)) {
console.log(
`${fmtMB(t.bytes).padStart(10)} MB ${String(t.count).padStart(9)} ${typeNames[typeIdx]}`,
);
}
console.log(`\n--- Top ${top} by (type, name) self size ---`);
console.log(
`${"MB".padStart(10)} ${"%".padStart(5)} ${"count".padStart(9)} group`,
);
for (const g of all.slice(0, top)) {
const name =
g.nameIdx < 0
? `(${typeNames[g.typeIdx]} data)`
: (names.get(g.nameIdx) ?? `<string #${g.nameIdx}>`);
console.log(
`${fmtMB(g.bytes).padStart(10)} ${((g.bytes * 100) / totalBytes)
.toFixed(1)
.padStart(
5,
)} ${String(g.count).padStart(9)} ${typeNames[g.typeIdx]} ${name}`,
);
}
}
main().catch((err) => {
console.error(err);
process.exit(1);
});