perf(map-generator): major CPU and memory optimisations (#3860)

## Description: Five performance improvements to the map generator, measured on three maps of increasing size. End-to-end time on `world` improved ~15×, heap allocations ~19×. | Map | Before | After | Speedup | |-----|--------|-------|---------| | bosphorusstraits (~612K tiles) | 578ms / 594MB | 45ms / 134MB | 13× / 4.4× | | world (~2M tiles) | 2333ms / 2128MB | 150ms / 553MB | 15× / 3.8× | | giantworldmap (~8M tiles) | 10701ms / 9300MB | 635ms / 2509MB | 17× / 3.7× | Changes (one commit each): - **`--workers` flag**: bounds concurrent map processing to limit peak memory - **Flat `[]bool` visited sets**: replaced `map[string]bool` keyed by `fmt.Sprintf` with flat `[]bool` indexed `x*height+y` — the dominant cost - **`neighborCoords` with stack buffer**: eliminates per-call slice allocation for neighbour lookups - **`Terrain` struct 24→16 bytes**: field reorder + `uint8` type for `TerrainType` - **Nil buffers early**: releases image/terrain arrays as soon as they're no longer needed - **BFS mark-visited on push**: each tile enters the queue once instead of up to 4×, halving queue memory also fixes a bug (according to Claude): Here's the bug: createMiniMap downscales by averaging/sampling 2x2 blocks, copying field values across — including Ocean=true from the parent scale. When a single connected ocean at 1x splits into multiple disconnected bodies at 4x (because narrow water channels disappear when you halve resolution), those smaller fragments still carry Ocean=true from the carryover. The 4x processWater call picks the new largest fragment and sets it to Ocean=true, but never clears the others — so multiple disconnected bodies end up flagged as Ocean. This PR's fix: before the new BFS pass, zero out every Ocean flag, so only the truly-largest body at the current scale ends up marked. It's incidental to the perf work but it's a real semantic change — the on-disk .bin files will differ from main on any map where ocean splits across downscaling. The PR doesn't mention it, which is why I flagged it. --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-06-21 07:40:43 +00:00 · 2026-05-07 04:38:49 +08:00
parent 6afedd8fd2
commit b0572ae83a
3 changed files with 80 additions and 51 deletions
@@ -108,6 +108,9 @@ var maps = []struct {
 // mapsFlag holds the comma-separated list of map names passed via the --maps command-line argument.
 var mapsFlag string

+// workersFlag controls how many maps are processed concurrently, bounding peak memory usage.
+var workersFlag int
+
 // logFlags holds all the flags related to configuring the map-generator logging
 var logFlags LogFlags

@@ -249,15 +252,20 @@ func parseMapsFlag() (map[string]bool, error) {

 // loadTerrainMaps manages the concurrent generation of all selected maps.
 // It spins up goroutines for each map and aggregates any errors.
+// Concurrency is bounded by --workers to cap peak memory usage.
 func loadTerrainMaps() error {
+	if workersFlag < 1 {
+		return fmt.Errorf("--workers must be >= 1, got %d", workersFlag)
+	}
 	selectedMaps, err := parseMapsFlag()
 	if err != nil {
 		return err
 	}
 	var wg sync.WaitGroup
 	errChan := make(chan error, len(maps))
+	sem := make(chan struct{}, workersFlag)

-	// Process maps concurrently
+	// Process maps concurrently, bounded by the semaphore
 	for _, mapItem := range maps {
 		if selectedMaps != nil && !selectedMaps[mapItem.Name] {
 			continue
@@ -266,6 +274,8 @@ func loadTerrainMaps() error {
 		mapItem := mapItem
 		go func() {
 			defer wg.Done()
+			sem <- struct{}{}
+			defer func() { <-sem }()
 			mapLogTag := slog.String("map", mapItem.Name)
 			testLogTag := slog.Bool("isTest", mapItem.IsTest)
 			logger := slog.Default().With(mapLogTag).With(testLogTag)
@@ -294,6 +304,7 @@ func loadTerrainMaps() error {
 // It parses flags and triggers the map generation process.
 func main() {
 	flag.StringVar(&mapsFlag, "maps", "", "optional comma-separated list of maps to process. ex: --maps=world,eastasia,big_plains")
+	flag.IntVar(&workersFlag, "workers", 4, "number of maps to process concurrently. reduce to lower peak memory usage.")
 	flag.StringVar(&logFlags.logLevel, "log-level", "", "Explicitly sets the log level to one of: ALL, DEBUG, INFO (default), WARN, ERROR.")
 	flag.BoolVar(&logFlags.verbose, "verbose", false, "Adds additional logging and prefixes logs with the [mapname].  Alias of log-level=DEBUG.")
 	flag.BoolVar(&logFlags.verbose, "v", false, "-verbose shorthand")
@@ -36,7 +36,7 @@ type Coord struct {
 }

 // TerrainType represents the classification of a map tile (e.g., Land or Water).
-type TerrainType int
+type TerrainType uint8

 // Enumeration of possible TerrainType values.
 const (
@@ -46,10 +46,13 @@ const (

 // Terrain represents the properties of a single map tile.
 // Magnitude represents elevation for Land (0-30) or distance to land for Water.
+// Fields are ordered to minimise alignment padding: float64 first (8 bytes,
+// offset 0), then three 1-byte fields, giving 16 bytes total vs 24 with the
+// original layout.
 type Terrain struct {
+	Magnitude float64
 	Type      TerrainType
 	Shoreline bool
-	Magnitude float64
 	Ocean     bool
 }

@@ -147,6 +150,9 @@ func GenerateMap(ctx context.Context, args GeneratorArgs) (MapResult, error) {
 			}
 		}
 	}
+	// Image data is no longer needed; release it for GC.
+	img = nil
+	args.ImageBuffer = nil

 	removeSmallIslands(ctx, terrain, minIslandSize, args.RemoveSmall)
 	processWater(ctx, terrain, args.RemoveSmall)
@@ -169,8 +175,11 @@ func GenerateMap(ctx context.Context, args GeneratorArgs) (MapResult, error) {
 	}

 	mapData, mapNumLandTiles := packTerrain(ctx, terrain)
+	terrain = nil
 	mapData4x, numLandTiles4x := packTerrain(ctx, terrain4x)
+	terrain4x = nil
 	mapData16x, numLandTiles16x := packTerrain(ctx, terrain16x)
+	terrain16x = nil

 	logger.Debug(fmt.Sprintf("Land Tile Count (1x): %d", mapNumLandTiles))
 	logger.Debug(fmt.Sprintf("Land Tile Count (4x): %d", numLandTiles4x))
@@ -272,24 +281,25 @@ func processShore(ctx context.Context, terrain [][]Terrain) []Coord {
 	width := len(terrain)
 	height := len(terrain[0])

+	var buf [4]Coord
 	for x := 0; x < width; x++ {
 		for y := 0; y < height; y++ {
 			tile := &terrain[x][y]
-			neighbors := getNeighbors(x, y, terrain)
 			tile.Shoreline = false
+			n := neighborCoords(x, y, width, height, &buf)

 			if tile.Type == Land {
 				// Land tile adjacent to water is shoreline
-				for _, n := range neighbors {
-					if n.Type == Water {
+				for _, c := range buf[:n] {
+					if terrain[c.X][c.Y].Type == Water {
 						tile.Shoreline = true
 						break
 					}
 				}
 			} else {
 				// Water tile adjacent to land is shoreline
-				for _, n := range neighbors {
-					if n.Type == Land {
+				for _, c := range buf[:n] {
+					if terrain[c.X][c.Y].Type == Land {
 						tile.Shoreline = true
 						shorelineWaters = append(shorelineWaters, Coord{X: x, Y: y})
 						break
@@ -351,37 +361,30 @@ func processDistToLand(ctx context.Context, shorelineWaters []Coord, terrain [][
 	}
 }

-// getNeighbors returns a list of Terrain tiles adjacent to the specified coordinates.
-func getNeighbors(x, y int, terrain [][]Terrain) []Terrain {
-	coords := getNeighborCoords(x, y, terrain)
-	neighbors := make([]Terrain, len(coords))
-	for i, coord := range coords {
-		neighbors[i] = terrain[coord.X][coord.Y]
-	}
-	return neighbors
-}
-
-// getNeighborCoords returns a list of valid adjacent coordinates (up, down, left, right).
-// It ensures that the returned coordinates are within the bounds of the terrain grid.
-func getNeighborCoords(x, y int, terrain [][]Terrain) []Coord {
-	width := len(terrain)
-	height := len(terrain[0])
-	var coords []Coord
-
+// neighborCoords fills out with the valid orthogonal neighbours of (x, y) and
+// returns the count. out must be a caller-allocated [4]Coord buffer; by
+// reusing the same buffer across calls the caller avoids any heap allocation.
+// Neighbours that would fall outside [0,width) × [0,height) are omitted, so
+// the count is 2 at corners, 3 on edges, and 4 in the interior.
+func neighborCoords(x, y, width, height int, out *[4]Coord) int {
+	n := 0
 	if x > 0 {
-		coords = append(coords, Coord{X: x - 1, Y: y})
+		out[n] = Coord{X: x - 1, Y: y}
+		n++
 	}
 	if x < width-1 {
-		coords = append(coords, Coord{X: x + 1, Y: y})
+		out[n] = Coord{X: x + 1, Y: y}
+		n++
 	}
 	if y > 0 {
-		coords = append(coords, Coord{X: x, Y: y - 1})
+		out[n] = Coord{X: x, Y: y - 1}
+		n++
 	}
 	if y < height-1 {
-		coords = append(coords, Coord{X: x, Y: y + 1})
+		out[n] = Coord{X: x, Y: y + 1}
+		n++
 	}
-
-	return coords
+	return n
 }

 // processWater identifies and processes bodies of water in the terrain.
@@ -391,7 +394,16 @@ func getNeighborCoords(x, y int, terrain [][]Terrain) []Coord {
 func processWater(ctx context.Context, terrain [][]Terrain, removeSmall bool) {
 	logger := LoggerFromContext(ctx)
 	logger.Info("Processing water bodies")
-	visited := make(map[string]bool)
+	width := len(terrain)
+	height := len(terrain[0])
+	visited := make([]bool, width*height)
+
+	// Clear any Ocean flags inherited from a previous scale's struct copy.
+	for x := 0; x < width; x++ {
+		for y := 0; y < height; y++ {
+			terrain[x][y].Ocean = false
+		}
+	}

 	type waterBody struct {
 		coords []Coord
@@ -401,11 +413,10 @@ func processWater(ctx context.Context, terrain [][]Terrain, removeSmall bool) {
 	var waterBodies []waterBody

 	// Find all distinct water bodies
-	for x := 0; x < len(terrain); x++ {
-		for y := 0; y < len(terrain[0]); y++ {
+	for x := 0; x < width; x++ {
+		for y := 0; y < height; y++ {
 			if terrain[x][y].Type == Water {
-				key := fmt.Sprintf("%d,%d", x, y)
-				if visited[key] {
+				if visited[x*height+y] {
 					continue
 				}

@@ -463,27 +474,32 @@ func processWater(ctx context.Context, terrain [][]Terrain, removeSmall bool) {

 // getArea performs a Breadth-First Search (BFS) to find a contiguous area of tiles
 // sharing the same TerrainType as the passed x,y coordinates.
-// The visited map is updated to prevent reprocessing tiles.
-func getArea(x, y int, terrain [][]Terrain, visited map[string]bool) []Coord {
+// visited is a flat bool slice of size width*height indexed by x*height+y
+// (column-major, matching the terrain[x][y] grid layout); it is updated to
+// prevent reprocessing tiles across multiple getArea calls.
+func getArea(x, y int, terrain [][]Terrain, visited []bool) []Coord {
+	width := len(terrain)
+	height := len(terrain[0])
 	targetType := terrain[x][y].Type
 	var area []Coord
+
+	visited[x*height+y] = true
 	queue := []Coord{{X: x, Y: y}}

+	var buf [4]Coord
 	for len(queue) > 0 {
 		coord := queue[0]
 		queue = queue[1:]

-		key := fmt.Sprintf("%d,%d", coord.X, coord.Y)
-		if visited[key] {
-			continue
-		}
-		visited[key] = true
-
 		if terrain[coord.X][coord.Y].Type == targetType {
 			area = append(area, coord)
-
-			neighborCoords := getNeighborCoords(coord.X, coord.Y, terrain)
-			queue = append(queue, neighborCoords...)
+			n := neighborCoords(coord.X, coord.Y, width, height, &buf)
+			for _, c := range buf[:n] {
+				if !visited[c.X*height+c.Y] {
+					visited[c.X*height+c.Y] = true
+					queue = append(queue, c)
+				}
+			}
 		}
 	}

@@ -499,7 +515,7 @@ func removeSmallIslands(ctx context.Context, terrain [][]Terrain, minSize int, r
 		return
 	}

-	visited := make(map[string]bool)
+	visited := make([]bool, len(terrain)*len(terrain[0]))

 	type landBody struct {
 		coords []Coord
@@ -509,11 +525,11 @@ func removeSmallIslands(ctx context.Context, terrain [][]Terrain, minSize int, r
 	var landBodies []landBody

 	// Find all distinct land bodies
+	height := len(terrain[0])
 	for x := 0; x < len(terrain); x++ {
-		for y := 0; y < len(terrain[0]); y++ {
+		for y := 0; y < height; y++ {
 			if terrain[x][y].Type == Land {
-				key := fmt.Sprintf("%d,%d", x, y)
-				if visited[key] {
+				if visited[x*height+y] {
 					continue
 				}

@@ -543,6 +559,8 @@ func removeSmallIslands(ctx context.Context, terrain [][]Terrain, minSize int, r
 }

 // packTerrain serializes the terrain grid into a byte slice.
+// The output buffer is row-major (y*width+x), matching the expected
+// raster scan order of the binary map format.
 // Each byte represents a single tile with bit flags:
 //   - Bit 7: Land (1) / Water (0)
 //   - Bit 6: Shoreline