mirror of
https://github.com/openfrontio/OpenFrontIO.git
synced 2026-07-28 13:00:29 +00:00
perf(map-generator): major CPU and memory optimisations (#3860)
## Description: Five performance improvements to the map generator, measured on three maps of increasing size. End-to-end time on `world` improved ~15×, heap allocations ~19×. | Map | Before | After | Speedup | |-----|--------|-------|---------| | bosphorusstraits (~612K tiles) | 578ms / 594MB | 45ms / 134MB | 13× / 4.4× | | world (~2M tiles) | 2333ms / 2128MB | 150ms / 553MB | 15× / 3.8× | | giantworldmap (~8M tiles) | 10701ms / 9300MB | 635ms / 2509MB | 17× / 3.7× | Changes (one commit each): - **`--workers` flag**: bounds concurrent map processing to limit peak memory - **Flat `[]bool` visited sets**: replaced `map[string]bool` keyed by `fmt.Sprintf` with flat `[]bool` indexed `x*height+y` — the dominant cost - **`neighborCoords` with stack buffer**: eliminates per-call slice allocation for neighbour lookups - **`Terrain` struct 24→16 bytes**: field reorder + `uint8` type for `TerrainType` - **Nil buffers early**: releases image/terrain arrays as soon as they're no longer needed - **BFS mark-visited on push**: each tile enters the queue once instead of up to 4×, halving queue memory also fixes a bug (according to Claude): Here's the bug: createMiniMap downscales by averaging/sampling 2x2 blocks, copying field values across — including Ocean=true from the parent scale. When a single connected ocean at 1x splits into multiple disconnected bodies at 4x (because narrow water channels disappear when you halve resolution), those smaller fragments still carry Ocean=true from the carryover. The 4x processWater call picks the new largest fragment and sets it to Ocean=true, but never clears the others — so multiple disconnected bodies end up flagged as Ocean. This PR's fix: before the new BFS pass, zero out every Ocean flag, so only the truly-largest body at the current scale ends up marked. It's incidental to the perf work but it's a real semantic change — the on-disk .bin files will differ from main on any map where ocean splits across downscaling. The PR doesn't mention it, which is why I flagged it. --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
committed by
evanpelle
co-authored by
Copilot
parent
30caea0c40
commit
365402bbc7
+12
-1
@@ -107,6 +107,9 @@ var maps = []struct {
|
||||
// mapsFlag holds the comma-separated list of map names passed via the --maps command-line argument.
|
||||
var mapsFlag string
|
||||
|
||||
// workersFlag controls how many maps are processed concurrently, bounding peak memory usage.
|
||||
var workersFlag int
|
||||
|
||||
// logFlags holds all the flags related to configuring the map-generator logging
|
||||
var logFlags LogFlags
|
||||
|
||||
@@ -248,15 +251,20 @@ func parseMapsFlag() (map[string]bool, error) {
|
||||
|
||||
// loadTerrainMaps manages the concurrent generation of all selected maps.
|
||||
// It spins up goroutines for each map and aggregates any errors.
|
||||
// Concurrency is bounded by --workers to cap peak memory usage.
|
||||
func loadTerrainMaps() error {
|
||||
if workersFlag < 1 {
|
||||
return fmt.Errorf("--workers must be >= 1, got %d", workersFlag)
|
||||
}
|
||||
selectedMaps, err := parseMapsFlag()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
errChan := make(chan error, len(maps))
|
||||
sem := make(chan struct{}, workersFlag)
|
||||
|
||||
// Process maps concurrently
|
||||
// Process maps concurrently, bounded by the semaphore
|
||||
for _, mapItem := range maps {
|
||||
if selectedMaps != nil && !selectedMaps[mapItem.Name] {
|
||||
continue
|
||||
@@ -265,6 +273,8 @@ func loadTerrainMaps() error {
|
||||
mapItem := mapItem
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
mapLogTag := slog.String("map", mapItem.Name)
|
||||
testLogTag := slog.Bool("isTest", mapItem.IsTest)
|
||||
logger := slog.Default().With(mapLogTag).With(testLogTag)
|
||||
@@ -293,6 +303,7 @@ func loadTerrainMaps() error {
|
||||
// It parses flags and triggers the map generation process.
|
||||
func main() {
|
||||
flag.StringVar(&mapsFlag, "maps", "", "optional comma-separated list of maps to process. ex: --maps=world,eastasia,big_plains")
|
||||
flag.IntVar(&workersFlag, "workers", 4, "number of maps to process concurrently. reduce to lower peak memory usage.")
|
||||
flag.StringVar(&logFlags.logLevel, "log-level", "", "Explicitly sets the log level to one of: ALL, DEBUG, INFO (default), WARN, ERROR.")
|
||||
flag.BoolVar(&logFlags.verbose, "verbose", false, "Adds additional logging and prefixes logs with the [mapname]. Alias of log-level=DEBUG.")
|
||||
flag.BoolVar(&logFlags.verbose, "v", false, "-verbose shorthand")
|
||||
|
||||
Reference in New Issue
Block a user