  1200→                    for (auto& t : threads) t.join();
  1201→                }
  1202→                log_phase("    Dedup: per-cell sort+dedup", _dt, _dc);
  1203→
  1204→                // Step 4: Count total pairs
  1205→                size_t total_pairs = 0;
  1206→                for (auto& [_, ids] : data.cell_to_addrs) total_pairs += ids.size();
  1207→                log_phase("    Dedup: count pairs", _dt, _dc);
  1208→
  1209→                // Step 5: Build sorted pairs
  1210→                std::vector<CellItemPair> pairs;
  1211→                pairs.reserve(total_pairs);
  1212→                for (uint64_t cell_id : sorted_cells) {
  1213→                    auto& ids = data.cell_to_addrs[cell_id];
  1214→                    for (auto id : ids) pairs.push_back({cell_id, id});
  1215→                }
  1216→                data.sorted_addr_cells = std::move(pairs);
  1217→                log_phase("    Dedup: build sorted pairs", _dt, _dc);
  1218→            });
  1219→            auto f4 = std::async(std::launch::async, [&]{ deduplicate(data.cell_to_admin); });
  1220→            f2.get(); f4.get();
  1221→        }
  1222→
  1223→        // Save cache if requested
  1224→        if (!save_cache_path.empty()) {
  1225→            serialize_cache(data, save_cache_path);
  1226→        }
  1227→    }
  1228→
  1229→    // --- Rebuild hash maps from sorted pairs if needed for cache saving ---
  1230→    // (Continent filtering now uses sorted pairs directly, so no rebuild needed for that.)
  1231→    if (!save_cache_path.empty()) {
  1232→        // Parallel rebuild: split sorted pairs into chunks, each thread builds
  1233→        // a partial map, then merge. Sorted pairs are grouped by cell_id so we
  1234→        // can split at cell boundaries for zero-conflict parallel insertion.
  1235→        auto rebuild_map_parallel = [](const std::vector<CellItemPair>& sorted,
  1236→                                        std::unordered_map<uint64_t, std::vector<uint32_t>>& map) {
  1237→            if (sorted.empty() || !map.empty()) return;
  1238→            unsigned nthreads = std::thread::hardware_concurrency();
  1239→            if (nthreads == 0) nthreads = 4;
  1240→            size_t chunk = (sorted.size() + nthreads - 1) / nthreads;
  1241→
  1242→            // Find cell boundaries for clean splits
  1243→            std::vector<size_t> boundaries = {0};
  1244→            for (unsigned t = 1; t < nthreads; t++) {
  1245→                size_t target = t * chunk;
  1246→                if (target >= sorted.size()) break;
  1247→                // Advance to next cell boundary
  1248→                while (target < sorted.size() && sorted[target].cell_id == sorted[target-1].cell_id)
  1249→                    target++;
  1250→                if (target < sorted.size()) boundaries.push_back(target);
  1251→            }
  1252→            boundaries.push_back(sorted.size());
  1253→
  1254→            // Each thread builds its own sub-map
  1255→            std::vector<std::unordered_map<uint64_t, std::vector<uint32_t>>> sub_maps(boundaries.size() - 1);
  1256→            std::vector<std::thread> threads;
  1257→            for (size_t t = 0; t + 1 < boundaries.size(); t++) {
  1258→                threads.emplace_back([&, t]() {
  1259→                    auto& sub = sub_maps[t];
  1260→                    // Pre-estimate capacity
  1261→                    size_t n = boundaries[t+1] - boundaries[t];
  1262→                    sub.reserve(n / 3); // rough estimate of unique cells
  1263→                    for (size_t i = boundaries[t]; i < boundaries[t+1]; i++) {
  1264→                        sub[sorted[i].cell_id].push_back(sorted[i].item_id);
  1265→                    }
  1266→                });
  1267→            }
  1268→            for (auto& t : threads) t.join();
  1269→
  1270→            // Merge sub-maps into main map (no conflicts since splits are at cell boundaries)
  1271→            size_t total_cells = 0;
  1272→            for (auto& sub : sub_maps) total_cells += sub.size();
  1273→            map.reserve(total_cells);
  1274→            for (auto& sub : sub_maps) {
  1275→                for (auto& [k, v] : sub) {
  1276→                    map[k] = std::move(v);
  1277→                }
  1278→            }
  1279→        };
  1280→
  1281→        std::cerr << "Rebuilding cell maps for continent filtering..." << std::endl;
  1282→        auto f1 = std::async(std::launch::async, [&]{ rebuild_map_parallel(data.sorted_way_cells, data.cell_to_ways); });
  1283→        auto f2 = std::async(std::launch::async, [&]{ rebuild_map_parallel(data.sorted_interp_cells, data.cell_to_interps); });
  1284→        f1.get(); f2.get();
  1285→        log_phase("Rebuild cell maps", _pt, _cpu);
  1286→    }
  1287→
  1288→    // --- Write index files ---
  1289→    log_phase("Deduplication", _pt, _cpu);
  1290→    std::cerr << "Writing index files to " << output_dir << "..." << std::endl;
  1291→
  1292→    auto quality_dir_name = [](double scale) -> std::string {
  1293→        if (scale == 0) return "uncapped";
  1294→        char buf[32]; snprintf(buf, sizeof(buf), "q%.4g", scale);
  1295→        return buf;
  1296→    };
  1297→
  1298→    // Write quality variants in parallel (bounded concurrency).
  1299→    // Each variant does parallel simplification internally, so limit
  1300→    // concurrency to avoid over-subscribing CPU.
  1301→    auto write_qualities = [&](const ParsedData& d, const std::string& admin_dir) {
  1302→        constexpr unsigned max_concurrent_qualities = 3;
  1303→        std::atomic<unsigned> active{0};
  1304→        std::mutex qmtx;
  1305→        std::condition_variable qcv;
  1306→        std::vector<std::future<void>> qfutures;
  1307→
  1308→        for (double scale : quality_scales) {
  1309→            {
  1310→                std::unique_lock<std::mutex> lock(qmtx);
  1311→                qcv.wait(lock, [&]{ return active.load() < max_concurrent_qualities; });
  1312→            }
  1313→            active.fetch_add(1);
  1314→
  1315→            std::string qname = quality_dir_name(scale);
  1316→            std::string qdir = admin_dir + "/" + qname;
  1317→            qfutures.push_back(std::async(std::launch::async, [&, qdir, scale]() {
  1318→                write_quality_variant(d, admin_dir, qdir, scale);
  1319→                active.fetch_sub(1);
  1320→                qcv.notify_one();
  1321→            }));
  1322→        }
  1323→        for (auto& f : qfutures) f.get();
  1324→    };
  1325→
  1326→    // Write one region: modes + quality variants
  1327→    auto write_region = [&](const ParsedData& d, const std::string& base_dir) {
  1328→        ensure_dir(base_dir);
  1329→
  1330→        if (multi_output) {
  1331→            // Write all 3 modes in parallel (they read shared data, write to separate dirs)
  1332→            auto wf1 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/full", IndexMode::Full); });
  1333→            auto wf2 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/no-addresses", IndexMode::NoAddresses); });
  1334→            auto wf3 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/admin", IndexMode::AdminOnly); });
  1335→            wf1.get(); wf2.get(); wf3.get();
  1336→        } else {
  1337→            write_index(d, base_dir, mode);
  1338→        }
  1339→
  1340→        // Write quality variants under admin/ (or base_dir if not multi_output)
  1341→        if (multi_quality) {
  1342→            std::string admin_dir = multi_output ? base_dir + "/admin" : base_dir;
  1343→            std::cerr << "  Writing quality variants for " << base_dir << "..." << std::endl;
  1344→            write_qualities(d, admin_dir);
  1345→        }
  1346→    };
  1347→
  1348→    // Write planet (async — overlaps with continent filtering start)
  1349→    auto planet_future = std::async(std::launch::async, [&]() {
  1350→        write_region(data, output_dir);
  1351→    });
  1352→
  1353→    // Process continents with bounded concurrency, largest first.
  1354→    if (generate_continents) {
  1355→        // Pre-compute continent membership for each unique cell in sorted pairs.
  1356→        // One parallel scan replaces 8 × cell_in_bbox per cell during filtering.
  1357→        auto _pct = std::chrono::steady_clock::now();
  1358→        auto _pcc = CpuTicks::now();
  1359→
  1360→        // Load continent polygons for boundary-based filtering
  1361→        auto continent_polys = get_continent_polygons();
  1362→        std::cerr << "  Loaded " << continent_polys.size() << " continent boundary polygons" << std::endl;
  1363→
  1364→        // For each sorted pair array, build a parallel array of continent bitmasks.
  1365→        // Since pairs are sorted by cell_id, consecutive entries share the same mask.
  1366→        auto precompute_masks = [&continent_polys](const std::vector<CellItemPair>& sorted) -> std::vector<uint8_t> {
  1367→            if (sorted.empty()) return {};
  1368→            std::vector<uint8_t> masks(sorted.size(), 0);
  1369→
  1370→            unsigned nthreads = std::max(1u, std::thread::hardware_concurrency());
  1371→            size_t chunk = (sorted.size() + nthreads - 1) / nthreads;
  1372→            std::vector<size_t> bounds = {0};
  1373→            for (unsigned t = 1; t < nthreads; t++) {
  1374→                size_t target = t * chunk;
  1375→                if (target >= sorted.size()) break;
  1376→                while (target < sorted.size() && sorted[target].cell_id == sorted[target-1].cell_id)
  1377→                    target++;
  1378→                if (target < sorted.size()) bounds.push_back(target);
  1379→            }
  1380→            bounds.push_back(sorted.size());
  1381→
  1382→            std::vector<std::thread> threads;
  1383→            for (size_t th = 0; th + 1 < bounds.size(); th++) {
  1384→                threads.emplace_back([&, th]() {
  1385→                    for (size_t i = bounds[th]; i < bounds[th+1]; ) {
  1386→                        uint64_t cell_id = sorted[i].cell_id;
  1387→                        S2CellId cell(cell_id);
  1388→                        S2LatLng center = cell.ToLatLng();
  1389→                        double lat = center.lat().degrees();
  1390→                        double lng = center.lng().degrees();
  1391→                        uint8_t mask = 0;
  1392→                        // Test against continent polygons (not bboxes)
  1393→                        for (size_t ci = 0; ci < continent_polys.size() && ci < 8; ci++) {
  1394→                            if (point_in_polygon(lat, lng, continent_polys[ci].vertices))
  1395→                                mask |= (1u << ci);
  1396→                        }
  1397→                        while (i < bounds[th+1] && sorted[i].cell_id == cell_id) {
  1398→                            masks[i] = mask;
  1399→                            i++;

                    for (auto& t : threads) t.join();
                }
                log_phase("    Dedup: per-cell sort+dedup", _dt, _dc);

                // Step 4: Count total pairs
                size_t total_pairs = 0;
                for (auto& [_, ids] : data.cell_to_addrs) total_pairs += ids.size();
                log_phase("    Dedup: count pairs", _dt, _dc);

                // Step 5: Build sorted pairs
                std::vector<CellItemPair> pairs;
                pairs.reserve(total_pairs);
                for (uint64_t cell_id : sorted_cells) {
                    auto& ids = data.cell_to_addrs[cell_id];
                    for (auto id : ids) pairs.push_back({cell_id, id});
                }
                data.sorted_addr_cells = std::move(pairs);
                log_phase("    Dedup: build sorted pairs", _dt, _dc);
            });
            auto f4 = std::async(std::launch::async, [&]{ deduplicate(data.cell_to_admin); });
            f2.get(); f4.get();
        }

        // Save cache if requested
        if (!save_cache_path.empty()) {
            serialize_cache(data, save_cache_path);
        }
    }

    // --- Rebuild hash maps from sorted pairs if needed for cache saving ---
    // (Continent filtering now uses sorted pairs directly, so no rebuild needed for that.)
    if (!save_cache_path.empty()) {
        // Parallel rebuild: split sorted pairs into chunks, each thread builds
        // a partial map, then merge. Sorted pairs are grouped by cell_id so we
        // can split at cell boundaries for zero-conflict parallel insertion.
        auto rebuild_map_parallel = [](const std::vector<CellItemPair>& sorted,
                                        std::unordered_map<uint64_t, std::vector<uint32_t>>& map) {
            if (sorted.empty() || !map.empty()) return;
            unsigned nthreads = std::thread::hardware_concurrency();
            if (nthreads == 0) nthreads = 4;
            size_t chunk = (sorted.size() + nthreads - 1) / nthreads;

            // Find cell boundaries for clean splits
            std::vector<size_t> boundaries = {0};
            for (unsigned t = 1; t < nthreads; t++) {
                size_t target = t * chunk;
                if (target >= sorted.size()) break;
                // Advance to next cell boundary
                while (target < sorted.size() && sorted[target].cell_id == sorted[target-1].cell_id)
                    target++;
                if (target < sorted.size()) boundaries.push_back(target);
            }
            boundaries.push_back(sorted.size());

            // Each thread builds its own sub-map
            std::vector<std::unordered_map<uint64_t, std::vector<uint32_t>>> sub_maps(boundaries.size() - 1);
            std::vector<std::thread> threads;
            for (size_t t = 0; t + 1 < boundaries.size(); t++) {
                threads.emplace_back([&, t]() {
                    auto& sub = sub_maps[t];
                    // Pre-estimate capacity
                    size_t n = boundaries[t+1] - boundaries[t];
                    sub.reserve(n / 3); // rough estimate of unique cells
                    for (size_t i = boundaries[t]; i < boundaries[t+1]; i++) {
                        sub[sorted[i].cell_id].push_back(sorted[i].item_id);
                    }
                });
            }
            for (auto& t : threads) t.join();

            // Merge sub-maps into main map (no conflicts since splits are at cell boundaries)
            size_t total_cells = 0;
            for (auto& sub : sub_maps) total_cells += sub.size();
            map.reserve(total_cells);
            for (auto& sub : sub_maps) {
                for (auto& [k, v] : sub) {
                    map[k] = std::move(v);
                }
            }
        };

        std::cerr << "Rebuilding cell maps for continent filtering..." << std::endl;
        auto f1 = std::async(std::launch::async, [&]{ rebuild_map_parallel(data.sorted_way_cells, data.cell_to_ways); });
        auto f2 = std::async(std::launch::async, [&]{ rebuild_map_parallel(data.sorted_interp_cells, data.cell_to_interps); });
        f1.get(); f2.get();
        log_phase("Rebuild cell maps", _pt, _cpu);
    }

    // --- Write index files ---
    log_phase("Deduplication", _pt, _cpu);
    std::cerr << "Writing index files to " << output_dir << "..." << std::endl;

    auto quality_dir_name = [](double scale) -> std::string {
        if (scale == 0) return "uncapped";
        char buf[32]; snprintf(buf, sizeof(buf), "q%.4g", scale);
        return buf;
    };

    // Write quality variants in parallel (bounded concurrency).
    // Each variant does parallel simplification internally, so limit
    // concurrency to avoid over-subscribing CPU.
    auto write_qualities = [&](const ParsedData& d, const std::string& admin_dir) {
        constexpr unsigned max_concurrent_qualities = 3;
        std::atomic<unsigned> active{0};
        std::mutex qmtx;
        std::condition_variable qcv;
        std::vector<std::future<void>> qfutures;

        for (double scale : quality_scales) {
            {
                std::unique_lock<std::mutex> lock(qmtx);
                qcv.wait(lock, [&]{ return active.load() < max_concurrent_qualities; });
            }
            active.fetch_add(1);

            std::string qname = quality_dir_name(scale);
            std::string qdir = admin_dir + "/" + qname;
            qfutures.push_back(std::async(std::launch::async, [&, qdir, scale]() {
                write_quality_variant(d, admin_dir, qdir, scale);
                active.fetch_sub(1);
                qcv.notify_one();
            }));
        }
        for (auto& f : qfutures) f.get();
    };

    // Write one region: modes + quality variants
    auto write_region = [&](const ParsedData& d, const std::string& base_dir) {
        ensure_dir(base_dir);

        if (multi_output) {
            // Write all 3 modes in parallel (they read shared data, write to separate dirs)
            auto wf1 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/full", IndexMode::Full); });
            auto wf2 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/no-addresses", IndexMode::NoAddresses); });
            auto wf3 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/admin", IndexMode::AdminOnly); });
            wf1.get(); wf2.get(); wf3.get();
        } else {
            write_index(d, base_dir, mode);
        }

        // Write quality variants under admin/ (or base_dir if not multi_output)
        if (multi_quality) {
            std::string admin_dir = multi_output ? base_dir + "/admin" : base_dir;
            std::cerr << "  Writing quality variants for " << base_dir << "..." << std::endl;
            write_qualities(d, admin_dir);
        }
    };

    // Write planet (async — overlaps with continent filtering start)
    auto planet_future = std::async(std::launch::async, [&]() {
        write_region(data, output_dir);
    });

    // Process continents with bounded concurrency, largest first.
    if (generate_continents) {
        // Pre-compute continent membership for each unique cell in sorted pairs.
        // One parallel scan replaces 8 × cell_in_bbox per cell during filtering.
        auto _pct = std::chrono::steady_clock::now();
        auto _pcc = CpuTicks::now();

        // Load continent polygons for boundary-based filtering
        auto continent_polys = get_continent_polygons();
        std::cerr << "  Loaded " << continent_polys.size() << " continent boundary polygons" << std::endl;

        // For each sorted pair array, build a parallel array of continent bitmasks.
        // Since pairs are sorted by cell_id, consecutive entries share the same mask.
        auto precompute_masks = [&continent_polys](const std::vector<CellItemPair>& sorted) -> std::vector<uint8_t> {
            if (sorted.empty()) return {};
            std::vector<uint8_t> masks(sorted.size(), 0);

            unsigned nthreads = std::max(1u, std::thread::hardware_concurrency());
            size_t chunk = (sorted.size() + nthreads - 1) / nthreads;
            std::vector<size_t> bounds = {0};
            for (unsigned t = 1; t < nthreads; t++) {
                size_t target = t * chunk;
                if (target >= sorted.size()) break;
                while (target < sorted.size() && sorted[target].cell_id == sorted[target-1].cell_id)
                    target++;
                if (target < sorted.size()) bounds.push_back(target);
            }
            bounds.push_back(sorted.size());

            std::vector<std::thread> threads;
            for (size_t th = 0; th + 1 < bounds.size(); th++) {
                threads.emplace_back([&, th]() {
                    for (size_t i = bounds[th]; i < bounds[th+1]; ) {
                        uint64_t cell_id = sorted[i].cell_id;
                        S2CellId cell(cell_id);
                        S2LatLng center = cell.ToLatLng();
                        double lat = center.lat().degrees();
                        double lng = center.lng().degrees();
                        uint8_t mask = 0;
                        // Test against continent polygons (not bboxes)
                        for (size_t ci = 0; ci < continent_polys.size() && ci < 8; ci++) {
                            if (point_in_polygon(lat, lng, continent_polys[ci].vertices))
                                mask |= (1u << ci);
                        }
                        while (i < bounds[th+1] && sorted[i].cell_id == cell_id) {
                            masks[i] = mask;
                            i++;


Now I have a thorough understanding. Let me look at the types to understand the data structures.


     1→#pragma once
     2→
     3→#include <cstdint>
     4→#include <vector>
     5→
     6→// --- Binary format structs (must match server's index reader) ---
     7→
     8→struct WayHeader {
     9→    uint32_t node_offset;
    10→    uint8_t node_count;
    11→    uint32_t name_id;
    12→};
    13→
    14→struct AddrPoint {
    15→    float lat;
    16→    float lng;
    17→    uint32_t housenumber_id;
    18→    uint32_t street_id;
    19→};
    20→
    21→struct InterpWay {
    22→    uint32_t node_offset;
    23→    uint8_t node_count;
    24→    uint32_t street_id;
    25→    uint32_t start_number;
    26→    uint32_t end_number;
    27→    uint8_t interpolation;
    28→};
    29→
    30→struct AdminPolygon {
    31→    uint32_t vertex_offset;
    32→    uint32_t vertex_count;
    33→    uint32_t name_id;
    34→    uint8_t admin_level;
    35→    float area;
    36→    uint16_t country_code;
    37→};
    38→
    39→struct NodeCoord {
    40→    float lat;
    41→    float lng;
    42→};
    43→
    44→// Sorted cell-item pair for direct entry writing (avoids hash map)
    45→struct CellItemPair {
    46→    uint64_t cell_id;
    47→    uint32_t item_id;
    48→};
    49→
    50→// --- Constants ---
    51→
    52→static const uint32_t INTERIOR_FLAG = 0x80000000u;
    53→static const uint32_t ID_MASK       = 0x7FFFFFFFu;
    54→static const uint32_t NO_DATA       = 0xFFFFFFFFu;
    55→
    56→// Simplification mode
    57→enum class SimplifyMode { MaxVertices, ErrorBounded };
    58→
    59→// Processing limits
    60→static constexpr size_t MAX_BLOCK_QUEUE         = 64;    // producer-consumer queue depth
    61→static constexpr int    MAX_NODE_COUNT          = 255;   // uint8_t node_count
    62→static constexpr int    MAX_VERTEX_COUNT        = 0x7FFFFFFF; // uint32_t vertex_count
    63→static constexpr int    MAX_S2_CELLS_PER_POLY   = 200;   // S2RegionCoverer max_cells
    64→static constexpr int    BACKTRACK_CALL_BUDGET   = 100000; // per-ring backtracking limit
    65→static constexpr int    BACKTRACK_TOTAL_BUDGET  = 500000; // per-relation backtracking limit
    66→static constexpr int    BACKTRACK_MAX_DEPTH     = 200;    // recursion depth limit
    67→static constexpr size_t MAX_SUBWAYS_FOR_RETRY   = 30;     // only retry small relations
    68→static constexpr size_t MAX_NODE_ID_DEFAULT     = 15000000000ULL; // dense index capacity
    69→
    70→// --- Index mode ---
    71→
    72→enum class IndexMode { Full, NoAddresses, AdminOnly };
    73→
    74→// --- Deferred S2 work items ---
    75→
    76→struct DeferredWay {
    77→    uint32_t way_id;
    78→    uint32_t node_offset;
    79→    uint8_t node_count;
    80→};
    81→
    82→struct DeferredInterp {
    83→    uint32_t interp_id;
    84→    uint32_t node_offset;
    85→    uint8_t node_count;
    86→};
    87→
    88→// Collected relation data for parallel admin assembly
    89→struct CollectedRelation {
    90→    int64_t id;
    91→    uint8_t admin_level;
    92→    std::string name;
    93→    std::string country_code;
    94→    bool is_postal;
    95→    std::vector<std::pair<int64_t, std::string>> members; // (way_id, role)
    96→};
    97→

#pragma once

#include <cstdint>
#include <vector>

// --- Binary format structs (must match server's index reader) ---

struct WayHeader {
    uint32_t node_offset;
    uint8_t node_count;
    uint32_t name_id;
};

struct AddrPoint {
    float lat;
    float lng;
    uint32_t housenumber_id;
    uint32_t street_id;
};

struct InterpWay {
    uint32_t node_offset;
    uint8_t node_count;
    uint32_t street_id;
    uint32_t start_number;
    uint32_t end_number;
    uint8_t interpolation;
};

struct AdminPolygon {
    uint32_t vertex_offset;
    uint32_t vertex_count;
    uint32_t name_id;
    uint8_t admin_level;
    float area;
    uint16_t country_code;
};

struct NodeCoord {
    float lat;
    float lng;
};

// Sorted cell-item pair for direct entry writing (avoids hash map)
struct CellItemPair {
    uint64_t cell_id;
    uint32_t item_id;
};

// --- Constants ---

static const uint32_t INTERIOR_FLAG = 0x80000000u;
static const uint32_t ID_MASK       = 0x7FFFFFFFu;
static const uint32_t NO_DATA       = 0xFFFFFFFFu;

// Simplification mode
enum class SimplifyMode { MaxVertices, ErrorBounded };

// Processing limits
static constexpr size_t MAX_BLOCK_QUEUE         = 64;    // producer-consumer queue depth
static constexpr int    MAX_NODE_COUNT          = 255;   // uint8_t node_count
static constexpr int    MAX_VERTEX_COUNT        = 0x7FFFFFFF; // uint32_t vertex_count
static constexpr int    MAX_S2_CELLS_PER_POLY   = 200;   // S2RegionCoverer max_cells
static constexpr int    BACKTRACK_CALL_BUDGET   = 100000; // per-ring backtracking limit
static constexpr int    BACKTRACK_TOTAL_BUDGET  = 500000; // per-relation backtracking limit
static constexpr int    BACKTRACK_MAX_DEPTH     = 200;    // recursion depth limit
static constexpr size_t MAX_SUBWAYS_FOR_RETRY   = 30;     // only retry small relations
static constexpr size_t MAX_NODE_ID_DEFAULT     = 15000000000ULL; // dense index capacity

// --- Index mode ---

enum class IndexMode { Full, NoAddresses, AdminOnly };

// --- Deferred S2 work items ---

struct DeferredWay {
    uint32_t way_id;
    uint32_t node_offset;
    uint8_t node_count;
};

struct DeferredInterp {
    uint32_t interp_id;
    uint32_t node_offset;
    uint8_t node_count;
};

// Collected relation data for parallel admin assembly
struct CollectedRelation {
    int64_t id;
    uint8_t admin_level;
    std::string name;
    std::string country_code;
    bool is_postal;
    std::vector<std::pair<int64_t, std::string>> members; // (way_id, role)
};


Now let me look at how cell_index.cpp writes the files and how the string pool works.


     1→#pragma once
     2→
     3→#include <cstdint>
     4→#include <string>
     5→#include <unordered_map>
     6→#include <vector>
     7→
     8→class StringPool {
     9→public:
    10→    uint32_t intern(const std::string& s) {
    11→        auto it = index_.find(s);
    12→        if (it != index_.end()) {
    13→            return it->second;
    14→        }
    15→        uint32_t offset = static_cast<uint32_t>(data_.size());
    16→        index_[s] = offset;
    17→        data_.insert(data_.end(), s.begin(), s.end());
    18→        data_.push_back('\0');
    19→        return offset;
    20→    }
    21→
    22→    const std::vector<char>& data() const { return data_; }
    23→    std::vector<char>& mutable_data() { return data_; }
    24→
    25→private:
    26→    std::unordered_map<std::string, uint32_t> index_;
    27→    std::vector<char> data_;
    28→};
    29→

#pragma once

#include <cstdint>
#include <string>
#include <unordered_map>
#include <vector>

class StringPool {
public:
    uint32_t intern(const std::string& s) {
        auto it = index_.find(s);
        if (it != index_.end()) {
            return it->second;
        }
        uint32_t offset = static_cast<uint32_t>(data_.size());
        index_[s] = offset;
        data_.insert(data_.end(), s.begin(), s.end());
        data_.push_back('\0');
        return offset;
    }

    const std::vector<char>& data() const { return data_; }
    std::vector<char>& mutable_data() { return data_; }

private:
    std::unordered_map<std::string, uint32_t> index_;
    std::vector<char> data_;
};


     1→#pragma once
     2→
     3→#include <string>
     4→#include <unordered_map>
     5→#include <vector>
     6→
     7→#include "types.h"
     8→#include "parsed_data.h"
     9→
    10→std::vector<uint32_t> write_entries(
    11→    const std::string& path,
    12→    const std::vector<uint64_t>& sorted_cells,
    13→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map);
    14→
    15→std::vector<uint32_t> write_entries_from_sorted(
    16→    const std::string& path,
    17→    const std::vector<uint64_t>& sorted_cells,
    18→    const std::vector<CellItemPair>& sorted_pairs);
    19→
    20→void write_cell_index(
    21→    const std::string& cells_path,
    22→    const std::string& entries_path,
    23→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map);
    24→
    25→void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode);
    26→
    27→// Write admin polygon/vertex files with on-the-fly simplification at a given epsilon scale.
    28→// Scale 0 = uncapped (no simplification). Other files are symlinked/copied from source_dir.
    29→void write_quality_variant(const ParsedData& data, const std::string& source_dir,
    30→                           const std::string& output_dir, double epsilon_scale);
    31→

#pragma once

#include <string>
#include <unordered_map>
#include <vector>

#include "types.h"
#include "parsed_data.h"

std::vector<uint32_t> write_entries(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map);

std::vector<uint32_t> write_entries_from_sorted(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::vector<CellItemPair>& sorted_pairs);

void write_cell_index(
    const std::string& cells_path,
    const std::string& entries_path,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map);

void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode);

// Write admin polygon/vertex files with on-the-fly simplification at a given epsilon scale.
// Scale 0 = uncapped (no simplification). Other files are symlinked/copied from source_dir.
void write_quality_variant(const ParsedData& data, const std::string& source_dir,
                           const std::string& output_dir, double epsilon_scale);


     1→#include "cell_index.h"
     2→
     3→#include <algorithm>
     4→#include <atomic>
     5→#include <cstring>
     6→#include <fstream>
     7→#include <future>
     8→#include <iostream>
     9→#include <thread>
    10→#include <unordered_map>
    11→#include <unordered_set>
    12→
    13→#include <unistd.h>
    14→
    15→#include "geometry.h"
    16→#include "s2_helpers.h"
    17→
    18→std::vector<uint32_t> write_entries(
    19→    const std::string& path,
    20→    const std::vector<uint64_t>& sorted_cells,
    21→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
    22→) {
    23→    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
    24→    std::vector<CellRef> sorted_refs;
    25→    sorted_refs.reserve(cell_map.size());
    26→    for (auto& [id, ids] : cell_map) sorted_refs.push_back({id, &ids});
    27→    std::sort(sorted_refs.begin(), sorted_refs.end(),
    28→        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });
    29→
    30→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    31→    size_t total_size = 0;
    32→    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);
    33→
    34→    std::vector<char> buf;
    35→    buf.reserve(total_size);
    36→    uint32_t current = 0;
    37→    size_t ri = 0;
    38→    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
    39→        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
    40→        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
    41→        offsets[si] = current;
    42→        const auto& ids = *sorted_refs[ri].ids;
    43→        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
    44→        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
    45→                   reinterpret_cast<const char*>(&count) + sizeof(count));
    46→        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
    47→                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
    48→        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
    49→        ri++;
    50→    }
    51→    std::ofstream f(path, std::ios::binary);
    52→    f.write(buf.data(), buf.size());
    53→    return offsets;
    54→}
    55→
    56→std::vector<uint32_t> write_entries_from_sorted(
    57→    const std::string& path,
    58→    const std::vector<uint64_t>& sorted_cells,
    59→    const std::vector<CellItemPair>& sorted_pairs
    60→) {
    61→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    62→    if (sorted_pairs.empty()) {
    63→        std::ofstream f(path, std::ios::binary);
    64→        return offsets;
    65→    }
    66→
    67→    unsigned int nthreads = std::thread::hardware_concurrency();
    68→    if (nthreads == 0) nthreads = 4;
    69→    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;
    70→
    71→    struct ChunkResult {
    72→        std::vector<char> buf;
    73→        size_t cell_start, cell_end;
    74→        uint32_t local_size;
    75→    };
    76→    std::vector<ChunkResult> chunks(nthreads);
    77→
    78→    std::vector<std::thread> threads;
    79→    for (unsigned int t = 0; t < nthreads; t++) {
    80→        size_t cs = t * cells_per_chunk;
    81→        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
    82→        if (cs >= sorted_cells.size()) break;
    83→
    84→        threads.emplace_back([&, t, cs, ce]() {
    85→            auto& chunk = chunks[t];
    86→            chunk.cell_start = cs;
    87→            chunk.cell_end = ce;
    88→            chunk.local_size = 0;
    89→
    90→            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
    91→                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
    92→                    return p.cell_id < id;
    93→                }) - sorted_pairs.begin();
    94→
    95→            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
    96→                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
    97→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
    98→                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;
    99→
   100→                offsets[si] = chunk.local_size;
   101→                size_t start = pi;
   102→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
   103→                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(MAX_VERTEX_COUNT)));
   104→                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
   105→                size_t buf_pos = chunk.buf.size();
   106→                chunk.buf.resize(buf_pos + entry_size);
   107→                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
   108→                for (size_t k = start; k < pi; k++) {
   109→                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
   110→                           &sorted_pairs[k].item_id, sizeof(uint32_t));
   111→                }
   112→                chunk.local_size += entry_size;
   113→            }
   114→        });
   115→    }
   116→    for (auto& t : threads) t.join();
   117→
   118→    uint32_t global_offset = 0;
   119→    for (auto& chunk : chunks) {
   120→        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
   121→            if (offsets[si] != NO_DATA) offsets[si] += global_offset;
   122→        }
   123→        global_offset += chunk.local_size;
   124→    }
   125→
   126→    std::vector<char> buf;
   127→    buf.reserve(global_offset);
   128→    for (auto& chunk : chunks) buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());
   129→
   130→    std::ofstream f(path, std::ios::binary);
   131→    f.write(buf.data(), buf.size());
   132→    return offsets;
   133→}
   134→
   135→void write_cell_index(
   136→    const std::string& cells_path,
   137→    const std::string& entries_path,
   138→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
   139→) {
   140→    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(cell_map.begin(), cell_map.end());
   141→    std::sort(sorted.begin(), sorted.end());
   142→
   143→    { std::ofstream f(cells_path, std::ios::binary);
   144→      uint32_t current_offset = 0;
   145→      for (const auto& [cell_id, ids] : sorted) {
   146→          f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
   147→          f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
   148→          current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
   149→      } }
   150→

#include "cell_index.h"

#include <algorithm>
#include <atomic>
#include <cstring>
#include <fstream>
#include <future>
#include <iostream>
#include <thread>
#include <unordered_map>
#include <unordered_set>

#include <unistd.h>

#include "geometry.h"
#include "s2_helpers.h"

std::vector<uint32_t> write_entries(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
    std::vector<CellRef> sorted_refs;
    sorted_refs.reserve(cell_map.size());
    for (auto& [id, ids] : cell_map) sorted_refs.push_back({id, &ids});
    std::sort(sorted_refs.begin(), sorted_refs.end(),
        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });

    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    size_t total_size = 0;
    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);

    std::vector<char> buf;
    buf.reserve(total_size);
    uint32_t current = 0;
    size_t ri = 0;
    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
        offsets[si] = current;
        const auto& ids = *sorted_refs[ri].ids;
        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
                   reinterpret_cast<const char*>(&count) + sizeof(count));
        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
        ri++;
    }
    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

std::vector<uint32_t> write_entries_from_sorted(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::vector<CellItemPair>& sorted_pairs
) {
    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    if (sorted_pairs.empty()) {
        std::ofstream f(path, std::ios::binary);
        return offsets;
    }

    unsigned int nthreads = std::thread::hardware_concurrency();
    if (nthreads == 0) nthreads = 4;
    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;

    struct ChunkResult {
        std::vector<char> buf;
        size_t cell_start, cell_end;
        uint32_t local_size;
    };
    std::vector<ChunkResult> chunks(nthreads);

    std::vector<std::thread> threads;
    for (unsigned int t = 0; t < nthreads; t++) {
        size_t cs = t * cells_per_chunk;
        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
        if (cs >= sorted_cells.size()) break;

        threads.emplace_back([&, t, cs, ce]() {
            auto& chunk = chunks[t];
            chunk.cell_start = cs;
            chunk.cell_end = ce;
            chunk.local_size = 0;

            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
                    return p.cell_id < id;
                }) - sorted_pairs.begin();

            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;

                offsets[si] = chunk.local_size;
                size_t start = pi;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(MAX_VERTEX_COUNT)));
                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
                size_t buf_pos = chunk.buf.size();
                chunk.buf.resize(buf_pos + entry_size);
                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
                for (size_t k = start; k < pi; k++) {
                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
                           &sorted_pairs[k].item_id, sizeof(uint32_t));
                }
                chunk.local_size += entry_size;
            }
        });
    }
    for (auto& t : threads) t.join();

    uint32_t global_offset = 0;
    for (auto& chunk : chunks) {
        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
            if (offsets[si] != NO_DATA) offsets[si] += global_offset;
        }
        global_offset += chunk.local_size;
    }

    std::vector<char> buf;
    buf.reserve(global_offset);
    for (auto& chunk : chunks) buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());

    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

void write_cell_index(
    const std::string& cells_path,
    const std::string& entries_path,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(cell_map.begin(), cell_map.end());
    std::sort(sorted.begin(), sorted.end());

    { std::ofstream f(cells_path, std::ios::binary);
      uint32_t current_offset = 0;
      for (const auto& [cell_id, ids] : sorted) {
          f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
          f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
          current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
      } }


   150→
   151→    { std::ofstream f(entries_path, std::ios::binary);
   152→      for (const auto& [cell_id, ids] : sorted) {
   153→          uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
   154→          f.write(reinterpret_cast<const char*>(&count), sizeof(count));
   155→          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
   156→      } }
   157→}
   158→
   159→void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
   160→    ensure_dir(output_dir);
   161→    auto _wt = std::chrono::steady_clock::now();
   162→
   163→    bool write_streets = (mode != IndexMode::AdminOnly);
   164→    bool write_addresses = (mode == IndexMode::Full);
   165→
   166→    if (write_streets) {
   167→        std::vector<uint64_t> sorted_geo_cells;
   168→        {
   169→            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
   170→                std::vector<uint64_t> cells;
   171→                cells.reserve(pairs.size() / 2);
   172→                for (size_t i = 0; i < pairs.size(); ) {
   173→                    cells.push_back(pairs[i].cell_id);
   174→                    uint64_t cur = pairs[i].cell_id;
   175→                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
   176→                }
   177→                return cells;
   178→            };
   179→            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
   180→                std::vector<uint64_t> cells;
   181→                cells.reserve(m.size());
   182→                for (auto& [id, _] : m) cells.push_back(id);
   183→                std::sort(cells.begin(), cells.end());
   184→                return cells;
   185→            };
   186→
   187→            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
   188→            {
   189→                auto f1 = std::async(std::launch::async, [&] {
   190→                    return !data.sorted_way_cells.empty()
   191→                        ? extract_unique_cells(data.sorted_way_cells)
   192→                        : extract_from_map(data.cell_to_ways);
   193→                });
   194→                if (write_addresses) {
   195→                    auto f2 = std::async(std::launch::async, [&] {
   196→                        return !data.sorted_addr_cells.empty()
   197→                            ? extract_unique_cells(data.sorted_addr_cells)
   198→                            : extract_from_map(data.cell_to_addrs);
   199→                    });
   200→                    auto f3 = std::async(std::launch::async, [&] {
   201→                        return !data.sorted_interp_cells.empty()
   202→                            ? extract_unique_cells(data.sorted_interp_cells)
   203→                            : extract_from_map(data.cell_to_interps);
   204→                    });
   205→                    addr_cells = f2.get();
   206→                    interp_cells = f3.get();
   207→                }
   208→                way_cells = f1.get();
   209→            }
   210→
   211→            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
   212→            std::merge(way_cells.begin(), way_cells.end(), addr_cells.begin(), addr_cells.end(),
   213→                       std::back_inserter(sorted_geo_cells));
   214→            if (!interp_cells.empty()) {
   215→                std::vector<uint64_t> tmp;
   216→                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
   217→                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
   218→                           interp_cells.begin(), interp_cells.end(), std::back_inserter(tmp));
   219→                sorted_geo_cells = std::move(tmp);
   220→            }
   221→            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
   222→                                    sorted_geo_cells.end());
   223→        }
   224→
   225→        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
   226→        {
   227→            auto f1 = std::async(std::launch::async, [&]() {
   228→                if (!data.sorted_way_cells.empty())
   229→                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
   230→                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
   231→            });
   232→            if (write_addresses) {
   233→                auto f2 = std::async(std::launch::async, [&]() {
   234→                    if (!data.sorted_addr_cells.empty())
   235→                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
   236→                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
   237→                });
   238→                auto f3 = std::async(std::launch::async, [&]() {
   239→                    if (!data.sorted_interp_cells.empty())
   240→                        return write_entries_from_sorted(output_dir + "/interp_entries.bin", sorted_geo_cells, data.sorted_interp_cells);
   241→                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
   242→                });
   243→                addr_offsets = f2.get();
   244→                interp_offsets = f3.get();
   245→            }
   246→            street_offsets = f1.get();
   247→        }
   248→
   249→        log_phase("  Write: entry files", _wt);
   250→        {
   251→            size_t n = sorted_geo_cells.size();
   252→            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
   253→            std::vector<char> buf(n * row_size);
   254→            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
   255→            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);
   256→
   257→            unsigned int nthreads = std::thread::hardware_concurrency();
   258→            if (nthreads == 0) nthreads = 4;
   259→            size_t chunk = (n + nthreads - 1) / nthreads;
   260→            std::vector<std::thread> fill_threads;
   261→            for (unsigned int t = 0; t < nthreads; t++) {
   262→                size_t start = t * chunk;
   263→                size_t end = std::min(start + chunk, n);
   264→                if (start >= n) break;
   265→                fill_threads.emplace_back([&, start, end]() {
   266→                    char* ptr = buf.data() + start * row_size;
   267→                    for (size_t i = start; i < end; i++) {
   268→                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
   269→                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   270→                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   271→                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   272→                    }
   273→                });
   274→            }
   275→            for (auto& t : fill_threads) t.join();
   276→
   277→            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
   278→            f.write(buf.data(), buf.size());
   279→        }
   280→
   281→        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
   282→                  << data.ways.size() << " ways, " << data.addr_points.size() << " addrs, "
   283→                  << data.interp_ways.size() << " interps)" << std::endl;
   284→    }
   285→
   286→    log_phase("  Write: geo_cells.bin", _wt);
   287→    auto admin_future = std::async(std::launch::async, [&] {
   288→        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
   289→        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
   290→    });
   291→
   292→    std::vector<std::future<void>> write_futures;
   293→    if (write_streets) {
   294→        write_futures.push_back(std::async(std::launch::async, [&] {
   295→            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
   296→            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
   297→        }));
   298→        write_futures.push_back(std::async(std::launch::async, [&] {
   299→            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
   300→            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
   301→        }));
   302→        if (write_addresses) {
   303→            write_futures.push_back(std::async(std::launch::async, [&] {
   304→                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
   305→                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
   306→            }));
   307→            write_futures.push_back(std::async(std::launch::async, [&] {
   308→                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
   309→                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
   310→            }));
   311→            write_futures.push_back(std::async(std::launch::async, [&] {
   312→                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
   313→                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
   314→            }));
   315→        }
   316→    }
   317→    // If not writing all data types, compact string pool and remap references
   318→    // before writing admin_polygons and strings.bin.
   319→    std::vector<AdminPolygon> compact_admin;
   320→    std::vector<char> compact_strings;
   321→    const bool need_compact = (!write_streets || !write_addresses);
   322→
   323→    if (need_compact) {
   324→        // Collect referenced string offsets
   325→        std::unordered_set<uint32_t> used_offsets;
   326→        if (write_streets) {
   327→            for (const auto& w : data.ways) used_offsets.insert(w.name_id);
   328→            if (write_addresses) {
   329→                for (const auto& a : data.addr_points) {
   330→                    used_offsets.insert(a.housenumber_id);
   331→                    used_offsets.insert(a.street_id);
   332→                }
   333→                for (const auto& iw : data.interp_ways) used_offsets.insert(iw.street_id);
   334→            }
   335→        }
   336→        for (const auto& ap : data.admin_polygons) used_offsets.insert(ap.name_id);
   337→
   338→        // Build compact pool and remap
   339→        const auto& old_sp = data.string_pool.data();
   340→        std::vector<uint32_t> sorted_offs(used_offsets.begin(), used_offsets.end());
   341→        std::sort(sorted_offs.begin(), sorted_offs.end());
   342→        std::unordered_map<uint32_t, uint32_t> remap;
   343→        compact_strings.reserve(used_offsets.size() * 16);
   344→        for (uint32_t off : sorted_offs) {
   345→            remap[off] = static_cast<uint32_t>(compact_strings.size());
   346→            size_t len = std::strlen(old_sp.data() + off);
   347→            compact_strings.insert(compact_strings.end(), old_sp.data() + off,
   348→                old_sp.data() + off + len + 1);
   349→        }

    { std::ofstream f(entries_path, std::ios::binary);
      for (const auto& [cell_id, ids] : sorted) {
          uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
          f.write(reinterpret_cast<const char*>(&count), sizeof(count));
          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
      } }
}

void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
    ensure_dir(output_dir);
    auto _wt = std::chrono::steady_clock::now();

    bool write_streets = (mode != IndexMode::AdminOnly);
    bool write_addresses = (mode == IndexMode::Full);

    if (write_streets) {
        std::vector<uint64_t> sorted_geo_cells;
        {
            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
                std::vector<uint64_t> cells;
                cells.reserve(pairs.size() / 2);
                for (size_t i = 0; i < pairs.size(); ) {
                    cells.push_back(pairs[i].cell_id);
                    uint64_t cur = pairs[i].cell_id;
                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
                }
                return cells;
            };
            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
                std::vector<uint64_t> cells;
                cells.reserve(m.size());
                for (auto& [id, _] : m) cells.push_back(id);
                std::sort(cells.begin(), cells.end());
                return cells;
            };

            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
            {
                auto f1 = std::async(std::launch::async, [&] {
                    return !data.sorted_way_cells.empty()
                        ? extract_unique_cells(data.sorted_way_cells)
                        : extract_from_map(data.cell_to_ways);
                });
                if (write_addresses) {
                    auto f2 = std::async(std::launch::async, [&] {
                        return !data.sorted_addr_cells.empty()
                            ? extract_unique_cells(data.sorted_addr_cells)
                            : extract_from_map(data.cell_to_addrs);
                    });
                    auto f3 = std::async(std::launch::async, [&] {
                        return !data.sorted_interp_cells.empty()
                            ? extract_unique_cells(data.sorted_interp_cells)
                            : extract_from_map(data.cell_to_interps);
                    });
                    addr_cells = f2.get();
                    interp_cells = f3.get();
                }
                way_cells = f1.get();
            }

            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
            std::merge(way_cells.begin(), way_cells.end(), addr_cells.begin(), addr_cells.end(),
                       std::back_inserter(sorted_geo_cells));
            if (!interp_cells.empty()) {
                std::vector<uint64_t> tmp;
                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
                           interp_cells.begin(), interp_cells.end(), std::back_inserter(tmp));
                sorted_geo_cells = std::move(tmp);
            }
            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
                                    sorted_geo_cells.end());
        }

        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
        {
            auto f1 = std::async(std::launch::async, [&]() {
                if (!data.sorted_way_cells.empty())
                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
            });
            if (write_addresses) {
                auto f2 = std::async(std::launch::async, [&]() {
                    if (!data.sorted_addr_cells.empty())
                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
                });
                auto f3 = std::async(std::launch::async, [&]() {
                    if (!data.sorted_interp_cells.empty())
                        return write_entries_from_sorted(output_dir + "/interp_entries.bin", sorted_geo_cells, data.sorted_interp_cells);
                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
                });
                addr_offsets = f2.get();
                interp_offsets = f3.get();
            }
            street_offsets = f1.get();
        }

        log_phase("  Write: entry files", _wt);
        {
            size_t n = sorted_geo_cells.size();
            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
            std::vector<char> buf(n * row_size);
            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);

            unsigned int nthreads = std::thread::hardware_concurrency();
            if (nthreads == 0) nthreads = 4;
            size_t chunk = (n + nthreads - 1) / nthreads;
            std::vector<std::thread> fill_threads;
            for (unsigned int t = 0; t < nthreads; t++) {
                size_t start = t * chunk;
                size_t end = std::min(start + chunk, n);
                if (start >= n) break;
                fill_threads.emplace_back([&, start, end]() {
                    char* ptr = buf.data() + start * row_size;
                    for (size_t i = start; i < end; i++) {
                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                    }
                });
            }
            for (auto& t : fill_threads) t.join();

            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
            f.write(buf.data(), buf.size());
        }

        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
                  << data.ways.size() << " ways, " << data.addr_points.size() << " addrs, "
                  << data.interp_ways.size() << " interps)" << std::endl;
    }

    log_phase("  Write: geo_cells.bin", _wt);
    auto admin_future = std::async(std::launch::async, [&] {
        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
    });

    std::vector<std::future<void>> write_futures;
    if (write_streets) {
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
        }));
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
        }));
        if (write_addresses) {
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
            }));
        }
    }
    // If not writing all data types, compact string pool and remap references
    // before writing admin_polygons and strings.bin.
    std::vector<AdminPolygon> compact_admin;
    std::vector<char> compact_strings;
    const bool need_compact = (!write_streets || !write_addresses);

    if (need_compact) {
        // Collect referenced string offsets
        std::unordered_set<uint32_t> used_offsets;
        if (write_streets) {
            for (const auto& w : data.ways) used_offsets.insert(w.name_id);
            if (write_addresses) {
                for (const auto& a : data.addr_points) {
                    used_offsets.insert(a.housenumber_id);
                    used_offsets.insert(a.street_id);
                }
                for (const auto& iw : data.interp_ways) used_offsets.insert(iw.street_id);
            }
        }
        for (const auto& ap : data.admin_polygons) used_offsets.insert(ap.name_id);

        // Build compact pool and remap
        const auto& old_sp = data.string_pool.data();
        std::vector<uint32_t> sorted_offs(used_offsets.begin(), used_offsets.end());
        std::sort(sorted_offs.begin(), sorted_offs.end());
        std::unordered_map<uint32_t, uint32_t> remap;
        compact_strings.reserve(used_offsets.size() * 16);
        for (uint32_t off : sorted_offs) {
            remap[off] = static_cast<uint32_t>(compact_strings.size());
            size_t len = std::strlen(old_sp.data() + off);
            compact_strings.insert(compact_strings.end(), old_sp.data() + off,
                old_sp.data() + off + len + 1);
        }


   350→
   351→        // Remap admin polygon name_ids
   352→        compact_admin = data.admin_polygons;
   353→        for (auto& ap : compact_admin) ap.name_id = remap[ap.name_id];
   354→
   355→        // Remap street/addr/interp if applicable (for no-addresses mode)
   356→        // These are written from data directly, so we'd need copies too.
   357→        // For simplicity, only compact when we have fewer types.
   358→    }
   359→
   360→    const auto& admin_to_write = need_compact ? compact_admin : data.admin_polygons;
   361→    write_futures.push_back(std::async(std::launch::async, [&] {
   362→        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
   363→        f.write(reinterpret_cast<const char*>(admin_to_write.data()), admin_to_write.size() * sizeof(AdminPolygon));
   364→    }));
   365→    write_futures.push_back(std::async(std::launch::async, [&] {
   366→        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
   367→        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
   368→    }));
   369→    write_futures.push_back(std::async(std::launch::async, [&] {
   370→        if (need_compact) {
   371→            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
   372→            f.write(compact_strings.data(), compact_strings.size());
   373→            std::cerr << "strings.bin: " << compact_strings.size() << " bytes (compacted from "
   374→                      << data.string_pool.data().size() << ")" << std::endl;
   375→        } else {
   376→            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
   377→            f.write(data.string_pool.data().data(), data.string_pool.data().size());
   378→            std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
   379→        }
   380→    }));
   381→
   382→    log_phase("  Write: parallel data files launched", _wt);
   383→    admin_future.get();
   384→    for (auto& f : write_futures) f.get();
   385→}
   386→
   387→void write_quality_variant(const ParsedData& data, const std::string& source_dir,
   388→                           const std::string& output_dir, double epsilon_scale) {
   389→    ensure_dir(output_dir);
   390→
   391→    // Re-simplify admin polygons at the given epsilon scale
   392→    std::vector<AdminPolygon> new_polys;
   393→    std::vector<NodeCoord> new_verts;
   394→    new_polys.reserve(data.admin_polygons.size());
   395→
   396→    // Parallel simplification
   397→    struct SimplifiedPoly {
   398→        std::vector<std::pair<double,double>> verts;
   399→    };
   400→    std::vector<SimplifiedPoly> simplified(data.admin_polygons.size());
   401→
   402→    {
   403→        std::atomic<size_t> idx{0};
   404→        unsigned nthreads = std::thread::hardware_concurrency();
   405→        if (nthreads == 0) nthreads = 4;
   406→        std::vector<std::thread> workers;
   407→        for (unsigned t = 0; t < nthreads; t++) {
   408→            workers.emplace_back([&]() {
   409→                while (true) {
   410→                    size_t i = idx.fetch_add(1);
   411→                    if (i >= data.admin_polygons.size()) break;
   412→                    const auto& ap = data.admin_polygons[i];
   413→
   414→                    // Extract original vertices
   415→                    std::vector<std::pair<double,double>> pts;
   416→                    pts.reserve(ap.vertex_count);
   417→                    for (uint32_t j = 0; j < ap.vertex_count; j++) {
   418→                        const auto& v = data.admin_vertices[ap.vertex_offset + j];
   419→                        pts.emplace_back(v.lat, v.lng);
   420→                    }
   421→
   422→                    if (epsilon_scale > 0) {
   423→                        double eps_m = admin_epsilon_meters(ap.admin_level) * epsilon_scale;
   424→                        double lat = pts.empty() ? 0.0 : pts[0].first;
   425→                        double eps_deg = meters_to_degrees(eps_m, lat);
   426→                        simplified[i].verts = simplify_polygon_epsilon(pts, eps_deg);
   427→                    } else {
   428→                        simplified[i].verts = std::move(pts);
   429→                    }
   430→                }
   431→            });
   432→        }
   433→        for (auto& w : workers) w.join();
   434→    }
   435→
   436→    // Sequential: build new polygon and vertex arrays
   437→    for (size_t i = 0; i < data.admin_polygons.size(); i++) {
   438→        auto& sv = simplified[i].verts;
   439→        if (sv.size() < 3) continue;
   440→
   441→        AdminPolygon np = data.admin_polygons[i];
   442→        np.vertex_offset = static_cast<uint32_t>(new_verts.size());
   443→        np.vertex_count = static_cast<uint32_t>(sv.size());
   444→        np.area = polygon_area(sv);
   445→
   446→        for (const auto& [lat, lng] : sv) {
   447→            new_verts.push_back({static_cast<float>(lat), static_cast<float>(lng)});
   448→        }
   449→        new_polys.push_back(np);
   450→    }
   451→
   452→    std::cerr << "Quality " << epsilon_scale << "x: " << new_polys.size()
   453→              << " polygons, " << new_verts.size() << " vertices ("
   454→              << new_verts.size() * 8 / 1024 / 1024 << " MiB)" << std::endl;
   455→
   456→    // Write admin files
   457→    {
   458→        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
   459→        f.write(reinterpret_cast<const char*>(new_polys.data()), new_polys.size() * sizeof(AdminPolygon));
   460→    }
   461→    {
   462→        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
   463→        f.write(reinterpret_cast<const char*>(new_verts.data()), new_verts.size() * sizeof(NodeCoord));
   464→    }
   465→
   466→    // Quality directories only contain the two files that change.
   467→    // Shared files (admin_cells.bin, admin_entries.bin, strings.bin)
   468→    // stay in the parent directory.
   469→}
   470→

        // Remap admin polygon name_ids
        compact_admin = data.admin_polygons;
        for (auto& ap : compact_admin) ap.name_id = remap[ap.name_id];

        // Remap street/addr/interp if applicable (for no-addresses mode)
        // These are written from data directly, so we'd need copies too.
        // For simplicity, only compact when we have fewer types.
    }

    const auto& admin_to_write = need_compact ? compact_admin : data.admin_polygons;
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(admin_to_write.data()), admin_to_write.size() * sizeof(AdminPolygon));
    }));
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
    }));
    write_futures.push_back(std::async(std::launch::async, [&] {
        if (need_compact) {
            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
            f.write(compact_strings.data(), compact_strings.size());
            std::cerr << "strings.bin: " << compact_strings.size() << " bytes (compacted from "
                      << data.string_pool.data().size() << ")" << std::endl;
        } else {
            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
            f.write(data.string_pool.data().data(), data.string_pool.data().size());
            std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
        }
    }));

    log_phase("  Write: parallel data files launched", _wt);
    admin_future.get();
    for (auto& f : write_futures) f.get();
}

void write_quality_variant(const ParsedData& data, const std::string& source_dir,
                           const std::string& output_dir, double epsilon_scale) {
    ensure_dir(output_dir);

    // Re-simplify admin polygons at the given epsilon scale
    std::vector<AdminPolygon> new_polys;
    std::vector<NodeCoord> new_verts;
    new_polys.reserve(data.admin_polygons.size());

    // Parallel simplification
    struct SimplifiedPoly {
        std::vector<std::pair<double,double>> verts;
    };
    std::vector<SimplifiedPoly> simplified(data.admin_polygons.size());

    {
        std::atomic<size_t> idx{0};
        unsigned nthreads = std::thread::hardware_concurrency();
        if (nthreads == 0) nthreads = 4;
        std::vector<std::thread> workers;
        for (unsigned t = 0; t < nthreads; t++) {
            workers.emplace_back([&]() {
                while (true) {
                    size_t i = idx.fetch_add(1);
                    if (i >= data.admin_polygons.size()) break;
                    const auto& ap = data.admin_polygons[i];

                    // Extract original vertices
                    std::vector<std::pair<double,double>> pts;
                    pts.reserve(ap.vertex_count);
                    for (uint32_t j = 0; j < ap.vertex_count; j++) {
                        const auto& v = data.admin_vertices[ap.vertex_offset + j];
                        pts.emplace_back(v.lat, v.lng);
                    }

                    if (epsilon_scale > 0) {
                        double eps_m = admin_epsilon_meters(ap.admin_level) * epsilon_scale;
                        double lat = pts.empty() ? 0.0 : pts[0].first;
                        double eps_deg = meters_to_degrees(eps_m, lat);
                        simplified[i].verts = simplify_polygon_epsilon(pts, eps_deg);
                    } else {
                        simplified[i].verts = std::move(pts);
                    }
                }
            });
        }
        for (auto& w : workers) w.join();
    }

    // Sequential: build new polygon and vertex arrays
    for (size_t i = 0; i < data.admin_polygons.size(); i++) {
        auto& sv = simplified[i].verts;
        if (sv.size() < 3) continue;

        AdminPolygon np = data.admin_polygons[i];
        np.vertex_offset = static_cast<uint32_t>(new_verts.size());
        np.vertex_count = static_cast<uint32_t>(sv.size());
        np.area = polygon_area(sv);

        for (const auto& [lat, lng] : sv) {
            new_verts.push_back({static_cast<float>(lat), static_cast<float>(lng)});
        }
        new_polys.push_back(np);
    }

    std::cerr << "Quality " << epsilon_scale << "x: " << new_polys.size()
              << " polygons, " << new_verts.size() << " vertices ("
              << new_verts.size() * 8 / 1024 / 1024 << " MiB)" << std::endl;

    // Write admin files
    {
        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(new_polys.data()), new_polys.size() * sizeof(AdminPolygon));
    }
    {
        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(new_verts.data()), new_verts.size() * sizeof(NodeCoord));
    }

    // Quality directories only contain the two files that change.
    // Shared files (admin_cells.bin, admin_entries.bin, strings.bin)
    // stay in the parent directory.
}


Now I have a complete picture of the format and pipeline. Let me implement stable build ordering — this is the key to making patches small. The approach:

1. Sort string pool alphabetically (stabilizes all string references)
2. Sort addr_points by S2 cell (stabilizes the biggest file)
3. Sort ways by primary S2 cell + name (stabilizes way ordering)
4. Sort interps by primary S2 cell (stabilizes interp ordering)
5. All downstream indexes (entries, geo_cells) automatically stabilize

Let me implement this.


     1→#pragma once
     2→
     3→#include <algorithm>
     4→#include <chrono>
     5→#include <iostream>
     6→#include <string>
     7→#include <unordered_map>
     8→#include <vector>
     9→
    10→#include <cstdio>
    11→#include <cstring>
    12→#include <dirent.h>
    13→#include <iomanip>
    14→#include <thread>
    15→#include <unistd.h>
    16→#include <sys/stat.h>
    17→
    18→#include "types.h"
    19→#include "string_pool.h"
    20→
    21→// --- Directory creation ---
    22→
    23→inline void ensure_dir(const std::string& path) {
    24→    // Recursive mkdir — create all parent directories
    25→    for (size_t i = 1; i < path.size(); i++) {
    26→        if (path[i] == '/') {
    27→            mkdir(path.substr(0, i).c_str(), 0755);
    28→        }
    29→    }
    30→    mkdir(path.c_str(), 0755);
    31→}
    32→
    33→// --- CPU tick reading (all threads via getrusage) ---
    34→
    35→#include <sys/resource.h>
    36→
    37→struct CpuTicks {
    38→    long long process_us = 0;  // user+system microseconds (all threads)
    39→
    40→    static CpuTicks now() {
    41→        CpuTicks ct;
    42→        struct rusage ru;
    43→        if (getrusage(RUSAGE_SELF, &ru) == 0) {
    44→            ct.process_us = (long long)ru.ru_utime.tv_sec * 1000000 + ru.ru_utime.tv_usec
    45→                          + (long long)ru.ru_stime.tv_sec * 1000000 + ru.ru_stime.tv_usec;
    46→        }
    47→        return ct;
    48→    }
    49→};
    50→
    51→// --- Phase timer with CPU utilization ---
    52→
    53→inline long get_rss_mb() {
    54→    long rss_pages = 0;
    55→    FILE* f = fopen("/proc/self/statm", "r");
    56→    if (f) {
    57→        long size;
    58→        if (fscanf(f, "%ld %ld", &size, &rss_pages) != 2) rss_pages = 0;
    59→        fclose(f);
    60→    }
    61→    return rss_pages * sysconf(_SC_PAGESIZE) / (1024 * 1024);
    62→}
    63→
    64→inline void log_phase(const char* name, std::chrono::steady_clock::time_point& t,
    65→                       CpuTicks& prev_cpu) {
    66→    auto now = std::chrono::steady_clock::now();
    67→    auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(now - t).count();
    68→    auto cpu_now = CpuTicks::now();
    69→    double cpu_sec = (cpu_now.process_us - prev_cpu.process_us) / 1e6;
    70→    double wall_sec = ms / 1000.0;
    71→    double cores_used = (wall_sec > 0.1) ? cpu_sec / wall_sec : 0;
    72→    unsigned ncpu = std::thread::hardware_concurrency();
    73→    int pct = (wall_sec > 0.1 && ncpu > 0) ? (int)(cores_used * 100.0 / ncpu) : 0;
    74→    long rss = get_rss_mb();
    75→    std::cerr << "  [" << ms/1000 << "." << (ms%1000)/100 << "s"
    76→              << " " << std::fixed << std::setprecision(1) << cores_used << "/" << ncpu << "cores"
    77→              << " " << pct << "%"
    78→              << " " << rss << "MiB] " << name << std::endl;
    79→    t = now;
    80→    prev_cpu = cpu_now;
    81→}
    82→
    83→// Backward compat overload (no CPU tracking)
    84→inline void log_phase(const char* name, std::chrono::steady_clock::time_point& t) {
    85→    auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(
    86→        std::chrono::steady_clock::now() - t).count();
    87→    std::cerr << "  [" << ms/1000 << "." << (ms%1000)/100 << "s] " << name << std::endl;
    88→    t = std::chrono::steady_clock::now();
    89→}
    90→
    91→// --- Parsed data container ---
    92→
    93→struct ParsedData {
    94→    StringPool string_pool;
    95→    std::vector<WayHeader> ways;
    96→    std::vector<NodeCoord> street_nodes;
    97→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_ways;
    98→    std::vector<AddrPoint> addr_points;
    99→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_addrs;
   100→    std::vector<InterpWay> interp_ways;
   101→    std::vector<NodeCoord> interp_nodes;
   102→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_interps;
   103→    std::vector<AdminPolygon> admin_polygons;
   104→    std::vector<NodeCoord> admin_vertices;
   105→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_admin;
   106→
   107→    // Deferred work for parallel S2 computation (ways + interps)
   108→    std::vector<DeferredWay> deferred_ways;
   109→    std::vector<DeferredInterp> deferred_interps;
   110→
   111→    // Sorted (cell_id, item_id) pairs — kept for direct entry writing
   112→    std::vector<CellItemPair> sorted_way_cells;
   113→    std::vector<CellItemPair> sorted_addr_cells;
   114→    std::vector<CellItemPair> sorted_interp_cells;
   115→
   116→    // Collected data for parallel admin assembly
   117→    std::vector<CollectedRelation> collected_relations;
   118→    struct WayGeometry {
   119→        std::vector<std::pair<double,double>> coords;
   120→        int64_t first_node_id;
   121→        int64_t last_node_id;
   122→    };
   123→    std::unordered_map<int64_t, WayGeometry> way_geometries;
   124→};
   125→
   126→// --- Deduplicate IDs per cell ---
   127→
   128→template<typename Map>
   129→inline void deduplicate(Map& cell_map) {
   130→    for (auto& [cell_id, ids] : cell_map) {
   131→        std::sort(ids.begin(), ids.end());
   132→        ids.erase(std::unique(ids.begin(), ids.end()), ids.end());
   133→    }
   134→}
   135→

#pragma once

#include <algorithm>
#include <chrono>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>

#include <cstdio>
#include <cstring>
#include <dirent.h>
#include <iomanip>
#include <thread>
#include <unistd.h>
#include <sys/stat.h>

#include "types.h"
#include "string_pool.h"

// --- Directory creation ---

inline void ensure_dir(const std::string& path) {
    // Recursive mkdir — create all parent directories
    for (size_t i = 1; i < path.size(); i++) {
        if (path[i] == '/') {
            mkdir(path.substr(0, i).c_str(), 0755);
        }
    }
    mkdir(path.c_str(), 0755);
}

// --- CPU tick reading (all threads via getrusage) ---

#include <sys/resource.h>

struct CpuTicks {
    long long process_us = 0;  // user+system microseconds (all threads)

    static CpuTicks now() {
        CpuTicks ct;
        struct rusage ru;
        if (getrusage(RUSAGE_SELF, &ru) == 0) {
            ct.process_us = (long long)ru.ru_utime.tv_sec * 1000000 + ru.ru_utime.tv_usec
                          + (long long)ru.ru_stime.tv_sec * 1000000 + ru.ru_stime.tv_usec;
        }
        return ct;
    }
};

// --- Phase timer with CPU utilization ---

inline long get_rss_mb() {
    long rss_pages = 0;
    FILE* f = fopen("/proc/self/statm", "r");
    if (f) {
        long size;
        if (fscanf(f, "%ld %ld", &size, &rss_pages) != 2) rss_pages = 0;
        fclose(f);
    }
    return rss_pages * sysconf(_SC_PAGESIZE) / (1024 * 1024);
}

inline void log_phase(const char* name, std::chrono::steady_clock::time_point& t,
                       CpuTicks& prev_cpu) {
    auto now = std::chrono::steady_clock::now();
    auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(now - t).count();
    auto cpu_now = CpuTicks::now();
    double cpu_sec = (cpu_now.process_us - prev_cpu.process_us) / 1e6;
    double wall_sec = ms / 1000.0;
    double cores_used = (wall_sec > 0.1) ? cpu_sec / wall_sec : 0;
    unsigned ncpu = std::thread::hardware_concurrency();
    int pct = (wall_sec > 0.1 && ncpu > 0) ? (int)(cores_used * 100.0 / ncpu) : 0;
    long rss = get_rss_mb();
    std::cerr << "  [" << ms/1000 << "." << (ms%1000)/100 << "s"
              << " " << std::fixed << std::setprecision(1) << cores_used << "/" << ncpu << "cores"
              << " " << pct << "%"
              << " " << rss << "MiB] " << name << std::endl;
    t = now;
    prev_cpu = cpu_now;
}

// Backward compat overload (no CPU tracking)
inline void log_phase(const char* name, std::chrono::steady_clock::time_point& t) {
    auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(
        std::chrono::steady_clock::now() - t).count();
    std::cerr << "  [" << ms/1000 << "." << (ms%1000)/100 << "s] " << name << std::endl;
    t = std::chrono::steady_clock::now();
}

// --- Parsed data container ---

struct ParsedData {
    StringPool string_pool;
    std::vector<WayHeader> ways;
    std::vector<NodeCoord> street_nodes;
    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_ways;
    std::vector<AddrPoint> addr_points;
    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_addrs;
    std::vector<InterpWay> interp_ways;
    std::vector<NodeCoord> interp_nodes;
    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_interps;
    std::vector<AdminPolygon> admin_polygons;
    std::vector<NodeCoord> admin_vertices;
    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_admin;

    // Deferred work for parallel S2 computation (ways + interps)
    std::vector<DeferredWay> deferred_ways;
    std::vector<DeferredInterp> deferred_interps;

    // Sorted (cell_id, item_id) pairs — kept for direct entry writing
    std::vector<CellItemPair> sorted_way_cells;
    std::vector<CellItemPair> sorted_addr_cells;
    std::vector<CellItemPair> sorted_interp_cells;

    // Collected data for parallel admin assembly
    std::vector<CollectedRelation> collected_relations;
    struct WayGeometry {
        std::vector<std::pair<double,double>> coords;
        int64_t first_node_id;
        int64_t last_node_id;
    };
    std::unordered_map<int64_t, WayGeometry> way_geometries;
};

// --- Deduplicate IDs per cell ---

template<typename Map>
inline void deduplicate(Map& cell_map) {
    for (auto& [cell_id, ids] : cell_map) {
        std::sort(ids.begin(), ids.end());
        ids.erase(std::unique(ids.begin(), ids.end()), ids.end());
    }
}


Now I'll implement the stable ordering. I'll add it to build_index.cpp right after the dedup phase.


   130→        } else if (arg == "--epsilon-levels" && i + 1 < argc) {
   131→            // Parse comma-separated values: L2,L3,L4,L5,L6,L7,L8
   132→            std::string vals = argv[++i];
   133→            int level = 2;
   134→            size_t pos = 0;
   135→            while (pos < vals.size() && level <= 8) {
   136→                size_t next = vals.find(',', pos);
   137→                if (next == std::string::npos) next = vals.size();
   138→                double v = std::strtod(vals.substr(pos, next - pos).c_str(), nullptr);
   139→                if (v > 0) kAdminEpsilonMeters[level] = v;
   140→                level++;
   141→                pos = next + 1;
   142→            }
   143→            // Also set levels 9-11 to same as level 8
   144→            for (int l = level; l <= 11; l++) kAdminEpsilonMeters[l] = kAdminEpsilonMeters[8];
   145→        } else if (arg == "--mode" && i + 1 < argc) {
   146→            std::string mode_str = argv[++i];
   147→            if (mode_str == "full") {
   148→                mode = IndexMode::Full;
   149→            } else if (mode_str == "no-addresses") {
   150→                mode = IndexMode::NoAddresses;
   151→            } else if (mode_str == "admin-only") {
   152→                mode = IndexMode::AdminOnly;
   153→            } else {
   154→                std::cerr << "Error: unknown mode '" << mode_str << "'" << std::endl;
   155→                return 1;
   156→            }
   157→        } else {
   158→            input_files.push_back(arg);
   159→        }

        } else if (arg == "--epsilon-levels" && i + 1 < argc) {
            // Parse comma-separated values: L2,L3,L4,L5,L6,L7,L8
            std::string vals = argv[++i];
            int level = 2;
            size_t pos = 0;
            while (pos < vals.size() && level <= 8) {
                size_t next = vals.find(',', pos);
                if (next == std::string::npos) next = vals.size();
                double v = std::strtod(vals.substr(pos, next - pos).c_str(), nullptr);
                if (v > 0) kAdminEpsilonMeters[level] = v;
                level++;
                pos = next + 1;
            }
            // Also set levels 9-11 to same as level 8
            for (int l = level; l <= 11; l++) kAdminEpsilonMeters[l] = kAdminEpsilonMeters[8];
        } else if (arg == "--mode" && i + 1 < argc) {
            std::string mode_str = argv[++i];
            if (mode_str == "full") {
                mode = IndexMode::Full;
            } else if (mode_str == "no-addresses") {
                mode = IndexMode::NoAddresses;
            } else if (mode_str == "admin-only") {
                mode = IndexMode::AdminOnly;
            } else {
                std::cerr << "Error: unknown mode '" << mode_str << "'" << std::endl;
                return 1;
            }
        } else {
            input_files.push_back(arg);
        }


   160→    }
   161→
   162→    // Set global simplification mode
   163→    kSimplifyMode = simplify_mode;
   164→    kSimplifyEpsilonOverride = simplify_epsilon_override;
   165→    if (simplify_mode == SimplifyMode::ErrorBounded) {
   166→        if (simplify_epsilon_override > 0) {
   167→            std::cerr << "Simplification: error-bounded, fixed " << simplify_epsilon_override << "m epsilon" << std::endl;
   168→        } else {
   169→            std::cerr << "Simplification: error-bounded, scale=" << kEpsilonScale
   170→                      << ", epsilons: L2=" << admin_epsilon_meters(2)
   171→                      << "m L4=" << admin_epsilon_meters(4)
   172→                      << "m L6=" << admin_epsilon_meters(6)
   173→                      << "m L8=" << admin_epsilon_meters(8) << "m" << std::endl;
   174→        }
   175→    }
   176→
   177→    ParsedData data;
   178→    auto _pt = std::chrono::steady_clock::now();
   179→    auto _cpu = CpuTicks::now();
   180→
   181→    if (!load_cache_path.empty()) {
   182→        // Load from cache
   183→        if (!deserialize_cache(data, load_cache_path)) {
   184→            std::cerr << "Error: failed to load cache" << std::endl;
   185→            return 1;
   186→        }
   187→    } else {
   188→        // Parse PBF files (always collect everything)
   189→        if (input_files.empty()) {
   190→            std::cerr << "Error: no input files specified and no --load-cache" << std::endl;
   191→            return 1;
   192→        }
   193→
   194→        // Create thread pool for concurrent admin polygon S2 covering
   195→        unsigned int num_threads = std::max(1u, std::thread::hardware_concurrency() - 1);
   196→        std::cerr << "Using " << num_threads << " worker threads." << std::endl;
   197→        AdminCoverPool admin_pool(num_threads);
   198→        // BuildHandler no longer used — parallel processing handles everything
   199→

    }

    // Set global simplification mode
    kSimplifyMode = simplify_mode;
    kSimplifyEpsilonOverride = simplify_epsilon_override;
    if (simplify_mode == SimplifyMode::ErrorBounded) {
        if (simplify_epsilon_override > 0) {
            std::cerr << "Simplification: error-bounded, fixed " << simplify_epsilon_override << "m epsilon" << std::endl;
        } else {
            std::cerr << "Simplification: error-bounded, scale=" << kEpsilonScale
                      << ", epsilons: L2=" << admin_epsilon_meters(2)
                      << "m L4=" << admin_epsilon_meters(4)
                      << "m L6=" << admin_epsilon_meters(6)
                      << "m L8=" << admin_epsilon_meters(8) << "m" << std::endl;
        }
    }

    ParsedData data;
    auto _pt = std::chrono::steady_clock::now();
    auto _cpu = CpuTicks::now();

    if (!load_cache_path.empty()) {
        // Load from cache
        if (!deserialize_cache(data, load_cache_path)) {
            std::cerr << "Error: failed to load cache" << std::endl;
            return 1;
        }
    } else {
        // Parse PBF files (always collect everything)
        if (input_files.empty()) {
            std::cerr << "Error: no input files specified and no --load-cache" << std::endl;
            return 1;
        }

        // Create thread pool for concurrent admin polygon S2 covering
        unsigned int num_threads = std::max(1u, std::thread::hardware_concurrency() - 1);
        std::cerr << "Using " << num_threads << " worker threads." << std::endl;
        AdminCoverPool admin_pool(num_threads);
        // BuildHandler no longer used — parallel processing handles everything


Good. Now let me find the exact insertion point — right after dedup and before writing.