I need to understand what makes the geocoder build non-deterministic. Examine /root/traccar-geocoder/builder/src/build_index.cpp focusing on:

1. **String pool ordering**: How does StringPool::intern() assign offsets? Is it first-come-first-served based on thread encounter order?

2. **Record ordering**: How are addr_points, street_ways, interp_ways ordered in the final output? Are they in PBF encounter order merged by thread index?

3. **Thread-local data merging**: Look at the thread-local merge sections (around lines 390-670). How are thread-local buffers merged? Does the merge order depend on thread scheduling?

4. **S2 cell computation**: Is the sorted_way_cells / sorted_addr_cells ordering deterministic given the same input?

5. **Admin polygon assembly**: Is the parallel admin assembly deterministic?

For each source of non-determinism, note:
- What causes it (thread scheduling, hash map iteration, PBF block assignment)
- How severe it is (affects ordering, affects content, affects both)
- What would be needed to make it deterministic

The goal: if we run the same build twice on the same PBF, do we get byte-identical output? If not, what needs to change?

Be thorough - check every parallel section and hash map usage.


I'll conduct a thorough analysis of the build_index.cpp file to identify non-determinism sources. Let me start by examining the structure and key sections.


Error: File content (22670 tokens) exceeds maximum allowed tokens (10000). Use offset and limit parameters to read specific portions of the file, or search for specific content instead of reading the whole file.


Let me read the file in sections to analyze the key areas mentioned:


     1→#include <algorithm>
     2→#include <atomic>
     3→#include <chrono>
     4→#include <condition_variable>
     5→#include <cstring>
     6→#include <deque>
     7→#include <fstream>
     8→#include <future>
     9→#include <iostream>
    10→#include <mutex>
    11→#include <numeric>
    12→#include <string>
    13→#include <thread>
    14→#include <unordered_set>
    15→#include <vector>
    16→
    17→#include <fcntl.h>
    18→#include <sys/mman.h>
    19→
    20→#include "pbf_reader.h"
    21→
    22→#include <s2/s2cell_id.h>
    23→#include <s2/s2latlng.h>
    24→
    25→#include "types.h"
    26→#include "string_pool.h"
    27→#include "geometry.h"
    28→#include "parsed_data.h"
    29→#include "s2_helpers.h"
    30→#include "ring_assembly.h"
    31→#include "continent_boundaries.h"
    32→#include "interpolation.h"
    33→#include "cache.h"
    34→#include "continent_filter.h"
    35→#include "cell_index.h"
    36→
    37→
    38→// --- Main ---
    39→
    40→int main(int argc, char* argv[]) {
    41→    if (argc < 2) {
    42→        std::cerr << "Usage: build-index <output-dir> <input.osm.pbf> [options]" << std::endl;
    43→        std::cerr << "       build-index <output-dir> --load-cache <path> [options]" << std::endl;
    44→        std::cerr << std::endl;
    45→        std::cerr << "Options:" << std::endl;
    46→        std::cerr << "  --street-level N       S2 cell level for streets (default: 17)" << std::endl;
    47→        std::cerr << "  --admin-level N        S2 cell level for admin (default: 10)" << std::endl;
    48→        std::cerr << "  --max-admin-level N    Only include admin levels <= N" << std::endl;
    49→        std::cerr << "  --save-cache <path>    Save parsed data to cache file" << std::endl;
    50→        std::cerr << "  --load-cache <path>    Load from cache instead of PBF" << std::endl;
    51→        std::cerr << "  --multi-output         Write full, no-addresses, admin-only indexes" << std::endl;
    52→        std::cerr << "  --continents           Also generate per-continent indexes" << std::endl;
    53→        std::cerr << "  --mode <mode>          Index mode: full, no-addresses, admin-only (default: full)" << std::endl;
    54→        std::cerr << "  --admin-only           Shorthand for --mode admin-only" << std::endl;
    55→        std::cerr << "  --no-addresses         Shorthand for --mode no-addresses" << std::endl;
    56→        std::cerr << "  --simplify-epsilon     Use error-bounded simplification (per-level defaults)" << std::endl;
    57→        std::cerr << "  --simplify-epsilon N   Use fixed epsilon of N meters for all levels" << std::endl;
    58→        std::cerr << "  --epsilon-scale F      Multiply all per-level epsilons by F (e.g. 2.0 = 2x coarser)" << std::endl;
    59→        std::cerr << "  --epsilon-levels L2,L3,...,L8  Set epsilon in meters per level (7 values)" << std::endl;
    60→        std::cerr << "  --multi-quality [S,S,...]  Write quality variants at given scales (default: 0,0.2,0.5,1,1.5,2,2.5)" << std::endl;
    61→        return 1;
    62→    }
    63→
    64→    std::string output_dir = argv[1];
    65→    std::vector<std::string> input_files;
    66→    std::string save_cache_path;
    67→    std::string load_cache_path;
    68→    bool multi_output = false;
    69→    bool generate_continents = false;
    70→    IndexMode mode = IndexMode::Full;
    71→    SimplifyMode simplify_mode = SimplifyMode::MaxVertices;
    72→    double simplify_epsilon_override = 0; // 0 = use per-level defaults
    73→    bool multi_quality = false;
    74→    std::vector<double> quality_scales;
    75→
    76→    for (int i = 2; i < argc; i++) {
    77→        std::string arg = argv[i];
    78→        if (arg == "--street-level" && i + 1 < argc) {
    79→            kStreetCellLevel = std::atoi(argv[++i]);
    80→        } else if (arg == "--admin-level" && i + 1 < argc) {
    81→            kAdminCellLevel = std::atoi(argv[++i]);
    82→        } else if (arg == "--admin-only") {
    83→            mode = IndexMode::AdminOnly;
    84→        } else if (arg == "--no-addresses") {
    85→            mode = IndexMode::NoAddresses;
    86→        } else if (arg == "--max-admin-level" && i + 1 < argc) {
    87→            kMaxAdminLevel = std::atoi(argv[++i]);
    88→        } else if (arg == "--save-cache" && i + 1 < argc) {
    89→            save_cache_path = argv[++i];
    90→        } else if (arg == "--load-cache" && i + 1 < argc) {
    91→            load_cache_path = argv[++i];
    92→        } else if (arg == "--multi-output") {
    93→            multi_output = true;
    94→        } else if (arg == "--continents") {
    95→            generate_continents = true;
    96→        } else if (arg == "--simplify-epsilon") {
    97→            simplify_mode = SimplifyMode::ErrorBounded;
    98→            // Optional: next arg might be a number (fixed epsilon in meters)
    99→            if (i + 1 < argc) {
   100→                char* end;


Bash

Get total line count


1604 /root/traccar-geocoder/builder/src/build_index.cpp


Now let me search for the key sections mentioned - StringPool, thread-local data merging, and the main processing logic:


Found 10 files
src/string_pool.h
src/build_index.cpp
src/pbf_reader.cpp
src/pbf_reader.h
src/cell_index.cpp
src/continent_filter.cpp
src/interpolation.cpp
src/parsed_data.h
src/s2_helpers.cpp
src/types.h


1059:            // Parallel sort each thread's pairs, then k-way merge directly into
1111:                    std::vector<std::thread> merge_threads;
1116:                        merge_threads.emplace_back([&, left, mid, right] {
1127:                    for (auto& t : merge_threads) t.join();
1136:            // Sort each thread's pairs in parallel, then k-way merge directly
1139:                parallel_sort_and_build(way_pairs, data.cell_to_ways, data.sorted_way_cells);
1216:                data.sorted_addr_cells = std::move(pairs);
1282:        auto f1 = std::async(std::launch::async, [&]{ rebuild_map_parallel(data.sorted_way_cells, data.cell_to_ways); });
1409:        auto way_masks = std::async(std::launch::async, [&]{ return precompute_masks(data.sorted_way_cells); });
1410:        auto addr_masks = std::async(std::launch::async, [&]{ return precompute_masks(data.sorted_addr_cells); });


grep -n "thread_local\|#pragma omp\|std::thread" /root/traccar-geocoder/builder/src/build_index.cpp | head -30

Find all parallel/threading constructs


195:        unsigned int num_threads = std::max(1u, std::thread::hardware_concurrency() - 1);
347:                // Use thread_local for streaming callback (no thread index available)
348:                static thread_local NodeThreadLocal* tl_node_data = nullptr;
386:                // Reset thread_local for next use
443:                static thread_local ThreadLocalData* tl_way_data = nullptr;
496:                    thread_local std::vector<PackedLocation> resolved_locs;
714:                    std::vector<std::thread> admin_workers;
840:                        std::vector<std::thread> prep_workers;
992:                std::vector<std::thread> threads;
1027:                std::vector<std::thread> threads;
1072:                    std::vector<std::thread> sort_threads;
1111:                    std::vector<std::thread> merge_threads;
1185:                    unsigned nthreads = std::thread::hardware_concurrency();
1187:                    std::vector<std::thread> threads;
1238:            unsigned nthreads = std::thread::hardware_concurrency();
1256:            std::vector<std::thread> threads;
1370:            unsigned nthreads = std::max(1u, std::thread::hardware_concurrency());
1382:            std::vector<std::thread> threads;
1429:            std::thread::hardware_concurrency() / 8));


Let me examine the StringPool implementation and the thread-local data merging sections in detail:


     1→#pragma once
     2→
     3→#include <cstdint>
     4→#include <string>
     5→#include <unordered_map>
     6→#include <vector>
     7→
     8→class StringPool {
     9→public:
    10→    uint32_t intern(const std::string& s) {
    11→        auto it = index_.find(s);
    12→        if (it != index_.end()) {
    13→            return it->second;
    14→        }
    15→        uint32_t offset = static_cast<uint32_t>(data_.size());
    16→        index_[s] = offset;
    17→        data_.insert(data_.end(), s.begin(), s.end());
    18→        data_.push_back('\0');
    19→        return offset;
    20→    }
    21→
    22→    const std::vector<char>& data() const { return data_; }
    23→    std::vector<char>& mutable_data() { return data_; }
    24→
    25→private:
    26→    std::unordered_map<std::string, uint32_t> index_;
    27→    std::vector<char> data_;
    28→};
    29→


   340→            std::cerr << "  Pass 2: processing nodes with " << num_threads << " threads..." << std::endl;
   341→            {
   342→                struct NodeThreadLocal {
   343→                    std::vector<std::pair<double,double>> addr_coords;
   344→                    std::vector<std::pair<std::string,std::string>> addr_strings;
   345→                    uint64_t count = 0;
   346→                };
   347→                // Use thread_local for streaming callback (no thread index available)
   348→                static thread_local NodeThreadLocal* tl_node_data = nullptr;
   349→                std::vector<NodeThreadLocal> ntld(num_threads);
   350→                std::atomic<unsigned> next_tl{0};
   351→
   352→                pbf.read_nodes_streaming([&](int64_t id, double lat, double lng,
   353→                    const uint32_t* tag_keys, const uint32_t* tag_vals, size_t ntags,
   354→                    const std::vector<std::string>& st) {
   355→
   356→                    // Lazy init thread-local pointer
   357→                    if (!tl_node_data) {
   358→                        unsigned idx = next_tl.fetch_add(1);
   359→                        tl_node_data = &ntld[idx % ntld.size()];
   360→                    }
   361→
   362→                    if (id > 0) {
   363→                        index.set(static_cast<uint64_t>(id), lat, lng);
   364→                    }
   365→
   366→                    // Check for address tags (fast — no string copies for tagless nodes)
   367→                    if (ntags > 0) {
   368→                        const char* housenumber = nullptr;
   369→                        const char* street = nullptr;
   370→                        for (size_t i = 0; i < ntags; i++) {
   371→                            if (tag_keys[i] < st.size()) {
   372→                                if (st[tag_keys[i]] == "addr:housenumber")
   373→                                    housenumber = tag_vals[i] < st.size() ? st[tag_vals[i]].c_str() : nullptr;
   374→                                else if (st[tag_keys[i]] == "addr:street")
   375→                                    street = tag_vals[i] < st.size() ? st[tag_vals[i]].c_str() : nullptr;
   376→                            }
   377→                        }
   378→                        if (housenumber && street) {
   379→                            tl_node_data->addr_coords.push_back({lat, lng});
   380→                            tl_node_data->addr_strings.push_back({housenumber, street});
   381→                            tl_node_data->count++;
   382→                        }
   383→                    }
   384→                });
   385→
   386→                // Reset thread_local for next use
   387→                tl_node_data = nullptr;
   388→
   389→                // Merge address points
   390→                uint64_t total_addrs = 0;
   391→                for (auto& local : ntld) {
   392→                    for (size_t j = 0; j < local.addr_coords.size(); j++) {
   393→                        uint64_t dummy = 0;
   394→                        add_addr_point(data, local.addr_coords[j].first, local.addr_coords[j].second,
   395→                                       local.addr_strings[j].first.c_str(),
   396→                                       local.addr_strings[j].second.c_str(), dummy);
   397→                    }
   398→                    total_addrs += local.count;
   399→                }
   400→                std::cerr << "  Node processing complete: " << total_addrs
   401→                          << " address points collected." << std::endl;
   402→            }
   403→            log_phase("Pass 2: node processing", _pt, _cpu);
   404→            pbf.release_pages(); // free PBF mmap pages, will re-fault for way pass
   405→
   406→            // --- Pass 2b: Way processing (fully parallel) ---
   407→            std::cerr << "  Processing ways with " << num_threads << " threads..." << std::endl;
   408→            {
   409→                // Thread-local data for parallel way processing
   410→                struct ThreadLocalData {
   411→                    std::vector<WayHeader> ways;
   412→                    std::vector<NodeCoord> street_nodes;
   413→                    std::vector<DeferredWay> deferred_ways;
   414→                    std::vector<InterpWay> interp_ways;
   415→                    std::vector<NodeCoord> interp_nodes;
   416→                    std::vector<DeferredInterp> deferred_interps;
   417→                    std::vector<AddrPoint> building_addrs;
   418→                    std::vector<std::pair<double, double>> building_addr_coords; // lat,lng for S2 cell
   419→                    std::vector<std::string> way_strings;      // way name strings
   420→                    std::vector<std::pair<std::string,std::string>> addr_strings; // building addr {hn, street}
   421→                    std::vector<std::string> interp_strings;   // interp street name strings
   422→                    uint64_t way_count = 0;
   423→                    uint64_t building_addr_count = 0;
   424→                    uint64_t interp_count = 0;
   425→                    // Way geometries for parallel admin assembly
   426→                    struct WayGeomEntry {
   427→                        int64_t way_id;
   428→                        std::vector<std::pair<double,double>> coords;
   429→                        int64_t first_node_id;
   430→                        int64_t last_node_id;
   431→                    };
   432→                    std::vector<WayGeomEntry> way_geoms;
   433→                    // Closed-way admin polygons (ways with boundary=administrative)
   434→                    struct ClosedWayAdmin {
   435→                        std::vector<std::pair<double,double>> vertices;
   436→                        std::string name;
   437→                        uint8_t admin_level;
   438→                        std::string country_code;
   439→                    };
   440→                    std::vector<ClosedWayAdmin> closed_way_admins;
   441→                };
   442→
   443→                static thread_local ThreadLocalData* tl_way_data = nullptr;
   444→                std::vector<ThreadLocalData> tld(num_threads);
   445→                std::atomic<unsigned> next_tl_way{0};
   446→
   447→                pbf.read_ways_streaming([&](int64_t way_id,
   448→                    const int64_t* refs_data, size_t refs_size,
   449→                    const uint32_t* tag_keys, const uint32_t* tag_vals, size_t ntags,
   450→                    const std::vector<std::string>& st) {
   451→
   452→                    if (!tl_way_data) {
   453→                        unsigned idx = next_tl_way.fetch_add(1);
   454→                        tl_way_data = &tld[idx % tld.size()];
   455→                    }
   456→                    auto& local = *tl_way_data;
   457→
   458→                    // Single-pass tag extraction — avoid repeated linear scans
   459→                    const char* t_interpolation = nullptr;
   460→                    const char* t_housenumber = nullptr;
   461→                    const char* t_street = nullptr;
   462→                    const char* t_highway = nullptr;
   463→                    const char* t_name = nullptr;
   464→                    const char* t_boundary = nullptr;
   465→                    const char* t_admin_level = nullptr;
   466→                    const char* t_postal_code = nullptr;
   467→                    const char* t_iso = nullptr;
   468→                    for (size_t i = 0; i < ntags; i++) {
   469→                        if (tag_keys[i] >= st.size()) continue;
   470→                        const auto& k = st[tag_keys[i]];
   471→                        const char* v = tag_vals[i] < st.size() ? st[tag_vals[i]].c_str() : nullptr;
   472→                        // Fast dispatch by first character to avoid 9 string comparisons
   473→                        switch (k.size() > 0 ? k[0] : 0) {
   474→                            case 'a':
   475→                                if (k == "addr:interpolation") t_interpolation = v;
   476→                                else if (k == "addr:housenumber") t_housenumber = v;
   477→                                else if (k == "addr:street") t_street = v;
   478→                                else if (k == "admin_level") t_admin_level = v;
   479→                                break;
   480→                            case 'h': if (k == "highway") t_highway = v; break;
   481→                            case 'n': if (k == "name") t_name = v; break;
   482→                            case 'b': if (k == "boundary") t_boundary = v; break;
   483→                            case 'p': if (k == "postal_code") t_postal_code = v; break;
   484→                            case 'I': if (k == "ISO3166-1:alpha2") t_iso = v; break;
   485→                        }
   486→                    }
   487→
   488→                    // Early exit: if no relevant tags, skip expensive node resolution
   489→                    bool need_nodes = t_interpolation || t_housenumber ||


Good. Now let me continue reading through the thread-local merge section and the main data processing:


   650→                        di.interp_id += interp_base;
   651→                        di.node_offset += interp_node_base;
   652→                        data.deferred_interps.push_back(di);
   653→                    }
   654→
   655→                    total_ways += local.way_count;
   656→                    total_building_addrs += local.building_addr_count;
   657→                    total_interps += local.interp_count;
   658→
   659→                    // Merge way geometries for admin assembly
   660→                    {
   661→                        for (auto& wg : local.way_geoms) {
   662→                            ParsedData::WayGeometry g;
   663→                            g.coords = std::move(wg.coords);
   664→                            g.first_node_id = wg.first_node_id;
   665→                            g.last_node_id = wg.last_node_id;
   666→                            data.way_geometries[wg.way_id] = std::move(g);
   667→                        }
   668→                        local.way_geoms.clear();
   669→                        local.way_geoms.shrink_to_fit();
   670→                    }
   671→                }
   672→                std::cerr << "  Merged: " << total_ways << " ways, "
   673→                          << total_building_addrs << " building addrs, "
   674→                          << total_interps << " interps from parallel processing." << std::endl;
   675→
   676→                // --- Merge closed-way admin polygons ---
   677→                {
   678→                    uint64_t closed_way_admin_count = 0;
   679→                    for (auto& local : tld) {
   680→                        for (auto& cwa : local.closed_way_admins) {
   681→                            const char* cc = cwa.country_code.empty() ? nullptr : cwa.country_code.c_str();
   682→                            add_admin_polygon(data, cwa.vertices, cwa.name.c_str(),
   683→                                              cwa.admin_level, cc, &admin_pool);
   684→                            closed_way_admin_count++;
   685→                        }
   686→                        local.closed_way_admins.clear();
   687→                    }
   688→                    if (closed_way_admin_count > 0) {
   689→                        std::cerr << "  Added " << closed_way_admin_count
   690→                                  << " admin polygons from closed ways." << std::endl;
   691→                    }
   692→                }
   693→
   694→                // --- Parallel admin boundary assembly ---
   695→                {
   696→                    log_phase("Pass 2b: way processing", _pt, _cpu);
   697→                    pbf.unmap(); // release 86 GiB PBF mmap before admin/S2 phases
   698→                    std::cerr << "  Assembling admin polygons in parallel ("
   699→                              << data.collected_relations.size() << " relations, "
   700→                              << data.way_geometries.size() << " way geometries)..." << std::endl;
   701→
   702→                    // Thread-local admin results (to avoid locking add_admin_polygon)
   703→                    struct AdminResult {
   704→                        std::vector<std::pair<double,double>> vertices;
   705→                        std::string name;
   706→                        uint8_t admin_level;
   707→                        std::string country_code;
   708→                    };
   709→
   710→                    std::vector<std::vector<AdminResult>> thread_admin_results(num_threads);
   711→                    std::atomic<size_t> rel_idx{0};
   712→                    std::atomic<uint64_t> assembled_count{0};
   713→
   714→                    std::vector<std::thread> admin_workers;
   715→                    for (unsigned int t = 0; t < num_threads; t++) {
   716→                        admin_workers.emplace_back([&, t]() {
   717→                            auto& local_results = thread_admin_results[t];
   718→                            while (true) {
   719→                                size_t i = rel_idx.fetch_add(1);
   720→                                if (i >= data.collected_relations.size()) break;
   721→
   722→                                const auto& rel = data.collected_relations[i];
   723→
   724→                                // Skip level 2 border-line relations — these have names like
   725→                                // "Deutschland - Österreich" (two countries separated by
   726→                                // " - " or " — "). They are boundary lines, not country
   727→                                // polygons, and produce self-intersecting geometry.
   728→                                if (rel.admin_level == 2 && rel.country_code.empty() &&
   729→                                    (rel.name.find(" - ") != std::string::npos ||
   730→                                     rel.name.find(" \xe2\x80\x94 ") != std::string::npos)) { // " — " (em dash)
   731→                                    assembled_count.fetch_add(1);
   732→                                    continue;
   733→                                }
   734→
   735→                                // Skip relations with missing member ways —
   736→                                // these cross the extract boundary and would produce
   737→                                // incorrect partial polygons
   738→                                bool has_missing = false;
   739→                                for (const auto& [way_id, role] : rel.members) {
   740→                                    if (data.way_geometries.find(way_id) == data.way_geometries.end()) {
   741→                                        has_missing = true;
   742→                                        break;
   743→                                    }
   744→                                }
   745→                                if (has_missing) {
   746→                                    assembled_count.fetch_add(1);
   747→                                    continue;
   748→                                }
   749→
   750→                                auto rings = assemble_outer_rings(rel.members, data.way_geometries);
   751→                                // If outer-only assembly failed, retry with all ways.
   752→                                // Osmium ignores roles during assembly (check_roles=false) —
   753→                                // inner ways sometimes form the boundary or bridge gaps.
   754→                                // Filter retry results: discard rings with duplicate coords
   755→                                // (figure-8 shapes from merging holes with outer boundary).
   756→                                if (rings.empty()) {
   757→                                    auto retry = assemble_outer_rings(rel.members, data.way_geometries, true);
   758→                                    for (auto& ring : retry) {
   759→                                        if (!ring_has_duplicate_coords(ring)) {
   760→                                            rings.push_back(std::move(ring));
   761→                                        }
   762→                                    }
   763→                                }
   764→
   765→                                // Diagnostic: log relations with 0 rings produced
   766→                                if (rings.empty() && !rel.members.empty()) {
   767→                                    int total_ways = 0, found = 0, missing = 0;
   768→                                    for (const auto& [way_id, role] : rel.members) {
   769→                                        total_ways++;
   770→                                        auto it = data.way_geometries.find(way_id);
   771→                                        if (it != data.way_geometries.end() && !it->second.coords.empty()) {
   772→                                            found++;
   773→                                        } else {
   774→                                            missing++;
   775→                                        }
   776→                                    }
   777→                                    if (missing == 0 && found > 0) {
   778→                                        std::string detail = "  DEBUG '" + rel.name + "': ways:";
   779→                                        for (const auto& [way_id, role] : rel.members) {
   780→                                            auto it = data.way_geometries.find(way_id);
   781→                                            if (it == data.way_geometries.end()) continue;
   782→                                            detail += " [w" + std::to_string(way_id) +
   783→                                                      " first=" + std::to_string(it->second.first_node_id) +
   784→                                                      " last=" + std::to_string(it->second.last_node_id) +
   785→                                                      " n=" + std::to_string(it->second.coords.size()) + "]";
   786→                                        }
   787→                                        std::cerr << detail << std::endl;
   788→                                    }
   789→                                    std::cerr << "  WARN: relation '" << rel.name
   790→                                              << "' (level " << (int)rel.admin_level
   791→                                              << ", " << rel.members.size() << " members): "
   792→                                              << "0 rings from " << total_ways << " ways ("
   793→                                              << found << " found, " << missing << " missing)" << std::endl;
   794→                                }
   795→
   796→                                for (auto& ring : rings) {
   797→                                    if (ring.size() >= 3) {
   798→                                        AdminResult ar;
   799→                                        ar.vertices = std::move(ring);
   800→                                        ar.name = rel.name;
   801→                                        ar.admin_level = rel.admin_level;
   802→                                        ar.country_code = rel.country_code;
   803→                                        local_results.push_back(std::move(ar));
   804→                                    }
   805→                                }
   806→
   807→                                uint64_t done = assembled_count.fetch_add(1) + 1;
   808→                                if (done % 10000 == 0) {
   809→                                    std::cerr << "  Assembled " << done / 1000
   810→                                              << "K admin relations..." << std::endl;
   811→                                }
   812→                            }
   813→                        });
   814→                    }
   815→                    for (auto& w : admin_workers) w.join();
   816→                    log_phase("    Admin: ring assembly", _pt, _cpu);
   817→
   818→                    // Flatten all admin results into a single vector for parallel processing
   819→                    std::vector<AdminResult> all_admin_results;
   820→                    for (auto& local_results : thread_admin_results) {
   821→                        all_admin_results.insert(all_admin_results.end(),
   822→                            std::make_move_iterator(local_results.begin()),
   823→                            std::make_move_iterator(local_results.end()));
   824→                        local_results.clear();
   825→                    }
   826→                    uint64_t total_admin_rings = all_admin_results.size();
   827→
   828→                    // Parallel simplification: the expensive work (normalize, simplify,
   829→                    // compute area) is done per-polygon with no shared state.
   830→                    struct PreparedPolygon {
   831→                        std::vector<std::pair<double,double>> simplified;
   832→                        std::string name;
   833→                        uint8_t admin_level;
   834→                        std::string country_code;
   835→                        float area;
   836→                    };
   837→                    std::vector<PreparedPolygon> prepared(total_admin_rings);
   838→                    {
   839→                        std::atomic<size_t> prep_idx{0};
   840→                        std::vector<std::thread> prep_workers;
   841→                        for (unsigned int t = 0; t < num_threads; t++) {
   842→                            prep_workers.emplace_back([&]() {
   843→                                while (true) {
   844→                                    size_t i = prep_idx.fetch_add(1);
   845→                                    if (i >= total_admin_rings) break;
   846→                                    auto& ar = all_admin_results[i];
   847→                                    auto& pp = prepared[i];
   848→
   849→                                    // Normalize ring rotation
   850→                                    auto& vertices = ar.vertices;
   851→                                    if (vertices.size() >= 4 &&
   852→                                        std::fabs(vertices.front().first - vertices.back().first) < 1e-7 &&
   853→                                        std::fabs(vertices.front().second - vertices.back().second) < 1e-7) {
   854→                                        vertices.pop_back();
   855→                                        auto min_it = std::min_element(vertices.begin(), vertices.end());
   856→                                        std::rotate(vertices.begin(), min_it, vertices.end());
   857→                                        vertices.push_back(vertices.front());
   858→                                    }
   859→
   860→                                    pp.simplified = simplify_admin_polygon(vertices, ar.admin_level);
   861→                                    if (pp.simplified.size() >= 3) {
   862→                                        pp.area = polygon_area(pp.simplified);
   863→                                    }
   864→                                    pp.name = std::move(ar.name);
   865→                                    pp.admin_level = ar.admin_level;
   866→                                    pp.country_code = std::move(ar.country_code);
   867→                                }
   868→                            });
   869→                        }
   870→                        for (auto& w : prep_workers) w.join();
   871→                    }
   872→                    all_admin_results.clear();
   873→                    log_phase("    Admin: parallel simplify", _pt, _cpu);
   874→
   875→                    // Sequential append + submit to S2 pool (cheap: just vector push + string intern)
   876→                    for (auto& pp : prepared) {
   877→                        if (pp.simplified.size() < 3) continue;
   878→
   879→                        uint32_t poly_id = static_cast<uint32_t>(data.admin_polygons.size());
   880→                        uint32_t vertex_offset = static_cast<uint32_t>(data.admin_vertices.size());
   881→
   882→                        for (const auto& [lat, lng] : pp.simplified) {
   883→                            data.admin_vertices.push_back({static_cast<float>(lat), static_cast<float>(lng)});
   884→                        }
   885→
   886→                        AdminPolygon poly{};
   887→                        poly.vertex_offset = vertex_offset;
   888→                        poly.vertex_count = static_cast<uint32_t>(pp.simplified.size());
   889→                        poly.name_id = data.string_pool.intern(pp.name);
   890→                        poly.admin_level = pp.admin_level;
   891→                        poly.area = pp.area;
   892→                        const char* cc = pp.country_code.empty() ? nullptr : pp.country_code.c_str();
   893→                        poly.country_code = (cc && cc[0] && cc[1])
   894→                            ? static_cast<uint16_t>((cc[0] << 8) | cc[1]) : 0;
   895→                        data.admin_polygons.push_back(poly);
   896→
   897→                        admin_pool.submit(poly_id, std::move(pp.simplified));
   898→                    }
   899→
   900→                    std::cerr << "  Parallel admin assembly complete: "
   901→                              << total_admin_rings << " polygon rings from "
   902→                              << data.collected_relations.size() << " relations." << std::endl;
   903→                    log_phase("    Admin: append + submit S2", _pt, _cpu);
   904→
   905→                    // Free collected data
   906→                    data.collected_relations.clear();
   907→                    data.collected_relations.shrink_to_fit();
   908→                    data.way_geometries.clear();
   909→                    std::unordered_map<int64_t, ParsedData::WayGeometry>().swap(data.way_geometries);
   910→                }
   911→            }
   912→        }
   913→
   914→        std::cerr << "Done reading:" << std::endl;
   915→        std::cerr << "  " << data.ways.size() << " street ways" << std::endl;
   916→        std::cerr << "  " << data.addr_points.size() << " address points" << std::endl;
   917→        std::cerr << "  " << data.interp_ways.size() << " interpolation ways" << std::endl;
   918→        std::cerr << "  " << data.admin_polygons.size() << " admin polygon rings" << std::endl;
   919→
   920→        // Drain admin polygon thread pool (may still be processing)
   921→        std::cerr << "Waiting for admin polygon S2 covering to complete..." << std::endl;
   922→        auto admin_results = admin_pool.drain();
   923→        log_phase("    Admin: S2 covering drain", _pt, _cpu);
   924→        for (auto& [cell_id, ids] : admin_results) {
   925→            auto& target = data.cell_to_admin[cell_id];
   926→            target.insert(target.end(), ids.begin(), ids.end());
   927→        }
   928→        std::cerr << "Admin polygon S2 covering complete (" << data.cell_to_admin.size() << " cells)." << std::endl;
   929→


Now let me look at the critical sections with sorting and ordered output:


  1050→                    });
  1051→                }
  1052→                for (auto& t : threads) t.join();
  1053→            }
  1054→            log_phase("  S2: interp ways (parallel)", _s2t, _s2cpu);
  1055→
  1056→            // Merge thread-local pairs into single vectors, sort, build cell maps
  1057→            std::cerr << "  Sorting and grouping cell pairs..." << std::endl;
  1058→            // Concatenate + sort helper for flat CellItem pairs
  1059→            // Parallel sort each thread's pairs, then k-way merge directly into
  1060→            // hash map + sorted output. No intermediate merged vector needed.
  1061→            auto parallel_sort_and_build = [](
  1062→                std::vector<std::vector<CellItem>>& thread_pairs,
  1063→                std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map,
  1064→                std::vector<CellItemPair>& sorted_out
  1065→            ) {
  1066→                // Step 1: Convert + sort each thread's data in parallel
  1067→                size_t total = 0;
  1068→                for (auto& v : thread_pairs) total += v.size();
  1069→
  1070→                std::vector<std::vector<CellItemPair>> chunks(thread_pairs.size());
  1071→                {
  1072→                    std::vector<std::thread> sort_threads;
  1073→                    for (size_t t = 0; t < thread_pairs.size(); t++) {
  1074→                        sort_threads.emplace_back([&, t]() {
  1075→                            auto& src = thread_pairs[t];
  1076→                            auto& dst = chunks[t];
  1077→                            dst.reserve(src.size());
  1078→                            for (auto& ci : src) dst.push_back({ci.cell_id, ci.item_id});
  1079→                            src.clear(); src.shrink_to_fit();
  1080→                            std::sort(dst.begin(), dst.end(), [](const CellItemPair& a, const CellItemPair& b) {
  1081→                                return a.cell_id < b.cell_id || (a.cell_id == b.cell_id && a.item_id < b.item_id);
  1082→                            });
  1083→                        });
  1084→                    }
  1085→                    for (auto& t : sort_threads) t.join();
  1086→                }
  1087→
  1088→                // Step 2: Concatenate sorted chunks and merge using parallel tree.
  1089→                // Each chunk is already sorted. Record run boundaries, concatenate,
  1090→                // then merge adjacent runs pairwise in parallel at each level.
  1091→                auto cmp = [](const CellItemPair& a, const CellItemPair& b) {
  1092→                    return a.cell_id < b.cell_id || (a.cell_id == b.cell_id && a.item_id < b.item_id);
  1093→                };
  1094→
  1095→                // Record run boundaries before concatenation
  1096→                std::vector<size_t> run_bounds = {0};
  1097→                sorted_out.clear();
  1098→                sorted_out.reserve(total);
  1099→                for (auto& chunk : chunks) {
  1100→                    sorted_out.insert(sorted_out.end(), chunk.begin(), chunk.end());
  1101→                    run_bounds.push_back(sorted_out.size());
  1102→                    chunk.clear(); chunk.shrink_to_fit();
  1103→                }
  1104→
  1105→                // Parallel tree merge: at each level, merge adjacent run pairs in parallel.
  1106→                // Level 0: merge runs (0,1), (2,3), ... → N/2 runs
  1107→                // Level 1: merge runs (01,23), (45,67), ... → N/4 runs
  1108→                // ... until one sorted run remains.
  1109→                while (run_bounds.size() > 2) {
  1110→                    std::vector<size_t> new_bounds = {0};
  1111→                    std::vector<std::thread> merge_threads;
  1112→                    for (size_t i = 0; i + 2 < run_bounds.size(); i += 2) {
  1113→                        size_t left = run_bounds[i];
  1114→                        size_t mid = run_bounds[i + 1];
  1115→                        size_t right = run_bounds[i + 2];
  1116→                        merge_threads.emplace_back([&, left, mid, right] {
  1117→                            std::inplace_merge(sorted_out.begin() + left,
  1118→                                               sorted_out.begin() + mid,
  1119→                                               sorted_out.begin() + right, cmp);
  1120→                        });
  1121→                        new_bounds.push_back(right);
  1122→                    }
  1123→                    // If odd number of runs, carry the last one forward
  1124→                    if (run_bounds.size() % 2 == 0) {
  1125→                        new_bounds.push_back(run_bounds.back());
  1126→                    }
  1127→                    for (auto& t : merge_threads) t.join();
  1128→                    run_bounds = std::move(new_bounds);
  1129→                }
  1130→
  1131→                // Skip building hash map — sorted_out is used directly for writing.
  1132→                // cell_map stays empty (only needed for cache/continent modes).
  1133→            };
  1134→
  1135→
  1136→            // Sort each thread's pairs in parallel, then k-way merge directly
  1137→            // into hash map + sorted output. No intermediate merged vector.
  1138→            auto f_ways = std::async(std::launch::async, [&] {
  1139→                parallel_sort_and_build(way_pairs, data.cell_to_ways, data.sorted_way_cells);
  1140→            });
  1141→            auto f_interps = std::async(std::launch::async, [&] {
  1142→                parallel_sort_and_build(interp_pairs, data.cell_to_interps, data.sorted_interp_cells);
  1143→            });
  1144→            f_ways.get();
  1145→            f_interps.get();
  1146→            log_phase("  S2: sort + group into cell maps", _s2t, _s2cpu);
  1147→
  1148→            // Free deferred work items
  1149→            data.deferred_ways.clear();
  1150→            data.deferred_ways.shrink_to_fit();
  1151→            data.deferred_interps.clear();
  1152→            data.deferred_interps.shrink_to_fit();
  1153→
  1154→            std::cerr << "S2 cell computation complete." << std::endl;
  1155→        }
  1156→
  1157→        // Resolve interpolation endpoints
  1158→        std::cerr << "Resolving interpolation endpoints..." << std::endl;
  1159→        resolve_interpolation_endpoints(data);
  1160→
  1161→        // Deduplicate + convert to sorted pairs for fast writing
  1162→        log_phase("S2 cell computation", _pt, _cpu);
  1163→        std::cerr << "Deduplicating + sorting for write..." << std::endl;
  1164→        {
  1165→            // Convert addr hash map to sorted pairs.
  1166→            // Sort cell IDs only (30M unique), then build pairs in sorted order.
  1167→            // Much faster than sorting all 160M pairs.
  1168→            auto f2 = std::async(std::launch::async, [&] {
  1169→                auto _dt = std::chrono::steady_clock::now();
  1170→                auto _dc = CpuTicks::now();
  1171→
  1172→                // Step 1: Extract cell IDs from hash map
  1173→                std::vector<uint64_t> sorted_cells;
  1174→                sorted_cells.reserve(data.cell_to_addrs.size());
  1175→                for (auto& [cell_id, ids] : data.cell_to_addrs)
  1176→                    sorted_cells.push_back(cell_id);
  1177→                log_phase("    Dedup: extract cell IDs", _dt, _dc);
  1178→
  1179→                // Step 2: Sort cell IDs
  1180→                std::sort(sorted_cells.begin(), sorted_cells.end());
  1181→                log_phase("    Dedup: sort cell IDs", _dt, _dc);
  1182→
  1183→                // Step 3: Parallel dedup within each cell
  1184→                {
  1185→                    unsigned nthreads = std::thread::hardware_concurrency();
  1186→                    size_t chunk = (sorted_cells.size() + nthreads - 1) / nthreads;
  1187→                    std::vector<std::thread> threads;
  1188→                    for (unsigned t = 0; t < nthreads; t++) {
  1189→                        size_t start = t * chunk;
  1190→                        size_t end = std::min(start + chunk, sorted_cells.size());
  1191→                        if (start >= sorted_cells.size()) break;
  1192→                        threads.emplace_back([&, start, end]() {
  1193→                            for (size_t i = start; i < end; i++) {
  1194→                                auto& ids = data.cell_to_addrs[sorted_cells[i]];
  1195→                                std::sort(ids.begin(), ids.end());
  1196→                                ids.erase(std::unique(ids.begin(), ids.end()), ids.end());
  1197→                            }
  1198→                        });
  1199→                    }
  1200→                    for (auto& t : threads) t.join();
  1201→                }
  1202→                log_phase("    Dedup: per-cell sort+dedup", _dt, _dc);
  1203→
  1204→                // Step 4: Count total pairs
  1205→                size_t total_pairs = 0;
  1206→                for (auto& [_, ids] : data.cell_to_addrs) total_pairs += ids.size();
  1207→                log_phase("    Dedup: count pairs", _dt, _dc);
  1208→
  1209→                // Step 5: Build sorted pairs
  1210→                std::vector<CellItemPair> pairs;
  1211→                pairs.reserve(total_pairs);
  1212→                for (uint64_t cell_id : sorted_cells) {
  1213→                    auto& ids = data.cell_to_addrs[cell_id];
  1214→                    for (auto id : ids) pairs.push_back({cell_id, id});
  1215→                }
  1216→                data.sorted_addr_cells = std::move(pairs);
  1217→                log_phase("    Dedup: build sorted pairs", _dt, _dc);
  1218→            });
  1219→            auto f4 = std::async(std::launch::async, [&]{ deduplicate(data.cell_to_admin); });
  1220→            f2.get(); f4.get();
  1221→        }
  1222→
  1223→        // Save cache if requested
  1224→        if (!save_cache_path.empty()) {
  1225→            serialize_cache(data, save_cache_path);
  1226→        }
  1227→    }
  1228→
  1229→    // --- Rebuild hash maps from sorted pairs if needed for cache saving ---
  1230→    // (Continent filtering now uses sorted pairs directly, so no rebuild needed for that.)
  1231→    if (!save_cache_path.empty()) {
  1232→        // Parallel rebuild: split sorted pairs into chunks, each thread builds
  1233→        // a partial map, then merge. Sorted pairs are grouped by cell_id so we
  1234→        // can split at cell boundaries for zero-conflict parallel insertion.
  1235→        auto rebuild_map_parallel = [](const std::vector<CellItemPair>& sorted,
  1236→                                        std::unordered_map<uint64_t, std::vector<uint32_t>>& map) {
  1237→            if (sorted.empty() || !map.empty()) return;
  1238→            unsigned nthreads = std::thread::hardware_concurrency();
  1239→            if (nthreads == 0) nthreads = 4;
  1240→            size_t chunk = (sorted.size() + nthreads - 1) / nthreads;
  1241→
  1242→            // Find cell boundaries for clean splits
  1243→            std::vector<size_t> boundaries = {0};
  1244→            for (unsigned t = 1; t < nthreads; t++) {
  1245→                size_t target = t * chunk;
  1246→                if (target >= sorted.size()) break;
  1247→                // Advance to next cell boundary
  1248→                while (target < sorted.size() && sorted[target].cell_id == sorted[target-1].cell_id)
  1249→                    target++;


Now let me check how the addresses and ways are merged into the final output:


   600→                tl_way_data = nullptr;
   601→                std::cerr << "  Parallel way processing complete." << std::endl;
   602→
   603→                // Process areas/multipolygons — sequential fallback path
   604→                // Merge thread-local way/interp data into main ParsedData
   605→                std::cerr << "  Merging thread-local data..." << std::endl;
   606→                uint64_t total_ways = 0, total_building_addrs = 0, total_interps = 0;
   607→                for (auto& local : tld) {
   608→                    uint32_t way_base = static_cast<uint32_t>(data.ways.size());
   609→                    uint32_t node_base = static_cast<uint32_t>(data.street_nodes.size());
   610→                    uint32_t interp_base = static_cast<uint32_t>(data.interp_ways.size());
   611→                    uint32_t interp_node_base = static_cast<uint32_t>(data.interp_nodes.size());
   612→
   613→                    // Merge street ways
   614→                    for (size_t i = 0; i < local.ways.size(); i++) {
   615→                        auto h = local.ways[i];
   616→                        h.node_offset += node_base;
   617→                        h.name_id = data.string_pool.intern(local.way_strings[i]);
   618→                        data.ways.push_back(h);
   619→                    }
   620→                    data.street_nodes.insert(data.street_nodes.end(),
   621→                        local.street_nodes.begin(), local.street_nodes.end());
   622→
   623→                    // Remap deferred ways
   624→                    for (auto dw : local.deferred_ways) {
   625→                        dw.way_id += way_base;
   626→                        dw.node_offset += node_base;
   627→                        data.deferred_ways.push_back(dw);
   628→                    }
   629→
   630→                    // Merge building addresses
   631→                    for (size_t i = 0; i < local.building_addrs.size(); i++) {
   632→                        uint64_t dummy = 0;
   633→                        add_addr_point(data, local.building_addrs[i].lat, local.building_addrs[i].lng,
   634→                                       local.addr_strings[i].first.c_str(),
   635→                                       local.addr_strings[i].second.c_str(), dummy);
   636→                    }
   637→
   638→                    // Merge interpolation ways
   639→                    for (size_t i = 0; i < local.interp_ways.size(); i++) {
   640→                        auto iw = local.interp_ways[i];
   641→                        iw.node_offset += interp_node_base;
   642→                        iw.street_id = data.string_pool.intern(local.interp_strings[i]);
   643→                        data.interp_ways.push_back(iw);
   644→                    }
   645→                    data.interp_nodes.insert(data.interp_nodes.end(),
   646→                        local.interp_nodes.begin(), local.interp_nodes.end());
   647→
   648→                    // Remap deferred interps
   649→                    for (auto di : local.deferred_interps) {
   650→                        di.interp_id += interp_base;
   651→                        di.node_offset += interp_node_base;
   652→                        data.deferred_interps.push_back(di);
   653→                    }
   654→
   655→                    total_ways += local.way_count;
   656→                    total_building_addrs += local.building_addr_count;
   657→                    total_interps += local.interp_count;
   658→
   659→                    // Merge way geometries for admin assembly
   660→                    {
   661→                        for (auto& wg : local.way_geoms) {
   662→                            ParsedData::WayGeometry g;
   663→                            g.coords = std::move(wg.coords);
   664→                            g.first_node_id = wg.first_node_id;
   665→                            g.last_node_id = wg.last_node_id;
   666→                            data.way_geometries[wg.way_id] = std::move(g);
   667→                        }
   668→                        local.way_geoms.clear();
   669→                        local.way_geoms.shrink_to_fit();
   670→                    }
   671→                }
   672→                std::cerr << "  Merged: " << total_ways << " ways, "
   673→                          << total_building_addrs << " building addrs, "
   674→                          << total_interps << " interps from parallel processing." << std::endl;
   675→
   676→                // --- Merge closed-way admin polygons ---
   677→                {
   678→                    uint64_t closed_way_admin_count = 0;
   679→                    for (auto& local : tld) {
   680→                        for (auto& cwa : local.closed_way_admins) {
   681→                            const char* cc = cwa.country_code.empty() ? nullptr : cwa.country_code.c_str();
   682→                            add_admin_polygon(data, cwa.vertices, cwa.name.c_str(),
   683→                                              cwa.admin_level, cc, &admin_pool);
   684→                            closed_way_admin_count++;
   685→                        }
   686→                        local.closed_way_admins.clear();
   687→                    }
   688→                    if (closed_way_admin_count > 0) {
   689→                        std::cerr << "  Added " << closed_way_admin_count
   690→                                  << " admin polygons from closed ways." << std::endl;
   691→                    }
   692→                }
   693→
   694→                // --- Parallel admin boundary assembly ---
   695→                {
   696→                    log_phase("Pass 2b: way processing", _pt, _cpu);
   697→                    pbf.unmap(); // release 86 GiB PBF mmap before admin/S2 phases
   698→                    std::cerr << "  Assembling admin polygons in parallel ("
   699→                              << data.collected_relations.size() << " relations, "


Let me check for hash map usage and how data is written:


909:                    std::unordered_map<int64_t, ParsedData::WayGeometry>().swap(data.way_geometries);
925:            auto& target = data.cell_to_admin[cell_id];
928:        std::cerr << "Admin polygon S2 covering complete (" << data.cell_to_admin.size() << " cells)." << std::endl;
962:            // Remap poly_ids in cell_to_admin (preserving INTERIOR_FLAG)
963:            for (auto& [cell_id, ids] : data.cell_to_admin) {
1063:                std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map,
1139:                parallel_sort_and_build(way_pairs, data.cell_to_ways, data.sorted_way_cells);
1142:                parallel_sort_and_build(interp_pairs, data.cell_to_interps, data.sorted_interp_cells);
1174:                sorted_cells.reserve(data.cell_to_addrs.size());
1175:                for (auto& [cell_id, ids] : data.cell_to_addrs)
1194:                                auto& ids = data.cell_to_addrs[sorted_cells[i]];
1206:                for (auto& [_, ids] : data.cell_to_addrs) total_pairs += ids.size();
1213:                    auto& ids = data.cell_to_addrs[cell_id];
1219:            auto f4 = std::async(std::launch::async, [&]{ deduplicate(data.cell_to_admin); });
1236:                                        std::unordered_map<uint64_t, std::vector<uint32_t>>& map) {
1255:            std::vector<std::unordered_map<uint64_t, std::vector<uint32_t>>> sub_maps(boundaries.size() - 1);
1282:        auto f1 = std::async(std::launch::async, [&]{ rebuild_map_parallel(data.sorted_way_cells, data.cell_to_ways); });
1283:        auto f2 = std::async(std::launch::async, [&]{ rebuild_map_parallel(data.sorted_interp_cells, data.cell_to_interps); });


  1300→    // concurrency to avoid over-subscribing CPU.
  1301→    auto write_qualities = [&](const ParsedData& d, const std::string& admin_dir) {
  1302→        constexpr unsigned max_concurrent_qualities = 3;
  1303→        std::atomic<unsigned> active{0};
  1304→        std::mutex qmtx;
  1305→        std::condition_variable qcv;
  1306→        std::vector<std::future<void>> qfutures;
  1307→
  1308→        for (double scale : quality_scales) {
  1309→            {
  1310→                std::unique_lock<std::mutex> lock(qmtx);
  1311→                qcv.wait(lock, [&]{ return active.load() < max_concurrent_qualities; });
  1312→            }
  1313→            active.fetch_add(1);
  1314→
  1315→            std::string qname = quality_dir_name(scale);
  1316→            std::string qdir = admin_dir + "/" + qname;
  1317→            qfutures.push_back(std::async(std::launch::async, [&, qdir, scale]() {
  1318→                write_quality_variant(d, admin_dir, qdir, scale);
  1319→                active.fetch_sub(1);
  1320→                qcv.notify_one();
  1321→            }));
  1322→        }
  1323→        for (auto& f : qfutures) f.get();
  1324→    };
  1325→
  1326→    // Write one region: modes + quality variants
  1327→    auto write_region = [&](const ParsedData& d, const std::string& base_dir) {
  1328→        ensure_dir(base_dir);
  1329→
  1330→        if (multi_output) {
  1331→            // Write all 3 modes in parallel (they read shared data, write to separate dirs)
  1332→            auto wf1 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/full", IndexMode::Full); });
  1333→            auto wf2 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/no-addresses", IndexMode::NoAddresses); });
  1334→            auto wf3 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/admin", IndexMode::AdminOnly); });
  1335→            wf1.get(); wf2.get(); wf3.get();
  1336→        } else {
  1337→            write_index(d, base_dir, mode);
  1338→        }
  1339→
  1340→        // Write quality variants under admin/ (or base_dir if not multi_output)
  1341→        if (multi_quality) {
  1342→            std::string admin_dir = multi_output ? base_dir + "/admin" : base_dir;
  1343→            std::cerr << "  Writing quality variants for " << base_dir << "..." << std::endl;
  1344→            write_qualities(d, admin_dir);
  1345→        }
  1346→    };
  1347→
  1348→    // Write planet (async — overlaps with continent filtering start)
  1349→    auto planet_future = std::async(std::launch::async, [&]() {
  1350→        write_region(data, output_dir);
  1351→    });
  1352→
  1353→    // Process continents with bounded concurrency, largest first.
  1354→    if (generate_continents) {
  1355→        // Pre-compute continent membership for each unique cell in sorted pairs.
  1356→        // One parallel scan replaces 8 × cell_in_bbox per cell during filtering.
  1357→        auto _pct = std::chrono::steady_clock::now();
  1358→        auto _pcc = CpuTicks::now();
  1359→
  1360→        // Load continent polygons for boundary-based filtering
  1361→        auto continent_polys = get_continent_polygons();
  1362→        std::cerr << "  Loaded " << continent_polys.size() << " continent boundary polygons" << std::endl;
  1363→
  1364→        // For each sorted pair array, build a parallel array of continent bitmasks.
  1365→        // Since pairs are sorted by cell_id, consecutive entries share the same mask.
  1366→        auto precompute_masks = [&continent_polys](const std::vector<CellItemPair>& sorted) -> std::vector<uint8_t> {
  1367→            if (sorted.empty()) return {};
  1368→            std::vector<uint8_t> masks(sorted.size(), 0);
  1369→
  1370→            unsigned nthreads = std::max(1u, std::thread::hardware_concurrency());
  1371→            size_t chunk = (sorted.size() + nthreads - 1) / nthreads;
  1372→            std::vector<size_t> bounds = {0};
  1373→            for (unsigned t = 1; t < nthreads; t++) {
  1374→                size_t target = t * chunk;
  1375→                if (target >= sorted.size()) break;
  1376→                while (target < sorted.size() && sorted[target].cell_id == sorted[target-1].cell_id)
  1377→                    target++;
  1378→                if (target < sorted.size()) bounds.push_back(target);
  1379→            }
  1380→            bounds.push_back(sorted.size());
  1381→
  1382→            std::vector<std::thread> threads;
  1383→            for (size_t th = 0; th + 1 < bounds.size(); th++) {
  1384→                threads.emplace_back([&, th]() {
  1385→                    for (size_t i = bounds[th]; i < bounds[th+1]; ) {
  1386→                        uint64_t cell_id = sorted[i].cell_id;
  1387→                        S2CellId cell(cell_id);
  1388→                        S2LatLng center = cell.ToLatLng();
  1389→                        double lat = center.lat().degrees();
  1390→                        double lng = center.lng().degrees();
  1391→                        uint8_t mask = 0;
  1392→                        // Test against continent polygons (not bboxes)
  1393→                        for (size_t ci = 0; ci < continent_polys.size() && ci < 8; ci++) {
  1394→                            if (point_in_polygon(lat, lng, continent_polys[ci].vertices))
  1395→                                mask |= (1u << ci);
  1396→                        }
  1397→                        while (i < bounds[th+1] && sorted[i].cell_id == cell_id) {
  1398→                            masks[i] = mask;
  1399→                            i++;


Now let me check the write_index function to see how data is written:


grep -n "void write_index\|write_index(" /root/traccar-geocoder/builder/src/build_index.cpp | head -5

Find write_index function


1332:            auto wf1 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/full", IndexMode::Full); });
1333:            auto wf2 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/no-addresses", IndexMode::NoAddresses); });
1334:            auto wf3 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/admin", IndexMode::AdminOnly); });
1337:            write_index(d, base_dir, mode);


Let me find the write_index function definition:


No files found


grep -r "void write_index" /root/traccar-geocoder/builder/src/ | head -3

Find write_index definition


/root/traccar-geocoder/builder/src/cell_index.cpp:void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
/root/traccar-geocoder/builder/src/cell_index.h:void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode);


Perfect! Now let me examine the cell_index.cpp for the write function:


     1→#include "cell_index.h"
     2→
     3→#include <algorithm>
     4→#include <atomic>
     5→#include <cstring>
     6→#include <fstream>
     7→#include <future>
     8→#include <iostream>
     9→#include <thread>
    10→#include <unordered_map>
    11→#include <unordered_set>
    12→
    13→#include <unistd.h>
    14→
    15→#include "geometry.h"
    16→#include "s2_helpers.h"
    17→
    18→std::vector<uint32_t> write_entries(
    19→    const std::string& path,
    20→    const std::vector<uint64_t>& sorted_cells,
    21→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
    22→) {
    23→    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
    24→    std::vector<CellRef> sorted_refs;
    25→    sorted_refs.reserve(cell_map.size());
    26→    for (auto& [id, ids] : cell_map) sorted_refs.push_back({id, &ids});
    27→    std::sort(sorted_refs.begin(), sorted_refs.end(),
    28→        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });
    29→
    30→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    31→    size_t total_size = 0;
    32→    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);
    33→
    34→    std::vector<char> buf;
    35→    buf.reserve(total_size);
    36→    uint32_t current = 0;
    37→    size_t ri = 0;
    38→    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
    39→        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
    40→        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
    41→        offsets[si] = current;
    42→        const auto& ids = *sorted_refs[ri].ids;
    43→        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
    44→        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
    45→                   reinterpret_cast<const char*>(&count) + sizeof(count));
    46→        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
    47→                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
    48→        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
    49→        ri++;
    50→    }
    51→    std::ofstream f(path, std::ios::binary);
    52→    f.write(buf.data(), buf.size());
    53→    return offsets;
    54→}
    55→
    56→std::vector<uint32_t> write_entries_from_sorted(
    57→    const std::string& path,
    58→    const std::vector<uint64_t>& sorted_cells,
    59→    const std::vector<CellItemPair>& sorted_pairs
    60→) {
    61→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    62→    if (sorted_pairs.empty()) {
    63→        std::ofstream f(path, std::ios::binary);
    64→        return offsets;
    65→    }
    66→
    67→    unsigned int nthreads = std::thread::hardware_concurrency();
    68→    if (nthreads == 0) nthreads = 4;
    69→    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;
    70→
    71→    struct ChunkResult {
    72→        std::vector<char> buf;
    73→        size_t cell_start, cell_end;
    74→        uint32_t local_size;
    75→    };
    76→    std::vector<ChunkResult> chunks(nthreads);
    77→
    78→    std::vector<std::thread> threads;
    79→    for (unsigned int t = 0; t < nthreads; t++) {
    80→        size_t cs = t * cells_per_chunk;
    81→        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
    82→        if (cs >= sorted_cells.size()) break;
    83→
    84→        threads.emplace_back([&, t, cs, ce]() {
    85→            auto& chunk = chunks[t];
    86→            chunk.cell_start = cs;
    87→            chunk.cell_end = ce;
    88→            chunk.local_size = 0;
    89→
    90→            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
    91→                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
    92→                    return p.cell_id < id;
    93→                }) - sorted_pairs.begin();
    94→
    95→            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
    96→                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
    97→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
    98→                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;
    99→
   100→                offsets[si] = chunk.local_size;


Let me continue reading the write_entries_from_sorted function and look for the actual write_index:


   100→                offsets[si] = chunk.local_size;
   101→                size_t start = pi;
   102→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
   103→                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(MAX_VERTEX_COUNT)));
   104→                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
   105→                size_t buf_pos = chunk.buf.size();
   106→                chunk.buf.resize(buf_pos + entry_size);
   107→                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
   108→                for (size_t k = start; k < pi; k++) {
   109→                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
   110→                           &sorted_pairs[k].item_id, sizeof(uint32_t));
   111→                }
   112→                chunk.local_size += entry_size;
   113→            }
   114→        });
   115→    }
   116→    for (auto& t : threads) t.join();
   117→
   118→    uint32_t global_offset = 0;
   119→    for (auto& chunk : chunks) {
   120→        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
   121→            if (offsets[si] != NO_DATA) offsets[si] += global_offset;
   122→        }
   123→        global_offset += chunk.local_size;
   124→    }
   125→
   126→    std::vector<char> buf;
   127→    buf.reserve(global_offset);
   128→    for (auto& chunk : chunks) buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());
   129→
   130→    std::ofstream f(path, std::ios::binary);
   131→    f.write(buf.data(), buf.size());
   132→    return offsets;
   133→}
   134→
   135→void write_cell_index(
   136→    const std::string& cells_path,
   137→    const std::string& entries_path,
   138→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
   139→) {
   140→    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(cell_map.begin(), cell_map.end());
   141→    std::sort(sorted.begin(), sorted.end());
   142→
   143→    { std::ofstream f(cells_path, std::ios::binary);
   144→      uint32_t current_offset = 0;
   145→      for (const auto& [cell_id, ids] : sorted) {
   146→          f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
   147→          f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
   148→          current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
   149→      } }
   150→
   151→    { std::ofstream f(entries_path, std::ios::binary);
   152→      for (const auto& [cell_id, ids] : sorted) {
   153→          uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
   154→          f.write(reinterpret_cast<const char*>(&count), sizeof(count));
   155→          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
   156→      } }
   157→}
   158→
   159→void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
   160→    ensure_dir(output_dir);
   161→    auto _wt = std::chrono::steady_clock::now();
   162→
   163→    bool write_streets = (mode != IndexMode::AdminOnly);
   164→    bool write_addresses = (mode == IndexMode::Full);
   165→
   166→    if (write_streets) {
   167→        std::vector<uint64_t> sorted_geo_cells;
   168→        {
   169→            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
   170→                std::vector<uint64_t> cells;
   171→                cells.reserve(pairs.size() / 2);
   172→                for (size_t i = 0; i < pairs.size(); ) {
   173→                    cells.push_back(pairs[i].cell_id);
   174→                    uint64_t cur = pairs[i].cell_id;
   175→                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
   176→                }
   177→                return cells;
   178→            };
   179→            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
   180→                std::vector<uint64_t> cells;
   181→                cells.reserve(m.size());
   182→                for (auto& [id, _] : m) cells.push_back(id);
   183→                std::sort(cells.begin(), cells.end());
   184→                return cells;
   185→            };
   186→
   187→            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
   188→            {
   189→                auto f1 = std::async(std::launch::async, [&] {
   190→                    return !data.sorted_way_cells.empty()
   191→                        ? extract_unique_cells(data.sorted_way_cells)
   192→                        : extract_from_map(data.cell_to_ways);
   193→                });
   194→                if (write_addresses) {
   195→                    auto f2 = std::async(std::launch::async, [&] {
   196→                        return !data.sorted_addr_cells.empty()
   197→                            ? extract_unique_cells(data.sorted_addr_cells)
   198→                            : extract_from_map(data.cell_to_addrs);
   199→                    });
   200→                    auto f3 = std::async(std::launch::async, [&] {
   201→                        return !data.sorted_interp_cells.empty()
   202→                            ? extract_unique_cells(data.sorted_interp_cells)
   203→                            : extract_from_map(data.cell_to_interps);
   204→                    });
   205→                    addr_cells = f2.get();
   206→                    interp_cells = f3.get();
   207→                }
   208→                way_cells = f1.get();
   209→            }
   210→
   211→            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
   212→            std::merge(way_cells.begin(), way_cells.end(), addr_cells.begin(), addr_cells.end(),
   213→                       std::back_inserter(sorted_geo_cells));
   214→            if (!interp_cells.empty()) {
   215→                std::vector<uint64_t> tmp;
   216→                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
   217→                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
   218→                           interp_cells.begin(), interp_cells.end(), std::back_inserter(tmp));
   219→                sorted_geo_cells = std::move(tmp);
   220→            }
   221→            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
   222→                                    sorted_geo_cells.end());
   223→        }
   224→
   225→        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
   226→        {
   227→            auto f1 = std::async(std::launch::async, [&]() {
   228→                if (!data.sorted_way_cells.empty())
   229→                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
   230→                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
   231→            });
   232→            if (write_addresses) {
   233→                auto f2 = std::async(std::launch::async, [&]() {
   234→                    if (!data.sorted_addr_cells.empty())
   235→                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
   236→                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
   237→                });
   238→                auto f3 = std::async(std::launch::async, [&]() {
   239→                    if (!data.sorted_interp_cells.empty())
   240→                        return write_entries_from_sorted(output_dir + "/interp_entries.bin", sorted_geo_cells, data.sorted_interp_cells);
   241→                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
   242→                });
   243→                addr_offsets = f2.get();
   244→                interp_offsets = f3.get();
   245→            }
   246→            street_offsets = f1.get();
   247→        }
   248→
   249→        log_phase("  Write: entry files", _wt);


Get line count for cell_index


469 /root/traccar-geocoder/builder/src/cell_index.cpp


Let me continue reading the write_index function:


   249→        log_phase("  Write: entry files", _wt);
   250→        {
   251→            size_t n = sorted_geo_cells.size();
   252→            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
   253→            std::vector<char> buf(n * row_size);
   254→            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
   255→            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);
   256→
   257→            unsigned int nthreads = std::thread::hardware_concurrency();
   258→            if (nthreads == 0) nthreads = 4;
   259→            size_t chunk = (n + nthreads - 1) / nthreads;
   260→            std::vector<std::thread> fill_threads;
   261→            for (unsigned int t = 0; t < nthreads; t++) {
   262→                size_t start = t * chunk;
   263→                size_t end = std::min(start + chunk, n);
   264→                if (start >= n) break;
   265→                fill_threads.emplace_back([&, start, end]() {
   266→                    char* ptr = buf.data() + start * row_size;
   267→                    for (size_t i = start; i < end; i++) {
   268→                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
   269→                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   270→                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   271→                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   272→                    }
   273→                });
   274→            }
   275→            for (auto& t : fill_threads) t.join();
   276→
   277→            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
   278→            f.write(buf.data(), buf.size());
   279→        }
   280→
   281→        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
   282→                  << data.ways.size() << " ways, " << data.addr_points.size() << " addrs, "
   283→                  << data.interp_ways.size() << " interps)" << std::endl;
   284→    }
   285→
   286→    log_phase("  Write: geo_cells.bin", _wt);
   287→    auto admin_future = std::async(std::launch::async, [&] {
   288→        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
   289→        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
   290→    });
   291→
   292→    std::vector<std::future<void>> write_futures;
   293→    if (write_streets) {
   294→        write_futures.push_back(std::async(std::launch::async, [&] {
   295→            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
   296→            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
   297→        }));
   298→        write_futures.push_back(std::async(std::launch::async, [&] {
   299→            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
   300→            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
   301→        }));
   302→        if (write_addresses) {
   303→            write_futures.push_back(std::async(std::launch::async, [&] {
   304→                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
   305→                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
   306→            }));
   307→            write_futures.push_back(std::async(std::launch::async, [&] {
   308→                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
   309→                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
   310→            }));
   311→            write_futures.push_back(std::async(std::launch::async, [&] {
   312→                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
   313→                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
   314→            }));
   315→        }
   316→    }
   317→    // If not writing all data types, compact string pool and remap references
   318→    // before writing admin_polygons and strings.bin.
   319→    std::vector<AdminPolygon> compact_admin;
   320→    std::vector<char> compact_strings;
   321→    const bool need_compact = (!write_streets || !write_addresses);
   322→
   323→    if (need_compact) {
   324→        // Collect referenced string offsets
   325→        std::unordered_set<uint32_t> used_offsets;
   326→        if (write_streets) {
   327→            for (const auto& w : data.ways) used_offsets.insert(w.name_id);
   328→            if (write_addresses) {
   329→                for (const auto& a : data.addr_points) {
   330→                    used_offsets.insert(a.housenumber_id);
   331→                    used_offsets.insert(a.street_id);
   332→                }
   333→                for (const auto& iw : data.interp_ways) used_offsets.insert(iw.street_id);
   334→            }
   335→        }
   336→        for (const auto& ap : data.admin_polygons) used_offsets.insert(ap.name_id);
   337→
   338→        // Build compact pool and remap
   339→        const auto& old_sp = data.string_pool.data();
   340→        std::vector<uint32_t> sorted_offs(used_offsets.begin(), used_offsets.end());
   341→        std::sort(sorted_offs.begin(), sorted_offs.end());
   342→        std::unordered_map<uint32_t, uint32_t> remap;
   343→        compact_strings.reserve(used_offsets.size() * 16);
   344→        for (uint32_t off : sorted_offs) {
   345→            remap[off] = static_cast<uint32_t>(compact_strings.size());
   346→            size_t len = std::strlen(old_sp.data() + off);
   347→            compact_strings.insert(compact_strings.end(), old_sp.data() + off,
   348→                old_sp.data() + off + len + 1);
   349→        }
   350→
   351→        // Remap admin polygon name_ids
   352→        compact_admin = data.admin_polygons;
   353→        for (auto& ap : compact_admin) ap.name_id = remap[ap.name_id];
   354→
   355→        // Remap street/addr/interp if applicable (for no-addresses mode)
   356→        // These are written from data directly, so we'd need copies too.
   357→        // For simplicity, only compact when we have fewer types.
   358→    }
   359→
   360→    const auto& admin_to_write = need_compact ? compact_admin : data.admin_polygons;
   361→    write_futures.push_back(std::async(std::launch::async, [&] {
   362→        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
   363→        f.write(reinterpret_cast<const char*>(admin_to_write.data()), admin_to_write.size() * sizeof(AdminPolygon));
   364→    }));
   365→    write_futures.push_back(std::async(std::launch::async, [&] {
   366→        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
   367→        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
   368→    }));
   369→    write_futures.push_back(std::async(std::launch::async, [&] {
   370→        if (need_compact) {
   371→            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
   372→            f.write(compact_strings.data(), compact_strings.size());
   373→            std::cerr << "strings.bin: " << compact_strings.size() << " bytes (compacted from "
   374→                      << data.string_pool.data().size() << ")" << std::endl;
   375→        } else {
   376→            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
   377→            f.write(data.string_pool.data().data(), data.string_pool.data().size());
   378→            std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
   379→        }
   380→    }));
   381→
   382→    log_phase("  Write: parallel data files launched", _wt);
   383→    admin_future.get();
   384→    for (auto& f : write_futures) f.get();
   385→}
   386→
   387→void write_quality_variant(const ParsedData& data, const std::string& source_dir,
   388→                           const std::string& output_dir, double epsilon_scale) {
   389→    ensure_dir(output_dir);
   390→
   391→    // Re-simplify admin polygons at the given epsilon scale
   392→    std::vector<AdminPolygon> new_polys;
   393→    std::vector<NodeCoord> new_verts;
   394→    new_polys.reserve(data.admin_polygons.size());
   395→
   396→    // Parallel simplification
   397→    struct SimplifiedPoly {
   398→        std::vector<std::pair<double,double>> verts;


Now let me check the parsed_data.h to see the structure of these data containers:


     1→#pragma once
     2→
     3→#include <algorithm>
     4→#include <chrono>
     5→#include <iostream>
     6→#include <string>
     7→#include <unordered_map>
     8→#include <vector>
     9→
    10→#include <cstdio>
    11→#include <cstring>
    12→#include <dirent.h>
    13→#include <iomanip>
    14→#include <thread>
    15→#include <unistd.h>
    16→#include <sys/stat.h>
    17→
    18→#include "types.h"
    19→#include "string_pool.h"
    20→
    21→// --- Directory creation ---
    22→
    23→inline void ensure_dir(const std::string& path) {
    24→    // Recursive mkdir — create all parent directories
    25→    for (size_t i = 1; i < path.size(); i++) {
    26→        if (path[i] == '/') {
    27→            mkdir(path.substr(0, i).c_str(), 0755);
    28→        }
    29→    }
    30→    mkdir(path.c_str(), 0755);
    31→}
    32→
    33→// --- CPU tick reading (all threads via getrusage) ---
    34→
    35→#include <sys/resource.h>
    36→
    37→struct CpuTicks {
    38→    long long process_us = 0;  // user+system microseconds (all threads)
    39→
    40→    static CpuTicks now() {
    41→        CpuTicks ct;
    42→        struct rusage ru;
    43→        if (getrusage(RUSAGE_SELF, &ru) == 0) {
    44→            ct.process_us = (long long)ru.ru_utime.tv_sec * 1000000 + ru.ru_utime.tv_usec
    45→                          + (long long)ru.ru_stime.tv_sec * 1000000 + ru.ru_stime.tv_usec;
    46→        }
    47→        return ct;
    48→    }
    49→};
    50→
    51→// --- Phase timer with CPU utilization ---
    52→
    53→inline long get_rss_mb() {
    54→    long rss_pages = 0;
    55→    FILE* f = fopen("/proc/self/statm", "r");
    56→    if (f) {
    57→        long size;
    58→        if (fscanf(f, "%ld %ld", &size, &rss_pages) != 2) rss_pages = 0;
    59→        fclose(f);
    60→    }
    61→    return rss_pages * sysconf(_SC_PAGESIZE) / (1024 * 1024);
    62→}
    63→
    64→inline void log_phase(const char* name, std::chrono::steady_clock::time_point& t,
    65→                       CpuTicks& prev_cpu) {
    66→    auto now = std::chrono::steady_clock::now();
    67→    auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(now - t).count();
    68→    auto cpu_now = CpuTicks::now();
    69→    double cpu_sec = (cpu_now.process_us - prev_cpu.process_us) / 1e6;
    70→    double wall_sec = ms / 1000.0;
    71→    double cores_used = (wall_sec > 0.1) ? cpu_sec / wall_sec : 0;
    72→    unsigned ncpu = std::thread::hardware_concurrency();
    73→    int pct = (wall_sec > 0.1 && ncpu > 0) ? (int)(cores_used * 100.0 / ncpu) : 0;
    74→    long rss = get_rss_mb();
    75→    std::cerr << "  [" << ms/1000 << "." << (ms%1000)/100 << "s"
    76→              << " " << std::fixed << std::setprecision(1) << cores_used << "/" << ncpu << "cores"
    77→              << " " << pct << "%"
    78→              << " " << rss << "MiB] " << name << std::endl;
    79→    t = now;
    80→    prev_cpu = cpu_now;
    81→}
    82→
    83→// Backward compat overload (no CPU tracking)
    84→inline void log_phase(const char* name, std::chrono::steady_clock::time_point& t) {
    85→    auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(
    86→        std::chrono::steady_clock::now() - t).count();
    87→    std::cerr << "  [" << ms/1000 << "." << (ms%1000)/100 << "s] " << name << std::endl;
    88→    t = std::chrono::steady_clock::now();
    89→}
    90→
    91→// --- Parsed data container ---
    92→
    93→struct ParsedData {
    94→    StringPool string_pool;
    95→    std::vector<WayHeader> ways;
    96→    std::vector<NodeCoord> street_nodes;
    97→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_ways;
    98→    std::vector<AddrPoint> addr_points;
    99→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_addrs;
   100→    std::vector<InterpWay> interp_ways;
   101→    std::vector<NodeCoord> interp_nodes;
   102→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_interps;
   103→    std::vector<AdminPolygon> admin_polygons;
   104→    std::vector<NodeCoord> admin_vertices;
   105→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_admin;
   106→
   107→    // Deferred work for parallel S2 computation (ways + interps)
   108→    std::vector<DeferredWay> deferred_ways;
   109→    std::vector<DeferredInterp> deferred_interps;
   110→
   111→    // Sorted (cell_id, item_id) pairs — kept for direct entry writing
   112→    std::vector<CellItemPair> sorted_way_cells;
   113→    std::vector<CellItemPair> sorted_addr_cells;
   114→    std::vector<CellItemPair> sorted_interp_cells;
   115→
   116→    // Collected data for parallel admin assembly
   117→    std::vector<CollectedRelation> collected_relations;
   118→    struct WayGeometry {
   119→        std::vector<std::pair<double,double>> coords;
   120→        int64_t first_node_id;
   121→        int64_t last_node_id;
   122→    };
   123→    std::unordered_map<int64_t, WayGeometry> way_geometries;
   124→};
   125→
   126→// --- Deduplicate IDs per cell ---
   127→
   128→template<typename Map>
   129→inline void deduplicate(Map& cell_map) {
   130→    for (auto& [cell_id, ids] : cell_map) {
   131→        std::sort(ids.begin(), ids.end());
   132→        ids.erase(std::unique(ids.begin(), ids.end()), ids.end());
   133→    }
   134→}
   135→


Excellent! Now let me search for the add_addr_point function and look at how threading is handled in PBF reading:


grep -n "void add_addr_point\|add_addr_point(" /root/traccar-geocoder/builder/src/build_index.cpp | head -10

Find add_addr_point function


394:                        add_addr_point(data, local.addr_coords[j].first, local.addr_coords[j].second,
633:                        add_addr_point(data, local.building_addrs[i].lat, local.building_addrs[i].lng,


Let me search for the definition:


grep -r "void add_addr_point\|inline.*add_addr_point" /root/traccar-geocoder/builder/src/

Find add_addr_point definition


/root/traccar-geocoder/builder/src/s2_helpers.h:void add_addr_point(ParsedData& data, double lat, double lng,
/root/traccar-geocoder/builder/src/s2_helpers.cpp:void add_addr_point(ParsedData& data, double lat, double lng,


     1→#include "s2_helpers.h"
     2→#include "geometry.h"
     3→
     4→#include <algorithm>
     5→#include <cmath>
     6→#include <iostream>
     7→#include <memory>
     8→
     9→#include <s2/s2latlng.h>
    10→#include <s2/s2region_coverer.h>
    11→#include <s2/s2polyline.h>
    12→#include <s2/s2polygon.h>
    13→#include <s2/s2loop.h>
    14→
    15→// --- S2 cell level globals ---
    16→
    17→int kStreetCellLevel = 17;
    18→int kAdminCellLevel = 10;
    19→int kMaxAdminLevel = 0;  // 0 means no filtering
    20→
    21→SimplifyMode kSimplifyMode = SimplifyMode::MaxVertices;
    22→double kSimplifyEpsilonOverride = 0;
    23→
    24→// --- cover_edge ---
    25→
    26→void cover_edge(double lat1, double lng1, double lat2, double lng2,
    27→                std::vector<S2CellId>& out) {
    28→    out.clear();
    29→
    30→    S2CellId c1 = S2CellId(S2LatLng::FromDegrees(lat1, lng1)).parent(kStreetCellLevel);
    31→    S2CellId c2 = S2CellId(S2LatLng::FromDegrees(lat2, lng2)).parent(kStreetCellLevel);
    32→
    33→    // Same cell — most common case for short edges
    34→    if (c1 == c2) {
    35→        out.push_back(c1);
    36→        return;
    37→    }
    38→
    39→    // Adjacent cells — check midpoint
    40→    S2CellId cm = S2CellId(S2LatLng::FromDegrees((lat1+lat2)/2, (lng1+lng2)/2)).parent(kStreetCellLevel);
    41→    if (cm == c1 || cm == c2) {
    42→        out.push_back(c1);
    43→        out.push_back(c2);
    44→        return;
    45→    }
    46→
    47→    // Longer edge — use S2RegionCoverer for correctness (handles geodesic curves)
    48→    thread_local S2RegionCoverer coverer = []() {
    49→        S2RegionCoverer::Options options;
    50→        options.set_fixed_level(kStreetCellLevel);
    51→        return S2RegionCoverer(options);
    52→    }();
    53→
    54→    S2Point p1 = S2LatLng::FromDegrees(lat1, lng1).ToPoint();
    55→    S2Point p2 = S2LatLng::FromDegrees(lat2, lng2).ToPoint();
    56→    if (p1 == p2) { out.push_back(c1); return; }
    57→
    58→    std::vector<S2Point> points = {p1, p2};
    59→    S2Polyline polyline(points);
    60→    S2CellUnion covering = coverer.GetCovering(polyline);
    61→    out = std::move(covering.cell_ids());
    62→}
    63→
    64→std::vector<S2CellId> cover_edge(double lat1, double lng1, double lat2, double lng2) {
    65→    std::vector<S2CellId> result;
    66→    cover_edge(lat1, lng1, lat2, lng2, result);
    67→    return result;
    68→}
    69→
    70→S2CellId point_to_cell(double lat, double lng) {
    71→    return S2CellId(S2LatLng::FromDegrees(lat, lng)).parent(kStreetCellLevel);
    72→}
    73→
    74→// --- cover_polygon ---
    75→
    76→std::vector<std::pair<S2CellId, bool>> cover_polygon(
    77→    const std::vector<std::pair<double,double>>& vertices) {
    78→    std::vector<S2Point> points;
    79→    points.reserve(vertices.size());
    80→    for (const auto& [lat, lng] : vertices) {
    81→        S2Point p = S2LatLng::FromDegrees(lat, lng).ToPoint();
    82→        if (!points.empty() && points.back() == p) continue;
    83→        points.push_back(p);
    84→    }
    85→    if (points.size() > 1 && points.front() == points.back()) {
    86→        points.pop_back();
    87→    }
    88→    if (points.size() < 3) return {};
    89→
    90→    S2Error error;
    91→    auto loop = std::make_unique<S2Loop>(points, S2Debug::DISABLE);
    92→    loop->Normalize();
    93→    if (loop->FindValidationError(&error)) return {};
    94→
    95→    S2Polygon polygon(std::move(loop));
    96→
    97→    S2RegionCoverer::Options options;
    98→    options.set_max_level(kAdminCellLevel);
    99→    options.set_max_cells(MAX_S2_CELLS_PER_POLY);
   100→


Let me continue reading to find add_addr_point:


grep -n "void add_addr_point" /root/traccar-geocoder/builder/src/s2_helpers.cpp

Find add_addr_point line number


229:void add_addr_point(ParsedData& data, double lat, double lng,


   229→void add_addr_point(ParsedData& data, double lat, double lng,
   230→                    const char* housenumber, const char* street,
   231→                    uint64_t& addr_count_total) {
   232→    uint32_t addr_id = static_cast<uint32_t>(data.addr_points.size());
   233→    data.addr_points.push_back({
   234→        static_cast<float>(lat),
   235→        static_cast<float>(lng),
   236→        data.string_pool.intern(housenumber),
   237→        data.string_pool.intern(street)
   238→    });
   239→
   240→    S2CellId cell = point_to_cell(lat, lng);
   241→    data.cell_to_addrs[cell.id()].push_back(addr_id);
   242→
   243→    addr_count_total++;
   244→    if (addr_count_total % 1000000 == 0) {
   245→        std::cerr << "Collected " << addr_count_total / 1000000 << "M addresses..." << std::endl;
   246→    }
   247→}
   248→
   249→// --- add_admin_polygon ---
   250→
   251→void add_admin_polygon(ParsedData& data,
   252→                       const std::vector<std::pair<double,double>>& vertices_in,
   253→                       const char* name, uint8_t admin_level,
   254→                       const char* country_code,
   255→                       AdminCoverPool* admin_pool) {
   256→    auto vertices = vertices_in;
   257→    if (vertices.size() >= 4 &&
   258→        std::fabs(vertices.front().first - vertices.back().first) < 1e-7 &&
   259→        std::fabs(vertices.front().second - vertices.back().second) < 1e-7) {
   260→        vertices.pop_back();
   261→        auto min_it = std::min_element(vertices.begin(), vertices.end());
   262→        std::rotate(vertices.begin(), min_it, vertices.end());
   263→        vertices.push_back(vertices.front());
   264→    }
   265→
   266→    auto simplified = simplify_admin_polygon(vertices, admin_level);
   267→    if (simplified.size() < 3) return;
   268→
   269→    uint32_t poly_id = static_cast<uint32_t>(data.admin_polygons.size());
   270→    uint32_t vertex_offset = static_cast<uint32_t>(data.admin_vertices.size());
   271→
   272→    for (const auto& [lat, lng] : simplified) {
   273→        data.admin_vertices.push_back({static_cast<float>(lat), static_cast<float>(lng)});
   274→    }
   275→
   276→    AdminPolygon poly{};
   277→    poly.vertex_offset = vertex_offset;
   278→    poly.vertex_count = static_cast<uint32_t>(simplified.size());


Now let me check the PBF reader to understand threading at that level:


grep -n "read_nodes_streaming\|read_ways_streaming\|thread" /root/traccar-geocoder/builder/src/pbf_reader.h | head -20

Check PBF reader streaming methods


5:// to a thread pool with zero sequential bottleneck.
129:// `num_threads` controls parallelism (0 = hardware_concurrency).
133:                       unsigned num_threads = 0,
139:    explicit PbfFile(const std::string& filename, unsigned num_threads = 0);
147:    // Callback is called from worker threads with (block, thread_index).
148:    // The callback must be thread-safe OR use the thread_index for thread-local storage.
153:    void read_blocks(std::function<void(PbfBlock&, unsigned thread_idx)> callback,
158:    unsigned thread_count() const { return num_threads_; }
161:    void read_nodes_streaming(const NodeCallback& callback);
162:    void read_ways_streaming(const WayCallback& callback);
168:    unsigned num_threads_;