In /root/traccar-geocoder/builder/src/build_index.cpp, find how address points get their IDs assigned. Specifically:
1. How are addr_points collected during Pass 2 (node scanning)?
2. How are they stored/sorted before writing to addr_points.bin?
3. What determines the order of records in addr_points.bin?
4. Similarly, how are street_ways ordered — what determines a way's position/ID in street_ways.bin?

I want to understand whether the ordering is deterministic and what it depends on (e.g., PBF file order, hash map iteration order, S2 cell order, etc.)


I'll analyze the build_index.cpp file to understand how address points and street ways get their IDs and ordering. Let me start by reading the file.


Error: File content (22670 tokens) exceeds maximum allowed tokens (10000). Use offset and limit parameters to read specific portions of the file, or search for specific content instead of reading the whole file.


The file is large. Let me search for key sections related to Pass 2 node scanning and address point collection.


336-            };
337-            std::cerr << "  Admin assembly needs " << admin_way_count << " way geometries." << std::endl;
338-
339:            // --- Pass 2: Node processing (streaming — no PbfNode objects) ---
340:            std::cerr << "  Pass 2: processing nodes with " << num_threads << " threads..." << std::endl;
341-            {
342-                struct NodeThreadLocal {
343-                    std::vector<std::pair<double,double>> addr_coords;
--
400-                std::cerr << "  Node processing complete: " << total_addrs
401-                          << " address points collected." << std::endl;
402-            }
403:            log_phase("Pass 2: node processing", _pt, _cpu);
404-            pbf.release_pages(); // free PBF mmap pages, will re-fault for way pass
405-
406:            // --- Pass 2b: Way processing (fully parallel) ---
407-            std::cerr << "  Processing ways with " << num_threads << " threads..." << std::endl;
408-            {
409-                // Thread-local data for parallel way processing
--
693-
694-                // --- Parallel admin boundary assembly ---
695-                {
696:                    log_phase("Pass 2b: way processing", _pt, _cpu);
697-                    pbf.unmap(); // release 86 GiB PBF mmap before admin/S2 phases
698-                    std::cerr << "  Assembling admin polygons in parallel ("
699-                              << data.collected_relations.size() << " relations, "
--
913-
914-        std::cerr << "Done reading:" << std::endl;
915-        std::cerr << "  " << data.ways.size() << " street ways" << std::endl;
916:        std::cerr << "  " << data.addr_points.size() << " address points" << std::endl;
917-        std::cerr << "  " << data.interp_ways.size() << " interpolation ways" << std::endl;
918-        std::cerr << "  " << data.admin_polygons.size() << " admin polygon rings" << std::endl;
919-
--
1491-            "admin_polygons.bin", "admin_vertices.bin"
1492-        };
1493-        std::vector<std::string> street_files = {
1494:            "street_ways.bin", "street_nodes.bin", "street_entries.bin"
1495-        };
1496-        std::vector<std::string> address_files = {
1497:            "addr_points.bin", "addr_entries.bin",
1498-            "interp_ways.bin", "interp_nodes.bin", "interp_entries.bin"
1499-        };
1500-        std::vector<std::string> geo_files = {"geo_cells.bin"};


1489-        };
1490-        std::vector<std::string> admin_quality_files = {
1491-            "admin_polygons.bin", "admin_vertices.bin"
1492-        };
1493-        std::vector<std::string> street_files = {
1494:            "street_ways.bin", "street_nodes.bin", "street_entries.bin"
1495-        };
1496-        std::vector<std::string> address_files = {
1497:            "addr_points.bin", "addr_entries.bin",
1498-            "interp_ways.bin", "interp_nodes.bin", "interp_entries.bin"
1499-        };
1500-        std::vector<std::string> geo_files = {"geo_cells.bin"};
1501-
1502-        // Build list of regions


Good. Now let me search for where addr_points are collected in Pass 2 and how they're sorted before writing.


339-            // --- Pass 2: Node processing (streaming — no PbfNode objects) ---
340-            std::cerr << "  Pass 2: processing nodes with " << num_threads << " threads..." << std::endl;
341-            {
342-                struct NodeThreadLocal {
343:                    std::vector<std::pair<double,double>> addr_coords;
344-                    std::vector<std::pair<std::string,std::string>> addr_strings;
345-                    uint64_t count = 0;
346-                };
347-                // Use thread_local for streaming callback (no thread index available)
--
375-                                    street = tag_vals[i] < st.size() ? st[tag_vals[i]].c_str() : nullptr;
376-                            }
377-                        }
378-                        if (housenumber && street) {
379:                            tl_node_data->addr_coords.push_back({lat, lng});
380-                            tl_node_data->addr_strings.push_back({housenumber, street});
381-                            tl_node_data->count++;
382-                        }
383-                    }
--
388-
389-                // Merge address points
390-                uint64_t total_addrs = 0;
391-                for (auto& local : ntld) {
392:                    for (size_t j = 0; j < local.addr_coords.size(); j++) {
393-                        uint64_t dummy = 0;
394:                        add_addr_point(data, local.addr_coords[j].first, local.addr_coords[j].second,
395-                                       local.addr_strings[j].first.c_str(),
396-                                       local.addr_strings[j].second.c_str(), dummy);
397-                    }
398-                    total_addrs += local.count;
--
414-                    std::vector<InterpWay> interp_ways;
415-                    std::vector<NodeCoord> interp_nodes;
416-                    std::vector<DeferredInterp> deferred_interps;
417-                    std::vector<AddrPoint> building_addrs;
418:                    std::vector<std::pair<double, double>> building_addr_coords; // lat,lng for S2 cell
419-                    std::vector<std::string> way_strings;      // way name strings
420-                    std::vector<std::pair<std::string,std::string>> addr_strings; // building addr {hn, street}
421-                    std::vector<std::string> interp_strings;   // interp street name strings
422-                    uint64_t way_count = 0;
--
536-                            for (const auto& loc : resolved_locs) {
537-                                if (loc.valid()) { sum_lat += loc.lat(); sum_lng += loc.lon(); valid++; }
538-                            }
539-                            if (valid > 0) {
540:                                local.building_addr_coords.push_back({sum_lat/valid, sum_lng/valid});
541-                                local.building_addrs.push_back({static_cast<float>(sum_lat/valid), static_cast<float>(sum_lng/valid), 0, 0});
542-                                local.addr_strings.push_back({housenumber, street});
543-                                local.building_addr_count++;
544-                            }
--
912-        }
913-
914-        std::cerr << "Done reading:" << std::endl;
915-        std::cerr << "  " << data.ways.size() << " street ways" << std::endl;
916:        std::cerr << "  " << data.addr_points.size() << " address points" << std::endl;
917-        std::cerr << "  " << data.interp_ways.size() << " interpolation ways" << std::endl;
918-        std::cerr << "  " << data.admin_polygons.size() << " admin polygon rings" << std::endl;
919-
920-        // Drain admin polygon thread pool (may still be processing)
--
1493-        std::vector<std::string> street_files = {
1494-            "street_ways.bin", "street_nodes.bin", "street_entries.bin"
1495-        };
1496-        std::vector<std::string> address_files = {
1497:            "addr_points.bin", "addr_entries.bin",
1498-            "interp_ways.bin", "interp_nodes.bin", "interp_entries.bin"
1499-        };
1500-        std::vector<std::string> geo_files = {"geo_cells.bin"};
1501-


Now let me search for the add_addr_point function and how addr_points are stored:


389-                // Merge address points
390-                uint64_t total_addrs = 0;
391-                for (auto& local : ntld) {
392-                    for (size_t j = 0; j < local.addr_coords.size(); j++) {
393-                        uint64_t dummy = 0;
394:                        add_addr_point(data, local.addr_coords[j].first, local.addr_coords[j].second,
395-                                       local.addr_strings[j].first.c_str(),
396-                                       local.addr_strings[j].second.c_str(), dummy);
397-                    }
398-                    total_addrs += local.count;
399-                }
--
628-                    }
629-
630-                    // Merge building addresses
631-                    for (size_t i = 0; i < local.building_addrs.size(); i++) {
632-                        uint64_t dummy = 0;
633:                        add_addr_point(data, local.building_addrs[i].lat, local.building_addrs[i].lng,
634-                                       local.addr_strings[i].first.c_str(),
635-                                       local.addr_strings[i].second.c_str(), dummy);
636-                    }
637-
638-                    // Merge interpolation ways
--
911-            }
912-        }
913-
914-        std::cerr << "Done reading:" << std::endl;
915-        std::cerr << "  " << data.ways.size() << " street ways" << std::endl;
916:        std::cerr << "  " << data.addr_points.size() << " address points" << std::endl;
917-        std::cerr << "  " << data.interp_ways.size() << " interpolation ways" << std::endl;
918-        std::cerr << "  " << data.admin_polygons.size() << " admin polygon rings" << std::endl;
919-
920-        // Drain admin polygon thread pool (may still be processing)
921-        std::cerr << "Waiting for admin polygon S2 covering to complete..." << std::endl;


Let me search for where add_addr_point is defined:


384-                });
385-
386-                // Reset thread_local for next use
387-                tl_node_data = nullptr;
388-
389-                // Merge address points
390-                uint64_t total_addrs = 0;
391-                for (auto& local : ntld) {
392-                    for (size_t j = 0; j < local.addr_coords.size(); j++) {
393-                        uint64_t dummy = 0;
394:                        add_addr_point(data, local.addr_coords[j].first, local.addr_coords[j].second,
395-                                       local.addr_strings[j].first.c_str(),
396-                                       local.addr_strings[j].second.c_str(), dummy);
397-                    }
398-                    total_addrs += local.count;
399-                }
400-                std::cerr << "  Node processing complete: " << total_addrs
401-                          << " address points collected." << std::endl;
402-            }
403-            log_phase("Pass 2: node processing", _pt, _cpu);
404-            pbf.release_pages(); // free PBF mmap pages, will re-fault for way pass
--
623-                    // Remap deferred ways
624-                    for (auto dw : local.deferred_ways) {
625-                        dw.way_id += way_base;
626-                        dw.node_offset += node_base;
627-                        data.deferred_ways.push_back(dw);
628-                    }
629-
630-                    // Merge building addresses
631-                    for (size_t i = 0; i < local.building_addrs.size(); i++) {
632-                        uint64_t dummy = 0;
633:                        add_addr_point(data, local.building_addrs[i].lat, local.building_addrs[i].lng,
634-                                       local.addr_strings[i].first.c_str(),
635-                                       local.addr_strings[i].second.c_str(), dummy);
636-                    }
637-
638-                    // Merge interpolation ways
639-                    for (size_t i = 0; i < local.interp_ways.size(); i++) {
640-                        auto iw = local.interp_ways[i];
641-                        iw.node_offset += interp_node_base;
642-                        iw.street_id = data.string_pool.intern(local.interp_strings[i]);
643-                        data.interp_ways.push_back(iw);


Let me search for the function definition in header files or earlier in the cpp file:


Bash

grep -n "^.*add_addr_point" /root/traccar-geocoder/builder/src/build_index.cpp | head -20


394:                        add_addr_point(data, local.addr_coords[j].first, local.addr_coords[j].second,
633:                        add_addr_point(data, local.building_addrs[i].lat, local.building_addrs[i].lng,


Let me check the header files:


find /root/traccar-geocoder/builder/src -name "*.h" -o -name "*.hpp" | head -20


/root/traccar-geocoder/builder/src/ring_assembly.h
/root/traccar-geocoder/builder/src/string_pool.h
/root/traccar-geocoder/builder/src/interpolation.h
/root/traccar-geocoder/builder/src/cache.h
/root/traccar-geocoder/builder/src/s2_helpers.h
/root/traccar-geocoder/builder/src/continent_boundaries.h
/root/traccar-geocoder/builder/src/cell_index.h
/root/traccar-geocoder/builder/src/pbf_reader.h
/root/traccar-geocoder/builder/src/relation_collector.h
/root/traccar-geocoder/builder/src/continent_filter.h
/root/traccar-geocoder/builder/src/parsed_data.h
/root/traccar-geocoder/builder/src/types.h
/root/traccar-geocoder/builder/src/geometry.h


Let me check the parsed_data.h file as that's where the data structures are likely defined:


     1→#pragma once
     2→
     3→#include <algorithm>
     4→#include <chrono>
     5→#include <iostream>
     6→#include <string>
     7→#include <unordered_map>
     8→#include <vector>
     9→
    10→#include <cstdio>
    11→#include <cstring>
    12→#include <dirent.h>
    13→#include <iomanip>
    14→#include <thread>
    15→#include <unistd.h>
    16→#include <sys/stat.h>
    17→
    18→#include "types.h"
    19→#include "string_pool.h"
    20→
    21→// --- Directory creation ---
    22→
    23→inline void ensure_dir(const std::string& path) {
    24→    // Recursive mkdir — create all parent directories
    25→    for (size_t i = 1; i < path.size(); i++) {
    26→        if (path[i] == '/') {
    27→            mkdir(path.substr(0, i).c_str(), 0755);
    28→        }
    29→    }
    30→    mkdir(path.c_str(), 0755);
    31→}
    32→
    33→// --- CPU tick reading (all threads via getrusage) ---
    34→
    35→#include <sys/resource.h>
    36→
    37→struct CpuTicks {
    38→    long long process_us = 0;  // user+system microseconds (all threads)
    39→
    40→    static CpuTicks now() {
    41→        CpuTicks ct;
    42→        struct rusage ru;
    43→        if (getrusage(RUSAGE_SELF, &ru) == 0) {
    44→            ct.process_us = (long long)ru.ru_utime.tv_sec * 1000000 + ru.ru_utime.tv_usec
    45→                          + (long long)ru.ru_stime.tv_sec * 1000000 + ru.ru_stime.tv_usec;
    46→        }
    47→        return ct;
    48→    }
    49→};
    50→
    51→// --- Phase timer with CPU utilization ---
    52→
    53→inline long get_rss_mb() {
    54→    long rss_pages = 0;
    55→    FILE* f = fopen("/proc/self/statm", "r");
    56→    if (f) {
    57→        long size;
    58→        if (fscanf(f, "%ld %ld", &size, &rss_pages) != 2) rss_pages = 0;
    59→        fclose(f);
    60→    }
    61→    return rss_pages * sysconf(_SC_PAGESIZE) / (1024 * 1024);
    62→}
    63→
    64→inline void log_phase(const char* name, std::chrono::steady_clock::time_point& t,
    65→                       CpuTicks& prev_cpu) {
    66→    auto now = std::chrono::steady_clock::now();
    67→    auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(now - t).count();
    68→    auto cpu_now = CpuTicks::now();
    69→    double cpu_sec = (cpu_now.process_us - prev_cpu.process_us) / 1e6;
    70→    double wall_sec = ms / 1000.0;
    71→    double cores_used = (wall_sec > 0.1) ? cpu_sec / wall_sec : 0;
    72→    unsigned ncpu = std::thread::hardware_concurrency();
    73→    int pct = (wall_sec > 0.1 && ncpu > 0) ? (int)(cores_used * 100.0 / ncpu) : 0;
    74→    long rss = get_rss_mb();
    75→    std::cerr << "  [" << ms/1000 << "." << (ms%1000)/100 << "s"
    76→              << " " << std::fixed << std::setprecision(1) << cores_used << "/" << ncpu << "cores"
    77→              << " " << pct << "%"
    78→              << " " << rss << "MiB] " << name << std::endl;
    79→    t = now;
    80→    prev_cpu = cpu_now;
    81→}
    82→
    83→// Backward compat overload (no CPU tracking)
    84→inline void log_phase(const char* name, std::chrono::steady_clock::time_point& t) {
    85→    auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(
    86→        std::chrono::steady_clock::now() - t).count();
    87→    std::cerr << "  [" << ms/1000 << "." << (ms%1000)/100 << "s] " << name << std::endl;
    88→    t = std::chrono::steady_clock::now();
    89→}
    90→
    91→// --- Parsed data container ---
    92→
    93→struct ParsedData {
    94→    StringPool string_pool;
    95→    std::vector<WayHeader> ways;
    96→    std::vector<NodeCoord> street_nodes;
    97→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_ways;
    98→    std::vector<AddrPoint> addr_points;
    99→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_addrs;
   100→    std::vector<InterpWay> interp_ways;
   101→    std::vector<NodeCoord> interp_nodes;
   102→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_interps;
   103→    std::vector<AdminPolygon> admin_polygons;
   104→    std::vector<NodeCoord> admin_vertices;
   105→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_admin;
   106→
   107→    // Deferred work for parallel S2 computation (ways + interps)
   108→    std::vector<DeferredWay> deferred_ways;
   109→    std::vector<DeferredInterp> deferred_interps;
   110→
   111→    // Sorted (cell_id, item_id) pairs — kept for direct entry writing
   112→    std::vector<CellItemPair> sorted_way_cells;
   113→    std::vector<CellItemPair> sorted_addr_cells;
   114→    std::vector<CellItemPair> sorted_interp_cells;
   115→
   116→    // Collected data for parallel admin assembly
   117→    std::vector<CollectedRelation> collected_relations;
   118→    struct WayGeometry {
   119→        std::vector<std::pair<double,double>> coords;
   120→        int64_t first_node_id;
   121→        int64_t last_node_id;
   122→    };
   123→    std::unordered_map<int64_t, WayGeometry> way_geometries;
   124→};
   125→
   126→// --- Deduplicate IDs per cell ---
   127→
   128→template<typename Map>
   129→inline void deduplicate(Map& cell_map) {
   130→    for (auto& [cell_id, ids] : cell_map) {
   131→        std::sort(ids.begin(), ids.end());
   132→        ids.erase(std::unique(ids.begin(), ids.end()), ids.end());
   133→    }
   134→}
   135→


Good! Now let me check the types.h file to see the AddrPoint structure definition:


     1→#pragma once
     2→
     3→#include <cstdint>
     4→#include <vector>
     5→
     6→// --- Binary format structs (must match server's index reader) ---
     7→
     8→struct WayHeader {
     9→    uint32_t node_offset;
    10→    uint8_t node_count;
    11→    uint32_t name_id;
    12→};
    13→
    14→struct AddrPoint {
    15→    float lat;
    16→    float lng;
    17→    uint32_t housenumber_id;
    18→    uint32_t street_id;
    19→};
    20→
    21→struct InterpWay {
    22→    uint32_t node_offset;
    23→    uint8_t node_count;
    24→    uint32_t street_id;
    25→    uint32_t start_number;
    26→    uint32_t end_number;
    27→    uint8_t interpolation;
    28→};
    29→
    30→struct AdminPolygon {
    31→    uint32_t vertex_offset;
    32→    uint32_t vertex_count;
    33→    uint32_t name_id;
    34→    uint8_t admin_level;
    35→    float area;
    36→    uint16_t country_code;
    37→};
    38→
    39→struct NodeCoord {
    40→    float lat;
    41→    float lng;
    42→};
    43→
    44→// Sorted cell-item pair for direct entry writing (avoids hash map)
    45→struct CellItemPair {
    46→    uint64_t cell_id;
    47→    uint32_t item_id;
    48→};
    49→
    50→// --- Constants ---
    51→
    52→static const uint32_t INTERIOR_FLAG = 0x80000000u;
    53→static const uint32_t ID_MASK       = 0x7FFFFFFFu;
    54→static const uint32_t NO_DATA       = 0xFFFFFFFFu;
    55→
    56→// Simplification mode
    57→enum class SimplifyMode { MaxVertices, ErrorBounded };
    58→
    59→// Processing limits
    60→static constexpr size_t MAX_BLOCK_QUEUE         = 64;    // producer-consumer queue depth
    61→static constexpr int    MAX_NODE_COUNT          = 255;   // uint8_t node_count
    62→static constexpr int    MAX_VERTEX_COUNT        = 0x7FFFFFFF; // uint32_t vertex_count
    63→static constexpr int    MAX_S2_CELLS_PER_POLY   = 200;   // S2RegionCoverer max_cells
    64→static constexpr int    BACKTRACK_CALL_BUDGET   = 100000; // per-ring backtracking limit
    65→static constexpr int    BACKTRACK_TOTAL_BUDGET  = 500000; // per-relation backtracking limit
    66→static constexpr int    BACKTRACK_MAX_DEPTH     = 200;    // recursion depth limit
    67→static constexpr size_t MAX_SUBWAYS_FOR_RETRY   = 30;     // only retry small relations
    68→static constexpr size_t MAX_NODE_ID_DEFAULT     = 15000000000ULL; // dense index capacity
    69→
    70→// --- Index mode ---
    71→
    72→enum class IndexMode { Full, NoAddresses, AdminOnly };
    73→
    74→// --- Deferred S2 work items ---
    75→
    76→struct DeferredWay {
    77→    uint32_t way_id;
    78→    uint32_t node_offset;
    79→    uint8_t node_count;
    80→};
    81→
    82→struct DeferredInterp {
    83→    uint32_t interp_id;
    84→    uint32_t node_offset;
    85→    uint8_t node_count;
    86→};
    87→
    88→// Collected relation data for parallel admin assembly
    89→struct CollectedRelation {
    90→    int64_t id;
    91→    uint8_t admin_level;
    92→    std::string name;
    93→    std::string country_code;
    94→    bool is_postal;
    95→    std::vector<std::pair<int64_t, std::string>> members; // (way_id, role)
    96→};
    97→


Now let me search for where add_addr_point is actually defined. Let me look at the beginning of the build_index.cpp file:


     1→#include <algorithm>
     2→#include <atomic>
     3→#include <chrono>
     4→#include <condition_variable>
     5→#include <cstring>
     6→#include <deque>
     7→#include <fstream>
     8→#include <future>
     9→#include <iostream>
    10→#include <mutex>
    11→#include <numeric>
    12→#include <string>
    13→#include <thread>
    14→#include <unordered_set>
    15→#include <vector>
    16→
    17→#include <fcntl.h>
    18→#include <sys/mman.h>
    19→
    20→#include "pbf_reader.h"
    21→
    22→#include <s2/s2cell_id.h>
    23→#include <s2/s2latlng.h>
    24→
    25→#include "types.h"
    26→#include "string_pool.h"
    27→#include "geometry.h"
    28→#include "parsed_data.h"
    29→#include "s2_helpers.h"
    30→#include "ring_assembly.h"
    31→#include "continent_boundaries.h"
    32→#include "interpolation.h"
    33→#include "cache.h"
    34→#include "continent_filter.h"
    35→#include "cell_index.h"
    36→
    37→
    38→// --- Main ---
    39→
    40→int main(int argc, char* argv[]) {
    41→    if (argc < 2) {
    42→        std::cerr << "Usage: build-index <output-dir> <input.osm.pbf> [options]" << std::endl;
    43→        std::cerr << "       build-index <output-dir> --load-cache <path> [options]" << std::endl;
    44→        std::cerr << std::endl;
    45→        std::cerr << "Options:" << std::endl;
    46→        std::cerr << "  --street-level N       S2 cell level for streets (default: 17)" << std::endl;
    47→        std::cerr << "  --admin-level N        S2 cell level for admin (default: 10)" << std::endl;
    48→        std::cerr << "  --max-admin-level N    Only include admin levels <= N" << std::endl;
    49→        std::cerr << "  --save-cache <path>    Save parsed data to cache file" << std::endl;
    50→        std::cerr << "  --load-cache <path>    Load from cache instead of PBF" << std::endl;
    51→        std::cerr << "  --multi-output         Write full, no-addresses, admin-only indexes" << std::endl;
    52→        std::cerr << "  --continents           Also generate per-continent indexes" << std::endl;
    53→        std::cerr << "  --mode <mode>          Index mode: full, no-addresses, admin-only (default: full)" << std::endl;
    54→        std::cerr << "  --admin-only           Shorthand for --mode admin-only" << std::endl;
    55→        std::cerr << "  --no-addresses         Shorthand for --mode no-addresses" << std::endl;
    56→        std::cerr << "  --simplify-epsilon     Use error-bounded simplification (per-level defaults)" << std::endl;
    57→        std::cerr << "  --simplify-epsilon N   Use fixed epsilon of N meters for all levels" << std::endl;
    58→        std::cerr << "  --epsilon-scale F      Multiply all per-level epsilons by F (e.g. 2.0 = 2x coarser)" << std::endl;
    59→        std::cerr << "  --epsilon-levels L2,L3,...,L8  Set epsilon in meters per level (7 values)" << std::endl;
    60→        std::cerr << "  --multi-quality [S,S,...]  Write quality variants at given scales (default: 0,0.2,0.5,1,1.5,2,2.5)" << std::endl;
    61→        return 1;
    62→    }
    63→
    64→    std::string output_dir = argv[1];
    65→    std::vector<std::string> input_files;
    66→    std::string save_cache_path;
    67→    std::string load_cache_path;
    68→    bool multi_output = false;
    69→    bool generate_continents = false;
    70→    IndexMode mode = IndexMode::Full;
    71→    SimplifyMode simplify_mode = SimplifyMode::MaxVertices;
    72→    double simplify_epsilon_override = 0; // 0 = use per-level defaults
    73→    bool multi_quality = false;
    74→    std::vector<double> quality_scales;
    75→
    76→    for (int i = 2; i < argc; i++) {
    77→        std::string arg = argv[i];
    78→        if (arg == "--street-level" && i + 1 < argc) {
    79→            kStreetCellLevel = std::atoi(argv[++i]);
    80→        } else if (arg == "--admin-level" && i + 1 < argc) {
    81→            kAdminCellLevel = std::atoi(argv[++i]);
    82→        } else if (arg == "--admin-only") {
    83→            mode = IndexMode::AdminOnly;
    84→        } else if (arg == "--no-addresses") {
    85→            mode = IndexMode::NoAddresses;
    86→        } else if (arg == "--max-admin-level" && i + 1 < argc) {
    87→            kMaxAdminLevel = std::atoi(argv[++i]);
    88→        } else if (arg == "--save-cache" && i + 1 < argc) {
    89→            save_cache_path = argv[++i];
    90→        } else if (arg == "--load-cache" && i + 1 < argc) {
    91→            load_cache_path = argv[++i];
    92→        } else if (arg == "--multi-output") {
    93→            multi_output = true;
    94→        } else if (arg == "--continents") {
    95→            generate_continents = true;
    96→        } else if (arg == "--simplify-epsilon") {
    97→            simplify_mode = SimplifyMode::ErrorBounded;
    98→            // Optional: next arg might be a number (fixed epsilon in meters)
    99→            if (i + 1 < argc) {
   100→                char* end;
   101→                double val = std::strtod(argv[i+1], &end);
   102→                if (*end == '\0' && val > 0) {
   103→                    simplify_epsilon_override = val;
   104→                    i++;
   105→                }
   106→            }
   107→        } else if (arg == "--epsilon-scale" && i + 1 < argc) {
   108→            kEpsilonScale = std::strtod(argv[++i], nullptr);
   109→        } else if (arg == "--multi-quality") {
   110→            multi_quality = true;
   111→            // Optional: next arg might be comma-separated scales
   112→            if (i + 1 < argc && argv[i+1][0] != '-') {
   113→                std::string vals = argv[++i];
   114→                size_t pos = 0;
   115→                while (pos < vals.size()) {
   116→                    size_t next = vals.find(',', pos);
   117→                    if (next == std::string::npos) next = vals.size();
   118→                    double v = std::strtod(vals.substr(pos, next - pos).c_str(), nullptr);
   119→                    quality_scales.push_back(v);
   120→                    pos = next + 1;
   121→                }
   122→            }
   123→            if (quality_scales.empty()) {
   124→                quality_scales = {0, 0.2, 0.5, 1.0, 1.5, 2.0, 2.5};
   125→            }
   126→            // Build with uncapped polygons — quality variants simplify at write time.
   127→            // We need full vertex detail to produce all quality levels accurately.
   128→            simplify_mode = SimplifyMode::ErrorBounded;
   129→            simplify_epsilon_override = 0.1; // effectively uncapped
   130→        } else if (arg == "--epsilon-levels" && i + 1 < argc) {
   131→            // Parse comma-separated values: L2,L3,L4,L5,L6,L7,L8
   132→            std::string vals = argv[++i];
   133→            int level = 2;
   134→            size_t pos = 0;
   135→            while (pos < vals.size() && level <= 8) {
   136→                size_t next = vals.find(',', pos);
   137→                if (next == std::string::npos) next = vals.size();
   138→                double v = std::strtod(vals.substr(pos, next - pos).c_str(), nullptr);
   139→                if (v > 0) kAdminEpsilonMeters[level] = v;
   140→                level++;
   141→                pos = next + 1;
   142→            }
   143→            // Also set levels 9-11 to same as level 8
   144→            for (int l = level; l <= 11; l++) kAdminEpsilonMeters[l] = kAdminEpsilonMeters[8];
   145→        } else if (arg == "--mode" && i + 1 < argc) {
   146→            std::string mode_str = argv[++i];
   147→            if (mode_str == "full") {
   148→                mode = IndexMode::Full;
   149→            } else if (mode_str == "no-addresses") {
   150→                mode = IndexMode::NoAddresses;
   151→            } else if (mode_str == "admin-only") {
   152→                mode = IndexMode::AdminOnly;
   153→            } else {
   154→                std::cerr << "Error: unknown mode '" << mode_str << "'" << std::endl;
   155→                return 1;
   156→            }
   157→        } else {
   158→            input_files.push_back(arg);
   159→        }
   160→    }
   161→
   162→    // Set global simplification mode
   163→    kSimplifyMode = simplify_mode;
   164→    kSimplifyEpsilonOverride = simplify_epsilon_override;
   165→    if (simplify_mode == SimplifyMode::ErrorBounded) {
   166→        if (simplify_epsilon_override > 0) {
   167→            std::cerr << "Simplification: error-bounded, fixed " << simplify_epsilon_override << "m epsilon" << std::endl;
   168→        } else {
   169→            std::cerr << "Simplification: error-bounded, scale=" << kEpsilonScale
   170→                      << ", epsilons: L2=" << admin_epsilon_meters(2)
   171→                      << "m L4=" << admin_epsilon_meters(4)
   172→                      << "m L6=" << admin_epsilon_meters(6)
   173→                      << "m L8=" << admin_epsilon_meters(8) << "m" << std::endl;
   174→        }
   175→    }
   176→
   177→    ParsedData data;
   178→    auto _pt = std::chrono::steady_clock::now();
   179→    auto _cpu = CpuTicks::now();
   180→
   181→    if (!load_cache_path.empty()) {
   182→        // Load from cache
   183→        if (!deserialize_cache(data, load_cache_path)) {
   184→            std::cerr << "Error: failed to load cache" << std::endl;
   185→            return 1;
   186→        }
   187→    } else {
   188→        // Parse PBF files (always collect everything)
   189→        if (input_files.empty()) {
   190→            std::cerr << "Error: no input files specified and no --load-cache" << std::endl;
   191→            return 1;
   192→        }
   193→
   194→        // Create thread pool for concurrent admin polygon S2 covering
   195→        unsigned int num_threads = std::max(1u, std::thread::hardware_concurrency() - 1);
   196→        std::cerr << "Using " << num_threads << " worker threads." << std::endl;
   197→        AdminCoverPool admin_pool(num_threads);
   198→        // BuildHandler no longer used — parallel processing handles everything
   199→
   200→        // Dense array node location index — lockless parallel writes


Now let me search for the add_addr_point function definition more carefully:


grep -B 5 -A 15 "inline.*add_addr_point\|auto.*add_addr_point\s*=" /root/traccar-geocoder/builder/src/build_index.cpp | head -60


(Bash completed with no output)


Let me search for where the function is defined, it might be a lambda:


No matches found


Let me read a section around where add_addr_point is first called:


   150→                mode = IndexMode::NoAddresses;
   151→            } else if (mode_str == "admin-only") {
   152→                mode = IndexMode::AdminOnly;
   153→            } else {
   154→                std::cerr << "Error: unknown mode '" << mode_str << "'" << std::endl;
   155→                return 1;
   156→            }
   157→        } else {
   158→            input_files.push_back(arg);
   159→        }
   160→    }
   161→
   162→    // Set global simplification mode
   163→    kSimplifyMode = simplify_mode;
   164→    kSimplifyEpsilonOverride = simplify_epsilon_override;
   165→    if (simplify_mode == SimplifyMode::ErrorBounded) {
   166→        if (simplify_epsilon_override > 0) {
   167→            std::cerr << "Simplification: error-bounded, fixed " << simplify_epsilon_override << "m epsilon" << std::endl;
   168→        } else {
   169→            std::cerr << "Simplification: error-bounded, scale=" << kEpsilonScale
   170→                      << ", epsilons: L2=" << admin_epsilon_meters(2)
   171→                      << "m L4=" << admin_epsilon_meters(4)
   172→                      << "m L6=" << admin_epsilon_meters(6)
   173→                      << "m L8=" << admin_epsilon_meters(8) << "m" << std::endl;
   174→        }
   175→    }
   176→
   177→    ParsedData data;
   178→    auto _pt = std::chrono::steady_clock::now();
   179→    auto _cpu = CpuTicks::now();
   180→
   181→    if (!load_cache_path.empty()) {
   182→        // Load from cache
   183→        if (!deserialize_cache(data, load_cache_path)) {
   184→            std::cerr << "Error: failed to load cache" << std::endl;
   185→            return 1;
   186→        }
   187→    } else {
   188→        // Parse PBF files (always collect everything)
   189→        if (input_files.empty()) {
   190→            std::cerr << "Error: no input files specified and no --load-cache" << std::endl;
   191→            return 1;
   192→        }
   193→
   194→        // Create thread pool for concurrent admin polygon S2 covering
   195→        unsigned int num_threads = std::max(1u, std::thread::hardware_concurrency() - 1);
   196→        std::cerr << "Using " << num_threads << " worker threads." << std::endl;
   197→        AdminCoverPool admin_pool(num_threads);
   198→        // BuildHandler no longer used — parallel processing handles everything
   199→
   200→        // Dense array node location index — lockless parallel writes
   201→        // Planet OSM node IDs max ~12.5 billion. 8 bytes per entry = 100GB virtual.
   202→        // MAP_NORESERVE means OS only allocates pages on write (~80GB for 10B nodes).
   203→        static const size_t MAX_NODE_ID = MAX_NODE_ID_DEFAULT;
   204→        struct PackedLocation {
   205→            int32_t lat_e7;  // latitude * 10^7
   206→            int32_t lon_e7;  // longitude * 10^7
   207→            bool valid() const { return lat_e7 != 0 || lon_e7 != 0; }
   208→            double lat() const { return lat_e7 / 10000000.0; }
   209→            double lon() const { return lon_e7 / 10000000.0; }
   210→        };
   211→        struct DenseIndex {
   212→            PackedLocation* data;
   213→            size_t capacity;
   214→
   215→            DenseIndex() : capacity(MAX_NODE_ID) {
   216→                size_t byte_size = capacity * sizeof(PackedLocation);
   217→                void* ptr = mmap(nullptr, byte_size,
   218→                    PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE,
   219→                    -1, 0);
   220→                if (ptr == MAP_FAILED) {
   221→                    std::cerr << "Error: failed to mmap dense node index ("
   222→                              << (byte_size / (1024*1024*1024)) << "GB)" << std::endl;
   223→                    std::exit(1);
   224→                }
   225→                madvise(ptr, byte_size, MADV_HUGEPAGE);
   226→                data = static_cast<PackedLocation*>(ptr);
   227→                std::cerr << "Allocated dense node index: " << (byte_size / (1024*1024*1024))
   228→                          << "GB virtual address space" << std::endl;
   229→            }
   230→
   231→            ~DenseIndex() {
   232→                munmap(data, capacity * sizeof(PackedLocation));
   233→            }
   234→
   235→            // Lockless — each node ID maps to a unique array slot
   236→            void set(uint64_t id, double lat, double lng) {
   237→                if (id >= capacity) return;
   238→                data[id] = {static_cast<int32_t>(lat * 10000000.0 + (lat >= 0 ? 0.5 : -0.5)),
   239→                            static_cast<int32_t>(lng * 10000000.0 + (lng >= 0 ? 0.5 : -0.5))};
   240→            }
   241→
   242→            PackedLocation get(uint64_t id) const {
   243→                if (id >= capacity) return {0, 0};
   244→                return data[id];
   245→            }
   246→        } index;
   247→
   248→        for (const auto& input_file : input_files) {
   249→            std::cerr << "Processing " << input_file << "..." << std::endl;
   250→
   251→            // --- Pass 1: collect relation members for parallel admin assembly ---
   252→            std::cerr << "  Pass 1: scanning relations..." << std::endl;
   253→            PbfFile pbf(input_file, num_threads);
   254→
   255→            {
   256→                std::mutex rel_mutex;
   257→                pbf.read_blocks([&](PbfBlock& block, unsigned) {
   258→                    for (auto& rel : block.relations) {
   259→                        const char* boundary = rel.tag("boundary");
   260→                        if (!boundary) continue;
   261→
   262→                        bool is_admin = (std::strcmp(boundary, "administrative") == 0);
   263→                        bool is_postal = (std::strcmp(boundary, "postal_code") == 0);
   264→                        if (!is_admin && !is_postal) continue;
   265→
   266→                        uint8_t admin_level = 0;
   267→                        if (is_admin) {
   268→                            const char* level_str = rel.tag("admin_level");
   269→                            if (!level_str) continue;
   270→                            admin_level = static_cast<uint8_t>(std::atoi(level_str));
   271→                            if (admin_level < 2 || admin_level > 10) continue;
   272→                        } else {
   273→                            admin_level = 11;
   274→                        }
   275→                        if (kMaxAdminLevel > 0 && admin_level > kMaxAdminLevel) continue;
   276→
   277→                        const char* name = rel.tag("name");
   278→                        if (!name && is_admin) continue;
   279→
   280→                        std::string name_str;
   281→                        if (is_postal) {
   282→                            const char* postal_code = rel.tag("postal_code");
   283→                            if (!postal_code) postal_code = name;
   284→                            if (!postal_code) continue;
   285→                            name_str = postal_code;
   286→                        } else {
   287→                            name_str = name;
   288→                        }
   289→
   290→                        std::string country_code;
   291→                        if (admin_level == 2) {
   292→                            const char* cc = rel.tag("ISO3166-1:alpha2");
   293→                            if (cc) country_code = cc;
   294→                        }
   295→
   296→                        CollectedRelation cr;
   297→                        cr.id = rel.id;
   298→                        cr.admin_level = admin_level;
   299→                        cr.name = std::move(name_str);
   300→                        cr.country_code = std::move(country_code);
   301→                        cr.is_postal = is_postal;
   302→
   303→                        for (size_t mi = 0; mi < rel.members.size(); mi++) {
   304→                            if (rel.members[mi].type == 'w') {
   305→                                cr.members.emplace_back(rel.members[mi].ref, rel.member_role(mi));
   306→                            }
   307→                        }
   308→
   309→                        if (!cr.members.empty()) {
   310→                            std::lock_guard<std::mutex> lock(rel_mutex);
   311→                            data.collected_relations.push_back(std::move(cr));
   312→                        }
   313→                    }
   314→                }, "r");
   315→            }
   316→            std::cerr << "  Collected " << data.collected_relations.size()
   317→                      << " admin/postal relations for parallel assembly." << std::endl;
   318→            log_phase("Pass 1: relation scanning", _pt, _cpu);
   319→
   320→            // Build admin_way_ids as a bitset for O(1) lookup
   321→            // Max way ID ~1.3B → ~162 MiB bitset (much faster than unordered_set)
   322→            static constexpr size_t MAX_WAY_ID = 2000000000ULL; // 2B, ~250 MiB
   323→            std::vector<uint8_t> admin_way_bits(MAX_WAY_ID / 8 + 1, 0);
   324→            size_t admin_way_count = 0;
   325→            for (const auto& rel : data.collected_relations) {
   326→                for (const auto& [way_id, role] : rel.members) {
   327→                    if (way_id > 0 && static_cast<size_t>(way_id) < MAX_WAY_ID) {
   328→                        admin_way_bits[way_id / 8] |= (1 << (way_id % 8));
   329→                        admin_way_count++;
   330→                    }
   331→                }
   332→            }
   333→            auto is_admin_way = [&](int64_t id) -> bool {
   334→                if (id <= 0 || static_cast<size_t>(id) >= MAX_WAY_ID) return false;
   335→                return admin_way_bits[id / 8] & (1 << (id % 8));
   336→            };
   337→            std::cerr << "  Admin assembly needs " << admin_way_count << " way geometries." << std::endl;
   338→
   339→            // --- Pass 2: Node processing (streaming — no PbfNode objects) ---
   340→            std::cerr << "  Pass 2: processing nodes with " << num_threads << " threads..." << std::endl;
   341→            {
   342→                struct NodeThreadLocal {
   343→                    std::vector<std::pair<double,double>> addr_coords;
   344→                    std::vector<std::pair<std::string,std::string>> addr_strings;
   345→                    uint64_t count = 0;
   346→                };
   347→                // Use thread_local for streaming callback (no thread index available)
   348→                static thread_local NodeThreadLocal* tl_node_data = nullptr;
   349→                std::vector<NodeThreadLocal> ntld(num_threads);
   350→                std::atomic<unsigned> next_tl{0};
   351→
   352→                pbf.read_nodes_streaming([&](int64_t id, double lat, double lng,
   353→                    const uint32_t* tag_keys, const uint32_t* tag_vals, size_t ntags,
   354→                    const std::vector<std::string>& st) {
   355→
   356→                    // Lazy init thread-local pointer
   357→                    if (!tl_node_data) {
   358→                        unsigned idx = next_tl.fetch_add(1);
   359→                        tl_node_data = &ntld[idx % ntld.size()];
   360→                    }
   361→
   362→                    if (id > 0) {
   363→                        index.set(static_cast<uint64_t>(id), lat, lng);
   364→                    }
   365→
   366→                    // Check for address tags (fast — no string copies for tagless nodes)
   367→                    if (ntags > 0) {
   368→                        const char* housenumber = nullptr;
   369→                        const char* street = nullptr;
   370→                        for (size_t i = 0; i < ntags; i++) {
   371→                            if (tag_keys[i] < st.size()) {
   372→                                if (st[tag_keys[i]] == "addr:housenumber")
   373→                                    housenumber = tag_vals[i] < st.size() ? st[tag_vals[i]].c_str() : nullptr;
   374→                                else if (st[tag_keys[i]] == "addr:street")
   375→                                    street = tag_vals[i] < st.size() ? st[tag_vals[i]].c_str() : nullptr;
   376→                            }
   377→                        }
   378→                        if (housenumber && street) {
   379→                            tl_node_data->addr_coords.push_back({lat, lng});
   380→                            tl_node_data->addr_strings.push_back({housenumber, street});
   381→                            tl_node_data->count++;
   382→                        }
   383→                    }
   384→                });
   385→
   386→                // Reset thread_local for next use
   387→                tl_node_data = nullptr;
   388→
   389→                // Merge address points
   390→                uint64_t total_addrs = 0;
   391→                for (auto& local : ntld) {
   392→                    for (size_t j = 0; j < local.addr_coords.size(); j++) {
   393→                        uint64_t dummy = 0;
   394→                        add_addr_point(data, local.addr_coords[j].first, local.addr_coords[j].second,
   395→                                       local.addr_strings[j].first.c_str(),
   396→                                       local.addr_strings[j].second.c_str(), dummy);
   397→                    }
   398→                    total_addrs += local.count;
   399→                }


Now I need to find the definition of add_addr_point. Let me search more systematically:


Let me search for it in the included headers:


grep -r "add_addr_point" /root/traccar-geocoder/builder/src/*.h 2>/dev/null


/root/traccar-geocoder/builder/src/s2_helpers.h:void add_addr_point(ParsedData& data, double lat, double lng,


Perfect! Let me read that file:


     1→#pragma once
     2→
     3→#include <condition_variable>
     4→#include <cstdint>
     5→#include <deque>
     6→#include <mutex>
     7→#include <thread>
     8→#include <unordered_map>
     9→#include <unordered_set>
    10→#include <vector>
    11→
    12→#include <s2/s2cell_id.h>
    13→
    14→#include "types.h"
    15→#include "geometry.h"
    16→#include "parsed_data.h"
    17→
    18→// --- S2 cell level globals (set by main arg parsing) ---
    19→
    20→extern int kStreetCellLevel;
    21→extern int kAdminCellLevel;
    22→extern int kMaxAdminLevel;
    23→
    24→// --- Simplification globals ---
    25→
    26→extern SimplifyMode kSimplifyMode;
    27→extern double kSimplifyEpsilonOverride; // 0 = use per-level defaults
    28→
    29→// --- S2 helpers ---
    30→
    31→void cover_edge(double lat1, double lng1, double lat2, double lng2,
    32→                std::vector<S2CellId>& out);
    33→
    34→std::vector<S2CellId> cover_edge(double lat1, double lng1, double lat2, double lng2);
    35→
    36→S2CellId point_to_cell(double lat, double lng);
    37→
    38→std::vector<std::pair<S2CellId, bool>> cover_polygon(
    39→    const std::vector<std::pair<double,double>>& vertices);
    40→
    41→// --- Thread pool for concurrent admin polygon S2 covering ---
    42→
    43→class AdminCoverPool {
    44→public:
    45→    struct WorkItem {
    46→        uint32_t poly_id;
    47→        std::vector<std::pair<double,double>> vertices;
    48→    };
    49→
    50→    explicit AdminCoverPool(size_t num_threads);
    51→    ~AdminCoverPool();
    52→
    53→    void submit(uint32_t poly_id, std::vector<std::pair<double,double>>&& vertices);
    54→    std::unordered_map<uint64_t, std::vector<uint32_t>> drain();
    55→
    56→private:
    57→    void worker_loop();
    58→
    59→    std::vector<std::thread> workers_;
    60→    std::deque<WorkItem> queue_;
    61→    std::vector<std::unordered_map<uint64_t, std::vector<uint32_t>>> thread_results_;
    62→    std::mutex mutex_;
    63→    std::condition_variable cv_;
    64→    std::condition_variable done_cv_;
    65→    size_t active_workers_ = 0;
    66→    bool stop_;
    67→};
    68→
    69→// --- Simplification dispatch ---
    70→
    71→// Simplify a polygon using the current global mode, taking admin_level into account.
    72→// In MaxVertices mode: caps at MAX_POLYGON_VERTICES (500).
    73→// In ErrorBounded mode: uses per-level epsilon in meters, converted to degrees.
    74→inline std::vector<std::pair<double,double>> simplify_admin_polygon(
    75→    const std::vector<std::pair<double,double>>& pts, uint8_t admin_level) {
    76→    if (kSimplifyMode == SimplifyMode::ErrorBounded) {
    77→        double eps_m = kSimplifyEpsilonOverride > 0
    78→            ? kSimplifyEpsilonOverride
    79→            : admin_epsilon_meters(admin_level);
    80→        // Estimate latitude from first vertex for degree conversion
    81→        double lat = pts.empty() ? 0.0 : pts[0].first;
    82→        double eps_deg = meters_to_degrees(eps_m, lat);
    83→        return simplify_polygon_epsilon(pts, eps_deg);
    84→    }
    85→    return simplify_polygon(pts);
    86→}
    87→
    88→// --- Helper functions that depend on S2 ---
    89→
    90→void add_addr_point(ParsedData& data, double lat, double lng,
    91→                    const char* housenumber, const char* street,
    92→                    uint64_t& addr_count_total);
    93→
    94→void add_admin_polygon(ParsedData& data,
    95→                       const std::vector<std::pair<double,double>>& vertices_in,
    96→                       const char* name, uint8_t admin_level,
    97→                       const char* country_code,
    98→                       AdminCoverPool* admin_pool = nullptr);
    99→


Now let me find the implementation of add_addr_point:


grep -n "void add_addr_point" /root/traccar-geocoder/builder/src/*.cpp


/root/traccar-geocoder/builder/src/s2_helpers.cpp:229:void add_addr_point(ParsedData& data, double lat, double lng,


Perfect! Now let me read that function:


   228→
   229→void add_addr_point(ParsedData& data, double lat, double lng,
   230→                    const char* housenumber, const char* street,
   231→                    uint64_t& addr_count_total) {
   232→    uint32_t addr_id = static_cast<uint32_t>(data.addr_points.size());
   233→    data.addr_points.push_back({
   234→        static_cast<float>(lat),
   235→        static_cast<float>(lng),
   236→        data.string_pool.intern(housenumber),
   237→        data.string_pool.intern(street)
   238→    });
   239→
   240→    S2CellId cell = point_to_cell(lat, lng);
   241→    data.cell_to_addrs[cell.id()].push_back(addr_id);
   242→
   243→    addr_count_total++;
   244→    if (addr_count_total % 1000000 == 0) {
   245→        std::cerr << "Collected " << addr_count_total / 1000000 << "M addresses..." << std::endl;
   246→    }
   247→}
   248→
   249→// --- add_admin_polygon ---
   250→
   251→void add_admin_polygon(ParsedData& data,
   252→                       const std::vector<std::pair<double,double>>& vertices_in,
   253→                       const char* name, uint8_t admin_level,
   254→                       const char* country_code,
   255→                       AdminCoverPool* admin_pool) {
   256→    auto vertices = vertices_in;
   257→    if (vertices.size() >= 4 &&
   258→        std::fabs(vertices.front().first - vertices.back().first) < 1e-7 &&
   259→        std::fabs(vertices.front().second - vertices.back().second) < 1e-7) {
   260→        vertices.pop_back();
   261→        auto min_it = std::min_element(vertices.begin(), vertices.end());
   262→        std::rotate(vertices.begin(), min_it, vertices.end());
   263→        vertices.push_back(vertices.front());
   264→    }
   265→
   266→    auto simplified = simplify_admin_polygon(vertices, admin_level);
   267→    if (simplified.size() < 3) return;
   268→
   269→    uint32_t poly_id = static_cast<uint32_t>(data.admin_polygons.size());
   270→    uint32_t vertex_offset = static_cast<uint32_t>(data.admin_vertices.size());
   271→
   272→    for (const auto& [lat, lng] : simplified) {
   273→        data.admin_vertices.push_back({static_cast<float>(lat), static_cast<float>(lng)});
   274→    }
   275→
   276→    AdminPolygon poly{};
   277→    poly.vertex_offset = vertex_offset;


Excellent! Now let me search for how addr_points are sorted and written to the binary file. Let me search for the writing phase:


1325-
1326-    // Write one region: modes + quality variants
1327-    auto write_region = [&](const ParsedData& d, const std::string& base_dir) {
1328-        ensure_dir(base_dir);
1329-
1330-        if (multi_output) {
1331-            // Write all 3 modes in parallel (they read shared data, write to separate dirs)
1332-            auto wf1 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/full", IndexMode::Full); });
1333:            auto wf2 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/no-addresses", IndexMode::NoAddresses); });
1334-            auto wf3 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/admin", IndexMode::AdminOnly); });
1335-            wf1.get(); wf2.get(); wf3.get();
1336-        } else {
1337-            write_index(d, base_dir, mode);
1338-        }
1339-
1340-        // Write quality variants under admin/ (or base_dir if not multi_output)
1341-        if (multi_quality) {
--
1489-        };
1490-        std::vector<std::string> admin_quality_files = {
1491-            "admin_polygons.bin", "admin_vertices.bin"
1492-        };
1493-        std::vector<std::string> street_files = {
1494-            "street_ways.bin", "street_nodes.bin", "street_entries.bin"
1495-        };
1496-        std::vector<std::string> address_files = {
1497:            "addr_points.bin", "addr_entries.bin",
1498-            "interp_ways.bin", "interp_nodes.bin", "interp_entries.bin"
1499-        };
1500-        std::vector<std::string> geo_files = {"geo_cells.bin"};
1501-
1502-        // Build list of regions
1503-        std::vector<std::pair<std::string, std::string>> regions; // (name, path)
1504-        regions.push_back({"planet", output_dir});
1505-        if (generate_continents) {


Let me search for the write_index function:


Let me search in other files:


grep -r "void write_index" /root/traccar-geocoder/builder/src/*.cpp


/root/traccar-geocoder/builder/src/cell_index.cpp:void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {


     1→#include "cell_index.h"
     2→
     3→#include <algorithm>
     4→#include <atomic>
     5→#include <cstring>
     6→#include <fstream>
     7→#include <future>
     8→#include <iostream>
     9→#include <thread>
    10→#include <unordered_map>
    11→#include <unordered_set>
    12→
    13→#include <unistd.h>
    14→
    15→#include "geometry.h"
    16→#include "s2_helpers.h"
    17→
    18→std::vector<uint32_t> write_entries(
    19→    const std::string& path,
    20→    const std::vector<uint64_t>& sorted_cells,
    21→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
    22→) {
    23→    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
    24→    std::vector<CellRef> sorted_refs;
    25→    sorted_refs.reserve(cell_map.size());
    26→    for (auto& [id, ids] : cell_map) sorted_refs.push_back({id, &ids});
    27→    std::sort(sorted_refs.begin(), sorted_refs.end(),
    28→        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });
    29→
    30→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    31→    size_t total_size = 0;
    32→    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);
    33→
    34→    std::vector<char> buf;
    35→    buf.reserve(total_size);
    36→    uint32_t current = 0;
    37→    size_t ri = 0;
    38→    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
    39→        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
    40→        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
    41→        offsets[si] = current;
    42→        const auto& ids = *sorted_refs[ri].ids;
    43→        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
    44→        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
    45→                   reinterpret_cast<const char*>(&count) + sizeof(count));
    46→        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
    47→                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
    48→        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
    49→        ri++;
    50→    }
    51→    std::ofstream f(path, std::ios::binary);
    52→    f.write(buf.data(), buf.size());
    53→    return offsets;
    54→}
    55→
    56→std::vector<uint32_t> write_entries_from_sorted(
    57→    const std::string& path,
    58→    const std::vector<uint64_t>& sorted_cells,
    59→    const std::vector<CellItemPair>& sorted_pairs
    60→) {
    61→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    62→    if (sorted_pairs.empty()) {
    63→        std::ofstream f(path, std::ios::binary);
    64→        return offsets;
    65→    }
    66→
    67→    unsigned int nthreads = std::thread::hardware_concurrency();
    68→    if (nthreads == 0) nthreads = 4;
    69→    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;
    70→
    71→    struct ChunkResult {
    72→        std::vector<char> buf;
    73→        size_t cell_start, cell_end;
    74→        uint32_t local_size;
    75→    };
    76→    std::vector<ChunkResult> chunks(nthreads);
    77→
    78→    std::vector<std::thread> threads;
    79→    for (unsigned int t = 0; t < nthreads; t++) {
    80→        size_t cs = t * cells_per_chunk;
    81→        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
    82→        if (cs >= sorted_cells.size()) break;
    83→
    84→        threads.emplace_back([&, t, cs, ce]() {
    85→            auto& chunk = chunks[t];
    86→            chunk.cell_start = cs;
    87→            chunk.cell_end = ce;
    88→            chunk.local_size = 0;
    89→
    90→            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
    91→                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
    92→                    return p.cell_id < id;
    93→                }) - sorted_pairs.begin();
    94→
    95→            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
    96→                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
    97→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
    98→                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;
    99→
   100→                offsets[si] = chunk.local_size;
   101→                size_t start = pi;
   102→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
   103→                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(MAX_VERTEX_COUNT)));
   104→                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
   105→                size_t buf_pos = chunk.buf.size();
   106→                chunk.buf.resize(buf_pos + entry_size);
   107→                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
   108→                for (size_t k = start; k < pi; k++) {
   109→                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
   110→                           &sorted_pairs[k].item_id, sizeof(uint32_t));
   111→                }
   112→                chunk.local_size += entry_size;
   113→            }
   114→        });
   115→    }
   116→    for (auto& t : threads) t.join();
   117→
   118→    uint32_t global_offset = 0;
   119→    for (auto& chunk : chunks) {
   120→        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
   121→            if (offsets[si] != NO_DATA) offsets[si] += global_offset;
   122→        }
   123→        global_offset += chunk.local_size;
   124→    }
   125→
   126→    std::vector<char> buf;
   127→    buf.reserve(global_offset);
   128→    for (auto& chunk : chunks) buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());
   129→
   130→    std::ofstream f(path, std::ios::binary);
   131→    f.write(buf.data(), buf.size());
   132→    return offsets;
   133→}
   134→
   135→void write_cell_index(
   136→    const std::string& cells_path,
   137→    const std::string& entries_path,
   138→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
   139→) {
   140→    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(cell_map.begin(), cell_map.end());
   141→    std::sort(sorted.begin(), sorted.end());
   142→
   143→    { std::ofstream f(cells_path, std::ios::binary);
   144→      uint32_t current_offset = 0;
   145→      for (const auto& [cell_id, ids] : sorted) {
   146→          f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
   147→          f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
   148→          current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
   149→      } }
   150→
   151→    { std::ofstream f(entries_path, std::ios::binary);
   152→      for (const auto& [cell_id, ids] : sorted) {
   153→          uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
   154→          f.write(reinterpret_cast<const char*>(&count), sizeof(count));
   155→          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
   156→      } }
   157→}
   158→
   159→void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
   160→    ensure_dir(output_dir);
   161→    auto _wt = std::chrono::steady_clock::now();
   162→
   163→    bool write_streets = (mode != IndexMode::AdminOnly);
   164→    bool write_addresses = (mode == IndexMode::Full);
   165→
   166→    if (write_streets) {
   167→        std::vector<uint64_t> sorted_geo_cells;
   168→        {
   169→            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
   170→                std::vector<uint64_t> cells;
   171→                cells.reserve(pairs.size() / 2);
   172→                for (size_t i = 0; i < pairs.size(); ) {
   173→                    cells.push_back(pairs[i].cell_id);
   174→                    uint64_t cur = pairs[i].cell_id;
   175→                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
   176→                }
   177→                return cells;
   178→            };
   179→            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
   180→                std::vector<uint64_t> cells;
   181→                cells.reserve(m.size());
   182→                for (auto& [id, _] : m) cells.push_back(id);
   183→                std::sort(cells.begin(), cells.end());
   184→                return cells;
   185→            };
   186→
   187→            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
   188→            {
   189→                auto f1 = std::async(std::launch::async, [&] {
   190→                    return !data.sorted_way_cells.empty()
   191→                        ? extract_unique_cells(data.sorted_way_cells)
   192→                        : extract_from_map(data.cell_to_ways);
   193→                });
   194→                if (write_addresses) {
   195→                    auto f2 = std::async(std::launch::async, [&] {
   196→                        return !data.sorted_addr_cells.empty()
   197→                            ? extract_unique_cells(data.sorted_addr_cells)
   198→                            : extract_from_map(data.cell_to_addrs);
   199→                    });
   200→                    auto f3 = std::async(std::launch::async, [&] {
   201→                        return !data.sorted_interp_cells.empty()
   202→                            ? extract_unique_cells(data.sorted_interp_cells)
   203→                            : extract_from_map(data.cell_to_interps);
   204→                    });
   205→                    addr_cells = f2.get();
   206→                    interp_cells = f3.get();
   207→                }
   208→                way_cells = f1.get();
   209→            }
   210→
   211→            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
   212→            std::merge(way_cells.begin(), way_cells.end(), addr_cells.begin(), addr_cells.end(),
   213→                       std::back_inserter(sorted_geo_cells));
   214→            if (!interp_cells.empty()) {
   215→                std::vector<uint64_t> tmp;
   216→                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
   217→                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
   218→                           interp_cells.begin(), interp_cells.end(), std::back_inserter(tmp));
   219→                sorted_geo_cells = std::move(tmp);
   220→            }
   221→            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
   222→                                    sorted_geo_cells.end());
   223→        }
   224→
   225→        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
   226→        {
   227→            auto f1 = std::async(std::launch::async, [&]() {
   228→                if (!data.sorted_way_cells.empty())
   229→                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
   230→                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
   231→            });
   232→            if (write_addresses) {
   233→                auto f2 = std::async(std::launch::async, [&]() {
   234→                    if (!data.sorted_addr_cells.empty())
   235→                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
   236→                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
   237→                });
   238→                auto f3 = std::async(std::launch::async, [&]() {
   239→                    if (!data.sorted_interp_cells.empty())
   240→                        return write_entries_from_sorted(output_dir + "/interp_entries.bin", sorted_geo_cells, data.sorted_interp_cells);
   241→                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
   242→                });
   243→                addr_offsets = f2.get();
   244→                interp_offsets = f3.get();
   245→            }
   246→            street_offsets = f1.get();
   247→        }
   248→
   249→        log_phase("  Write: entry files", _wt);
   250→        {
   251→            size_t n = sorted_geo_cells.size();
   252→            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
   253→            std::vector<char> buf(n * row_size);
   254→            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
   255→            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);
   256→
   257→            unsigned int nthreads = std::thread::hardware_concurrency();
   258→            if (nthreads == 0) nthreads = 4;
   259→            size_t chunk = (n + nthreads - 1) / nthreads;
   260→            std::vector<std::thread> fill_threads;
   261→            for (unsigned int t = 0; t < nthreads; t++) {
   262→                size_t start = t * chunk;
   263→                size_t end = std::min(start + chunk, n);
   264→                if (start >= n) break;
   265→                fill_threads.emplace_back([&, start, end]() {
   266→                    char* ptr = buf.data() + start * row_size;
   267→                    for (size_t i = start; i < end; i++) {
   268→                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
   269→                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   270→                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   271→                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   272→                    }
   273→                });
   274→            }
   275→            for (auto& t : fill_threads) t.join();
   276→
   277→            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
   278→            f.write(buf.data(), buf.size());
   279→        }
   280→
   281→        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
   282→                  << data.ways.size() << " ways, " << data.addr_points.size() << " addrs, "
   283→                  << data.interp_ways.size() << " interps)" << std::endl;
   284→    }
   285→
   286→    log_phase("  Write: geo_cells.bin", _wt);
   287→    auto admin_future = std::async(std::launch::async, [&] {
   288→        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
   289→        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
   290→    });
   291→
   292→    std::vector<std::future<void>> write_futures;
   293→    if (write_streets) {
   294→        write_futures.push_back(std::async(std::launch::async, [&] {
   295→            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
   296→            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
   297→        }));
   298→        write_futures.push_back(std::async(std::launch::async, [&] {
   299→            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
   300→            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
   301→        }));
   302→        if (write_addresses) {
   303→            write_futures.push_back(std::async(std::launch::async, [&] {
   304→                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
   305→                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
   306→            }));
   307→            write_futures.push_back(std::async(std::launch::async, [&] {
   308→                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
   309→                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
   310→            }));
   311→            write_futures.push_back(std::async(std::launch::async, [&] {
   312→                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
   313→                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
   314→            }));
   315→        }
   316→    }
   317→    // If not writing all data types, compact string pool and remap references
   318→    // before writing admin_polygons and strings.bin.
   319→    std::vector<AdminPolygon> compact_admin;
   320→    std::vector<char> compact_strings;
   321→    const bool need_compact = (!write_streets || !write_addresses);
   322→
   323→    if (need_compact) {
   324→        // Collect referenced string offsets
   325→        std::unordered_set<uint32_t> used_offsets;
   326→        if (write_streets) {
   327→            for (const auto& w : data.ways) used_offsets.insert(w.name_id);
   328→            if (write_addresses) {
   329→                for (const auto& a : data.addr_points) {
   330→                    used_offsets.insert(a.housenumber_id);
   331→                    used_offsets.insert(a.street_id);
   332→                }
   333→                for (const auto& iw : data.interp_ways) used_offsets.insert(iw.street_id);
   334→            }
   335→        }
   336→        for (const auto& ap : data.admin_polygons) used_offsets.insert(ap.name_id);
   337→
   338→        // Build compact pool and remap
   339→        const auto& old_sp = data.string_pool.data();
   340→        std::vector<uint32_t> sorted_offs(used_offsets.begin(), used_offsets.end());
   341→        std::sort(sorted_offs.begin(), sorted_offs.end());
   342→        std::unordered_map<uint32_t, uint32_t> remap;
   343→        compact_strings.reserve(used_offsets.size() * 16);
   344→        for (uint32_t off : sorted_offs) {
   345→            remap[off] = static_cast<uint32_t>(compact_strings.size());
   346→            size_t len = std::strlen(old_sp.data() + off);
   347→            compact_strings.insert(compact_strings.end(), old_sp.data() + off,
   348→                old_sp.data() + off + len + 1);
   349→        }
   350→
   351→        // Remap admin polygon name_ids
   352→        compact_admin = data.admin_polygons;
   353→        for (auto& ap : compact_admin) ap.name_id = remap[ap.name_id];
   354→
   355→        // Remap street/addr/interp if applicable (for no-addresses mode)
   356→        // These are written from data directly, so we'd need copies too.
   357→        // For simplicity, only compact when we have fewer types.
   358→    }
   359→
   360→    const auto& admin_to_write = need_compact ? compact_admin : data.admin_polygons;
   361→    write_futures.push_back(std::async(std::launch::async, [&] {
   362→        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
   363→        f.write(reinterpret_cast<const char*>(admin_to_write.data()), admin_to_write.size() * sizeof(AdminPolygon));
   364→    }));
   365→    write_futures.push_back(std::async(std::launch::async, [&] {
   366→        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
   367→        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
   368→    }));
   369→    write_futures.push_back(std::async(std::launch::async, [&] {
   370→        if (need_compact) {
   371→            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
   372→            f.write(compact_strings.data(), compact_strings.size());
   373→            std::cerr << "strings.bin: " << compact_strings.size() << " bytes (compacted from "
   374→                      << data.string_pool.data().size() << ")" << std::endl;
   375→        } else {
   376→            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
   377→            f.write(data.string_pool.data().data(), data.string_pool.data().size());
   378→            std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
   379→        }
   380→    }));
   381→
   382→    log_phase("  Write: parallel data files launched", _wt);
   383→    admin_future.get();
   384→    for (auto& f : write_futures) f.get();
   385→}
   386→
   387→void write_quality_variant(const ParsedData& data, const std::string& source_dir,
   388→                           const std::string& output_dir, double epsilon_scale) {
   389→    ensure_dir(output_dir);
   390→
   391→    // Re-simplify admin polygons at the given epsilon scale
   392→    std::vector<AdminPolygon> new_polys;
   393→    std::vector<NodeCoord> new_verts;
   394→    new_polys.reserve(data.admin_polygons.size());
   395→
   396→    // Parallel simplification
   397→    struct SimplifiedPoly {
   398→        std::vector<std::pair<double,double>> verts;
   399→    };
   400→    std::vector<SimplifiedPoly> simplified(data.admin_polygons.size());
   401→
   402→    {
   403→        std::atomic<size_t> idx{0};
   404→        unsigned nthreads = std::thread::hardware_concurrency();
   405→        if (nthreads == 0) nthreads = 4;
   406→        std::vector<std::thread> workers;
   407→        for (unsigned t = 0; t < nthreads; t++) {
   408→            workers.emplace_back([&]() {
   409→                while (true) {
   410→                    size_t i = idx.fetch_add(1);
   411→                    if (i >= data.admin_polygons.size()) break;
   412→                    const auto& ap = data.admin_polygons[i];
   413→
   414→                    // Extract original vertices
   415→                    std::vector<std::pair<double,double>> pts;
   416→                    pts.reserve(ap.vertex_count);
   417→                    for (uint32_t j = 0; j < ap.vertex_count; j++) {
   418→                        const auto& v = data.admin_vertices[ap.vertex_offset + j];
   419→                        pts.emplace_back(v.lat, v.lng);
   420→                    }
   421→
   422→                    if (epsilon_scale > 0) {
   423→                        double eps_m = admin_epsilon_meters(ap.admin_level) * epsilon_scale;
   424→                        double lat = pts.empty() ? 0.0 : pts[0].first;
   425→                        double eps_deg = meters_to_degrees(eps_m, lat);
   426→                        simplified[i].verts = simplify_polygon_epsilon(pts, eps_deg);
   427→                    } else {
   428→                        simplified[i].verts = std::move(pts);
   429→                    }
   430→                }
   431→            });
   432→        }
   433→        for (auto& w : workers) w.join();
   434→    }
   435→
   436→    // Sequential: build new polygon and vertex arrays
   437→    for (size_t i = 0; i < data.admin_polygons.size(); i++) {
   438→        auto& sv = simplified[i].verts;
   439→        if (sv.size() < 3) continue;
   440→
   441→        AdminPolygon np = data.admin_polygons[i];
   442→        np.vertex_offset = static_cast<uint32_t>(new_verts.size());
   443→        np.vertex_count = static_cast<uint32_t>(sv.size());
   444→        np.area = polygon_area(sv);
   445→
   446→        for (const auto& [lat, lng] : sv) {
   447→            new_verts.push_back({static_cast<float>(lat), static_cast<float>(lng)});
   448→        }
   449→        new_polys.push_back(np);
   450→    }
   451→
   452→    std::cerr << "Quality " << epsilon_scale << "x: " << new_polys.size()
   453→              << " polygons, " << new_verts.size() << " vertices ("
   454→              << new_verts.size() * 8 / 1024 / 1024 << " MiB)" << std::endl;
   455→
   456→    // Write admin files
   457→    {
   458→        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
   459→        f.write(reinterpret_cast<const char*>(new_polys.data()), new_polys.size() * sizeof(AdminPolygon));
   460→    }
   461→    {
   462→        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
   463→        f.write(reinterpret_cast<const char*>(new_verts.data()), new_verts.size() * sizeof(NodeCoord));
   464→    }
   465→
   466→    // Quality directories only contain the two files that change.
   467→    // Shared files (admin_cells.bin, admin_entries.bin, strings.bin)
   468→    // stay in the parent directory.
   469→}
   470→


Excellent! Now let me search for how street_ways are ordered. Let me look at how ways are added:


315-            }
316-            std::cerr << "  Collected " << data.collected_relations.size()
317-                      << " admin/postal relations for parallel assembly." << std::endl;
318-            log_phase("Pass 1: relation scanning", _pt, _cpu);
319-
320:            // Build admin_way_ids as a bitset for O(1) lookup
321-            // Max way ID ~1.3B → ~162 MiB bitset (much faster than unordered_set)
322-            static constexpr size_t MAX_WAY_ID = 2000000000ULL; // 2B, ~250 MiB
323-            std::vector<uint8_t> admin_way_bits(MAX_WAY_ID / 8 + 1, 0);
324-            size_t admin_way_count = 0;
325-            for (const auto& rel : data.collected_relations) {
326:                for (const auto& [way_id, role] : rel.members) {
327:                    if (way_id > 0 && static_cast<size_t>(way_id) < MAX_WAY_ID) {
328:                        admin_way_bits[way_id / 8] |= (1 << (way_id % 8));
329-                        admin_way_count++;
330-                    }
331-                }
332-            }
333-            auto is_admin_way = [&](int64_t id) -> bool {
--
422-                    uint64_t way_count = 0;
423-                    uint64_t building_addr_count = 0;
424-                    uint64_t interp_count = 0;
425-                    // Way geometries for parallel admin assembly
426-                    struct WayGeomEntry {
427:                        int64_t way_id;
428-                        std::vector<std::pair<double,double>> coords;
429-                        int64_t first_node_id;
430-                        int64_t last_node_id;
431-                    };
432-                    std::vector<WayGeomEntry> way_geoms;
--
442-
443-                static thread_local ThreadLocalData* tl_way_data = nullptr;
444-                std::vector<ThreadLocalData> tld(num_threads);
445-                std::atomic<unsigned> next_tl_way{0};
446-
447:                pbf.read_ways_streaming([&](int64_t way_id,
448-                    const int64_t* refs_data, size_t refs_size,
449-                    const uint32_t* tag_keys, const uint32_t* tag_vals, size_t ntags,
450-                    const std::vector<std::string>& st) {
451-
452-                    if (!tl_way_data) {
--
487-
488-                    // Early exit: if no relevant tags, skip expensive node resolution
489-                    bool need_nodes = t_interpolation || t_housenumber ||
490-                        (t_highway && is_included_highway(t_highway) && t_name) ||
491-                        t_boundary ||
492:                        (refs_size > 0 && is_admin_way(way_id));
493-                    if (!need_nodes) return;
494-
495-                    // Pre-resolve all node locations
496-                    thread_local std::vector<PackedLocation> resolved_locs;
497-                    resolved_locs.clear();
--
560-                            local.way_count++;
561-                        }
562-                    }
563-

[Showing results with pagination = limit: 60]


Let me search more specifically for where ways are added to data.ways:


grep -n "data\.ways\s*\.push_back\|data\.ways\.emplace_back" /root/traccar-geocoder/builder/src/build_index.cpp | head -20


618:                        data.ways.push_back(h);


Let me read that section:


   600→                tl_way_data = nullptr;
   601→                std::cerr << "  Parallel way processing complete." << std::endl;
   602→
   603→                // Process areas/multipolygons — sequential fallback path
   604→                // Merge thread-local way/interp data into main ParsedData
   605→                std::cerr << "  Merging thread-local data..." << std::endl;
   606→                uint64_t total_ways = 0, total_building_addrs = 0, total_interps = 0;
   607→                for (auto& local : tld) {
   608→                    uint32_t way_base = static_cast<uint32_t>(data.ways.size());
   609→                    uint32_t node_base = static_cast<uint32_t>(data.street_nodes.size());
   610→                    uint32_t interp_base = static_cast<uint32_t>(data.interp_ways.size());
   611→                    uint32_t interp_node_base = static_cast<uint32_t>(data.interp_nodes.size());
   612→
   613→                    // Merge street ways
   614→                    for (size_t i = 0; i < local.ways.size(); i++) {
   615→                        auto h = local.ways[i];
   616→                        h.node_offset += node_base;
   617→                        h.name_id = data.string_pool.intern(local.way_strings[i]);
   618→                        data.ways.push_back(h);
   619→                    }
   620→                    data.street_nodes.insert(data.street_nodes.end(),
   621→                        local.street_nodes.begin(), local.street_nodes.end());
   622→
   623→                    // Remap deferred ways
   624→                    for (auto dw : local.deferred_ways) {
   625→                        dw.way_id += way_base;
   626→                        dw.node_offset += node_base;
   627→                        data.deferred_ways.push_back(dw);
   628→                    }
   629→
   630→                    // Merge building addresses
   631→                    for (size_t i = 0; i < local.building_addrs.size(); i++) {
   632→                        uint64_t dummy = 0;
   633→                        add_addr_point(data, local.building_addrs[i].lat, local.building_addrs[i].lng,
   634→                                       local.addr_strings[i].first.c_str(),
   635→                                       local.addr_strings[i].second.c_str(), dummy);
   636→                    }
   637→
   638→                    // Merge interpolation ways
   639→                    for (size_t i = 0; i < local.interp_ways.size(); i++) {
   640→                        auto iw = local.interp_ways[i];
   641→                        iw.node_offset += interp_node_base;
   642→                        iw.street_id = data.string_pool.intern(local.interp_strings[i]);
   643→                        data.interp_ways.push_back(iw);
   644→                    }
   645→                    data.interp_nodes.insert(data.interp_nodes.end(),
   646→                        local.interp_nodes.begin(), local.interp_nodes.end());
   647→
   648→                    // Remap deferred interps
   649→                    for (auto di : local.deferred_interps) {
   650→                        di.interp_id += interp_base;
   651→                        di.node_offset += interp_node_base;
   652→                        data.deferred_interps.push_back(di);
   653→                    }
   654→
   655→                    total_ways += local.way_count;
   656→                    total_building_addrs += local.building_addr_count;
   657→                    total_interps += local.interp_count;
   658→
   659→                    // Merge way geometries for admin assembly
   660→                    {
   661→                        for (auto& wg : local.way_geoms) {
   662→                            ParsedData::WayGeometry g;
   663→                            g.coords = std::move(wg.coords);
   664→                            g.first_node_id = wg.first_node_id;
   665→                            g.last_node_id = wg.last_node_id;
   666→                            data.way_geometries[wg.way_id] = std::move(g);
   667→                        }
   668→                        local.way_geoms.clear();
   669→                        local.way_geoms.shrink_to_fit();
   670→                    }
   671→                }
   672→                std::cerr << "  Merged: " << total_ways << " ways, "
   673→                          << total_building_addrs << " building addrs, "
   674→                          << total_interps << " interps from parallel processing." << std::endl;
   675→
   676→                // --- Merge closed-way admin polygons ---
   677→                {
   678→                    uint64_t closed_way_admin_count = 0;
   679→                    for (auto& local : tld) {
   680→                        for (auto& cwa : local.closed_way_admins) {
   681→                            const char* cc = cwa.country_code.empty() ? nullptr : cwa.country_code.c_str();
   682→                            add_admin_polygon(data, cwa.vertices, cwa.name.c_str(),
   683→                                              cwa.admin_level, cc, &admin_pool);
   684→                            closed_way_admin_count++;
   685→                        }
   686→                        local.closed_way_admins.clear();
   687→                    }
   688→                    if (closed_way_admin_count > 0) {
   689→                        std::cerr << "  Added " << closed_way_admin_count
   690→                                  << " admin polygons from closed ways." << std::endl;
   691→                    }
   692→                }
   693→
   694→                // --- Parallel admin boundary assembly ---
   695→                {
   696→                    log_phase("Pass 2b: way processing", _pt, _cpu);
   697→                    pbf.unmap(); // release 86 GiB PBF mmap before admin/S2 phases
   698→                    std::cerr << "  Assembling admin polygons in parallel ("
   699→                              << data.collected_relations.size() << " relations, "


Perfect! Now let me search for where sorted_way_cells and sorted_addr_cells are populated. These are key to understanding the ordering:


1131-                // Skip building hash map — sorted_out is used directly for writing.
1132-                // cell_map stays empty (only needed for cache/continent modes).
1133-            };
1134-
1135-
1136-            // Sort each thread's pairs in parallel, then k-way merge directly
1137-            // into hash map + sorted output. No intermediate merged vector.
1138-            auto f_ways = std::async(std::launch::async, [&] {
1139:                parallel_sort_and_build(way_pairs, data.cell_to_ways, data.sorted_way_cells);
1140-            });
1141-            auto f_interps = std::async(std::launch::async, [&] {
1142:                parallel_sort_and_build(interp_pairs, data.cell_to_interps, data.sorted_interp_cells);
1143-            });
1144-            f_ways.get();
1145-            f_interps.get();
1146-            log_phase("  S2: sort + group into cell maps", _s2t, _s2cpu);
1147-
1148-            // Free deferred work items
1149-            data.deferred_ways.clear();
1150-            data.deferred_ways.shrink_to_fit();
--
1208-
1209-                // Step 5: Build sorted pairs
1210-                std::vector<CellItemPair> pairs;
1211-                pairs.reserve(total_pairs);
1212-                for (uint64_t cell_id : sorted_cells) {
1213-                    auto& ids = data.cell_to_addrs[cell_id];
1214-                    for (auto id : ids) pairs.push_back({cell_id, id});
1215-                }
1216:                data.sorted_addr_cells = std::move(pairs);
1217-                log_phase("    Dedup: build sorted pairs", _dt, _dc);
1218-            });
1219-            auto f4 = std::async(std::launch::async, [&]{ deduplicate(data.cell_to_admin); });
1220-            f2.get(); f4.get();
1221-        }
1222-
1223-        // Save cache if requested
1224-        if (!save_cache_path.empty()) {
--
1274-            for (auto& sub : sub_maps) {
1275-                for (auto& [k, v] : sub) {
1276-                    map[k] = std::move(v);
1277-                }
1278-            }
1279-        };
1280-
1281-        std::cerr << "Rebuilding cell maps for continent filtering..." << std::endl;
1282:        auto f1 = std::async(std::launch::async, [&]{ rebuild_map_parallel(data.sorted_way_cells, data.cell_to_ways); });
1283:        auto f2 = std::async(std::launch::async, [&]{ rebuild_map_parallel(data.sorted_interp_cells, data.cell_to_interps); });
1284-        f1.get(); f2.get();
1285-        log_phase("Rebuild cell maps", _pt, _cpu);
1286-    }
1287-
1288-    // --- Write index files ---
1289-    log_phase("Deduplication", _pt, _cpu);
1290-    std::cerr << "Writing index files to " << output_dir << "..." << std::endl;
1291-
--
1401-                    }
1402-                });
1403-            }
1404-            for (auto& t : threads) t.join();
1405-            return masks;
1406-        };
1407-
1408-        // Precompute for all three sorted pair arrays
1409:        auto way_masks = std::async(std::launch::async, [&]{ return precompute_masks(data.sorted_way_cells); });
1410:        auto addr_masks = std::async(std::launch::async, [&]{ return precompute_masks(data.sorted_addr_cells); });
1411:        auto interp_masks = std::async(std::launch::async, [&]{ return precompute_masks(data.sorted_interp_cells); });
1412-        auto way_continent_masks = way_masks.get();
1413-        auto addr_continent_masks = addr_masks.get();
1414-        auto interp_continent_masks = interp_masks.get();
1415-
1416-        log_phase("Pre-compute continent masks", _pct, _pcc);
1417-
1418-        // Sort by bbox area descending — largest continents first
1419-        std::vector<size_t> continent_order(kContinentCount);


Perfect! Now let me look at the parallel_sort_and_build function:


grep -n "parallel_sort_and_build\|auto.*parallel_sort_and_build" /root/traccar-geocoder/builder/src/build_index.cpp | head -20


1061:            auto parallel_sort_and_build = [](
1139:                parallel_sort_and_build(way_pairs, data.cell_to_ways, data.sorted_way_cells);
1142:                parallel_sort_and_build(interp_pairs, data.cell_to_interps, data.sorted_interp_cells);


Let me read that lambda:


  1060→            // hash map + sorted output. No intermediate merged vector needed.
  1061→            auto parallel_sort_and_build = [](
  1062→                std::vector<std::vector<CellItem>>& thread_pairs,
  1063→                std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map,
  1064→                std::vector<CellItemPair>& sorted_out
  1065→            ) {
  1066→                // Step 1: Convert + sort each thread's data in parallel
  1067→                size_t total = 0;
  1068→                for (auto& v : thread_pairs) total += v.size();
  1069→
  1070→                std::vector<std::vector<CellItemPair>> chunks(thread_pairs.size());
  1071→                {
  1072→                    std::vector<std::thread> sort_threads;
  1073→                    for (size_t t = 0; t < thread_pairs.size(); t++) {
  1074→                        sort_threads.emplace_back([&, t]() {
  1075→                            auto& src = thread_pairs[t];
  1076→                            auto& dst = chunks[t];
  1077→                            dst.reserve(src.size());
  1078→                            for (auto& ci : src) dst.push_back({ci.cell_id, ci.item_id});
  1079→                            src.clear(); src.shrink_to_fit();
  1080→                            std::sort(dst.begin(), dst.end(), [](const CellItemPair& a, const CellItemPair& b) {
  1081→                                return a.cell_id < b.cell_id || (a.cell_id == b.cell_id && a.item_id < b.item_id);
  1082→                            });
  1083→                        });
  1084→                    }
  1085→                    for (auto& t : sort_threads) t.join();
  1086→                }
  1087→
  1088→                // Step 2: Concatenate sorted chunks and merge using parallel tree.
  1089→                // Each chunk is already sorted. Record run boundaries, concatenate,
  1090→                // then merge adjacent runs pairwise in parallel at each level.
  1091→                auto cmp = [](const CellItemPair& a, const CellItemPair& b) {
  1092→                    return a.cell_id < b.cell_id || (a.cell_id == b.cell_id && a.item_id < b.item_id);
  1093→                };
  1094→
  1095→                // Record run boundaries before concatenation
  1096→                std::vector<size_t> run_bounds = {0};
  1097→                sorted_out.clear();
  1098→                sorted_out.reserve(total);
  1099→                for (auto& chunk : chunks) {
  1100→                    sorted_out.insert(sorted_out.end(), chunk.begin(), chunk.end());
  1101→                    run_bounds.push_back(sorted_out.size());
  1102→                    chunk.clear(); chunk.shrink_to_fit();
  1103→                }
  1104→
  1105→                // Parallel tree merge: at each level, merge adjacent run pairs in parallel.
  1106→                // Level 0: merge runs (0,1), (2,3), ... → N/2 runs
  1107→                // Level 1: merge runs (01,23), (45,67), ... → N/4 runs
  1108→                // ... until one sorted run remains.
  1109→                while (run_bounds.size() > 2) {
  1110→                    std::vector<size_t> new_bounds = {0};
  1111→                    std::vector<std::thread> merge_threads;
  1112→                    for (size_t i = 0; i + 2 < run_bounds.size(); i += 2) {
  1113→                        size_t left = run_bounds[i];
  1114→                        size_t mid = run_bounds[i + 1];
  1115→                        size_t right = run_bounds[i + 2];
  1116→                        merge_threads.emplace_back([&, left, mid, right] {
  1117→                            std::inplace_merge(sorted_out.begin() + left,
  1118→                                               sorted_out.begin() + mid,
  1119→                                               sorted_out.begin() + right, cmp);
  1120→                        });
  1121→                        new_bounds.push_back(right);
  1122→                    }
  1123→                    // If odd number of runs, carry the last one forward
  1124→                    if (run_bounds.size() % 2 == 0) {
  1125→                        new_bounds.push_back(run_bounds.back());
  1126→                    }
  1127→                    for (auto& t : merge_threads) t.join();
  1128→                    run_bounds = std::move(new_bounds);
  1129→                }
  1130→
  1131→                // Skip building hash map — sorted_out is used directly for writing.
  1132→                // cell_map stays empty (only needed for cache/continent modes).
  1133→            };
  1134→
  1135→
  1136→            // Sort each thread's pairs in parallel, then k-way merge directly
  1137→            // into hash map + sorted output. No intermediate merged vector.
  1138→            auto f_ways = std::async(std::launch::async, [&] {
  1139→                parallel_sort_and_build(way_pairs, data.cell_to_ways, data.sorted_way_cells);
  1140→            });
  1141→            auto f_interps = std::async(std::launch::async, [&] {
  1142→                parallel_sort_and_build(interp_pairs, data.cell_to_interps, data.sorted_interp_cells);
  1143→            });
  1144→            f_ways.get();
  1145→            f_interps.get();
  1146→            log_phase("  S2: sort + group into cell maps", _s2t, _s2cpu);
  1147→
  1148→            // Free deferred work items
  1149→            data.deferred_ways.clear();


Now let me search for where way_pairs and addr_pairs are constructed: