     1→#pragma once
     2→
     3→#include <condition_variable>
     4→#include <cstdint>
     5→#include <deque>
     6→#include <mutex>
     7→#include <thread>
     8→#include <unordered_map>
     9→#include <unordered_set>
    10→#include <vector>
    11→
    12→#include <s2/s2cell_id.h>
    13→
    14→#include "types.h"
    15→#include "geometry.h"
    16→#include "parsed_data.h"
    17→
    18→// --- S2 cell level globals (set by main arg parsing) ---
    19→
    20→extern int kStreetCellLevel;
    21→extern int kAdminCellLevel;
    22→extern int kMaxAdminLevel;
    23→
    24→// --- Simplification globals ---
    25→
    26→extern SimplifyMode kSimplifyMode;
    27→extern double kSimplifyEpsilonOverride; // 0 = use per-level defaults
    28→
    29→// --- S2 helpers ---
    30→
    31→void cover_edge(double lat1, double lng1, double lat2, double lng2,
    32→                std::vector<S2CellId>& out);
    33→
    34→std::vector<S2CellId> cover_edge(double lat1, double lng1, double lat2, double lng2);
    35→
    36→S2CellId point_to_cell(double lat, double lng);
    37→
    38→std::vector<std::pair<S2CellId, bool>> cover_polygon(
    39→    const std::vector<std::pair<double,double>>& vertices);
    40→
    41→// --- Thread pool for concurrent admin polygon S2 covering ---
    42→
    43→class AdminCoverPool {
    44→public:
    45→    struct WorkItem {
    46→        uint32_t poly_id;
    47→        std::vector<std::pair<double,double>> vertices;
    48→    };
    49→
    50→    explicit AdminCoverPool(size_t num_threads);
    51→    ~AdminCoverPool();
    52→
    53→    void submit(uint32_t poly_id, std::vector<std::pair<double,double>>&& vertices);
    54→    std::unordered_map<uint64_t, std::vector<uint32_t>> drain();
    55→
    56→private:
    57→    void worker_loop();
    58→
    59→    std::vector<std::thread> workers_;
    60→    std::deque<WorkItem> queue_;
    61→    std::vector<std::unordered_map<uint64_t, std::vector<uint32_t>>> thread_results_;
    62→    std::mutex mutex_;
    63→    std::condition_variable cv_;
    64→    std::condition_variable done_cv_;
    65→    size_t active_workers_ = 0;
    66→    bool stop_;
    67→};
    68→
    69→// --- Simplification dispatch ---
    70→
    71→// Simplify a polygon using the current global mode, taking admin_level into account.
    72→// In MaxVertices mode: caps at MAX_POLYGON_VERTICES (500).
    73→// In ErrorBounded mode: uses per-level epsilon in meters, converted to degrees.
    74→inline std::vector<std::pair<double,double>> simplify_admin_polygon(
    75→    const std::vector<std::pair<double,double>>& pts, uint8_t admin_level) {
    76→    if (kSimplifyMode == SimplifyMode::ErrorBounded) {
    77→        double eps_m = kSimplifyEpsilonOverride > 0
    78→            ? kSimplifyEpsilonOverride
    79→            : admin_epsilon_meters(admin_level);
    80→        // Estimate latitude from first vertex for degree conversion
    81→        double lat = pts.empty() ? 0.0 : pts[0].first;
    82→        double eps_deg = meters_to_degrees(eps_m, lat);
    83→        return simplify_polygon_epsilon(pts, eps_deg);
    84→    }
    85→    return simplify_polygon(pts);
    86→}
    87→
    88→// --- Helper functions that depend on S2 ---
    89→
    90→void add_addr_point(ParsedData& data, double lat, double lng,
    91→                    const char* housenumber, const char* street,
    92→                    uint64_t& addr_count_total);
    93→
    94→void add_admin_polygon(ParsedData& data,
    95→                       const std::vector<std::pair<double,double>>& vertices_in,
    96→                       const char* name, uint8_t admin_level,
    97→                       const char* country_code,
    98→                       AdminCoverPool* admin_pool = nullptr);
    99→


Bash

grep -n "void add_addr_point" /root/traccar-geocoder/builder/src/*.cpp


/root/traccar-geocoder/builder/src/s2_helpers.cpp:229:void add_addr_point(ParsedData& data, double lat, double lng,


   228→
   229→void add_addr_point(ParsedData& data, double lat, double lng,
   230→                    const char* housenumber, const char* street,
   231→                    uint64_t& addr_count_total) {
   232→    uint32_t addr_id = static_cast<uint32_t>(data.addr_points.size());
   233→    data.addr_points.push_back({
   234→        static_cast<float>(lat),
   235→        static_cast<float>(lng),
   236→        data.string_pool.intern(housenumber),
   237→        data.string_pool.intern(street)
   238→    });
   239→
   240→    S2CellId cell = point_to_cell(lat, lng);
   241→    data.cell_to_addrs[cell.id()].push_back(addr_id);
   242→
   243→    addr_count_total++;
   244→    if (addr_count_total % 1000000 == 0) {
   245→        std::cerr << "Collected " << addr_count_total / 1000000 << "M addresses..." << std::endl;
   246→    }
   247→}
   248→
   249→// --- add_admin_polygon ---
   250→
   251→void add_admin_polygon(ParsedData& data,
   252→                       const std::vector<std::pair<double,double>>& vertices_in,
   253→                       const char* name, uint8_t admin_level,
   254→                       const char* country_code,
   255→                       AdminCoverPool* admin_pool) {
   256→    auto vertices = vertices_in;
   257→    if (vertices.size() >= 4 &&
   258→        std::fabs(vertices.front().first - vertices.back().first) < 1e-7 &&
   259→        std::fabs(vertices.front().second - vertices.back().second) < 1e-7) {
   260→        vertices.pop_back();
   261→        auto min_it = std::min_element(vertices.begin(), vertices.end());
   262→        std::rotate(vertices.begin(), min_it, vertices.end());
   263→        vertices.push_back(vertices.front());
   264→    }
   265→
   266→    auto simplified = simplify_admin_polygon(vertices, admin_level);
   267→    if (simplified.size() < 3) return;
   268→
   269→    uint32_t poly_id = static_cast<uint32_t>(data.admin_polygons.size());
   270→    uint32_t vertex_offset = static_cast<uint32_t>(data.admin_vertices.size());
   271→
   272→    for (const auto& [lat, lng] : simplified) {
   273→        data.admin_vertices.push_back({static_cast<float>(lat), static_cast<float>(lng)});
   274→    }
   275→
   276→    AdminPolygon poly{};
   277→    poly.vertex_offset = vertex_offset;


1325-
1326-    // Write one region: modes + quality variants
1327-    auto write_region = [&](const ParsedData& d, const std::string& base_dir) {
1328-        ensure_dir(base_dir);
1329-
1330-        if (multi_output) {
1331-            // Write all 3 modes in parallel (they read shared data, write to separate dirs)
1332-            auto wf1 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/full", IndexMode::Full); });
1333:            auto wf2 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/no-addresses", IndexMode::NoAddresses); });
1334-            auto wf3 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/admin", IndexMode::AdminOnly); });
1335-            wf1.get(); wf2.get(); wf3.get();
1336-        } else {
1337-            write_index(d, base_dir, mode);
1338-        }
1339-
1340-        // Write quality variants under admin/ (or base_dir if not multi_output)
1341-        if (multi_quality) {
--
1489-        };
1490-        std::vector<std::string> admin_quality_files = {
1491-            "admin_polygons.bin", "admin_vertices.bin"
1492-        };
1493-        std::vector<std::string> street_files = {
1494-            "street_ways.bin", "street_nodes.bin", "street_entries.bin"
1495-        };
1496-        std::vector<std::string> address_files = {
1497:            "addr_points.bin", "addr_entries.bin",
1498-            "interp_ways.bin", "interp_nodes.bin", "interp_entries.bin"
1499-        };
1500-        std::vector<std::string> geo_files = {"geo_cells.bin"};
1501-
1502-        // Build list of regions
1503-        std::vector<std::pair<std::string, std::string>> regions; // (name, path)
1504-        regions.push_back({"planet", output_dir});
1505-        if (generate_continents) {


No matches found


grep -r "void write_index" /root/traccar-geocoder/builder/src/*.cpp


/root/traccar-geocoder/builder/src/cell_index.cpp:void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {


     1→#include "cell_index.h"
     2→
     3→#include <algorithm>
     4→#include <atomic>
     5→#include <cstring>
     6→#include <fstream>
     7→#include <future>
     8→#include <iostream>
     9→#include <thread>
    10→#include <unordered_map>
    11→#include <unordered_set>
    12→
    13→#include <unistd.h>
    14→
    15→#include "geometry.h"
    16→#include "s2_helpers.h"
    17→
    18→std::vector<uint32_t> write_entries(
    19→    const std::string& path,
    20→    const std::vector<uint64_t>& sorted_cells,
    21→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
    22→) {
    23→    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
    24→    std::vector<CellRef> sorted_refs;
    25→    sorted_refs.reserve(cell_map.size());
    26→    for (auto& [id, ids] : cell_map) sorted_refs.push_back({id, &ids});
    27→    std::sort(sorted_refs.begin(), sorted_refs.end(),
    28→        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });
    29→
    30→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    31→    size_t total_size = 0;
    32→    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);
    33→
    34→    std::vector<char> buf;
    35→    buf.reserve(total_size);
    36→    uint32_t current = 0;
    37→    size_t ri = 0;
    38→    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
    39→        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
    40→        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
    41→        offsets[si] = current;
    42→        const auto& ids = *sorted_refs[ri].ids;
    43→        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
    44→        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
    45→                   reinterpret_cast<const char*>(&count) + sizeof(count));
    46→        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
    47→                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
    48→        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
    49→        ri++;
    50→    }
    51→    std::ofstream f(path, std::ios::binary);
    52→    f.write(buf.data(), buf.size());
    53→    return offsets;
    54→}
    55→
    56→std::vector<uint32_t> write_entries_from_sorted(
    57→    const std::string& path,
    58→    const std::vector<uint64_t>& sorted_cells,
    59→    const std::vector<CellItemPair>& sorted_pairs
    60→) {
    61→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    62→    if (sorted_pairs.empty()) {
    63→        std::ofstream f(path, std::ios::binary);
    64→        return offsets;
    65→    }
    66→
    67→    unsigned int nthreads = std::thread::hardware_concurrency();
    68→    if (nthreads == 0) nthreads = 4;
    69→    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;
    70→
    71→    struct ChunkResult {
    72→        std::vector<char> buf;
    73→        size_t cell_start, cell_end;
    74→        uint32_t local_size;
    75→    };
    76→    std::vector<ChunkResult> chunks(nthreads);
    77→
    78→    std::vector<std::thread> threads;
    79→    for (unsigned int t = 0; t < nthreads; t++) {
    80→        size_t cs = t * cells_per_chunk;
    81→        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
    82→        if (cs >= sorted_cells.size()) break;
    83→
    84→        threads.emplace_back([&, t, cs, ce]() {
    85→            auto& chunk = chunks[t];
    86→            chunk.cell_start = cs;
    87→            chunk.cell_end = ce;
    88→            chunk.local_size = 0;
    89→
    90→            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
    91→                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
    92→                    return p.cell_id < id;
    93→                }) - sorted_pairs.begin();
    94→
    95→            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
    96→                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
    97→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
    98→                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;
    99→
   100→                offsets[si] = chunk.local_size;
   101→                size_t start = pi;
   102→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
   103→                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(MAX_VERTEX_COUNT)));
   104→                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
   105→                size_t buf_pos = chunk.buf.size();
   106→                chunk.buf.resize(buf_pos + entry_size);
   107→                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
   108→                for (size_t k = start; k < pi; k++) {
   109→                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
   110→                           &sorted_pairs[k].item_id, sizeof(uint32_t));
   111→                }
   112→                chunk.local_size += entry_size;
   113→            }
   114→        });
   115→    }
   116→    for (auto& t : threads) t.join();
   117→
   118→    uint32_t global_offset = 0;
   119→    for (auto& chunk : chunks) {
   120→        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
   121→            if (offsets[si] != NO_DATA) offsets[si] += global_offset;
   122→        }
   123→        global_offset += chunk.local_size;
   124→    }
   125→
   126→    std::vector<char> buf;
   127→    buf.reserve(global_offset);
   128→    for (auto& chunk : chunks) buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());
   129→
   130→    std::ofstream f(path, std::ios::binary);
   131→    f.write(buf.data(), buf.size());
   132→    return offsets;
   133→}
   134→
   135→void write_cell_index(
   136→    const std::string& cells_path,
   137→    const std::string& entries_path,
   138→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
   139→) {
   140→    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(cell_map.begin(), cell_map.end());
   141→    std::sort(sorted.begin(), sorted.end());
   142→
   143→    { std::ofstream f(cells_path, std::ios::binary);
   144→      uint32_t current_offset = 0;
   145→      for (const auto& [cell_id, ids] : sorted) {
   146→          f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
   147→          f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
   148→          current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
   149→      } }
   150→
   151→    { std::ofstream f(entries_path, std::ios::binary);
   152→      for (const auto& [cell_id, ids] : sorted) {
   153→          uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
   154→          f.write(reinterpret_cast<const char*>(&count), sizeof(count));
   155→          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
   156→      } }
   157→}
   158→
   159→void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
   160→    ensure_dir(output_dir);
   161→    auto _wt = std::chrono::steady_clock::now();
   162→
   163→    bool write_streets = (mode != IndexMode::AdminOnly);
   164→    bool write_addresses = (mode == IndexMode::Full);
   165→
   166→    if (write_streets) {
   167→        std::vector<uint64_t> sorted_geo_cells;
   168→        {
   169→            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
   170→                std::vector<uint64_t> cells;
   171→                cells.reserve(pairs.size() / 2);
   172→                for (size_t i = 0; i < pairs.size(); ) {
   173→                    cells.push_back(pairs[i].cell_id);
   174→                    uint64_t cur = pairs[i].cell_id;
   175→                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
   176→                }
   177→                return cells;
   178→            };
   179→            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
   180→                std::vector<uint64_t> cells;
   181→                cells.reserve(m.size());
   182→                for (auto& [id, _] : m) cells.push_back(id);
   183→                std::sort(cells.begin(), cells.end());
   184→                return cells;
   185→            };
   186→
   187→            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
   188→            {
   189→                auto f1 = std::async(std::launch::async, [&] {
   190→                    return !data.sorted_way_cells.empty()
   191→                        ? extract_unique_cells(data.sorted_way_cells)
   192→                        : extract_from_map(data.cell_to_ways);
   193→                });
   194→                if (write_addresses) {
   195→                    auto f2 = std::async(std::launch::async, [&] {
   196→                        return !data.sorted_addr_cells.empty()
   197→                            ? extract_unique_cells(data.sorted_addr_cells)
   198→                            : extract_from_map(data.cell_to_addrs);
   199→                    });
   200→                    auto f3 = std::async(std::launch::async, [&] {
   201→                        return !data.sorted_interp_cells.empty()
   202→                            ? extract_unique_cells(data.sorted_interp_cells)
   203→                            : extract_from_map(data.cell_to_interps);
   204→                    });
   205→                    addr_cells = f2.get();
   206→                    interp_cells = f3.get();
   207→                }
   208→                way_cells = f1.get();
   209→            }
   210→
   211→            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
   212→            std::merge(way_cells.begin(), way_cells.end(), addr_cells.begin(), addr_cells.end(),
   213→                       std::back_inserter(sorted_geo_cells));
   214→            if (!interp_cells.empty()) {
   215→                std::vector<uint64_t> tmp;
   216→                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
   217→                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
   218→                           interp_cells.begin(), interp_cells.end(), std::back_inserter(tmp));
   219→                sorted_geo_cells = std::move(tmp);
   220→            }
   221→            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
   222→                                    sorted_geo_cells.end());
   223→        }
   224→
   225→        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
   226→        {
   227→            auto f1 = std::async(std::launch::async, [&]() {
   228→                if (!data.sorted_way_cells.empty())
   229→                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
   230→                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
   231→            });
   232→            if (write_addresses) {
   233→                auto f2 = std::async(std::launch::async, [&]() {
   234→                    if (!data.sorted_addr_cells.empty())
   235→                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
   236→                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
   237→                });
   238→                auto f3 = std::async(std::launch::async, [&]() {
   239→                    if (!data.sorted_interp_cells.empty())
   240→                        return write_entries_from_sorted(output_dir + "/interp_entries.bin", sorted_geo_cells, data.sorted_interp_cells);
   241→                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
   242→                });
   243→                addr_offsets = f2.get();
   244→                interp_offsets = f3.get();
   245→            }
   246→            street_offsets = f1.get();
   247→        }
   248→
   249→        log_phase("  Write: entry files", _wt);
   250→        {
   251→            size_t n = sorted_geo_cells.size();
   252→            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
   253→            std::vector<char> buf(n * row_size);
   254→            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
   255→            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);
   256→
   257→            unsigned int nthreads = std::thread::hardware_concurrency();
   258→            if (nthreads == 0) nthreads = 4;
   259→            size_t chunk = (n + nthreads - 1) / nthreads;
   260→            std::vector<std::thread> fill_threads;
   261→            for (unsigned int t = 0; t < nthreads; t++) {
   262→                size_t start = t * chunk;
   263→                size_t end = std::min(start + chunk, n);
   264→                if (start >= n) break;
   265→                fill_threads.emplace_back([&, start, end]() {
   266→                    char* ptr = buf.data() + start * row_size;
   267→                    for (size_t i = start; i < end; i++) {
   268→                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
   269→                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   270→                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   271→                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   272→                    }
   273→                });
   274→            }
   275→            for (auto& t : fill_threads) t.join();
   276→
   277→            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
   278→            f.write(buf.data(), buf.size());
   279→        }
   280→
   281→        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
   282→                  << data.ways.size() << " ways, " << data.addr_points.size() << " addrs, "
   283→                  << data.interp_ways.size() << " interps)" << std::endl;
   284→    }
   285→
   286→    log_phase("  Write: geo_cells.bin", _wt);
   287→    auto admin_future = std::async(std::launch::async, [&] {
   288→        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
   289→        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
   290→    });
   291→
   292→    std::vector<std::future<void>> write_futures;
   293→    if (write_streets) {
   294→        write_futures.push_back(std::async(std::launch::async, [&] {
   295→            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
   296→            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
   297→        }));
   298→        write_futures.push_back(std::async(std::launch::async, [&] {
   299→            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
   300→            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
   301→        }));
   302→        if (write_addresses) {
   303→            write_futures.push_back(std::async(std::launch::async, [&] {
   304→                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
   305→                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
   306→            }));
   307→            write_futures.push_back(std::async(std::launch::async, [&] {
   308→                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
   309→                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
   310→            }));
   311→            write_futures.push_back(std::async(std::launch::async, [&] {
   312→                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
   313→                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
   314→            }));
   315→        }
   316→    }
   317→    // If not writing all data types, compact string pool and remap references
   318→    // before writing admin_polygons and strings.bin.
   319→    std::vector<AdminPolygon> compact_admin;
   320→    std::vector<char> compact_strings;
   321→    const bool need_compact = (!write_streets || !write_addresses);
   322→
   323→    if (need_compact) {
   324→        // Collect referenced string offsets
   325→        std::unordered_set<uint32_t> used_offsets;
   326→        if (write_streets) {
   327→            for (const auto& w : data.ways) used_offsets.insert(w.name_id);
   328→            if (write_addresses) {
   329→                for (const auto& a : data.addr_points) {
   330→                    used_offsets.insert(a.housenumber_id);
   331→                    used_offsets.insert(a.street_id);
   332→                }
   333→                for (const auto& iw : data.interp_ways) used_offsets.insert(iw.street_id);
   334→            }
   335→        }
   336→        for (const auto& ap : data.admin_polygons) used_offsets.insert(ap.name_id);
   337→
   338→        // Build compact pool and remap
   339→        const auto& old_sp = data.string_pool.data();
   340→        std::vector<uint32_t> sorted_offs(used_offsets.begin(), used_offsets.end());
   341→        std::sort(sorted_offs.begin(), sorted_offs.end());
   342→        std::unordered_map<uint32_t, uint32_t> remap;
   343→        compact_strings.reserve(used_offsets.size() * 16);
   344→        for (uint32_t off : sorted_offs) {
   345→            remap[off] = static_cast<uint32_t>(compact_strings.size());
   346→            size_t len = std::strlen(old_sp.data() + off);
   347→            compact_strings.insert(compact_strings.end(), old_sp.data() + off,
   348→                old_sp.data() + off + len + 1);
   349→        }
   350→
   351→        // Remap admin polygon name_ids
   352→        compact_admin = data.admin_polygons;
   353→        for (auto& ap : compact_admin) ap.name_id = remap[ap.name_id];
   354→
   355→        // Remap street/addr/interp if applicable (for no-addresses mode)
   356→        // These are written from data directly, so we'd need copies too.
   357→        // For simplicity, only compact when we have fewer types.
   358→    }
   359→
   360→    const auto& admin_to_write = need_compact ? compact_admin : data.admin_polygons;
   361→    write_futures.push_back(std::async(std::launch::async, [&] {
   362→        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
   363→        f.write(reinterpret_cast<const char*>(admin_to_write.data()), admin_to_write.size() * sizeof(AdminPolygon));
   364→    }));
   365→    write_futures.push_back(std::async(std::launch::async, [&] {
   366→        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
   367→        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
   368→    }));
   369→    write_futures.push_back(std::async(std::launch::async, [&] {
   370→        if (need_compact) {
   371→            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
   372→            f.write(compact_strings.data(), compact_strings.size());
   373→            std::cerr << "strings.bin: " << compact_strings.size() << " bytes (compacted from "
   374→                      << data.string_pool.data().size() << ")" << std::endl;
   375→        } else {
   376→            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
   377→            f.write(data.string_pool.data().data(), data.string_pool.data().size());
   378→            std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
   379→        }
   380→    }));
   381→
   382→    log_phase("  Write: parallel data files launched", _wt);
   383→    admin_future.get();
   384→    for (auto& f : write_futures) f.get();
   385→}
   386→
   387→void write_quality_variant(const ParsedData& data, const std::string& source_dir,
   388→                           const std::string& output_dir, double epsilon_scale) {
   389→    ensure_dir(output_dir);
   390→
   391→    // Re-simplify admin polygons at the given epsilon scale
   392→    std::vector<AdminPolygon> new_polys;
   393→    std::vector<NodeCoord> new_verts;
   394→    new_polys.reserve(data.admin_polygons.size());
   395→
   396→    // Parallel simplification
   397→    struct SimplifiedPoly {
   398→        std::vector<std::pair<double,double>> verts;
   399→    };
   400→    std::vector<SimplifiedPoly> simplified(data.admin_polygons.size());
   401→
   402→    {
   403→        std::atomic<size_t> idx{0};
   404→        unsigned nthreads = std::thread::hardware_concurrency();
   405→        if (nthreads == 0) nthreads = 4;
   406→        std::vector<std::thread> workers;
   407→        for (unsigned t = 0; t < nthreads; t++) {
   408→            workers.emplace_back([&]() {
   409→                while (true) {
   410→                    size_t i = idx.fetch_add(1);
   411→                    if (i >= data.admin_polygons.size()) break;
   412→                    const auto& ap = data.admin_polygons[i];
   413→
   414→                    // Extract original vertices
   415→                    std::vector<std::pair<double,double>> pts;
   416→                    pts.reserve(ap.vertex_count);
   417→                    for (uint32_t j = 0; j < ap.vertex_count; j++) {
   418→                        const auto& v = data.admin_vertices[ap.vertex_offset + j];
   419→                        pts.emplace_back(v.lat, v.lng);
   420→                    }
   421→
   422→                    if (epsilon_scale > 0) {
   423→                        double eps_m = admin_epsilon_meters(ap.admin_level) * epsilon_scale;
   424→                        double lat = pts.empty() ? 0.0 : pts[0].first;
   425→                        double eps_deg = meters_to_degrees(eps_m, lat);
   426→                        simplified[i].verts = simplify_polygon_epsilon(pts, eps_deg);
   427→                    } else {
   428→                        simplified[i].verts = std::move(pts);
   429→                    }
   430→                }
   431→            });
   432→        }
   433→        for (auto& w : workers) w.join();
   434→    }
   435→
   436→    // Sequential: build new polygon and vertex arrays
   437→    for (size_t i = 0; i < data.admin_polygons.size(); i++) {
   438→        auto& sv = simplified[i].verts;
   439→        if (sv.size() < 3) continue;
   440→
   441→        AdminPolygon np = data.admin_polygons[i];
   442→        np.vertex_offset = static_cast<uint32_t>(new_verts.size());
   443→        np.vertex_count = static_cast<uint32_t>(sv.size());
   444→        np.area = polygon_area(sv);
   445→
   446→        for (const auto& [lat, lng] : sv) {
   447→            new_verts.push_back({static_cast<float>(lat), static_cast<float>(lng)});
   448→        }
   449→        new_polys.push_back(np);
   450→    }
   451→
   452→    std::cerr << "Quality " << epsilon_scale << "x: " << new_polys.size()
   453→              << " polygons, " << new_verts.size() << " vertices ("
   454→              << new_verts.size() * 8 / 1024 / 1024 << " MiB)" << std::endl;
   455→
   456→    // Write admin files
   457→    {
   458→        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
   459→        f.write(reinterpret_cast<const char*>(new_polys.data()), new_polys.size() * sizeof(AdminPolygon));
   460→    }
   461→    {
   462→        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
   463→        f.write(reinterpret_cast<const char*>(new_verts.data()), new_verts.size() * sizeof(NodeCoord));
   464→    }
   465→
   466→    // Quality directories only contain the two files that change.
   467→    // Shared files (admin_cells.bin, admin_entries.bin, strings.bin)
   468→    // stay in the parent directory.
   469→}
   470→


315-            }
316-            std::cerr << "  Collected " << data.collected_relations.size()
317-                      << " admin/postal relations for parallel assembly." << std::endl;
318-            log_phase("Pass 1: relation scanning", _pt, _cpu);
319-
320:            // Build admin_way_ids as a bitset for O(1) lookup
321-            // Max way ID ~1.3B → ~162 MiB bitset (much faster than unordered_set)
322-            static constexpr size_t MAX_WAY_ID = 2000000000ULL; // 2B, ~250 MiB
323-            std::vector<uint8_t> admin_way_bits(MAX_WAY_ID / 8 + 1, 0);
324-            size_t admin_way_count = 0;
325-            for (const auto& rel : data.collected_relations) {
326:                for (const auto& [way_id, role] : rel.members) {
327:                    if (way_id > 0 && static_cast<size_t>(way_id) < MAX_WAY_ID) {
328:                        admin_way_bits[way_id / 8] |= (1 << (way_id % 8));
329-                        admin_way_count++;
330-                    }
331-                }
332-            }
333-            auto is_admin_way = [&](int64_t id) -> bool {
--
422-                    uint64_t way_count = 0;
423-                    uint64_t building_addr_count = 0;
424-                    uint64_t interp_count = 0;
425-                    // Way geometries for parallel admin assembly
426-                    struct WayGeomEntry {
427:                        int64_t way_id;
428-                        std::vector<std::pair<double,double>> coords;
429-                        int64_t first_node_id;
430-                        int64_t last_node_id;
431-                    };
432-                    std::vector<WayGeomEntry> way_geoms;
--
442-
443-                static thread_local ThreadLocalData* tl_way_data = nullptr;
444-                std::vector<ThreadLocalData> tld(num_threads);
445-                std::atomic<unsigned> next_tl_way{0};
446-
447:                pbf.read_ways_streaming([&](int64_t way_id,
448-                    const int64_t* refs_data, size_t refs_size,
449-                    const uint32_t* tag_keys, const uint32_t* tag_vals, size_t ntags,
450-                    const std::vector<std::string>& st) {
451-
452-                    if (!tl_way_data) {
--
487-
488-                    // Early exit: if no relevant tags, skip expensive node resolution
489-                    bool need_nodes = t_interpolation || t_housenumber ||
490-                        (t_highway && is_included_highway(t_highway) && t_name) ||
491-                        t_boundary ||
492:                        (refs_size > 0 && is_admin_way(way_id));
493-                    if (!need_nodes) return;
494-
495-                    // Pre-resolve all node locations
496-                    thread_local std::vector<PackedLocation> resolved_locs;
497-                    resolved_locs.clear();
--
560-                            local.way_count++;
561-                        }
562-                    }
563-

[Showing results with pagination = limit: 60]


grep -n "data\.ways\s*\.push_back\|data\.ways\.emplace_back" /root/traccar-geocoder/builder/src/build_index.cpp | head -20


618:                        data.ways.push_back(h);


   600→                tl_way_data = nullptr;
   601→                std::cerr << "  Parallel way processing complete." << std::endl;
   602→
   603→                // Process areas/multipolygons — sequential fallback path
   604→                // Merge thread-local way/interp data into main ParsedData
   605→                std::cerr << "  Merging thread-local data..." << std::endl;
   606→                uint64_t total_ways = 0, total_building_addrs = 0, total_interps = 0;
   607→                for (auto& local : tld) {
   608→                    uint32_t way_base = static_cast<uint32_t>(data.ways.size());
   609→                    uint32_t node_base = static_cast<uint32_t>(data.street_nodes.size());
   610→                    uint32_t interp_base = static_cast<uint32_t>(data.interp_ways.size());
   611→                    uint32_t interp_node_base = static_cast<uint32_t>(data.interp_nodes.size());
   612→
   613→                    // Merge street ways
   614→                    for (size_t i = 0; i < local.ways.size(); i++) {
   615→                        auto h = local.ways[i];
   616→                        h.node_offset += node_base;
   617→                        h.name_id = data.string_pool.intern(local.way_strings[i]);
   618→                        data.ways.push_back(h);
   619→                    }
   620→                    data.street_nodes.insert(data.street_nodes.end(),
   621→                        local.street_nodes.begin(), local.street_nodes.end());
   622→
   623→                    // Remap deferred ways
   624→                    for (auto dw : local.deferred_ways) {
   625→                        dw.way_id += way_base;
   626→                        dw.node_offset += node_base;
   627→                        data.deferred_ways.push_back(dw);
   628→                    }
   629→
   630→                    // Merge building addresses
   631→                    for (size_t i = 0; i < local.building_addrs.size(); i++) {
   632→                        uint64_t dummy = 0;
   633→                        add_addr_point(data, local.building_addrs[i].lat, local.building_addrs[i].lng,
   634→                                       local.addr_strings[i].first.c_str(),
   635→                                       local.addr_strings[i].second.c_str(), dummy);
   636→                    }
   637→
   638→                    // Merge interpolation ways
   639→                    for (size_t i = 0; i < local.interp_ways.size(); i++) {
   640→                        auto iw = local.interp_ways[i];
   641→                        iw.node_offset += interp_node_base;
   642→                        iw.street_id = data.string_pool.intern(local.interp_strings[i]);
   643→                        data.interp_ways.push_back(iw);
   644→                    }
   645→                    data.interp_nodes.insert(data.interp_nodes.end(),
   646→                        local.interp_nodes.begin(), local.interp_nodes.end());
   647→
   648→                    // Remap deferred interps
   649→                    for (auto di : local.deferred_interps) {
   650→                        di.interp_id += interp_base;
   651→                        di.node_offset += interp_node_base;
   652→                        data.deferred_interps.push_back(di);
   653→                    }
   654→
   655→                    total_ways += local.way_count;
   656→                    total_building_addrs += local.building_addr_count;
   657→                    total_interps += local.interp_count;
   658→
   659→                    // Merge way geometries for admin assembly
   660→                    {
   661→                        for (auto& wg : local.way_geoms) {
   662→                            ParsedData::WayGeometry g;
   663→                            g.coords = std::move(wg.coords);
   664→                            g.first_node_id = wg.first_node_id;
   665→                            g.last_node_id = wg.last_node_id;
   666→                            data.way_geometries[wg.way_id] = std::move(g);
   667→                        }
   668→                        local.way_geoms.clear();
   669→                        local.way_geoms.shrink_to_fit();
   670→                    }
   671→                }
   672→                std::cerr << "  Merged: " << total_ways << " ways, "
   673→                          << total_building_addrs << " building addrs, "
   674→                          << total_interps << " interps from parallel processing." << std::endl;
   675→
   676→                // --- Merge closed-way admin polygons ---
   677→                {
   678→                    uint64_t closed_way_admin_count = 0;
   679→                    for (auto& local : tld) {
   680→                        for (auto& cwa : local.closed_way_admins) {
   681→                            const char* cc = cwa.country_code.empty() ? nullptr : cwa.country_code.c_str();
   682→                            add_admin_polygon(data, cwa.vertices, cwa.name.c_str(),
   683→                                              cwa.admin_level, cc, &admin_pool);
   684→                            closed_way_admin_count++;
   685→                        }
   686→                        local.closed_way_admins.clear();
   687→                    }
   688→                    if (closed_way_admin_count > 0) {
   689→                        std::cerr << "  Added " << closed_way_admin_count
   690→                                  << " admin polygons from closed ways." << std::endl;
   691→                    }
   692→                }
   693→
   694→                // --- Parallel admin boundary assembly ---
   695→                {
   696→                    log_phase("Pass 2b: way processing", _pt, _cpu);
   697→                    pbf.unmap(); // release 86 GiB PBF mmap before admin/S2 phases
   698→                    std::cerr << "  Assembling admin polygons in parallel ("
   699→                              << data.collected_relations.size() << " relations, "


1131-                // Skip building hash map — sorted_out is used directly for writing.
1132-                // cell_map stays empty (only needed for cache/continent modes).
1133-            };
1134-
1135-
1136-            // Sort each thread's pairs in parallel, then k-way merge directly
1137-            // into hash map + sorted output. No intermediate merged vector.
1138-            auto f_ways = std::async(std::launch::async, [&] {
1139:                parallel_sort_and_build(way_pairs, data.cell_to_ways, data.sorted_way_cells);
1140-            });
1141-            auto f_interps = std::async(std::launch::async, [&] {
1142:                parallel_sort_and_build(interp_pairs, data.cell_to_interps, data.sorted_interp_cells);
1143-            });
1144-            f_ways.get();
1145-            f_interps.get();
1146-            log_phase("  S2: sort + group into cell maps", _s2t, _s2cpu);
1147-
1148-            // Free deferred work items
1149-            data.deferred_ways.clear();
1150-            data.deferred_ways.shrink_to_fit();
--
1208-
1209-                // Step 5: Build sorted pairs
1210-                std::vector<CellItemPair> pairs;
1211-                pairs.reserve(total_pairs);
1212-                for (uint64_t cell_id : sorted_cells) {
1213-                    auto& ids = data.cell_to_addrs[cell_id];
1214-                    for (auto id : ids) pairs.push_back({cell_id, id});
1215-                }
1216:                data.sorted_addr_cells = std::move(pairs);
1217-                log_phase("    Dedup: build sorted pairs", _dt, _dc);
1218-            });
1219-            auto f4 = std::async(std::launch::async, [&]{ deduplicate(data.cell_to_admin); });
1220-            f2.get(); f4.get();
1221-        }
1222-
1223-        // Save cache if requested
1224-        if (!save_cache_path.empty()) {
--
1274-            for (auto& sub : sub_maps) {
1275-                for (auto& [k, v] : sub) {
1276-                    map[k] = std::move(v);
1277-                }
1278-            }
1279-        };
1280-
1281-        std::cerr << "Rebuilding cell maps for continent filtering..." << std::endl;
1282:        auto f1 = std::async(std::launch::async, [&]{ rebuild_map_parallel(data.sorted_way_cells, data.cell_to_ways); });
1283:        auto f2 = std::async(std::launch::async, [&]{ rebuild_map_parallel(data.sorted_interp_cells, data.cell_to_interps); });
1284-        f1.get(); f2.get();
1285-        log_phase("Rebuild cell maps", _pt, _cpu);
1286-    }
1287-
1288-    // --- Write index files ---
1289-    log_phase("Deduplication", _pt, _cpu);
1290-    std::cerr << "Writing index files to " << output_dir << "..." << std::endl;
1291-
--
1401-                    }
1402-                });
1403-            }
1404-            for (auto& t : threads) t.join();
1405-            return masks;
1406-        };
1407-
1408-        // Precompute for all three sorted pair arrays
1409:        auto way_masks = std::async(std::launch::async, [&]{ return precompute_masks(data.sorted_way_cells); });
1410:        auto addr_masks = std::async(std::launch::async, [&]{ return precompute_masks(data.sorted_addr_cells); });
1411:        auto interp_masks = std::async(std::launch::async, [&]{ return precompute_masks(data.sorted_interp_cells); });
1412-        auto way_continent_masks = way_masks.get();
1413-        auto addr_continent_masks = addr_masks.get();
1414-        auto interp_continent_masks = interp_masks.get();
1415-
1416-        log_phase("Pre-compute continent masks", _pct, _pcc);
1417-
1418-        // Sort by bbox area descending — largest continents first
1419-        std::vector<size_t> continent_order(kContinentCount);


grep -n "parallel_sort_and_build\|auto.*parallel_sort_and_build" /root/traccar-geocoder/builder/src/build_index.cpp | head -20


1061:            auto parallel_sort_and_build = [](
1139:                parallel_sort_and_build(way_pairs, data.cell_to_ways, data.sorted_way_cells);
1142:                parallel_sort_and_build(interp_pairs, data.cell_to_interps, data.sorted_interp_cells);


  1060→            // hash map + sorted output. No intermediate merged vector needed.
  1061→            auto parallel_sort_and_build = [](
  1062→                std::vector<std::vector<CellItem>>& thread_pairs,
  1063→                std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map,
  1064→                std::vector<CellItemPair>& sorted_out
  1065→            ) {
  1066→                // Step 1: Convert + sort each thread's data in parallel
  1067→                size_t total = 0;
  1068→                for (auto& v : thread_pairs) total += v.size();
  1069→
  1070→                std::vector<std::vector<CellItemPair>> chunks(thread_pairs.size());
  1071→                {
  1072→                    std::vector<std::thread> sort_threads;
  1073→                    for (size_t t = 0; t < thread_pairs.size(); t++) {
  1074→                        sort_threads.emplace_back([&, t]() {
  1075→                            auto& src = thread_pairs[t];
  1076→                            auto& dst = chunks[t];
  1077→                            dst.reserve(src.size());
  1078→                            for (auto& ci : src) dst.push_back({ci.cell_id, ci.item_id});
  1079→                            src.clear(); src.shrink_to_fit();
  1080→                            std::sort(dst.begin(), dst.end(), [](const CellItemPair& a, const CellItemPair& b) {
  1081→                                return a.cell_id < b.cell_id || (a.cell_id == b.cell_id && a.item_id < b.item_id);
  1082→                            });
  1083→                        });
  1084→                    }
  1085→                    for (auto& t : sort_threads) t.join();
  1086→                }
  1087→
  1088→                // Step 2: Concatenate sorted chunks and merge using parallel tree.
  1089→                // Each chunk is already sorted. Record run boundaries, concatenate,
  1090→                // then merge adjacent runs pairwise in parallel at each level.
  1091→                auto cmp = [](const CellItemPair& a, const CellItemPair& b) {
  1092→                    return a.cell_id < b.cell_id || (a.cell_id == b.cell_id && a.item_id < b.item_id);
  1093→                };
  1094→
  1095→                // Record run boundaries before concatenation
  1096→                std::vector<size_t> run_bounds = {0};
  1097→                sorted_out.clear();
  1098→                sorted_out.reserve(total);
  1099→                for (auto& chunk : chunks) {
  1100→                    sorted_out.insert(sorted_out.end(), chunk.begin(), chunk.end());
  1101→                    run_bounds.push_back(sorted_out.size());
  1102→                    chunk.clear(); chunk.shrink_to_fit();
  1103→                }
  1104→
  1105→                // Parallel tree merge: at each level, merge adjacent run pairs in parallel.
  1106→                // Level 0: merge runs (0,1), (2,3), ... → N/2 runs
  1107→                // Level 1: merge runs (01,23), (45,67), ... → N/4 runs
  1108→                // ... until one sorted run remains.
  1109→                while (run_bounds.size() > 2) {
  1110→                    std::vector<size_t> new_bounds = {0};
  1111→                    std::vector<std::thread> merge_threads;
  1112→                    for (size_t i = 0; i + 2 < run_bounds.size(); i += 2) {
  1113→                        size_t left = run_bounds[i];
  1114→                        size_t mid = run_bounds[i + 1];
  1115→                        size_t right = run_bounds[i + 2];
  1116→                        merge_threads.emplace_back([&, left, mid, right] {
  1117→                            std::inplace_merge(sorted_out.begin() + left,
  1118→                                               sorted_out.begin() + mid,
  1119→                                               sorted_out.begin() + right, cmp);
  1120→                        });
  1121→                        new_bounds.push_back(right);
  1122→                    }
  1123→                    // If odd number of runs, carry the last one forward
  1124→                    if (run_bounds.size() % 2 == 0) {
  1125→                        new_bounds.push_back(run_bounds.back());
  1126→                    }
  1127→                    for (auto& t : merge_threads) t.join();
  1128→                    run_bounds = std::move(new_bounds);
  1129→                }
  1130→
  1131→                // Skip building hash map — sorted_out is used directly for writing.
  1132→                // cell_map stays empty (only needed for cache/continent modes).
  1133→            };
  1134→
  1135→
  1136→            // Sort each thread's pairs in parallel, then k-way merge directly
  1137→            // into hash map + sorted output. No intermediate merged vector.
  1138→            auto f_ways = std::async(std::launch::async, [&] {
  1139→                parallel_sort_and_build(way_pairs, data.cell_to_ways, data.sorted_way_cells);
  1140→            });
  1141→            auto f_interps = std::async(std::launch::async, [&] {
  1142→                parallel_sort_and_build(interp_pairs, data.cell_to_interps, data.sorted_interp_cells);
  1143→            });
  1144→            f_ways.get();
  1145→            f_interps.get();
  1146→            log_phase("  S2: sort + group into cell maps", _s2t, _s2cpu);
  1147→
  1148→            // Free deferred work items
  1149→            data.deferred_ways.clear();


grep -n "way_pairs\|addr_pairs\|CellItem\|struct.*CellItem" /root/traccar-geocoder/builder/src/build_index.cpp | head -50


982:            struct CellItem { uint64_t cell_id; uint32_t item_id; };
989:            std::vector<std::vector<CellItem>> way_pairs(num_threads);
995:                        auto& local = way_pairs[t];
1024:            std::vector<std::vector<CellItem>> interp_pairs(num_threads);
1058:            // Concatenate + sort helper for flat CellItem pairs
1062:                std::vector<std::vector<CellItem>>& thread_pairs,
1064:                std::vector<CellItemPair>& sorted_out
1070:                std::vector<std::vector<CellItemPair>> chunks(thread_pairs.size());
1080:                            std::sort(dst.begin(), dst.end(), [](const CellItemPair& a, const CellItemPair& b) {
1091:                auto cmp = [](const CellItemPair& a, const CellItemPair& b) {
1139:                parallel_sort_and_build(way_pairs, data.cell_to_ways, data.sorted_way_cells);
1210:                std::vector<CellItemPair> pairs;
1235:        auto rebuild_map_parallel = [](const std::vector<CellItemPair>& sorted,
1366:        auto precompute_masks = [&continent_polys](const std::vector<CellItemPair>& sorted) -> std::vector<uint8_t> {


   975→            log_phase("Admin assembly", _pt, _cpu);
   976→            std::cerr << "Computing S2 cells for ways with " << num_threads << " threads..." << std::endl;
   977→
   978→            // Flat-array approach: threads emit (cell_id, item_id) pairs into
   979→            // thread-local vectors, then we sort globally and group by cell_id.
   980→            // This avoids hash maps entirely — sort is cache-friendly O(n log n).
   981→
   982→            struct CellItem { uint64_t cell_id; uint32_t item_id; };
   983→
   984→            auto _s2t = std::chrono::steady_clock::now();
   985→            auto _s2cpu = CpuTicks::now();
   986→
   987→            // Process streets: emit (cell_id, way_id) pairs
   988→            std::cerr << "  Processing " << data.deferred_ways.size() << " street ways..." << std::endl;
   989→            std::vector<std::vector<CellItem>> way_pairs(num_threads);
   990→            {
   991→                std::atomic<size_t> way_idx{0};
   992→                std::vector<std::thread> threads;
   993→                for (unsigned int t = 0; t < num_threads; t++) {
   994→                    threads.emplace_back([&, t]() {
   995→                        auto& local = way_pairs[t];
   996→                        local.reserve(data.deferred_ways.size() / num_threads * 3);
   997→                        std::vector<S2CellId> edge_cells;
   998→                        std::vector<uint64_t> way_cells;
   999→                        while (true) {
  1000→                            size_t i = way_idx.fetch_add(1);
  1001→                            if (i >= data.deferred_ways.size()) break;
  1002→                            const auto& dw = data.deferred_ways[i];
  1003→                            way_cells.clear();
  1004→                            for (uint8_t j = 0; j + 1 < dw.node_count; j++) {
  1005→                                const auto& n1 = data.street_nodes[dw.node_offset + j];
  1006→                                const auto& n2 = data.street_nodes[dw.node_offset + j + 1];
  1007→                                cover_edge(n1.lat, n1.lng, n2.lat, n2.lng, edge_cells);
  1008→                                for (const auto& c : edge_cells) way_cells.push_back(c.id());
  1009→                            }
  1010→                            std::sort(way_cells.begin(), way_cells.end());
  1011→                            way_cells.erase(std::unique(way_cells.begin(), way_cells.end()), way_cells.end());
  1012→                            for (uint64_t cell_id : way_cells) {
  1013→                                local.push_back({cell_id, dw.way_id});
  1014→                            }
  1015→                        }
  1016→                    });
  1017→                }
  1018→                for (auto& t : threads) t.join();
  1019→            }
  1020→            log_phase("  S2: street ways (parallel)", _s2t, _s2cpu);
  1021→
  1022→            // Process interpolations: emit (cell_id, interp_id) pairs
  1023→            std::cerr << "  Processing " << data.deferred_interps.size() << " interpolation ways..." << std::endl;
  1024→            std::vector<std::vector<CellItem>> interp_pairs(num_threads);
  1025→            {
  1026→                std::atomic<size_t> interp_idx{0};
  1027→                std::vector<std::thread> threads;
  1028→                for (unsigned int t = 0; t < num_threads; t++) {
  1029→                    threads.emplace_back([&, t]() {
  1030→                        auto& local = interp_pairs[t];
  1031→                        std::vector<S2CellId> edge_cells;
  1032→                        std::vector<uint64_t> way_cells;
  1033→                        while (true) {
  1034→                            size_t i = interp_idx.fetch_add(1);
  1035→                            if (i >= data.deferred_interps.size()) break;
  1036→                            const auto& di = data.deferred_interps[i];
  1037→                            way_cells.clear();
  1038→                            for (uint8_t j = 0; j + 1 < di.node_count; j++) {
  1039→                                const auto& n1 = data.interp_nodes[di.node_offset + j];
  1040→                                const auto& n2 = data.interp_nodes[di.node_offset + j + 1];
  1041→                                cover_edge(n1.lat, n1.lng, n2.lat, n2.lng, edge_cells);
  1042→                                for (const auto& c : edge_cells) way_cells.push_back(c.id());
  1043→                            }
  1044→                            std::sort(way_cells.begin(), way_cells.end());
  1045→                            way_cells.erase(std::unique(way_cells.begin(), way_cells.end()), way_cells.end());
  1046→                            for (uint64_t cell_id : way_cells) {
  1047→                                local.push_back({cell_id, di.interp_id});
  1048→                            }
  1049→                        }
  1050→                    });
  1051→                }
  1052→                for (auto& t : threads) t.join();
  1053→            }
  1054→            log_phase("  S2: interp ways (parallel)", _s2t, _s2cpu);
  1055→
  1056→            // Merge thread-local pairs into single vectors, sort, build cell maps
  1057→            std::cerr << "  Sorting and grouping cell pairs..." << std::endl;
  1058→            // Concatenate + sort helper for flat CellItem pairs
  1059→            // Parallel sort each thread's pairs, then k-way merge directly into
  1060→            // hash map + sorted output. No intermediate merged vector needed.
  1061→            auto parallel_sort_and_build = [](
  1062→                std::vector<std::vector<CellItem>>& thread_pairs,
  1063→                std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map,
  1064→                std::vector<CellItemPair>& sorted_out
  1065→            ) {
  1066→                // Step 1: Convert + sort each thread's data in parallel
  1067→                size_t total = 0;
  1068→                for (auto& v : thread_pairs) total += v.size();
  1069→
  1070→                std::vector<std::vector<CellItemPair>> chunks(thread_pairs.size());
  1071→                {
  1072→                    std::vector<std::thread> sort_threads;
  1073→                    for (size_t t = 0; t < thread_pairs.size(); t++) {
  1074→                        sort_threads.emplace_back([&, t]() {


grep -n "cell_to_addrs\|address.*cell\|addr.*S2" /root/traccar-geocoder/builder/src/build_index.cpp | head -40


418:                    std::vector<std::pair<double, double>> building_addr_coords; // lat,lng for S2 cell
1174:                sorted_cells.reserve(data.cell_to_addrs.size());
1175:                for (auto& [cell_id, ids] : data.cell_to_addrs)
1194:                                auto& ids = data.cell_to_addrs[sorted_cells[i]];
1206:                for (auto& [_, ids] : data.cell_to_addrs) total_pairs += ids.size();
1213:                    auto& ids = data.cell_to_addrs[cell_id];


  1165→            // Convert addr hash map to sorted pairs.
  1166→            // Sort cell IDs only (30M unique), then build pairs in sorted order.
  1167→            // Much faster than sorting all 160M pairs.
  1168→            auto f2 = std::async(std::launch::async, [&] {
  1169→                auto _dt = std::chrono::steady_clock::now();
  1170→                auto _dc = CpuTicks::now();
  1171→
  1172→                // Step 1: Extract cell IDs from hash map
  1173→                std::vector<uint64_t> sorted_cells;
  1174→                sorted_cells.reserve(data.cell_to_addrs.size());
  1175→                for (auto& [cell_id, ids] : data.cell_to_addrs)
  1176→                    sorted_cells.push_back(cell_id);
  1177→                log_phase("    Dedup: extract cell IDs", _dt, _dc);
  1178→
  1179→                // Step 2: Sort cell IDs
  1180→                std::sort(sorted_cells.begin(), sorted_cells.end());
  1181→                log_phase("    Dedup: sort cell IDs", _dt, _dc);
  1182→
  1183→                // Step 3: Parallel dedup within each cell
  1184→                {
  1185→                    unsigned nthreads = std::thread::hardware_concurrency();
  1186→                    size_t chunk = (sorted_cells.size() + nthreads - 1) / nthreads;
  1187→                    std::vector<std::thread> threads;
  1188→                    for (unsigned t = 0; t < nthreads; t++) {
  1189→                        size_t start = t * chunk;
  1190→                        size_t end = std::min(start + chunk, sorted_cells.size());
  1191→                        if (start >= sorted_cells.size()) break;
  1192→                        threads.emplace_back([&, start, end]() {
  1193→                            for (size_t i = start; i < end; i++) {
  1194→                                auto& ids = data.cell_to_addrs[sorted_cells[i]];
  1195→                                std::sort(ids.begin(), ids.end());
  1196→                                ids.erase(std::unique(ids.begin(), ids.end()), ids.end());
  1197→                            }
  1198→                        });
  1199→                    }
  1200→                    for (auto& t : threads) t.join();
  1201→                }
  1202→                log_phase("    Dedup: per-cell sort+dedup", _dt, _dc);
  1203→
  1204→                // Step 4: Count total pairs
  1205→                size_t total_pairs = 0;
  1206→                for (auto& [_, ids] : data.cell_to_addrs) total_pairs += ids.size();
  1207→                log_phase("    Dedup: count pairs", _dt, _dc);
  1208→
  1209→                // Step 5: Build sorted pairs
  1210→                std::vector<CellItemPair> pairs;
  1211→                pairs.reserve(total_pairs);
  1212→                for (uint64_t cell_id : sorted_cells) {
  1213→                    auto& ids = data.cell_to_addrs[cell_id];
  1214→                    for (auto id : ids) pairs.push_back({cell_id, id});
  1215→                }
  1216→                data.sorted_addr_cells = std::move(pairs);
  1217→                log_phase("    Dedup: build sorted pairs", _dt, _dc);
  1218→            });
  1219→            auto f4 = std::async(std::launch::async, [&]{ deduplicate(data.cell_to_admin); });
  1220→            f2.get(); f4.get();
  1221→        }
  1222→
  1223→        // Save cache if requested
  1224→        if (!save_cache_path.empty()) {
  1225→            serialize_cache(data, save_cache_path);
  1226→        }
  1227→    }
  1228→
  1229→    // --- Rebuild hash maps from sorted pairs if needed for cache saving ---
  1230→    // (Continent filtering now uses sorted pairs directly, so no rebuild needed for that.)
  1231→    if (!save_cache_path.empty()) {
  1232→        // Parallel rebuild: split sorted pairs into chunks, each thread builds
  1233→        // a partial map, then merge. Sorted pairs are grouped by cell_id so we
  1234→        // can split at cell boundaries for zero-conflict parallel insertion.


   302→        if (write_addresses) {
   303→            write_futures.push_back(std::async(std::launch::async, [&] {
   304→                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
   305→                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
   306→            }));
   307→            write_futures.push_back(std::async(std::launch::async, [&] {
   308→                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
   309→                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
   310→            }));
   311→            write_futures.push_back(std::async(std::launch::async, [&] {


   293→    if (write_streets) {
   294→        write_futures.push_back(std::async(std::launch::async, [&] {
   295→            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
   296→            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
   297→        }));
   298→        write_futures.push_back(std::async(std::launch::async, [&] {
   299→            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
   300→            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
   301→        }));
   302→        if (write_addresses) {


Perfect! Now I have all the information I need. Let me compile my findings into a comprehensive report.

## Summary: ID Assignment and Ordering in build_index.cpp

Based on my analysis of the code, here's how address points and street ways get their IDs and ordering:

### 1. **Address Points Collection (Pass 2 - Node Scanning)**

**Location:** `/root/traccar-geocoder/builder/src/build_index.cpp`, lines 339-401

During Pass 2, nodes are scanned in a streaming fashion with thread-local data collection:

- **Thread-local collection:** Each thread maintains a local vector of `addr_coords` and `addr_strings` (lines 342-346)
- **Streaming callback:** For each node with `addr:housenumber` and `addr:street` tags, coordinates and strings are appended to the thread-local vectors (lines 379-381)
- **Sequential merge:** After all PBF blocks are processed, the thread-local data from all threads is merged sequentially into `data.addr_points` (lines 389-399)

```cpp
// From lines 392-396:
for (size_t j = 0; j < local.addr_coords.size(); j++) {
    uint64_t dummy = 0;
    add_addr_point(data, local.addr_coords[j].first, local.addr_coords[j].second,
                   local.addr_strings[j].first.c_str(),
                   local.addr_strings[j].second.c_str(), dummy);
}
```

### 2. **Address Point ID Assignment**

**Location:** `/root/traccar-geocoder/builder/src/s2_helpers.cpp`, lines 229-247

Each address point gets its ID based on its position in `data.addr_points`:

```cpp
uint32_t addr_id = static_cast<uint32_t>(data.addr_points.size());
data.addr_points.push_back({
    static_cast<float>(lat),
    static_cast<float>(lng),
    data.string_pool.intern(housenumber),
    data.string_pool.intern(street)
});
```

The addr_id is simply the vector index at the time of insertion. Simultaneously, the address point is also added to the S2 cell index:
```cpp
S2CellId cell = point_to_cell(lat, lng);
data.cell_to_addrs[cell.id()].push_back(addr_id);
```

### 3. **Storage and Sorting Before Writing to addr_points.bin**

**Key finding:** Address points are **NOT sorted** before writing to the binary file.

- **No explicit sorting:** There is no sort operation on `data.addr_points` before writing
- **Direct write:** addr_points.bin is written directly from the vector as-is (cell_index.cpp, lines 304-305):
  ```cpp
  std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
  f.write(reinterpret_cast<const char*>(data.addr_points.data()),
          data.addr_points.size() * sizeof(AddrPoint));
  ```

- **Separate index:** The S2 cell mapping (`cell_to_addrs`) is converted to sorted pairs (`sorted_addr_cells`) for indexing purposes only, not for reordering addr_points.bin

### 4. **Ordering in addr_points.bin**

**Deterministic based on:**
- **PBF file processing order:** Nodes are processed sequentially through PBF blocks as they appear in the file
- **Thread assignment:** Thread-local buffers collect nodes in parallel, but the merge phase (lines 391-399) processes thread-local data in thread index order (for loop over `ntld`)
- **Thread pool iteration:** Within each thread's data, addresses appear in the order they were encountered during streaming

**NOT based on:**
- Hash map iteration (not used for ordering)
- S2 cell order (S2 cells are computed but used only for indexing, not for reordering records)

**Result:** The ordering is **deterministic** and depends on:
1. The order of nodes in the PBF input file
2. The order threads are assigned during multi-threaded processing
3. The order of thread data merging (thread-0 data, then thread-1, etc.)

### 5. **Street Ways Ordering**

**Location:** `/root/traccar-geocoder/builder/src/build_index.cpp`, lines 607-619

Street ways follow a similar pattern:

- **Collection:** Ways are collected in thread-local data structures during parallel way processing (Pass 2b)
- **Sequential merge:** Thread-local ways are merged sequentially:
  ```cpp
  for (auto& local : tld) {
      uint32_t way_base = static_cast<uint32_t>(data.ways.size());
      // ...
      for (size_t i = 0; i < local.ways.size(); i++) {
          auto h = local.ways[i];
          h.node_offset += node_base;
          h.name_id = data.string_pool.intern(local.way_strings[i]);
          data.ways.push_back(h);  // <-- way_id = data.ways.size() at insertion
      }
  }
  ```

- **Writing:** street_ways.bin is written directly without sorting (cell_index.cpp, lines 295-296):
  ```cpp
  std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
  f.write(reinterpret_cast<const char*>(data.ways.data()),
          data.ways.size() * sizeof(WayHeader));
  ```

**Ordering determinism:** Same as addr_points - **deterministic** based on PBF file order and thread merging order.

### 6. **S2 Cell Index Ordering**

The sorted pairs (`sorted_addr_cells`, `sorted_way_cells`) used for addr_entries.bin and street_entries.bin ARE sorted, but with a specific approach:

**Location:** `/root/traccar-geocoder/builder/src/build_index.cpp`, lines 1061-1133

- **Parallel sort:** Each thread sorts its (cell_id, item_id) pairs independently
- **Parallel tree merge:** Adjacent sorted runs are merged pairwise in parallel using `std::inplace_merge`
- **Sorting order:**
  ```cpp
  std::sort(dst.begin(), dst.end(), [](const CellItemPair& a, const CellItemPair& b) {
      return a.cell_id < b.cell_id || (a.cell_id == b.cell_id && a.item_id < b.item_id);
  });
  ```

This means the *index* files (addr_entries.bin and street_entries.bin) ARE sorted by S2 cell ID, but the *data* files (addr_points.bin and street_ways.bin) maintain their original order.

### Summary Table

| Aspect | addr_points.bin | street_ways.bin |
|--------|-----------------|-----------------|
| **ID assignment** | Vector index at push_back | Vector index at push_back |
| **Sorting before write** | NO | NO |
| **Deterministic** | YES | YES |
| **Depends on** | PBF order + thread merge order | PBF order + thread merge order |
| **S2 cell used for** | Indexing only (separate index files) | Indexing only (separate index files) |
| **Hash map iteration** | NOT used for ordering | NOT used for ordering |

Prompt:
In /root/traccar-geocoder/builder/src/build_index.cpp, find how address points get their IDs assigned. Specifically:
1. How are addr_points collected during Pass 2 (node scanning)?
2. How are they stored/sorted before writing to addr_points.bin?
3. What determines the order of records in addr_points.bin?
4. Similarly, how are street_ways ordered — what determines a way's position/ID in street_ways.bin?

I want to understand whether the ordering is deterministic and what it depends on (e.g., PBF file order, hash map iteration order, S2 cell order, etc.)


Here's the analysis of why zstd binary patching is poor for most files:

**Total zstd patch: ~1.9 GiB for 7 GiB dataset (27%)** — not great for daily updates.

The root cause is **unstable record ordering**. The builder writes records in PBF encounter order (which varies between PBF builds), merged by thread index. Adding one new address shifts every subsequent ID, which cascades through:
- Data files (addr_points, street_ways) → all records shift
- Entry files (addr_entries, street_entries) → all referenced IDs change
- Cell index (geo_cells) → all byte offsets into entries change
- String pool → new strings shift all subsequent offsets

Only pure coordinate files (street_nodes, admin_vertices) patch well because they're mostly append-only with small changes.

**The fix: Stabilize build ordering by sorting records geographically (by S2 cell ID)**

If we sort data files by S2 cell before writing:
- Same address at same location → same position in file across builds
- Insertions only shift the local region (one cell)
- zstd's 2GB window can match shifted content
- Estimated patch ratio: **2-5%** instead of 27% (~100-350 MB for Europe)

This requires no format change — same binary layout, same server code. Just a different write order in the builder.

The plan:
1. After collecting all addr_points, sort by (S2 cell, lat, lng) and remap IDs
2. After collecting all ways, sort by (S2 cell of first node, name) and remap IDs
3. Sort string pool alphabetically, remap all string offsets
4. Rebuild entry/cell indexes from the new stable ordering

Then zstd --patch-from should work dramatically better. If still not enough, we can layer a chunked delta system on top (only transmit 4KB blocks that changed).

Let me implement the stable ordering. First, let me read the current write pipeline in detail.


     1→#include <algorithm>
     2→#include <atomic>
     3→#include <chrono>
     4→#include <condition_variable>
     5→#include <cstring>
     6→#include <deque>
     7→#include <fstream>
     8→#include <future>
     9→#include <iostream>
    10→#include <mutex>
    11→#include <numeric>
    12→#include <string>
    13→#include <thread>
    14→#include <unordered_set>
    15→#include <vector>
    16→
    17→#include <fcntl.h>
    18→#include <sys/mman.h>
    19→
    20→#include "pbf_reader.h"
    21→
    22→#include <s2/s2cell_id.h>
    23→#include <s2/s2latlng.h>
    24→
    25→#include "types.h"
    26→#include "string_pool.h"
    27→#include "geometry.h"
    28→#include "parsed_data.h"
    29→#include "s2_helpers.h"
    30→#include "ring_assembly.h"
    31→#include "continent_boundaries.h"
    32→#include "interpolation.h"
    33→#include "cache.h"
    34→#include "continent_filter.h"
    35→#include "cell_index.h"
    36→
    37→
    38→// --- Main ---
    39→
    40→int main(int argc, char* argv[]) {
    41→    if (argc < 2) {
    42→        std::cerr << "Usage: build-index <output-dir> <input.osm.pbf> [options]" << std::endl;
    43→        std::cerr << "       build-index <output-dir> --load-cache <path> [options]" << std::endl;
    44→        std::cerr << std::endl;
    45→        std::cerr << "Options:" << std::endl;
    46→        std::cerr << "  --street-level N       S2 cell level for streets (default: 17)" << std::endl;
    47→        std::cerr << "  --admin-level N        S2 cell level for admin (default: 10)" << std::endl;
    48→        std::cerr << "  --max-admin-level N    Only include admin levels <= N" << std::endl;
    49→        std::cerr << "  --save-cache <path>    Save parsed data to cache file" << std::endl;
    50→        std::cerr << "  --load-cache <path>    Load from cache instead of PBF" << std::endl;
    51→        std::cerr << "  --multi-output         Write full, no-addresses, admin-only indexes" << std::endl;
    52→        std::cerr << "  --continents           Also generate per-continent indexes" << std::endl;
    53→        std::cerr << "  --mode <mode>          Index mode: full, no-addresses, admin-only (default: full)" << std::endl;
    54→        std::cerr << "  --admin-only           Shorthand for --mode admin-only" << std::endl;
    55→        std::cerr << "  --no-addresses         Shorthand for --mode no-addresses" << std::endl;
    56→        std::cerr << "  --simplify-epsilon     Use error-bounded simplification (per-level defaults)" << std::endl;
    57→        std::cerr << "  --simplify-epsilon N   Use fixed epsilon of N meters for all levels" << std::endl;
    58→        std::cerr << "  --epsilon-scale F      Multiply all per-level epsilons by F (e.g. 2.0 = 2x coarser)" << std::endl;
    59→        std::cerr << "  --epsilon-levels L2,L3,...,L8  Set epsilon in meters per level (7 values)" << std::endl;
    60→        std::cerr << "  --multi-quality [S,S,...]  Write quality variants at given scales (default: 0,0.2,0.5,1,1.5,2,2.5)" << std::endl;
    61→        return 1;
    62→    }
    63→
    64→    std::string output_dir = argv[1];
    65→    std::vector<std::string> input_files;
    66→    std::string save_cache_path;
    67→    std::string load_cache_path;
    68→    bool multi_output = false;
    69→    bool generate_continents = false;
    70→    IndexMode mode = IndexMode::Full;
    71→    SimplifyMode simplify_mode = SimplifyMode::MaxVertices;
    72→    double simplify_epsilon_override = 0; // 0 = use per-level defaults
    73→    bool multi_quality = false;
    74→    std::vector<double> quality_scales;
    75→
    76→    for (int i = 2; i < argc; i++) {
    77→        std::string arg = argv[i];
    78→        if (arg == "--street-level" && i + 1 < argc) {
    79→            kStreetCellLevel = std::atoi(argv[++i]);
    80→        } else if (arg == "--admin-level" && i + 1 < argc) {
    81→            kAdminCellLevel = std::atoi(argv[++i]);
    82→        } else if (arg == "--admin-only") {
    83→            mode = IndexMode::AdminOnly;
    84→        } else if (arg == "--no-addresses") {
    85→            mode = IndexMode::NoAddresses;
    86→        } else if (arg == "--max-admin-level" && i + 1 < argc) {
    87→            kMaxAdminLevel = std::atoi(argv[++i]);
    88→        } else if (arg == "--save-cache" && i + 1 < argc) {
    89→            save_cache_path = argv[++i];
    90→        } else if (arg == "--load-cache" && i + 1 < argc) {
    91→            load_cache_path = argv[++i];
    92→        } else if (arg == "--multi-output") {
    93→            multi_output = true;
    94→        } else if (arg == "--continents") {
    95→            generate_continents = true;
    96→        } else if (arg == "--simplify-epsilon") {
    97→            simplify_mode = SimplifyMode::ErrorBounded;
    98→            // Optional: next arg might be a number (fixed epsilon in meters)
    99→            if (i + 1 < argc) {
   100→                char* end;

#include <algorithm>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstring>
#include <deque>
#include <fstream>
#include <future>
#include <iostream>
#include <mutex>
#include <numeric>
#include <string>
#include <thread>
#include <unordered_set>
#include <vector>

#include <fcntl.h>
#include <sys/mman.h>

#include "pbf_reader.h"

#include <s2/s2cell_id.h>
#include <s2/s2latlng.h>

#include "types.h"
#include "string_pool.h"
#include "geometry.h"
#include "parsed_data.h"
#include "s2_helpers.h"
#include "ring_assembly.h"
#include "continent_boundaries.h"
#include "interpolation.h"
#include "cache.h"
#include "continent_filter.h"
#include "cell_index.h"

// --- Main ---

int main(int argc, char* argv[]) {
    if (argc < 2) {
        std::cerr << "Usage: build-index <output-dir> <input.osm.pbf> [options]" << std::endl;
        std::cerr << "       build-index <output-dir> --load-cache <path> [options]" << std::endl;
        std::cerr << std::endl;
        std::cerr << "Options:" << std::endl;
        std::cerr << "  --street-level N       S2 cell level for streets (default: 17)" << std::endl;
        std::cerr << "  --admin-level N        S2 cell level for admin (default: 10)" << std::endl;
        std::cerr << "  --max-admin-level N    Only include admin levels <= N" << std::endl;
        std::cerr << "  --save-cache <path>    Save parsed data to cache file" << std::endl;
        std::cerr << "  --load-cache <path>    Load from cache instead of PBF" << std::endl;
        std::cerr << "  --multi-output         Write full, no-addresses, admin-only indexes" << std::endl;
        std::cerr << "  --continents           Also generate per-continent indexes" << std::endl;
        std::cerr << "  --mode <mode>          Index mode: full, no-addresses, admin-only (default: full)" << std::endl;
        std::cerr << "  --admin-only           Shorthand for --mode admin-only" << std::endl;
        std::cerr << "  --no-addresses         Shorthand for --mode no-addresses" << std::endl;
        std::cerr << "  --simplify-epsilon     Use error-bounded simplification (per-level defaults)" << std::endl;
        std::cerr << "  --simplify-epsilon N   Use fixed epsilon of N meters for all levels" << std::endl;
        std::cerr << "  --epsilon-scale F      Multiply all per-level epsilons by F (e.g. 2.0 = 2x coarser)" << std::endl;
        std::cerr << "  --epsilon-levels L2,L3,...,L8  Set epsilon in meters per level (7 values)" << std::endl;
        std::cerr << "  --multi-quality [S,S,...]  Write quality variants at given scales (default: 0,0.2,0.5,1,1.5,2,2.5)" << std::endl;
        return 1;
    }

    std::string output_dir = argv[1];
    std::vector<std::string> input_files;
    std::string save_cache_path;
    std::string load_cache_path;
    bool multi_output = false;
    bool generate_continents = false;
    IndexMode mode = IndexMode::Full;
    SimplifyMode simplify_mode = SimplifyMode::MaxVertices;
    double simplify_epsilon_override = 0; // 0 = use per-level defaults
    bool multi_quality = false;
    std::vector<double> quality_scales;

    for (int i = 2; i < argc; i++) {
        std::string arg = argv[i];
        if (arg == "--street-level" && i + 1 < argc) {
            kStreetCellLevel = std::atoi(argv[++i]);
        } else if (arg == "--admin-level" && i + 1 < argc) {
            kAdminCellLevel = std::atoi(argv[++i]);
        } else if (arg == "--admin-only") {
            mode = IndexMode::AdminOnly;
        } else if (arg == "--no-addresses") {
            mode = IndexMode::NoAddresses;
        } else if (arg == "--max-admin-level" && i + 1 < argc) {
            kMaxAdminLevel = std::atoi(argv[++i]);
        } else if (arg == "--save-cache" && i + 1 < argc) {
            save_cache_path = argv[++i];
        } else if (arg == "--load-cache" && i + 1 < argc) {
            load_cache_path = argv[++i];
        } else if (arg == "--multi-output") {
            multi_output = true;
        } else if (arg == "--continents") {
            generate_continents = true;
        } else if (arg == "--simplify-epsilon") {
            simplify_mode = SimplifyMode::ErrorBounded;
            // Optional: next arg might be a number (fixed epsilon in meters)
            if (i + 1 < argc) {
                char* end;