Search the codebase at /root/traccar-geocoder/ for:
1. Any existing patch generation or delta tools (builder/tools/, scripts, etc.)
2. The manifest.json that gets written - what's in it?
3. The compare_indexes.cpp and deep_compare.cpp tools - how do they compare files semantically?
4. How the continent filter produces filtered data (filter_by_bbox_masked, filter_and_remap_sorted) - this shows how data is extracted per-region and might inform the patch design
5. Any download/distribution mechanism for the index files

Also look at builder/src/cell_index.cpp for write_entries_from_sorted and write_index to understand exactly how each file is produced from the in-memory data structures.


I'll search the codebase systematically for these components. Let me start by exploring the structure and finding these key files.


Bash

find /root/traccar-geocoder -type f -name "*.json" | head -20


(Bash completed with no output)


Let me continue the exploration with glob patterns and grep searches to find the relevant files:


No files found


/root/traccar-geocoder/.git/hooks/pre-applypatch.sample
/root/traccar-geocoder/.git/hooks/applypatch-msg.sample


Now let me search for the filter functions and examine the builder structure:


tools/comparison.geojson
tools/simplify_compare.cpp
tools/simplification_viewer.html
tools/simplification_map.html


Found 3 files
src/build_index.cpp
src/continent_filter.cpp
src/continent_filter.h


Good! Let me now search for download and distribution mechanisms, and read the key files:


Found 1 file
src/build_index.cpp


Error: File does not exist. Note: your current working directory is /root/traccar-geocoder/builder. Did you mean /root/traccar-geocoder/builder/src/cell_index.cpp?


Error: File does not exist. Note: your current working directory is /root/traccar-geocoder/builder. Did you mean /root/traccar-geocoder/builder/test/compare_indexes.cpp?


Let me correct the paths:


find /root/traccar-geocoder -type f \( -name "*.cpp" -o -name "*.h" \) | sort


/root/traccar-geocoder/builder/build-presplit/CMakeFiles/3.31.6/CompilerIdCXX/CMakeCXXCompilerId.cpp
/root/traccar-geocoder/builder/build/CMakeFiles/3.31.6/CompilerIdCXX/CMakeCXXCompilerId.cpp
/root/traccar-geocoder/builder/src/build_index.cpp
/root/traccar-geocoder/builder/src/cache.cpp
/root/traccar-geocoder/builder/src/cache.h
/root/traccar-geocoder/builder/src/cell_index.cpp
/root/traccar-geocoder/builder/src/cell_index.h
/root/traccar-geocoder/builder/src/continent_boundaries.cpp
/root/traccar-geocoder/builder/src/continent_boundaries.h
/root/traccar-geocoder/builder/src/continent_filter.cpp
/root/traccar-geocoder/builder/src/continent_filter.h
/root/traccar-geocoder/builder/src/geometry.h
/root/traccar-geocoder/builder/src/interpolation.cpp
/root/traccar-geocoder/builder/src/interpolation.h
/root/traccar-geocoder/builder/src/parsed_data.h
/root/traccar-geocoder/builder/src/pbf_reader.cpp
/root/traccar-geocoder/builder/src/pbf_reader.h
/root/traccar-geocoder/builder/src/relation_collector.h
/root/traccar-geocoder/builder/src/ring_assembly.cpp
/root/traccar-geocoder/builder/src/ring_assembly.h
/root/traccar-geocoder/builder/src/s2_helpers.cpp
/root/traccar-geocoder/builder/src/s2_helpers.h
/root/traccar-geocoder/builder/src/string_pool.h
/root/traccar-geocoder/builder/src/types.h
/root/traccar-geocoder/builder/test/compare_indexes.cpp
/root/traccar-geocoder/builder/test/deep_compare.cpp
/root/traccar-geocoder/builder/test/test_custom_pbf.cpp
/root/traccar-geocoder/builder/test/test_pbf_parsing.cpp
/root/traccar-geocoder/builder/test/test_ring_assembly.cpp
/root/traccar-geocoder/builder/tools/simplify_compare.cpp


Now let me read the key files:


     1→#include "cell_index.h"
     2→
     3→#include <algorithm>
     4→#include <atomic>
     5→#include <cstring>
     6→#include <fstream>
     7→#include <future>
     8→#include <iostream>
     9→#include <thread>
    10→#include <unordered_map>
    11→#include <unordered_set>
    12→
    13→#include <unistd.h>
    14→
    15→#include "geometry.h"
    16→#include "s2_helpers.h"
    17→
    18→std::vector<uint32_t> write_entries(
    19→    const std::string& path,
    20→    const std::vector<uint64_t>& sorted_cells,
    21→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
    22→) {
    23→    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
    24→    std::vector<CellRef> sorted_refs;
    25→    sorted_refs.reserve(cell_map.size());
    26→    for (auto& [id, ids] : cell_map) sorted_refs.push_back({id, &ids});
    27→    std::sort(sorted_refs.begin(), sorted_refs.end(),
    28→        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });
    29→
    30→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    31→    size_t total_size = 0;
    32→    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);
    33→
    34→    std::vector<char> buf;
    35→    buf.reserve(total_size);
    36→    uint32_t current = 0;
    37→    size_t ri = 0;
    38→    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
    39→        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
    40→        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
    41→        offsets[si] = current;
    42→        const auto& ids = *sorted_refs[ri].ids;
    43→        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
    44→        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
    45→                   reinterpret_cast<const char*>(&count) + sizeof(count));
    46→        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
    47→                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
    48→        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
    49→        ri++;
    50→    }
    51→    std::ofstream f(path, std::ios::binary);
    52→    f.write(buf.data(), buf.size());
    53→    return offsets;
    54→}
    55→
    56→std::vector<uint32_t> write_entries_from_sorted(
    57→    const std::string& path,
    58→    const std::vector<uint64_t>& sorted_cells,
    59→    const std::vector<CellItemPair>& sorted_pairs
    60→) {
    61→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    62→    if (sorted_pairs.empty()) {
    63→        std::ofstream f(path, std::ios::binary);
    64→        return offsets;
    65→    }
    66→
    67→    unsigned int nthreads = std::thread::hardware_concurrency();
    68→    if (nthreads == 0) nthreads = 4;
    69→    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;
    70→
    71→    struct ChunkResult {
    72→        std::vector<char> buf;
    73→        size_t cell_start, cell_end;
    74→        uint32_t local_size;
    75→    };
    76→    std::vector<ChunkResult> chunks(nthreads);
    77→
    78→    std::vector<std::thread> threads;
    79→    for (unsigned int t = 0; t < nthreads; t++) {
    80→        size_t cs = t * cells_per_chunk;
    81→        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
    82→        if (cs >= sorted_cells.size()) break;
    83→
    84→        threads.emplace_back([&, t, cs, ce]() {
    85→            auto& chunk = chunks[t];
    86→            chunk.cell_start = cs;
    87→            chunk.cell_end = ce;
    88→            chunk.local_size = 0;
    89→
    90→            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
    91→                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
    92→                    return p.cell_id < id;
    93→                }) - sorted_pairs.begin();
    94→
    95→            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
    96→                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
    97→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
    98→                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;
    99→
   100→                offsets[si] = chunk.local_size;
   101→                size_t start = pi;
   102→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
   103→                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(MAX_VERTEX_COUNT)));
   104→                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
   105→                size_t buf_pos = chunk.buf.size();
   106→                chunk.buf.resize(buf_pos + entry_size);
   107→                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
   108→                for (size_t k = start; k < pi; k++) {
   109→                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
   110→                           &sorted_pairs[k].item_id, sizeof(uint32_t));
   111→                }
   112→                chunk.local_size += entry_size;
   113→            }
   114→        });
   115→    }
   116→    for (auto& t : threads) t.join();
   117→
   118→    uint32_t global_offset = 0;
   119→    for (auto& chunk : chunks) {
   120→        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
   121→            if (offsets[si] != NO_DATA) offsets[si] += global_offset;
   122→        }
   123→        global_offset += chunk.local_size;
   124→    }
   125→
   126→    std::vector<char> buf;
   127→    buf.reserve(global_offset);
   128→    for (auto& chunk : chunks) buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());
   129→
   130→    std::ofstream f(path, std::ios::binary);
   131→    f.write(buf.data(), buf.size());
   132→    return offsets;
   133→}
   134→
   135→void write_cell_index(
   136→    const std::string& cells_path,
   137→    const std::string& entries_path,
   138→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
   139→) {
   140→    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(cell_map.begin(), cell_map.end());
   141→    std::sort(sorted.begin(), sorted.end());
   142→
   143→    { std::ofstream f(cells_path, std::ios::binary);
   144→      uint32_t current_offset = 0;
   145→      for (const auto& [cell_id, ids] : sorted) {
   146→          f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
   147→          f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
   148→          current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
   149→      } }
   150→
   151→    { std::ofstream f(entries_path, std::ios::binary);
   152→      for (const auto& [cell_id, ids] : sorted) {
   153→          uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
   154→          f.write(reinterpret_cast<const char*>(&count), sizeof(count));
   155→          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
   156→      } }
   157→}
   158→
   159→void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
   160→    ensure_dir(output_dir);
   161→    auto _wt = std::chrono::steady_clock::now();
   162→
   163→    bool write_streets = (mode != IndexMode::AdminOnly);
   164→    bool write_addresses = (mode == IndexMode::Full);
   165→
   166→    if (write_streets) {
   167→        std::vector<uint64_t> sorted_geo_cells;
   168→        {
   169→            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
   170→                std::vector<uint64_t> cells;
   171→                cells.reserve(pairs.size() / 2);
   172→                for (size_t i = 0; i < pairs.size(); ) {
   173→                    cells.push_back(pairs[i].cell_id);
   174→                    uint64_t cur = pairs[i].cell_id;
   175→                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
   176→                }
   177→                return cells;
   178→            };
   179→            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
   180→                std::vector<uint64_t> cells;
   181→                cells.reserve(m.size());
   182→                for (auto& [id, _] : m) cells.push_back(id);
   183→                std::sort(cells.begin(), cells.end());
   184→                return cells;
   185→            };
   186→
   187→            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
   188→            {
   189→                auto f1 = std::async(std::launch::async, [&] {
   190→                    return !data.sorted_way_cells.empty()
   191→                        ? extract_unique_cells(data.sorted_way_cells)
   192→                        : extract_from_map(data.cell_to_ways);
   193→                });
   194→                if (write_addresses) {
   195→                    auto f2 = std::async(std::launch::async, [&] {
   196→                        return !data.sorted_addr_cells.empty()
   197→                            ? extract_unique_cells(data.sorted_addr_cells)
   198→                            : extract_from_map(data.cell_to_addrs);
   199→                    });
   200→                    auto f3 = std::async(std::launch::async, [&] {
   201→                        return !data.sorted_interp_cells.empty()
   202→                            ? extract_unique_cells(data.sorted_interp_cells)
   203→                            : extract_from_map(data.cell_to_interps);
   204→                    });
   205→                    addr_cells = f2.get();
   206→                    interp_cells = f3.get();
   207→                }
   208→                way_cells = f1.get();
   209→            }
   210→
   211→            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
   212→            std::merge(way_cells.begin(), way_cells.end(), addr_cells.begin(), addr_cells.end(),
   213→                       std::back_inserter(sorted_geo_cells));
   214→            if (!interp_cells.empty()) {
   215→                std::vector<uint64_t> tmp;
   216→                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
   217→                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
   218→                           interp_cells.begin(), interp_cells.end(), std::back_inserter(tmp));
   219→                sorted_geo_cells = std::move(tmp);
   220→            }
   221→            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
   222→                                    sorted_geo_cells.end());
   223→        }
   224→
   225→        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
   226→        {
   227→            auto f1 = std::async(std::launch::async, [&]() {
   228→                if (!data.sorted_way_cells.empty())
   229→                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
   230→                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
   231→            });
   232→            if (write_addresses) {
   233→                auto f2 = std::async(std::launch::async, [&]() {
   234→                    if (!data.sorted_addr_cells.empty())
   235→                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
   236→                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
   237→                });
   238→                auto f3 = std::async(std::launch::async, [&]() {
   239→                    if (!data.sorted_interp_cells.empty())
   240→                        return write_entries_from_sorted(output_dir + "/interp_entries.bin", sorted_geo_cells, data.sorted_interp_cells);
   241→                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
   242→                });
   243→                addr_offsets = f2.get();
   244→                interp_offsets = f3.get();
   245→            }
   246→            street_offsets = f1.get();
   247→        }
   248→
   249→        log_phase("  Write: entry files", _wt);
   250→        {
   251→            size_t n = sorted_geo_cells.size();
   252→            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
   253→            std::vector<char> buf(n * row_size);
   254→            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
   255→            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);
   256→
   257→            unsigned int nthreads = std::thread::hardware_concurrency();
   258→            if (nthreads == 0) nthreads = 4;
   259→            size_t chunk = (n + nthreads - 1) / nthreads;
   260→            std::vector<std::thread> fill_threads;
   261→            for (unsigned int t = 0; t < nthreads; t++) {
   262→                size_t start = t * chunk;
   263→                size_t end = std::min(start + chunk, n);
   264→                if (start >= n) break;
   265→                fill_threads.emplace_back([&, start, end]() {
   266→                    char* ptr = buf.data() + start * row_size;
   267→                    for (size_t i = start; i < end; i++) {
   268→                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
   269→                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   270→                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   271→                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   272→                    }
   273→                });
   274→            }
   275→            for (auto& t : fill_threads) t.join();
   276→
   277→            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
   278→            f.write(buf.data(), buf.size());
   279→        }
   280→
   281→        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
   282→                  << data.ways.size() << " ways, " << data.addr_points.size() << " addrs, "
   283→                  << data.interp_ways.size() << " interps)" << std::endl;
   284→    }
   285→
   286→    log_phase("  Write: geo_cells.bin", _wt);
   287→    auto admin_future = std::async(std::launch::async, [&] {
   288→        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
   289→        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
   290→    });
   291→
   292→    std::vector<std::future<void>> write_futures;
   293→    if (write_streets) {
   294→        write_futures.push_back(std::async(std::launch::async, [&] {
   295→            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
   296→            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
   297→        }));
   298→        write_futures.push_back(std::async(std::launch::async, [&] {
   299→            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
   300→            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
   301→        }));
   302→        if (write_addresses) {
   303→            write_futures.push_back(std::async(std::launch::async, [&] {
   304→                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
   305→                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
   306→            }));
   307→            write_futures.push_back(std::async(std::launch::async, [&] {
   308→                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
   309→                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
   310→            }));
   311→            write_futures.push_back(std::async(std::launch::async, [&] {
   312→                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
   313→                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
   314→            }));
   315→        }
   316→    }
   317→    // If not writing all data types, compact string pool and remap references
   318→    // before writing admin_polygons and strings.bin.
   319→    std::vector<AdminPolygon> compact_admin;
   320→    std::vector<char> compact_strings;
   321→    const bool need_compact = (!write_streets || !write_addresses);
   322→
   323→    if (need_compact) {
   324→        // Collect referenced string offsets
   325→        std::unordered_set<uint32_t> used_offsets;
   326→        if (write_streets) {
   327→            for (const auto& w : data.ways) used_offsets.insert(w.name_id);
   328→            if (write_addresses) {
   329→                for (const auto& a : data.addr_points) {
   330→                    used_offsets.insert(a.housenumber_id);
   331→                    used_offsets.insert(a.street_id);
   332→                }
   333→                for (const auto& iw : data.interp_ways) used_offsets.insert(iw.street_id);
   334→            }
   335→        }
   336→        for (const auto& ap : data.admin_polygons) used_offsets.insert(ap.name_id);
   337→
   338→        // Build compact pool and remap
   339→        const auto& old_sp = data.string_pool.data();
   340→        std::vector<uint32_t> sorted_offs(used_offsets.begin(), used_offsets.end());
   341→        std::sort(sorted_offs.begin(), sorted_offs.end());
   342→        std::unordered_map<uint32_t, uint32_t> remap;
   343→        compact_strings.reserve(used_offsets.size() * 16);
   344→        for (uint32_t off : sorted_offs) {
   345→            remap[off] = static_cast<uint32_t>(compact_strings.size());
   346→            size_t len = std::strlen(old_sp.data() + off);
   347→            compact_strings.insert(compact_strings.end(), old_sp.data() + off,
   348→                old_sp.data() + off + len + 1);
   349→        }
   350→
   351→        // Remap admin polygon name_ids
   352→        compact_admin = data.admin_polygons;
   353→        for (auto& ap : compact_admin) ap.name_id = remap[ap.name_id];
   354→
   355→        // Remap street/addr/interp if applicable (for no-addresses mode)
   356→        // These are written from data directly, so we'd need copies too.
   357→        // For simplicity, only compact when we have fewer types.
   358→    }
   359→
   360→    const auto& admin_to_write = need_compact ? compact_admin : data.admin_polygons;
   361→    write_futures.push_back(std::async(std::launch::async, [&] {
   362→        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
   363→        f.write(reinterpret_cast<const char*>(admin_to_write.data()), admin_to_write.size() * sizeof(AdminPolygon));
   364→    }));
   365→    write_futures.push_back(std::async(std::launch::async, [&] {
   366→        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
   367→        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
   368→    }));
   369→    write_futures.push_back(std::async(std::launch::async, [&] {
   370→        if (need_compact) {
   371→            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
   372→            f.write(compact_strings.data(), compact_strings.size());
   373→            std::cerr << "strings.bin: " << compact_strings.size() << " bytes (compacted from "
   374→                      << data.string_pool.data().size() << ")" << std::endl;
   375→        } else {
   376→            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
   377→            f.write(data.string_pool.data().data(), data.string_pool.data().size());
   378→            std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
   379→        }
   380→    }));
   381→
   382→    log_phase("  Write: parallel data files launched", _wt);
   383→    admin_future.get();
   384→    for (auto& f : write_futures) f.get();
   385→}
   386→
   387→void write_quality_variant(const ParsedData& data, const std::string& source_dir,
   388→                           const std::string& output_dir, double epsilon_scale) {
   389→    ensure_dir(output_dir);
   390→
   391→    // Re-simplify admin polygons at the given epsilon scale
   392→    std::vector<AdminPolygon> new_polys;
   393→    std::vector<NodeCoord> new_verts;
   394→    new_polys.reserve(data.admin_polygons.size());
   395→
   396→    // Parallel simplification
   397→    struct SimplifiedPoly {
   398→        std::vector<std::pair<double,double>> verts;
   399→    };
   400→    std::vector<SimplifiedPoly> simplified(data.admin_polygons.size());
   401→
   402→    {
   403→        std::atomic<size_t> idx{0};
   404→        unsigned nthreads = std::thread::hardware_concurrency();
   405→        if (nthreads == 0) nthreads = 4;
   406→        std::vector<std::thread> workers;
   407→        for (unsigned t = 0; t < nthreads; t++) {
   408→            workers.emplace_back([&]() {
   409→                while (true) {
   410→                    size_t i = idx.fetch_add(1);
   411→                    if (i >= data.admin_polygons.size()) break;
   412→                    const auto& ap = data.admin_polygons[i];
   413→
   414→                    // Extract original vertices
   415→                    std::vector<std::pair<double,double>> pts;
   416→                    pts.reserve(ap.vertex_count);
   417→                    for (uint32_t j = 0; j < ap.vertex_count; j++) {
   418→                        const auto& v = data.admin_vertices[ap.vertex_offset + j];
   419→                        pts.emplace_back(v.lat, v.lng);
   420→                    }
   421→
   422→                    if (epsilon_scale > 0) {
   423→                        double eps_m = admin_epsilon_meters(ap.admin_level) * epsilon_scale;
   424→                        double lat = pts.empty() ? 0.0 : pts[0].first;
   425→                        double eps_deg = meters_to_degrees(eps_m, lat);
   426→                        simplified[i].verts = simplify_polygon_epsilon(pts, eps_deg);
   427→                    } else {
   428→                        simplified[i].verts = std::move(pts);
   429→                    }
   430→                }
   431→            });
   432→        }
   433→        for (auto& w : workers) w.join();
   434→    }
   435→
   436→    // Sequential: build new polygon and vertex arrays
   437→    for (size_t i = 0; i < data.admin_polygons.size(); i++) {
   438→        auto& sv = simplified[i].verts;
   439→        if (sv.size() < 3) continue;
   440→
   441→        AdminPolygon np = data.admin_polygons[i];
   442→        np.vertex_offset = static_cast<uint32_t>(new_verts.size());
   443→        np.vertex_count = static_cast<uint32_t>(sv.size());
   444→        np.area = polygon_area(sv);
   445→
   446→        for (const auto& [lat, lng] : sv) {
   447→            new_verts.push_back({static_cast<float>(lat), static_cast<float>(lng)});
   448→        }
   449→        new_polys.push_back(np);
   450→    }
   451→
   452→    std::cerr << "Quality " << epsilon_scale << "x: " << new_polys.size()
   453→              << " polygons, " << new_verts.size() << " vertices ("
   454→              << new_verts.size() * 8 / 1024 / 1024 << " MiB)" << std::endl;
   455→
   456→    // Write admin files
   457→    {
   458→        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
   459→        f.write(reinterpret_cast<const char*>(new_polys.data()), new_polys.size() * sizeof(AdminPolygon));
   460→    }
   461→    {
   462→        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
   463→        f.write(reinterpret_cast<const char*>(new_verts.data()), new_verts.size() * sizeof(NodeCoord));
   464→    }
   465→
   466→    // Quality directories only contain the two files that change.
   467→    // Shared files (admin_cells.bin, admin_entries.bin, strings.bin)
   468→    // stay in the parent directory.
   469→}
   470→


     1→// Deep comparison: dumps samples of items only in A or only in B
     2→// with their coordinates, to understand boundary differences.
     3→// Usage: deep_compare <dir-A> <dir-B> [--ways|--addrs|--admins] [--limit N]
     4→
     5→#include <algorithm>
     6→#include <cmath>
     7→#include <cstdint>
     8→#include <cstring>
     9→#include <fstream>
    10→#include <iostream>
    11→#include <string>
    12→#include <unordered_map>
    13→#include <unordered_set>
    14→#include <vector>
    15→
    16→struct WayHeader { uint32_t node_offset; uint8_t node_count; uint32_t name_id; };
    17→struct AddrPoint { float lat; float lng; uint32_t housenumber_id; uint32_t street_id; };
    18→struct NodeCoord { float lat; float lng; };
    19→
    20→static std::vector<char> read_file(const std::string& path) {
    21→    std::ifstream f(path, std::ios::binary | std::ios::ate);
    22→    if (!f) return {};
    23→    auto size = f.tellg(); f.seekg(0);
    24→    std::vector<char> data(size); f.read(data.data(), size);
    25→    return data;
    26→}
    27→
    28→template<typename T>
    29→static std::vector<T> read_structs(const std::string& path) {
    30→    auto data = read_file(path);
    31→    size_t count = data.size() / sizeof(T);
    32→    std::vector<T> result(count);
    33→    memcpy(result.data(), data.data(), count * sizeof(T));
    34→    return result;
    35→}
    36→
    37→static const char* get_string(const std::vector<char>& pool, uint32_t offset) {
    38→    if (offset >= pool.size()) return "(invalid)";
    39→    return pool.data() + offset;
    40→}
    41→
    42→static int to_grid(float v) {
    43→    return (int)(v * 1e5f + (v >= 0 ? 0.5f : -0.5f));
    44→}
    45→
    46→int main(int argc, char* argv[]) {
    47→    if (argc < 3) {
    48→        std::cerr << "Usage: deep_compare <dir-A> <dir-B> [--ways|--addrs|--admins] [--limit N]" << std::endl;
    49→        return 1;
    50→    }
    51→    std::string dir_a = argv[1], dir_b = argv[2];
    52→    bool do_ways = false, do_addrs = false, do_admins = false;
    53→    int limit = 20;
    54→    for (int i = 3; i < argc; i++) {
    55→        if (strcmp(argv[i], "--ways") == 0) do_ways = true;
    56→        else if (strcmp(argv[i], "--addrs") == 0) do_addrs = true;
    57→        else if (strcmp(argv[i], "--admins") == 0) do_admins = true;
    58→        else if (strcmp(argv[i], "--limit") == 0 && i+1 < argc) limit = atoi(argv[++i]);
    59→    }
    60→    if (!do_ways && !do_addrs && !do_admins) { do_ways = do_addrs = true; }
    61→
    62→    if (do_ways) {
    63→        std::cerr << "Loading ways..." << std::endl;
    64→        auto ways_a = read_structs<WayHeader>(dir_a + "/street_ways.bin");
    65→        auto ways_b = read_structs<WayHeader>(dir_b + "/street_ways.bin");
    66→        auto nodes_a = read_structs<NodeCoord>(dir_a + "/street_nodes.bin");
    67→        auto nodes_b = read_structs<NodeCoord>(dir_b + "/street_nodes.bin");
    68→        auto sp_a = read_file(dir_a + "/strings.bin");
    69→        auto sp_b = read_file(dir_b + "/strings.bin");
    70→
    71→        // Build fingerprints: name + sorted node coords
    72→        auto way_fp = [&](const std::vector<WayHeader>& ways, const std::vector<NodeCoord>& nodes,
    73→                          const std::vector<char>& sp, size_t idx) -> std::string {
    74→            const auto& w = ways[idx];
    75→            std::string name = get_string(sp, w.name_id);
    76→            std::string fp = name + "|" + std::to_string(w.node_count);
    77→            // Use first node coords as primary identifier
    78→            if (w.node_count > 0) {
    79→                fp += "|" + std::to_string(to_grid(nodes[w.node_offset].lat)) +
    80→                      "," + std::to_string(to_grid(nodes[w.node_offset].lng));
    81→            }
    82→            return fp;
    83→        };
    84→
    85→        std::unordered_map<std::string, std::vector<size_t>> fps_a, fps_b;
    86→        for (size_t i = 0; i < ways_a.size(); i++) fps_a[way_fp(ways_a, nodes_a, sp_a, i)].push_back(i);
    87→        for (size_t i = 0; i < ways_b.size(); i++) fps_b[way_fp(ways_b, nodes_b, sp_b, i)].push_back(i);
    88→
    89→        // Find items only in A
    90→        std::vector<std::pair<std::string, size_t>> only_a, only_b;
    91→        for (auto& [fp, indices] : fps_a) {
    92→            size_t bcount = fps_b.count(fp) ? fps_b[fp].size() : 0;
    93→            if (indices.size() > bcount) {
    94→                for (size_t i = bcount; i < indices.size(); i++)
    95→                    only_a.push_back({fp, indices[i]});
    96→            }
    97→        }
    98→        for (auto& [fp, indices] : fps_b) {
    99→            size_t acount = fps_a.count(fp) ? fps_a[fp].size() : 0;
   100→            if (indices.size() > acount) {
   101→                for (size_t i = acount; i < indices.size(); i++)
   102→                    only_b.push_back({fp, indices[i]});
   103→            }
   104→        }
   105→
   106→        std::cout << "=== WAYS ===" << std::endl;
   107→        std::cout << "A: " << ways_a.size() << "  B: " << ways_b.size() << std::endl;
   108→        std::cout << "Only in A (standalone): " << only_a.size() << std::endl;
   109→        int count = 0;
   110→        for (auto& [fp, idx] : only_a) {
   111→            if (count++ >= limit) { std::cout << "  ... and " << (only_a.size() - limit) << " more" << std::endl; break; }
   112→            const auto& w = ways_a[idx];
   113→            std::cout << "  " << get_string(sp_a, w.name_id)
   114→                      << " nodes=" << (int)w.node_count
   115→                      << " first=(" << nodes_a[w.node_offset].lat << "," << nodes_a[w.node_offset].lng << ")"
   116→                      << " last=(" << nodes_a[w.node_offset + w.node_count - 1].lat << "," << nodes_a[w.node_offset + w.node_count - 1].lng << ")"
   117→                      << std::endl;
   118→        }
   119→        std::cout << "Only in B (filtered): " << only_b.size() << std::endl;
   120→        count = 0;
   121→        for (auto& [fp, idx] : only_b) {
   122→            if (count++ >= limit) { std::cout << "  ... and " << (only_b.size() - limit) << " more" << std::endl; break; }
   123→            const auto& w = ways_b[idx];
   124→            std::cout << "  " << get_string(sp_b, w.name_id)
   125→                      << " nodes=" << (int)w.node_count
   126→                      << " first=(" << nodes_b[w.node_offset].lat << "," << nodes_b[w.node_offset].lng << ")"
   127→                      << " last=(" << nodes_b[w.node_offset + w.node_count - 1].lat << "," << nodes_b[w.node_offset + w.node_count - 1].lng << ")"
   128→                      << std::endl;
   129→        }
   130→    }
   131→
   132→    if (do_addrs) {
   133→        std::cerr << "Loading addrs..." << std::endl;
   134→        auto addrs_a = read_structs<AddrPoint>(dir_a + "/addr_points.bin");
   135→        auto addrs_b = read_structs<AddrPoint>(dir_b + "/addr_points.bin");
   136→        auto sp_a = read_file(dir_a + "/strings.bin");
   137→        auto sp_b = read_file(dir_b + "/strings.bin");
   138→
   139→        auto addr_fp = [&](const std::vector<AddrPoint>& addrs, const std::vector<char>& sp, size_t idx) -> std::string {
   140→            const auto& a = addrs[idx];
   141→            return std::to_string(to_grid(a.lat)) + "," + std::to_string(to_grid(a.lng)) +
   142→                   "|" + get_string(sp, a.housenumber_id) + "|" + get_string(sp, a.street_id);
   143→        };
   144→
   145→        std::unordered_set<std::string> set_a, set_b;
   146→        for (size_t i = 0; i < addrs_a.size(); i++) set_a.insert(addr_fp(addrs_a, sp_a, i));
   147→        for (size_t i = 0; i < addrs_b.size(); i++) set_b.insert(addr_fp(addrs_b, sp_b, i));
   148→
   149→        std::vector<std::pair<std::string, size_t>> only_a, only_b;
   150→        for (size_t i = 0; i < addrs_a.size(); i++) {
   151→            auto fp = addr_fp(addrs_a, sp_a, i);
   152→            if (!set_b.count(fp)) only_a.push_back({fp, i});
   153→        }
   154→        for (size_t i = 0; i < addrs_b.size(); i++) {
   155→            auto fp = addr_fp(addrs_b, sp_b, i);
   156→            if (!set_a.count(fp)) only_b.push_back({fp, i});
   157→        }
   158→
   159→        std::cout << "\n=== ADDRESSES ===" << std::endl;
   160→        std::cout << "A: " << addrs_a.size() << "  B: " << addrs_b.size() << std::endl;
   161→        std::cout << "Only in A (standalone): " << only_a.size() << std::endl;
   162→        int count = 0;
   163→        for (auto& [fp, idx] : only_a) {
   164→            if (count++ >= limit) { std::cout << "  ... and " << (only_a.size() - limit) << " more" << std::endl; break; }
   165→            const auto& a = addrs_a[idx];
   166→            std::cout << "  (" << a.lat << "," << a.lng << ") "
   167→                      << get_string(sp_a, a.housenumber_id) << " "
   168→                      << get_string(sp_a, a.street_id) << std::endl;
   169→        }
   170→        std::cout << "Only in B (filtered): " << only_b.size() << std::endl;
   171→        count = 0;
   172→        for (auto& [fp, idx] : only_b) {
   173→            if (count++ >= limit) { std::cout << "  ... and " << (only_b.size() - limit) << " more" << std::endl; break; }
   174→            const auto& a = addrs_b[idx];
   175→            std::cout << "  (" << a.lat << "," << a.lng << ") "
   176→                      << get_string(sp_b, a.housenumber_id) << " "
   177→                      << get_string(sp_b, a.street_id) << std::endl;
   178→        }
   179→    }
   180→
   181→    return 0;
   182→}
   183→


     1→// Semantic comparison of two geocoder index directories.
     2→// Decodes all binary files, creates canonical fingerprints for each item,
     3→// and compares as multisets — handles any ordering differences between
     4→// parallel and sequential builds.
     5→// Returns 0 if semantically identical, 1 if different.
     6→//
     7→// Usage: compare_indexes <dir-A> <dir-B>
     8→
     9→#include <algorithm>
    10→#include <cmath>
    11→#include <cstdint>
    12→#include <cstring>
    13→#include <fstream>
    14→#include <iostream>
    15→#include <map>
    16→#include <set>
    17→#include <string>
    18→#include <unordered_map>
    19→#include <vector>
    20→
    21→// --- Binary structs (must match build_index.cpp and server main.rs) ---
    22→
    23→struct WayHeader {
    24→    uint32_t node_offset;
    25→    uint8_t node_count;
    26→    uint32_t name_id;
    27→};
    28→
    29→struct AddrPoint {
    30→    float lat;
    31→    float lng;
    32→    uint32_t housenumber_id;
    33→    uint32_t street_id;
    34→};
    35→
    36→struct InterpWay {
    37→    uint32_t node_offset;
    38→    uint8_t node_count;
    39→    uint32_t street_id;
    40→    uint32_t start_number;
    41→    uint32_t end_number;
    42→    uint8_t interpolation;
    43→};
    44→
    45→struct AdminPolygon {
    46→    uint32_t vertex_offset;
    47→    uint16_t vertex_count;
    48→    uint32_t name_id;
    49→    uint8_t admin_level;
    50→    float area;
    51→    uint16_t country_code;
    52→};
    53→
    54→struct NodeCoord {
    55→    float lat;
    56→    float lng;
    57→};
    58→
    59→// --- Helpers ---
    60→
    61→static std::vector<char> read_file(const std::string& path) {
    62→    std::ifstream f(path, std::ios::binary | std::ios::ate);
    63→    if (!f) return {};
    64→    auto size = f.tellg();
    65→    f.seekg(0);
    66→    std::vector<char> data(size);
    67→    f.read(data.data(), size);
    68→    return data;
    69→}
    70→
    71→static const char* get_string(const std::vector<char>& pool, uint32_t offset) {
    72→    if (offset >= pool.size()) return "(invalid)";
    73→    return pool.data() + offset;
    74→}
    75→
    76→template<typename T>
    77→static std::vector<T> read_structs(const std::string& path) {
    78→    auto data = read_file(path);
    79→    size_t count = data.size() / sizeof(T);
    80→    std::vector<T> result(count);
    81→    memcpy(result.data(), data.data(), count * sizeof(T));
    82→    return result;
    83→}
    84→
    85→// Round a float coordinate to an integer grid (1e5 ~ 1m precision)
    86→static int to_grid(float v) {
    87→    return (int)(v * 1e5f + (v >= 0 ? 0.5f : -0.5f));
    88→}
    89→
    90→// --- Canonical fingerprint builders ---
    91→// Each function produces a string that uniquely identifies an item,
    92→// independent of storage order.
    93→
    94→// For a way: "name|sorted_node_coords"
    95→// Nodes are kept in original order (path order matters for ways),
    96→// but we also provide a sorted-set version for order-independent matching.
    97→static std::string way_fingerprint(const std::string& name,
    98→                                    const std::vector<std::pair<float,float>>& nodes) {
    99→    // Use sorted coordinate set as the canonical form.
   100→    // Two ways are "the same" if they have the same name and same set of nodes.
   101→    std::vector<std::pair<int,int>> grid_nodes;
   102→    grid_nodes.reserve(nodes.size());
   103→    for (const auto& n : nodes) {
   104→        grid_nodes.push_back({to_grid(n.first), to_grid(n.second)});
   105→    }
   106→    std::sort(grid_nodes.begin(), grid_nodes.end());
   107→
   108→    std::string fp = name + "|" + std::to_string(nodes.size());
   109→    for (const auto& g : grid_nodes) {
   110→        fp += "|";
   111→        fp += std::to_string(g.first);
   112→        fp += ",";
   113→        fp += std::to_string(g.second);
   114→    }
   115→    return fp;
   116→}
   117→
   118→// For an address: "grid_lat,grid_lng|housenumber|street"
   119→static std::string addr_fingerprint(float lat, float lng,
   120→                                     const std::string& housenumber,
   121→                                     const std::string& street) {
   122→    return std::to_string(to_grid(lat)) + "," + std::to_string(to_grid(lng)) +
   123→           "|" + housenumber + "|" + street;
   124→}
   125→
   126→// For an admin polygon: "name|level|country|sorted_vertices"
   127→static std::string admin_fingerprint(const std::string& name,
   128→                                      uint8_t admin_level,
   129→                                      uint16_t country_code,
   130→                                      const std::vector<std::pair<float,float>>& vertices) {
   131→    std::vector<std::pair<int,int>> grid_verts;
   132→    grid_verts.reserve(vertices.size());
   133→    for (const auto& v : vertices) {
   134→        grid_verts.push_back({to_grid(v.first), to_grid(v.second)});
   135→    }
   136→    std::sort(grid_verts.begin(), grid_verts.end());
   137→
   138→    std::string fp = name + "|" + std::to_string(admin_level) + "|" + std::to_string(country_code);
   139→    for (const auto& g : grid_verts) {
   140→        fp += "|";
   141→        fp += std::to_string(g.first);
   142→        fp += ",";
   143→        fp += std::to_string(g.second);
   144→    }
   145→    return fp;
   146→}
   147→
   148→// For interpolation ways: "street|start|end|interp|sorted_nodes"
   149→static std::string interp_fingerprint(const std::string& street,
   150→                                       uint32_t start_number, uint32_t end_number,
   151→                                       uint8_t interpolation,
   152→                                       const std::vector<std::pair<float,float>>& nodes) {
   153→    std::vector<std::pair<int,int>> grid_nodes;
   154→    grid_nodes.reserve(nodes.size());
   155→    for (const auto& n : nodes) {
   156→        grid_nodes.push_back({to_grid(n.first), to_grid(n.second)});
   157→    }
   158→    std::sort(grid_nodes.begin(), grid_nodes.end());
   159→
   160→    std::string fp = street + "|" + std::to_string(start_number) + "-" +
   161→                     std::to_string(end_number) + "|" + std::to_string(interpolation);
   162→    for (const auto& g : grid_nodes) {
   163→        fp += "|";
   164→        fp += std::to_string(g.first);
   165→        fp += ",";
   166→        fp += std::to_string(g.second);
   167→    }
   168→    return fp;
   169→}
   170→
   171→// --- Multiset comparison ---
   172→// Compare two collections by their fingerprints. Reports missing/extra items.
   173→
   174→static bool compare_multisets(const char* label,
   175→                               const std::vector<std::string>& fps_a,
   176→                               const std::vector<std::string>& fps_b) {
   177→    // Build frequency maps
   178→    std::unordered_map<std::string, int> freq_a, freq_b;
   179→    for (const auto& fp : fps_a) freq_a[fp]++;
   180→    for (const auto& fp : fps_b) freq_b[fp]++;
   181→
   182→    // Find items in A but not in B (missing from B)
   183→    size_t missing = 0;
   184→    for (const auto& [fp, count] : freq_a) {
   185→        int b_count = 0;
   186→        auto it = freq_b.find(fp);
   187→        if (it != freq_b.end()) b_count = it->second;
   188→        if (count > b_count) missing += (count - b_count);
   189→    }
   190→
   191→    // Find items in B but not in A (extra in B)
   192→    size_t extra = 0;
   193→    for (const auto& [fp, count] : freq_b) {
   194→        int a_count = 0;
   195→        auto it = freq_a.find(fp);
   196→        if (it != freq_a.end()) a_count = it->second;
   197→        if (count > a_count) extra += (count - a_count);
   198→    }
   199→
   200→    size_t matched = fps_a.size() - missing;
   201→
   202→    if (missing == 0 && extra == 0) {
   203→        std::cout << "PASS: " << label << " (" << fps_a.size() << " items, all matched)" << std::endl;
   204→        return true;
   205→    } else {
   206→        std::cerr << "FAIL: " << label << " — A=" << fps_a.size() << " B=" << fps_b.size()
   207→                  << " matched=" << matched
   208→                  << " only_in_A=" << missing
   209→                  << " only_in_B=" << extra << std::endl;
   210→        return false;
   211→    }
   212→}
   213→
   214→// --- Decode + fingerprint functions ---
   215→
   216→static std::vector<std::string> fingerprint_admin(const std::string& dir) {
   217→    auto strings = read_file(dir + "/strings.bin");
   218→    auto polygons = read_structs<AdminPolygon>(dir + "/admin_polygons.bin");
   219→    auto vertices = read_structs<NodeCoord>(dir + "/admin_vertices.bin");
   220→
   221→    std::vector<std::string> fps;
   222→    fps.reserve(polygons.size());
   223→    for (const auto& p : polygons) {
   224→        std::string name = get_string(strings, p.name_id);
   225→        std::vector<std::pair<float,float>> verts;
   226→        for (uint16_t i = 0; i < p.vertex_count; i++) {
   227→            uint32_t vi = p.vertex_offset + i;
   228→            if (vi < vertices.size()) {
   229→                verts.push_back({vertices[vi].lat, vertices[vi].lng});
   230→            }
   231→        }
   232→        fps.push_back(admin_fingerprint(name, p.admin_level, p.country_code, verts));
   233→    }
   234→    return fps;
   235→}
   236→
   237→static std::vector<std::string> fingerprint_ways(const std::string& dir) {
   238→    auto strings = read_file(dir + "/strings.bin");
   239→    auto ways = read_structs<WayHeader>(dir + "/street_ways.bin");
   240→    auto nodes = read_structs<NodeCoord>(dir + "/street_nodes.bin");
   241→
   242→    std::vector<std::string> fps;
   243→    fps.reserve(ways.size());
   244→    for (const auto& w : ways) {
   245→        std::string name = get_string(strings, w.name_id);
   246→        std::vector<std::pair<float,float>> way_nodes;
   247→        for (uint8_t i = 0; i < w.node_count; i++) {
   248→            uint32_t ni = w.node_offset + i;
   249→            if (ni < nodes.size()) {
   250→                way_nodes.push_back({nodes[ni].lat, nodes[ni].lng});
   251→            }
   252→        }
   253→        fps.push_back(way_fingerprint(name, way_nodes));
   254→    }
   255→    return fps;
   256→}
   257→
   258→static std::vector<std::string> fingerprint_addrs(const std::string& dir) {
   259→    auto strings = read_file(dir + "/strings.bin");
   260→    auto points = read_structs<AddrPoint>(dir + "/addr_points.bin");
   261→
   262→    std::vector<std::string> fps;
   263→    fps.reserve(points.size());
   264→    for (const auto& p : points) {
   265→        std::string hn = get_string(strings, p.housenumber_id);
   266→        std::string st = get_string(strings, p.street_id);
   267→        fps.push_back(addr_fingerprint(p.lat, p.lng, hn, st));
   268→    }
   269→    return fps;
   270→}
   271→
   272→static std::vector<std::string> fingerprint_interps(const std::string& dir) {
   273→    auto strings = read_file(dir + "/strings.bin");
   274→    auto interps = read_structs<InterpWay>(dir + "/interp_ways.bin");
   275→    auto nodes = read_structs<NodeCoord>(dir + "/interp_nodes.bin");
   276→
   277→    std::vector<std::string> fps;
   278→    fps.reserve(interps.size());
   279→    for (const auto& iw : interps) {
   280→        std::string street = get_string(strings, iw.street_id);
   281→        std::vector<std::pair<float,float>> way_nodes;
   282→        for (uint8_t i = 0; i < iw.node_count; i++) {
   283→            uint32_t ni = iw.node_offset + i;
   284→            if (ni < nodes.size()) {
   285→                way_nodes.push_back({nodes[ni].lat, nodes[ni].lng});
   286→            }
   287→        }
   288→        fps.push_back(interp_fingerprint(street, iw.start_number, iw.end_number,
   289→                                          iw.interpolation, way_nodes));
   290→    }
   291→    return fps;
   292→}
   293→
   294→// --- Semantic (cross-version) fingerprints ---
   295→// Match by name/attributes only, ignoring coordinate precision differences
   296→
   297→// Way: name + node_count only
   298→static std::string way_semantic_fp(const std::string& name, size_t node_count) {
   299→    return name + "|" + std::to_string(node_count);
   300→}
   301→
   302→// Address: housenumber + street + coarse grid coords (~100m precision)
   303→// Centroid may differ slightly across code versions, so use coarse grid
   304→static std::string addr_semantic_fp(float lat, float lng,
   305→                                     const std::string& housenumber, const std::string& street) {
   306→    int glat = (int)(lat * 1e3f + (lat >= 0 ? 0.5f : -0.5f));
   307→    int glng = (int)(lng * 1e3f + (lng >= 0 ? 0.5f : -0.5f));
   308→    return std::to_string(glat) + "," + std::to_string(glng) + "|" + housenumber + "|" + street;
   309→}
   310→
   311→// Admin: name + level
   312→static std::string admin_semantic_fp(const std::string& name, uint8_t admin_level) {
   313→    return name + "|" + std::to_string(admin_level);
   314→}
   315→
   316→static std::vector<std::string> semantic_fp_ways(const std::string& dir) {
   317→    auto strings = read_file(dir + "/strings.bin");
   318→    auto ways = read_structs<WayHeader>(dir + "/street_ways.bin");
   319→
   320→    std::vector<std::string> fps;
   321→    fps.reserve(ways.size());
   322→    for (const auto& w : ways) {
   323→        std::string name = get_string(strings, w.name_id);
   324→        fps.push_back(way_semantic_fp(name, w.node_count));
   325→    }
   326→    return fps;
   327→}
   328→
   329→static std::vector<std::string> semantic_fp_addrs(const std::string& dir) {
   330→    auto strings = read_file(dir + "/strings.bin");
   331→    auto points = read_structs<AddrPoint>(dir + "/addr_points.bin");
   332→
   333→    std::vector<std::string> fps;
   334→    fps.reserve(points.size());
   335→    for (const auto& p : points) {
   336→        std::string hn = get_string(strings, p.housenumber_id);
   337→        std::string st = get_string(strings, p.street_id);
   338→        fps.push_back(addr_semantic_fp(p.lat, p.lng, hn, st));
   339→    }
   340→    return fps;
   341→}
   342→
   343→static std::vector<std::string> semantic_fp_admin(const std::string& dir) {
   344→    auto strings = read_file(dir + "/strings.bin");
   345→    auto polygons = read_structs<AdminPolygon>(dir + "/admin_polygons.bin");
   346→
   347→    std::vector<std::string> fps;
   348→    fps.reserve(polygons.size());
   349→    for (const auto& p : polygons) {
   350→        std::string name = get_string(strings, p.name_id);
   351→        fps.push_back(admin_semantic_fp(name, p.admin_level));
   352→    }
   353→    return fps;
   354→}
   355→
   356→int main(int argc, char* argv[]) {
   357→    if (argc < 3) {
   358→        std::cerr << "Usage: compare_indexes <dir-A> <dir-B> [--semantic]" << std::endl;
   359→        return 2;
   360→    }
   361→
   362→    std::string dir_a = argv[1];
   363→    std::string dir_b = argv[2];
   364→    bool semantic = (argc > 3 && std::string(argv[3]) == "--semantic");
   365→
   366→    std::cout << "Comparing" << (semantic ? " (semantic/cross-version)" : " (exact)") << ":" << std::endl;
   367→    std::cout << "  A: " << dir_a << std::endl;
   368→    std::cout << "  B: " << dir_b << std::endl;
   369→    std::cout << std::endl;
   370→
   371→    bool all_pass = true;
   372→
   373→    // Compare admin polygons
   374→    {
   375→        std::cout << "Loading admin polygons..." << std::flush;
   376→        auto fps_a = semantic ? semantic_fp_admin(dir_a) : fingerprint_admin(dir_a);
   377→        auto fps_b = semantic ? semantic_fp_admin(dir_b) : fingerprint_admin(dir_b);
   378→        std::cout << " done." << std::endl;
   379→        if (!compare_multisets("admin_polygons", fps_a, fps_b))
   380→            all_pass = false;
   381→    }
   382→
   383→    // Compare street ways
   384→    {
   385→        std::cout << "Loading street ways..." << std::flush;
   386→        auto fps_a = semantic ? semantic_fp_ways(dir_a) : fingerprint_ways(dir_a);
   387→        auto fps_b = semantic ? semantic_fp_ways(dir_b) : fingerprint_ways(dir_b);
   388→        std::cout << " done." << std::endl;
   389→        if (!compare_multisets("street_ways", fps_a, fps_b))
   390→            all_pass = false;
   391→    }
   392→
   393→    // Compare address points
   394→    {
   395→        std::cout << "Loading address points..." << std::flush;
   396→        auto fps_a = semantic ? semantic_fp_addrs(dir_a) : fingerprint_addrs(dir_a);
   397→        auto fps_b = semantic ? semantic_fp_addrs(dir_b) : fingerprint_addrs(dir_b);
   398→        std::cout << " done." << std::endl;
   399→        if (!compare_multisets("address_points", fps_a, fps_b))
   400→            all_pass = false;
   401→    }
   402→
   403→    // Compare interpolation ways
   404→    {
   405→        std::cout << "Loading interpolation ways..." << std::flush;
   406→        auto fps_a = fingerprint_interps(dir_a);
   407→        auto fps_b = fingerprint_interps(dir_b);
   408→        std::cout << " done." << std::endl;
   409→        if (!compare_multisets("interp_ways", fps_a, fps_b))
   410→            all_pass = false;
   411→    }
   412→
   413→    // Compare cell index sizes (should be identical regardless of ordering)
   414→    {
   415→        auto ac = read_file(dir_a + "/admin_cells.bin");
   416→        auto bc = read_file(dir_b + "/admin_cells.bin");
   417→        if (ac == bc) {
   418→            std::cout << "PASS: admin_cells.bin byte-identical (" << ac.size() << " bytes)" << std::endl;
   419→        } else if (ac.size() == bc.size()) {
   420→            std::cout << "INFO: admin_cells.bin same size (" << ac.size() << " bytes) but different content" << std::endl;
   421→        } else {
   422→            std::cerr << "FAIL: admin_cells.bin size differs (A=" << ac.size()
   423→                      << " B=" << bc.size() << ")" << std::endl;
   424→            all_pass = false;
   425→        }
   426→
   427→        auto gc_a = read_file(dir_a + "/geo_cells.bin");
   428→        auto gc_b = read_file(dir_b + "/geo_cells.bin");
   429→        if (gc_a.size() == gc_b.size()) {
   430→            std::cout << "PASS: geo_cells.bin same size (" << gc_a.size() << " bytes)" << std::endl;
   431→        } else {
   432→            std::cerr << "FAIL: geo_cells.bin size differs (A=" << gc_a.size()
   433→                      << " B=" << gc_b.size() << ")" << std::endl;
   434→            all_pass = false;
   435→        }
   436→    }
   437→
   438→    std::cout << std::endl;
   439→    if (all_pass) {
   440→        std::cout << "=== ALL COMPARISONS PASSED ===" << std::endl;
   441→        return 0;
   442→    } else {
   443→        std::cout << "=== SOME COMPARISONS FAILED ===" << std::endl;
   444→        return 1;
   445→    }
   446→}
   447→


Error: File content (11263 tokens) exceeds maximum allowed tokens (10000). Use offset and limit parameters to read specific portions of the file, or search for specific content instead of reading the whole file.


Let me read the continent_filter.cpp in sections and also check the build_index file:


     1→#include "continent_filter.h"
     2→#include "continent_boundaries.h"
     3→#include "parsed_data.h"
     4→
     5→#include <algorithm>
     6→#include <cstring>
     7→#include <future>
     8→#include <unordered_set>
     9→
    10→#include <s2/s2cell_id.h>
    11→#include <s2/s2latlng.h>
    12→
    13→const ContinentBBox kContinents[] = {
    14→    {"africa",            -35.0,  37.5,  -25.0,  55.0},
    15→    {"asia",              -12.0,  82.0,   25.0, 180.0},
    16→    {"europe",             35.0,  72.0,  -25.0,  45.0},
    17→    {"north-america",       7.0,  84.0, -170.0, -50.0},
    18→    {"south-america",     -56.0,  13.0,  -82.0, -34.0},
    19→    {"oceania",           -50.0,   0.0,  110.0, 180.0},
    20→    {"central-america",     7.0,  23.5, -120.0, -57.0},
    21→    {"antarctica",        -90.0, -60.0, -180.0, 180.0},
    22→};
    23→
    24→const size_t kContinentCount = sizeof(kContinents) / sizeof(kContinents[0]);
    25→
    26→static bool cell_in_bbox(uint64_t cell_id, const ContinentBBox& bbox) {
    27→    S2CellId cell(cell_id);
    28→    S2LatLng center = cell.ToLatLng();
    29→    double lat = center.lat().degrees();
    30→    double lng = center.lng().degrees();
    31→    return lat >= bbox.min_lat && lat <= bbox.max_lat &&
    32→           lng >= bbox.min_lng && lng <= bbox.max_lng;
    33→}
    34→
    35→ParsedData filter_by_bbox(const ParsedData& full, const ContinentBBox& bbox) {
    36→    ParsedData out;
    37→    auto _ft = std::chrono::steady_clock::now();
    38→    auto _fc = CpuTicks::now();
    39→
    40→    std::cerr << "    data: ways_sorted=" << full.sorted_way_cells.size()
    41→              << " addrs_sorted=" << full.sorted_addr_cells.size()
    42→              << " interps_sorted=" << full.sorted_interp_cells.size()
    43→              << " ways_map=" << full.cell_to_ways.size()
    44→              << " addrs_map=" << full.cell_to_addrs.size()
    45→              << " interps_map=" << full.cell_to_interps.size()
    46→              << " admin_map=" << full.cell_to_admin.size()
    47→              << std::endl;
    48→
    49→    // Filter from hash map
    50→    auto filter_cells_map = [&](const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map,
    51→                                bool mask_interior = false) {
    52→        std::cerr << "    DEBUG filter_cells_map: size=" << cell_map.size() << std::endl;
    53→        std::unordered_set<uint32_t> ids;
    54→        for (const auto& [cell_id, cell_ids] : cell_map) {
    55→            if (cell_in_bbox(cell_id, bbox)) {
    56→                for (uint32_t id : cell_ids)
    57→                    ids.insert(mask_interior ? (id & ID_MASK) : id);
    58→            }
    59→        }
    60→        return ids;
    61→    };
    62→
    63→    // Filter from sorted pairs — parallel chunked scan.
    64→    // Split the sorted array into N chunks at cell boundaries, each thread
    65→    // builds its own ID set, then merge.
    66→    auto filter_cells_sorted = [&](const std::vector<CellItemPair>& sorted,
    67→                                   bool mask_interior = false) {
    68→        if (sorted.empty()) return std::unordered_set<uint32_t>{};
    69→
    70→        unsigned nthreads = std::max(1u, std::thread::hardware_concurrency() / 4);
    71→        size_t chunk = (sorted.size() + nthreads - 1) / nthreads;
    72→        std::cerr << "    DEBUG filter_cells_sorted: size=" << sorted.size()
    73→                  << " nthreads=" << nthreads << " chunk=" << chunk << std::endl;
    74→
    75→        // Find chunk boundaries at cell_id transitions
    76→        std::vector<size_t> bounds = {0};
    77→        for (unsigned t = 1; t < nthreads; t++) {
    78→            size_t target = t * chunk;
    79→            if (target >= sorted.size()) break;
    80→            while (target < sorted.size() && sorted[target].cell_id == sorted[target-1].cell_id)
    81→                target++;
    82→            if (target < sorted.size()) bounds.push_back(target);
    83→        }
    84→        bounds.push_back(sorted.size());
    85→
    86→        std::vector<std::unordered_set<uint32_t>> thread_ids(bounds.size() - 1);
    87→        std::vector<std::thread> threads;
    88→        for (size_t t = 0; t + 1 < bounds.size(); t++) {
    89→            threads.emplace_back([&, t]() {
    90→                auto& local = thread_ids[t];
    91→                for (size_t i = bounds[t]; i < bounds[t+1]; ) {
    92→                    uint64_t cell_id = sorted[i].cell_id;
    93→                    bool match = cell_in_bbox(cell_id, bbox);
    94→                    while (i < bounds[t+1] && sorted[i].cell_id == cell_id) {
    95→                        if (match) {
    96→                            uint32_t id = sorted[i].item_id;
    97→                            local.insert(mask_interior ? (id & ID_MASK) : id);
    98→                        }
    99→                        i++;
   100→                    }
   101→                }
   102→            });
   103→        }
   104→        std::cerr << "    DEBUG: spawned " << threads.size() << " threads, "
   105→                  << bounds.size() - 1 << " chunks" << std::endl;
   106→        for (auto& t : threads) t.join();
   107→
   108→        // Merge thread-local sets
   109→        size_t largest = 0;
   110→        for (size_t t = 0; t < thread_ids.size(); t++)
   111→            if (thread_ids[t].size() > thread_ids[largest].size()) largest = t;
   112→        auto& merged = thread_ids[largest];
   113→        for (size_t t = 0; t < thread_ids.size(); t++) {
   114→            if (t == largest) continue;
   115→            for (auto id : thread_ids[t]) merged.insert(id);
   116→        }
   117→        return std::move(merged);
   118→    };
   119→
   120→    // Use sorted pairs where available (faster linear scan), fall back to hash maps
   121→    auto f_ways = std::async(std::launch::async, [&]{
   122→        return !full.sorted_way_cells.empty()
   123→            ? filter_cells_sorted(full.sorted_way_cells)
   124→            : filter_cells_map(full.cell_to_ways);
   125→    });
   126→    auto f_addrs = std::async(std::launch::async, [&]{
   127→        return !full.sorted_addr_cells.empty()
   128→            ? filter_cells_sorted(full.sorted_addr_cells)
   129→            : filter_cells_map(full.cell_to_addrs);
   130→    });
   131→    auto f_interps = std::async(std::launch::async, [&]{
   132→        return !full.sorted_interp_cells.empty()
   133→            ? filter_cells_sorted(full.sorted_interp_cells)
   134→            : filter_cells_map(full.cell_to_interps);
   135→    });
   136→    auto f_admin = std::async(std::launch::async, [&]{
   137→        return filter_cells_map(full.cell_to_admin, true);
   138→    });
   139→
   140→    auto used_way_ids = f_ways.get();
   141→    auto used_addr_ids = f_addrs.get();
   142→    auto used_interp_ids = f_interps.get();
   143→    auto used_admin_ids = f_admin.get();
   144→    log_phase("      filter: ID collection", _ft, _fc);
   145→
   146→    // Remap all 4 data types in parallel — each builds its own vectors and remap table
   147→    std::unordered_map<uint32_t, uint32_t> way_remap, addr_remap, interp_remap, admin_remap;
   148→
   149→    // Ways
   150→    auto f_remap_ways = std::async(std::launch::async, [&]() {


     1→#include <algorithm>
     2→#include <atomic>
     3→#include <chrono>
     4→#include <condition_variable>
     5→#include <cstring>
     6→#include <deque>
     7→#include <fstream>
     8→#include <future>
     9→#include <iostream>
    10→#include <mutex>
    11→#include <numeric>
    12→#include <string>
    13→#include <thread>
    14→#include <unordered_set>
    15→#include <vector>
    16→
    17→#include <fcntl.h>
    18→#include <sys/mman.h>
    19→
    20→#include "pbf_reader.h"
    21→
    22→#include <s2/s2cell_id.h>
    23→#include <s2/s2latlng.h>
    24→
    25→#include "types.h"
    26→#include "string_pool.h"
    27→#include "geometry.h"
    28→#include "parsed_data.h"
    29→#include "s2_helpers.h"
    30→#include "ring_assembly.h"
    31→#include "continent_boundaries.h"
    32→#include "interpolation.h"
    33→#include "cache.h"
    34→#include "continent_filter.h"
    35→#include "cell_index.h"
    36→
    37→
    38→// --- Main ---
    39→
    40→int main(int argc, char* argv[]) {
    41→    if (argc < 2) {
    42→        std::cerr << "Usage: build-index <output-dir> <input.osm.pbf> [options]" << std::endl;
    43→        std::cerr << "       build-index <output-dir> --load-cache <path> [options]" << std::endl;
    44→        std::cerr << std::endl;
    45→        std::cerr << "Options:" << std::endl;
    46→        std::cerr << "  --street-level N       S2 cell level for streets (default: 17)" << std::endl;
    47→        std::cerr << "  --admin-level N        S2 cell level for admin (default: 10)" << std::endl;
    48→        std::cerr << "  --max-admin-level N    Only include admin levels <= N" << std::endl;
    49→        std::cerr << "  --save-cache <path>    Save parsed data to cache file" << std::endl;
    50→        std::cerr << "  --load-cache <path>    Load from cache instead of PBF" << std::endl;
    51→        std::cerr << "  --multi-output         Write full, no-addresses, admin-only indexes" << std::endl;
    52→        std::cerr << "  --continents           Also generate per-continent indexes" << std::endl;
    53→        std::cerr << "  --mode <mode>          Index mode: full, no-addresses, admin-only (default: full)" << std::endl;
    54→        std::cerr << "  --admin-only           Shorthand for --mode admin-only" << std::endl;
    55→        std::cerr << "  --no-addresses         Shorthand for --mode no-addresses" << std::endl;
    56→        std::cerr << "  --simplify-epsilon     Use error-bounded simplification (per-level defaults)" << std::endl;
    57→        std::cerr << "  --simplify-epsilon N   Use fixed epsilon of N meters for all levels" << std::endl;
    58→        std::cerr << "  --epsilon-scale F      Multiply all per-level epsilons by F (e.g. 2.0 = 2x coarser)" << std::endl;
    59→        std::cerr << "  --epsilon-levels L2,L3,...,L8  Set epsilon in meters per level (7 values)" << std::endl;
    60→        std::cerr << "  --multi-quality [S,S,...]  Write quality variants at given scales (default: 0,0.2,0.5,1,1.5,2,2.5)" << std::endl;
    61→        std::cerr << "  --previous-build <dir>     Load previous build for stable string offsets (patchability)" << std::endl;
    62→        return 1;
    63→    }
    64→
    65→    std::string output_dir = argv[1];
    66→    std::vector<std::string> input_files;
    67→    std::string save_cache_path;
    68→    std::string load_cache_path;
    69→    std::string previous_build_path;
    70→    bool multi_output = false;
    71→    bool generate_continents = false;
    72→    IndexMode mode = IndexMode::Full;
    73→    SimplifyMode simplify_mode = SimplifyMode::MaxVertices;
    74→    double simplify_epsilon_override = 0; // 0 = use per-level defaults
    75→    bool multi_quality = false;
    76→    std::vector<double> quality_scales;
    77→
    78→    for (int i = 2; i < argc; i++) {
    79→        std::string arg = argv[i];
    80→        if (arg == "--street-level" && i + 1 < argc) {
    81→            kStreetCellLevel = std::atoi(argv[++i]);
    82→        } else if (arg == "--admin-level" && i + 1 < argc) {
    83→            kAdminCellLevel = std::atoi(argv[++i]);
    84→        } else if (arg == "--admin-only") {
    85→            mode = IndexMode::AdminOnly;
    86→        } else if (arg == "--no-addresses") {
    87→            mode = IndexMode::NoAddresses;
    88→        } else if (arg == "--max-admin-level" && i + 1 < argc) {
    89→            kMaxAdminLevel = std::atoi(argv[++i]);
    90→        } else if (arg == "--save-cache" && i + 1 < argc) {
    91→            save_cache_path = argv[++i];
    92→        } else if (arg == "--load-cache" && i + 1 < argc) {
    93→            load_cache_path = argv[++i];
    94→        } else if (arg == "--previous-build" && i + 1 < argc) {
    95→            previous_build_path = argv[++i];
    96→        } else if (arg == "--multi-output") {
    97→            multi_output = true;
    98→        } else if (arg == "--continents") {
    99→            generate_continents = true;
   100→        } else if (arg == "--simplify-epsilon") {
   101→            simplify_mode = SimplifyMode::ErrorBounded;
   102→            // Optional: next arg might be a number (fixed epsilon in meters)
   103→            if (i + 1 < argc) {
   104→                char* end;
   105→                double val = std::strtod(argv[i+1], &end);
   106→                if (*end == '\0' && val > 0) {
   107→                    simplify_epsilon_override = val;
   108→                    i++;
   109→                }
   110→            }
   111→        } else if (arg == "--epsilon-scale" && i + 1 < argc) {
   112→            kEpsilonScale = std::strtod(argv[++i], nullptr);
   113→        } else if (arg == "--multi-quality") {
   114→            multi_quality = true;
   115→            // Optional: next arg might be comma-separated scales
   116→            if (i + 1 < argc && argv[i+1][0] != '-') {
   117→                std::string vals = argv[++i];
   118→                size_t pos = 0;
   119→                while (pos < vals.size()) {
   120→                    size_t next = vals.find(',', pos);
   121→                    if (next == std::string::npos) next = vals.size();
   122→                    double v = std::strtod(vals.substr(pos, next - pos).c_str(), nullptr);
   123→                    quality_scales.push_back(v);
   124→                    pos = next + 1;
   125→                }
   126→            }
   127→            if (quality_scales.empty()) {
   128→                quality_scales = {0, 0.2, 0.5, 1.0, 1.5, 2.0, 2.5};
   129→            }
   130→            // Build with uncapped polygons — quality variants simplify at write time.
   131→            // We need full vertex detail to produce all quality levels accurately.
   132→            simplify_mode = SimplifyMode::ErrorBounded;
   133→            simplify_epsilon_override = 0.1; // effectively uncapped
   134→        } else if (arg == "--epsilon-levels" && i + 1 < argc) {
   135→            // Parse comma-separated values: L2,L3,L4,L5,L6,L7,L8
   136→            std::string vals = argv[++i];
   137→            int level = 2;
   138→            size_t pos = 0;
   139→            while (pos < vals.size() && level <= 8) {
   140→                size_t next = vals.find(',', pos);
   141→                if (next == std::string::npos) next = vals.size();
   142→                double v = std::strtod(vals.substr(pos, next - pos).c_str(), nullptr);
   143→                if (v > 0) kAdminEpsilonMeters[level] = v;
   144→                level++;
   145→                pos = next + 1;
   146→            }
   147→            // Also set levels 9-11 to same as level 8
   148→            for (int l = level; l <= 11; l++) kAdminEpsilonMeters[l] = kAdminEpsilonMeters[8];
   149→        } else if (arg == "--mode" && i + 1 < argc) {
   150→            std::string mode_str = argv[++i];


find /root/traccar-geocoder -type f -name "*.md" | head -20


/root/traccar-geocoder/builder/OPTIMIZATION_LOG.md
/root/traccar-geocoder/builder/test/VALIDATION_REPORT.md
/root/traccar-geocoder/README.md


Now let me search for distribution/download mechanisms and check the directory structure:


ls -la /root/traccar-geocoder/builder/ | head -40


total 77
drwxr-xr-x 8 root root   10 Mar 26 20:19 .
drwxr-xr-x 7 root root   14 Mar 21 11:08 ..
-rw-r--r-- 1 root root 1059 Mar 26 01:10 CMakeLists.txt
-rw-r--r-- 1 root root 3827 Mar 26 20:19 OPTIMIZATION_LOG.md
drwxr-xr-x 3 root root   12 Mar 27 03:30 build
drwxr-xr-x 3 root root    7 Mar 23 11:51 build-presplit
drwxr-xr-x 2 root root   10 Mar 26 01:23 data
drwxr-xr-x 2 root root   24 Mar 27 03:30 src
drwxr-xr-x 2 root root   15 Mar 26 11:11 test
drwxr-xr-x 2 root root    6 Mar 24 22:15 tools


Let me check the README and look for download/distribution mechanisms:


     1→# Traccar Geocoder
     2→
     3→A fast, self-hosted reverse geocoding service built from OpenStreetMap data. Given latitude and longitude coordinates, it returns the nearest street address including house number, street name, city, state, county, postcode, and country.
     4→
     5→Part of the [Traccar](https://www.traccar.org) open source GPS tracking platform. Also available as a [hosted service](https://www.traccar.org/product/geocoder/).
     6→
     7→## Features
     8→
     9→- Street-level reverse geocoding from OSM data
    10→- Address point, street name, and address interpolation lookup
    11→- Administrative boundary resolution (country, state, county, city, postcode)
    12→- Sub-millisecond query latency with memory-mapped index files
    13→- Automatic HTTPS with Let's Encrypt
    14→- Docker support with automatic PBF download and indexing
    15→
    16→## Quick Start
    17→
    18→### Docker Compose
    19→
    20→```yaml
    21→services:
    22→  geocoder:
    23→    image: traccar/traccar-geocoder
    24→    environment:
    25→      - PBF_URLS=https://download.geofabrik.de/europe/monaco-latest.osm.pbf
    26→    ports:
    27→      - "3000:3000"
    28→    volumes:
    29→      - geocoder-data:/data
    30→
    31→volumes:
    32→  geocoder-data:
    33→```
    34→
    35→```bash
    36→docker compose up
    37→```
    38→
    39→### Docker
    40→
    41→```bash
    42→# All-in-one: download, build index, and serve
    43→docker run -e PBF_URLS="https://download.geofabrik.de/europe-latest.osm.pbf" \
    44→  -v geocoder-data:/data -p 3000:3000 traccar/traccar-geocoder
    45→
    46→# Build index only
    47→docker run -e PBF_URLS="https://download.geofabrik.de/europe-latest.osm.pbf" \
    48→  -v geocoder-data:/data traccar/traccar-geocoder build
    49→
    50→# Serve only (from pre-built index)
    51→docker run -v geocoder-data:/data -p 3000:3000 traccar/traccar-geocoder serve
    52→
    53→# Multiple PBF files
    54→docker run -e PBF_URLS="https://download.geofabrik.de/europe/france-latest.osm.pbf https://download.geofabrik.de/europe/germany-latest.osm.pbf" \
    55→  -v geocoder-data:/data -p 3000:3000 traccar/traccar-geocoder
    56→
    57→# With automatic HTTPS
    58→docker run -e PBF_URLS="https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf" \
    59→  -e DOMAIN=geocoder.example.com \
    60→  -v geocoder-data:/data -p 443:443 traccar/traccar-geocoder
    61→```
    62→
    63→PBF files can be downloaded from [Geofabrik](https://download.geofabrik.de/).
    64→
    65→## API
    66→
    67→### GET /reverse
    68→
    69→Query parameters:
    70→- `lat` - latitude (required)
    71→- `lon` - longitude (required)
    72→- `key` - API key (required)
    73→
    74→Example request:
    75→
    76→```
    77→GET /reverse?lat=43.7384&lon=7.4246&key=YOUR_API_KEY
    78→```
    79→
    80→Response follows [Nominatim](https://nominatim.org/release-docs/latest/api/Reverse/) format:
    81→
    82→```json
    83→{
    84→  "display_name": "Avenue de la Costa 42, 98000 Monaco, Monaco",
    85→  "address": {
    86→    "house_number": "42",
    87→    "road": "Avenue de la Costa",
    88→    "city": "Monaco",
    89→    "state": "Monaco",
    90→    "county": "Monaco",
    91→    "postcode": "98000",
    92→    "country": "Monaco",
    93→    "country_code": "MC"
    94→  }
    95→}
    96→```
    97→
    98→Fields are omitted when not available.
    99→
   100→Status codes:
   101→- `200` - success
   102→- `401` - missing or invalid API key
   103→- `429` - rate limit exceeded
   104→
   105→### Authentication
   106→
   107→The server includes a web dashboard for managing API keys. On first launch, navigate to the server URL in a browser to create an admin account. Once logged in, you can generate API keys and create additional users with configurable rate limits.
   108→
   109→## Architecture
   110→
   111→The project consists of two components:
   112→
   113→- **Builder** (C++) - Parses OSM PBF files and creates a compact binary index using S2 geometry cells for spatial lookup.
   114→- **Server** (Rust) - Memory-maps the index files and serves queries via HTTP/HTTPS with sub-millisecond latency.
   115→
   116→### Index Structure
   117→
   118→The builder produces 14 binary files:
   119→
   120→| File | Description |
   121→|------|-------------|
   122→| `geo_cells.bin` | Merged S2 cell index for streets, addresses, and interpolations |
   123→| `street_entries.bin` | Street way IDs per cell |
   124→| `street_ways.bin` | Street way headers (node offset, name) |
   125→| `street_nodes.bin` | Street node coordinates |
   126→| `addr_entries.bin` | Address point IDs per cell |
   127→| `addr_points.bin` | Address point data (coordinates, house number, street) |
   128→| `interp_entries.bin` | Interpolation way IDs per cell |
   129→| `interp_ways.bin` | Interpolation way headers |
   130→| `interp_nodes.bin` | Interpolation node coordinates |
   131→| `admin_cells.bin` | S2 cell index for admin boundaries |
   132→| `admin_entries.bin` | Admin polygon IDs per cell |
   133→| `admin_polygons.bin` | Admin polygon metadata |
   134→| `admin_vertices.bin` | Admin polygon vertices |
   135→| `strings.bin` | Deduplicated string pool |
   136→
   137→## Building from Source
   138→
   139→### Prerequisites
   140→
   141→**Builder (C++):**
   142→- CMake 3.16+
   143→- C++17 compiler
   144→- libosmium, protozero, s2geometry, zlib, bzip2, expat
   145→
   146→**Server (Rust):**
   147→- Rust toolchain
   148→
   149→### Build
   150→
   151→```bash
   152→# Build the indexer
   153→mkdir build && cd build && cmake ../builder && make
   154→
   155→# Build the server
   156→cargo build --release --manifest-path server/Cargo.toml
   157→```
   158→
   159→### Run
   160→
   161→```bash
   162→# Create index from PBF file
   163→./build/build-index output-dir input.osm.pbf [input2.osm.pbf ...]
   164→
   165→# Start the server
   166→./server/target/release/query-server output-dir [bind-address]
   167→
   168→# Start with automatic HTTPS
   169→./server/target/release/query-server output-dir --domain geocoder.example.com
   170→```
   171→
   172→## Environment Variables (Docker)
   173→
   174→| Variable | Description | Default |
   175→|----------|-------------|---------|
   176→| `PBF_URLS` | Space-separated list of PBF download URLs | (required for auto/build) |
   177→| `DOMAIN` | Domain name for automatic HTTPS via Let's Encrypt | (disabled) |
   178→| `BIND_ADDR` | HTTP bind address | `0.0.0.0:3000` |
   179→| `DATA_DIR` | Data directory for PBF files and index | `/data` |
   180→| `CACHE_DIR` | ACME certificate cache directory | `acme-cache` |
   181→
   182→## License
   183→
   184→    Apache License, Version 2.0
   185→
   186→    Licensed under the Apache License, Version 2.0 (the "License");
   187→    you may not use this file except in compliance with the License.
   188→    You may obtain a copy of the License at
   189→
   190→        http://www.apache.org/licenses/LICENSE-2.0
   191→
   192→    Unless required by applicable law or agreed to in writing, software
   193→    distributed under the License is distributed on an "AS IS" BASIS,
   194→    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   195→    See the License for the specific language governing permissions and
   196→    limitations under the License.
   197→


Found 6 files
src/build_index.cpp
src/pbf_reader.cpp
src/pbf_reader.h
src/continent_filter.h
src/s2_helpers.h
src/cache.cpp


   150→            std::string mode_str = argv[++i];
   151→            if (mode_str == "full") {
   152→                mode = IndexMode::Full;
   153→            } else if (mode_str == "no-addresses") {
   154→                mode = IndexMode::NoAddresses;
   155→            } else if (mode_str == "admin-only") {
   156→                mode = IndexMode::AdminOnly;
   157→            } else {
   158→                std::cerr << "Error: unknown mode '" << mode_str << "'" << std::endl;
   159→                return 1;
   160→            }
   161→        } else {
   162→            input_files.push_back(arg);
   163→        }
   164→    }
   165→
   166→    // Set global simplification mode
   167→    kSimplifyMode = simplify_mode;
   168→    kSimplifyEpsilonOverride = simplify_epsilon_override;
   169→    if (simplify_mode == SimplifyMode::ErrorBounded) {
   170→        if (simplify_epsilon_override > 0) {
   171→            std::cerr << "Simplification: error-bounded, fixed " << simplify_epsilon_override << "m epsilon" << std::endl;
   172→        } else {
   173→            std::cerr << "Simplification: error-bounded, scale=" << kEpsilonScale
   174→                      << ", epsilons: L2=" << admin_epsilon_meters(2)
   175→                      << "m L4=" << admin_epsilon_meters(4)
   176→                      << "m L6=" << admin_epsilon_meters(6)
   177→                      << "m L8=" << admin_epsilon_meters(8) << "m" << std::endl;
   178→        }
   179→    }
   180→
   181→    ParsedData data;
   182→    auto _pt = std::chrono::steady_clock::now();
   183→    auto _cpu = CpuTicks::now();
   184→
   185→    // Load previous build's string pool for stable offsets across builds
   186→    if (!previous_build_path.empty()) {
   187→        std::string prev_strings = previous_build_path + "/strings.bin";
   188→        if (data.string_pool.load_previous(prev_strings)) {
   189→            data.string_pool.mark_previous_loaded();
   190→            std::cerr << "Loaded previous string pool: " << data.string_pool.previous_size()
   191→                      << " bytes from " << prev_strings << std::endl;
   192→        } else {
   193→            std::cerr << "Warning: could not load previous string pool from " << prev_strings << std::endl;
   194→        }
   195→    }
   196→
   197→    if (!load_cache_path.empty()) {
   198→        // Load from cache
   199→        if (!deserialize_cache(data, load_cache_path)) {
   200→            std::cerr << "Error: failed to load cache" << std::endl;
   201→            return 1;
   202→        }
   203→    } else {
   204→        // Parse PBF files (always collect everything)
   205→        if (input_files.empty()) {
   206→            std::cerr << "Error: no input files specified and no --load-cache" << std::endl;
   207→            return 1;
   208→        }
   209→
   210→        // Create thread pool for concurrent admin polygon S2 covering
   211→        unsigned int num_threads = std::max(1u, std::thread::hardware_concurrency() - 1);
   212→        std::cerr << "Using " << num_threads << " worker threads." << std::endl;
   213→        AdminCoverPool admin_pool(num_threads);
   214→        // BuildHandler no longer used — parallel processing handles everything
   215→
   216→        // Dense array node location index — lockless parallel writes
   217→        // Planet OSM node IDs max ~12.5 billion. 8 bytes per entry = 100GB virtual.
   218→        // MAP_NORESERVE means OS only allocates pages on write (~80GB for 10B nodes).
   219→        static const size_t MAX_NODE_ID = MAX_NODE_ID_DEFAULT;
   220→        struct PackedLocation {
   221→            int32_t lat_e7;  // latitude * 10^7
   222→            int32_t lon_e7;  // longitude * 10^7
   223→            bool valid() const { return lat_e7 != 0 || lon_e7 != 0; }
   224→            double lat() const { return lat_e7 / 10000000.0; }
   225→            double lon() const { return lon_e7 / 10000000.0; }
   226→        };
   227→        struct DenseIndex {
   228→            PackedLocation* data;
   229→            size_t capacity;
   230→
   231→            DenseIndex() : capacity(MAX_NODE_ID) {
   232→                size_t byte_size = capacity * sizeof(PackedLocation);
   233→                void* ptr = mmap(nullptr, byte_size,
   234→                    PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE,
   235→                    -1, 0);
   236→                if (ptr == MAP_FAILED) {
   237→                    std::cerr << "Error: failed to mmap dense node index ("
   238→                              << (byte_size / (1024*1024*1024)) << "GB)" << std::endl;
   239→                    std::exit(1);
   240→                }
   241→                madvise(ptr, byte_size, MADV_HUGEPAGE);
   242→                data = static_cast<PackedLocation*>(ptr);
   243→                std::cerr << "Allocated dense node index: " << (byte_size / (1024*1024*1024))
   244→                          << "GB virtual address space" << std::endl;
   245→            }
   246→
   247→            ~DenseIndex() {
   248→                munmap(data, capacity * sizeof(PackedLocation));
   249→            }
   250→
   251→            // Lockless — each node ID maps to a unique array slot
   252→            void set(uint64_t id, double lat, double lng) {
   253→                if (id >= capacity) return;
   254→                data[id] = {static_cast<int32_t>(lat * 10000000.0 + (lat >= 0 ? 0.5 : -0.5)),
   255→                            static_cast<int32_t>(lng * 10000000.0 + (lng >= 0 ? 0.5 : -0.5))};
   256→            }
   257→
   258→            PackedLocation get(uint64_t id) const {
   259→                if (id >= capacity) return {0, 0};
   260→                return data[id];
   261→            }
   262→        } index;
   263→
   264→        for (const auto& input_file : input_files) {
   265→            std::cerr << "Processing " << input_file << "..." << std::endl;
   266→
   267→            // --- Pass 1: collect relation members for parallel admin assembly ---
   268→            std::cerr << "  Pass 1: scanning relations..." << std::endl;
   269→            PbfFile pbf(input_file, num_threads);
   270→
   271→            {
   272→                std::mutex rel_mutex;
   273→                pbf.read_blocks([&](PbfBlock& block, unsigned) {
   274→                    for (auto& rel : block.relations) {
   275→                        const char* boundary = rel.tag("boundary");
   276→                        if (!boundary) continue;
   277→
   278→                        bool is_admin = (std::strcmp(boundary, "administrative") == 0);
   279→                        bool is_postal = (std::strcmp(boundary, "postal_code") == 0);
   280→                        if (!is_admin && !is_postal) continue;
   281→
   282→                        uint8_t admin_level = 0;
   283→                        if (is_admin) {
   284→                            const char* level_str = rel.tag("admin_level");
   285→                            if (!level_str) continue;
   286→                            admin_level = static_cast<uint8_t>(std::atoi(level_str));
   287→                            if (admin_level < 2 || admin_level > 10) continue;
   288→                        } else {
   289→                            admin_level = 11;
   290→                        }
   291→                        if (kMaxAdminLevel > 0 && admin_level > kMaxAdminLevel) continue;
   292→
   293→                        const char* name = rel.tag("name");
   294→                        if (!name && is_admin) continue;
   295→
   296→                        std::string name_str;
   297→                        if (is_postal) {
   298→                            const char* postal_code = rel.tag("postal_code");
   299→                            if (!postal_code) postal_code = name;
   300→                            if (!postal_code) continue;
   301→                            name_str = postal_code;
   302→                        } else {
   303→                            name_str = name;
   304→                        }
   305→
   306→                        std::string country_code;
   307→                        if (admin_level == 2) {
   308→                            const char* cc = rel.tag("ISO3166-1:alpha2");
   309→                            if (cc) country_code = cc;
   310→                        }
   311→
   312→                        CollectedRelation cr;
   313→                        cr.id = rel.id;
   314→                        cr.admin_level = admin_level;
   315→                        cr.name = std::move(name_str);
   316→                        cr.country_code = std::move(country_code);
   317→                        cr.is_postal = is_postal;
   318→
   319→                        for (size_t mi = 0; mi < rel.members.size(); mi++) {
   320→                            if (rel.members[mi].type == 'w') {
   321→                                cr.members.emplace_back(rel.members[mi].ref, rel.member_role(mi));
   322→                            }
   323→                        }
   324→
   325→                        if (!cr.members.empty()) {
   326→                            std::lock_guard<std::mutex> lock(rel_mutex);
   327→                            data.collected_relations.push_back(std::move(cr));
   328→                        }
   329→                    }
   330→                }, "r");
   331→            }
   332→            std::cerr << "  Collected " << data.collected_relations.size()
   333→                      << " admin/postal relations for parallel assembly." << std::endl;
   334→            log_phase("Pass 1: relation scanning", _pt, _cpu);
   335→
   336→            // Build admin_way_ids as a bitset for O(1) lookup
   337→            // Max way ID ~1.3B → ~162 MiB bitset (much faster than unordered_set)
   338→            static constexpr size_t MAX_WAY_ID = 2000000000ULL; // 2B, ~250 MiB
   339→            std::vector<uint8_t> admin_way_bits(MAX_WAY_ID / 8 + 1, 0);
   340→            size_t admin_way_count = 0;
   341→            for (const auto& rel : data.collected_relations) {
   342→                for (const auto& [way_id, role] : rel.members) {
   343→                    if (way_id > 0 && static_cast<size_t>(way_id) < MAX_WAY_ID) {
   344→                        admin_way_bits[way_id / 8] |= (1 << (way_id % 8));
   345→                        admin_way_count++;
   346→                    }
   347→                }
   348→            }
   349→            auto is_admin_way = [&](int64_t id) -> bool {
   350→                if (id <= 0 || static_cast<size_t>(id) >= MAX_WAY_ID) return false;
   351→                return admin_way_bits[id / 8] & (1 << (id % 8));
   352→            };
   353→            std::cerr << "  Admin assembly needs " << admin_way_count << " way geometries." << std::endl;
   354→
   355→            // --- Pass 2: Node processing (streaming — no PbfNode objects) ---
   356→            std::cerr << "  Pass 2: processing nodes with " << num_threads << " threads..." << std::endl;
   357→            {
   358→                struct NodeThreadLocal {
   359→                    std::vector<std::pair<double,double>> addr_coords;
   360→                    std::vector<std::pair<std::string,std::string>> addr_strings;
   361→                    uint64_t count = 0;
   362→                };
   363→                // Use thread_local for streaming callback (no thread index available)
   364→                static thread_local NodeThreadLocal* tl_node_data = nullptr;
   365→                std::vector<NodeThreadLocal> ntld(num_threads);
   366→                std::atomic<unsigned> next_tl{0};
   367→
   368→                pbf.read_nodes_streaming([&](int64_t id, double lat, double lng,
   369→                    const uint32_t* tag_keys, const uint32_t* tag_vals, size_t ntags,
   370→                    const std::vector<std::string>& st) {
   371→
   372→                    // Lazy init thread-local pointer
   373→                    if (!tl_node_data) {
   374→                        unsigned idx = next_tl.fetch_add(1);
   375→                        tl_node_data = &ntld[idx % ntld.size()];
   376→                    }
   377→
   378→                    if (id > 0) {
   379→                        index.set(static_cast<uint64_t>(id), lat, lng);
   380→                    }
   381→
   382→                    // Check for address tags (fast — no string copies for tagless nodes)
   383→                    if (ntags > 0) {
   384→                        const char* housenumber = nullptr;
   385→                        const char* street = nullptr;
   386→                        for (size_t i = 0; i < ntags; i++) {
   387→                            if (tag_keys[i] < st.size()) {
   388→                                if (st[tag_keys[i]] == "addr:housenumber")
   389→                                    housenumber = tag_vals[i] < st.size() ? st[tag_vals[i]].c_str() : nullptr;
   390→                                else if (st[tag_keys[i]] == "addr:street")
   391→                                    street = tag_vals[i] < st.size() ? st[tag_vals[i]].c_str() : nullptr;
   392→                            }
   393→                        }
   394→                        if (housenumber && street) {
   395→                            tl_node_data->addr_coords.push_back({lat, lng});
   396→                            tl_node_data->addr_strings.push_back({housenumber, street});
   397→                            tl_node_data->count++;
   398→                        }
   399→                    }
   400→                });
   401→
   402→                // Reset thread_local for next use
   403→                tl_node_data = nullptr;
   404→
   405→                // Merge address points
   406→                uint64_t total_addrs = 0;
   407→                for (auto& local : ntld) {
   408→                    for (size_t j = 0; j < local.addr_coords.size(); j++) {
   409→                        uint64_t dummy = 0;
   410→                        add_addr_point(data, local.addr_coords[j].first, local.addr_coords[j].second,
   411→                                       local.addr_strings[j].first.c_str(),
   412→                                       local.addr_strings[j].second.c_str(), dummy);
   413→                    }
   414→                    total_addrs += local.count;
   415→                }
   416→                std::cerr << "  Node processing complete: " << total_addrs
   417→                          << " address points collected." << std::endl;
   418→            }
   419→            log_phase("Pass 2: node processing", _pt, _cpu);
   420→            pbf.release_pages(); // free PBF mmap pages, will re-fault for way pass
   421→
   422→            // --- Pass 2b: Way processing (fully parallel) ---
   423→            std::cerr << "  Processing ways with " << num_threads << " threads..." << std::endl;
   424→            {
   425→                // Thread-local data for parallel way processing
   426→                struct ThreadLocalData {
   427→                    std::vector<WayHeader> ways;
   428→                    std::vector<NodeCoord> street_nodes;
   429→                    std::vector<DeferredWay> deferred_ways;
   430→                    std::vector<InterpWay> interp_ways;
   431→                    std::vector<NodeCoord> interp_nodes;
   432→                    std::vector<DeferredInterp> deferred_interps;
   433→                    std::vector<AddrPoint> building_addrs;
   434→                    std::vector<std::pair<double, double>> building_addr_coords; // lat,lng for S2 cell
   435→                    std::vector<std::string> way_strings;      // way name strings
   436→                    std::vector<std::pair<std::string,std::string>> addr_strings; // building addr {hn, street}
   437→                    std::vector<std::string> interp_strings;   // interp street name strings
   438→                    uint64_t way_count = 0;
   439→                    uint64_t building_addr_count = 0;
   440→                    uint64_t interp_count = 0;
   441→                    // Way geometries for parallel admin assembly
   442→                    struct WayGeomEntry {
   443→                        int64_t way_id;
   444→                        std::vector<std::pair<double,double>> coords;
   445→                        int64_t first_node_id;
   446→                        int64_t last_node_id;
   447→                    };
   448→                    std::vector<WayGeomEntry> way_geoms;
   449→                    // Closed-way admin polygons (ways with boundary=administrative)


Now let me continue reading build_index.cpp to find the final output section and check the continent_filter more:


grep -n "write_index\|continents\|--continents\|generate_continents" /root/traccar-geocoder/builder/src/build_index.cpp | head -40