     1→#include <algorithm>
     2→#include <atomic>
     3→#include <chrono>
     4→#include <condition_variable>
     5→#include <cstring>
     6→#include <deque>
     7→#include <fstream>
     8→#include <future>
     9→#include <iostream>
    10→#include <mutex>
    11→#include <numeric>
    12→#include <string>
    13→#include <thread>
    14→#include <unordered_set>
    15→#include <vector>
    16→
    17→#include <fcntl.h>
    18→#include <sys/mman.h>
    19→
    20→#include "pbf_reader.h"
    21→
    22→#include <s2/s2cell_id.h>
    23→#include <s2/s2latlng.h>
    24→
    25→#include "types.h"
    26→#include "string_pool.h"
    27→#include "geometry.h"
    28→#include "parsed_data.h"
    29→#include "s2_helpers.h"
    30→#include "ring_assembly.h"
    31→#include "continent_boundaries.h"
    32→#include "interpolation.h"
    33→#include "cache.h"
    34→#include "continent_filter.h"
    35→#include "cell_index.h"
    36→
    37→
    38→// --- Main ---
    39→
    40→int main(int argc, char* argv[]) {
    41→    if (argc < 2) {
    42→        std::cerr << "Usage: build-index <output-dir> <input.osm.pbf> [options]" << std::endl;
    43→        std::cerr << "       build-index <output-dir> --load-cache <path> [options]" << std::endl;
    44→        std::cerr << std::endl;
    45→        std::cerr << "Options:" << std::endl;
    46→        std::cerr << "  --street-level N       S2 cell level for streets (default: 17)" << std::endl;
    47→        std::cerr << "  --admin-level N        S2 cell level for admin (default: 10)" << std::endl;
    48→        std::cerr << "  --max-admin-level N    Only include admin levels <= N" << std::endl;
    49→        std::cerr << "  --save-cache <path>    Save parsed data to cache file" << std::endl;
    50→        std::cerr << "  --load-cache <path>    Load from cache instead of PBF" << std::endl;
    51→        std::cerr << "  --multi-output         Write full, no-addresses, admin-only indexes" << std::endl;
    52→        std::cerr << "  --continents           Also generate per-continent indexes" << std::endl;
    53→        std::cerr << "  --mode <mode>          Index mode: full, no-addresses, admin-only (default: full)" << std::endl;
    54→        std::cerr << "  --admin-only           Shorthand for --mode admin-only" << std::endl;
    55→        std::cerr << "  --no-addresses         Shorthand for --mode no-addresses" << std::endl;
    56→        std::cerr << "  --simplify-epsilon     Use error-bounded simplification (per-level defaults)" << std::endl;
    57→        std::cerr << "  --simplify-epsilon N   Use fixed epsilon of N meters for all levels" << std::endl;
    58→        std::cerr << "  --epsilon-scale F      Multiply all per-level epsilons by F (e.g. 2.0 = 2x coarser)" << std::endl;
    59→        std::cerr << "  --epsilon-levels L2,L3,...,L8  Set epsilon in meters per level (7 values)" << std::endl;
    60→        std::cerr << "  --multi-quality [S,S,...]  Write quality variants at given scales (default: 0,0.2,0.5,1,1.5,2,2.5)" << std::endl;
    61→        std::cerr << "  --previous-build <dir>     Load previous build for stable string offsets (patchability)" << std::endl;
    62→        return 1;
    63→    }
    64→
    65→    std::string output_dir = argv[1];
    66→    std::vector<std::string> input_files;
    67→    std::string save_cache_path;
    68→    std::string load_cache_path;
    69→    std::string previous_build_path;
    70→    bool multi_output = false;
    71→    bool generate_continents = false;
    72→    IndexMode mode = IndexMode::Full;
    73→    SimplifyMode simplify_mode = SimplifyMode::MaxVertices;
    74→    double simplify_epsilon_override = 0; // 0 = use per-level defaults
    75→    bool multi_quality = false;
    76→    std::vector<double> quality_scales;
    77→
    78→    for (int i = 2; i < argc; i++) {
    79→        std::string arg = argv[i];
    80→        if (arg == "--street-level" && i + 1 < argc) {
    81→            kStreetCellLevel = std::atoi(argv[++i]);
    82→        } else if (arg == "--admin-level" && i + 1 < argc) {
    83→            kAdminCellLevel = std::atoi(argv[++i]);
    84→        } else if (arg == "--admin-only") {
    85→            mode = IndexMode::AdminOnly;
    86→        } else if (arg == "--no-addresses") {
    87→            mode = IndexMode::NoAddresses;
    88→        } else if (arg == "--max-admin-level" && i + 1 < argc) {
    89→            kMaxAdminLevel = std::atoi(argv[++i]);
    90→        } else if (arg == "--save-cache" && i + 1 < argc) {
    91→            save_cache_path = argv[++i];
    92→        } else if (arg == "--load-cache" && i + 1 < argc) {
    93→            load_cache_path = argv[++i];
    94→        } else if (arg == "--previous-build" && i + 1 < argc) {
    95→            previous_build_path = argv[++i];
    96→        } else if (arg == "--multi-output") {
    97→            multi_output = true;
    98→        } else if (arg == "--continents") {
    99→            generate_continents = true;
   100→        } else if (arg == "--simplify-epsilon") {


     1→#include "continent_filter.h"
     2→#include "continent_boundaries.h"
     3→#include "parsed_data.h"
     4→
     5→#include <algorithm>
     6→#include <cstring>
     7→#include <future>
     8→#include <unordered_set>
     9→
    10→#include <s2/s2cell_id.h>
    11→#include <s2/s2latlng.h>
    12→
    13→const ContinentBBox kContinents[] = {
    14→    {"africa",            -35.0,  37.5,  -25.0,  55.0},
    15→    {"asia",              -12.0,  82.0,   25.0, 180.0},
    16→    {"europe",             35.0,  72.0,  -25.0,  45.0},
    17→    {"north-america",       7.0,  84.0, -170.0, -50.0},
    18→    {"south-america",     -56.0,  13.0,  -82.0, -34.0},
    19→    {"oceania",           -50.0,   0.0,  110.0, 180.0},
    20→    {"central-america",     7.0,  23.5, -120.0, -57.0},
    21→    {"antarctica",        -90.0, -60.0, -180.0, 180.0},
    22→};
    23→
    24→const size_t kContinentCount = sizeof(kContinents) / sizeof(kContinents[0]);
    25→
    26→static bool cell_in_bbox(uint64_t cell_id, const ContinentBBox& bbox) {
    27→    S2CellId cell(cell_id);
    28→    S2LatLng center = cell.ToLatLng();
    29→    double lat = center.lat().degrees();
    30→    double lng = center.lng().degrees();
    31→    return lat >= bbox.min_lat && lat <= bbox.max_lat &&
    32→           lng >= bbox.min_lng && lng <= bbox.max_lng;
    33→}
    34→
    35→ParsedData filter_by_bbox(const ParsedData& full, const ContinentBBox& bbox) {
    36→    ParsedData out;
    37→    auto _ft = std::chrono::steady_clock::now();
    38→    auto _fc = CpuTicks::now();
    39→
    40→    std::cerr << "    data: ways_sorted=" << full.sorted_way_cells.size()
    41→              << " addrs_sorted=" << full.sorted_addr_cells.size()
    42→              << " interps_sorted=" << full.sorted_interp_cells.size()
    43→              << " ways_map=" << full.cell_to_ways.size()
    44→              << " addrs_map=" << full.cell_to_addrs.size()
    45→              << " interps_map=" << full.cell_to_interps.size()
    46→              << " admin_map=" << full.cell_to_admin.size()
    47→              << std::endl;
    48→
    49→    // Filter from hash map
    50→    auto filter_cells_map = [&](const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map,
    51→                                bool mask_interior = false) {
    52→        std::cerr << "    DEBUG filter_cells_map: size=" << cell_map.size() << std::endl;
    53→        std::unordered_set<uint32_t> ids;
    54→        for (const auto& [cell_id, cell_ids] : cell_map) {
    55→            if (cell_in_bbox(cell_id, bbox)) {
    56→                for (uint32_t id : cell_ids)
    57→                    ids.insert(mask_interior ? (id & ID_MASK) : id);
    58→            }
    59→        }
    60→        return ids;
    61→    };
    62→
    63→    // Filter from sorted pairs — parallel chunked scan.
    64→    // Split the sorted array into N chunks at cell boundaries, each thread
    65→    // builds its own ID set, then merge.
    66→    auto filter_cells_sorted = [&](const std::vector<CellItemPair>& sorted,
    67→                                   bool mask_interior = false) {
    68→        if (sorted.empty()) return std::unordered_set<uint32_t>{};
    69→
    70→        unsigned nthreads = std::max(1u, std::thread::hardware_concurrency() / 4);
    71→        size_t chunk = (sorted.size() + nthreads - 1) / nthreads;
    72→        std::cerr << "    DEBUG filter_cells_sorted: size=" << sorted.size()
    73→                  << " nthreads=" << nthreads << " chunk=" << chunk << std::endl;
    74→
    75→        // Find chunk boundaries at cell_id transitions
    76→        std::vector<size_t> bounds = {0};
    77→        for (unsigned t = 1; t < nthreads; t++) {
    78→            size_t target = t * chunk;
    79→            if (target >= sorted.size()) break;
    80→            while (target < sorted.size() && sorted[target].cell_id == sorted[target-1].cell_id)
    81→                target++;
    82→            if (target < sorted.size()) bounds.push_back(target);
    83→        }
    84→        bounds.push_back(sorted.size());
    85→
    86→        std::vector<std::unordered_set<uint32_t>> thread_ids(bounds.size() - 1);
    87→        std::vector<std::thread> threads;
    88→        for (size_t t = 0; t + 1 < bounds.size(); t++) {
    89→            threads.emplace_back([&, t]() {
    90→                auto& local = thread_ids[t];
    91→                for (size_t i = bounds[t]; i < bounds[t+1]; ) {
    92→                    uint64_t cell_id = sorted[i].cell_id;
    93→                    bool match = cell_in_bbox(cell_id, bbox);
    94→                    while (i < bounds[t+1] && sorted[i].cell_id == cell_id) {
    95→                        if (match) {
    96→                            uint32_t id = sorted[i].item_id;
    97→                            local.insert(mask_interior ? (id & ID_MASK) : id);
    98→                        }
    99→                        i++;
   100→                    }
   101→                }
   102→            });
   103→        }
   104→        std::cerr << "    DEBUG: spawned " << threads.size() << " threads, "
   105→                  << bounds.size() - 1 << " chunks" << std::endl;
   106→        for (auto& t : threads) t.join();
   107→
   108→        // Merge thread-local sets
   109→        size_t largest = 0;
   110→        for (size_t t = 0; t < thread_ids.size(); t++)
   111→            if (thread_ids[t].size() > thread_ids[largest].size()) largest = t;
   112→        auto& merged = thread_ids[largest];
   113→        for (size_t t = 0; t < thread_ids.size(); t++) {
   114→            if (t == largest) continue;
   115→            for (auto id : thread_ids[t]) merged.insert(id);
   116→        }
   117→        return std::move(merged);
   118→    };
   119→
   120→    // Use sorted pairs where available (faster linear scan), fall back to hash maps
   121→    auto f_ways = std::async(std::launch::async, [&]{
   122→        return !full.sorted_way_cells.empty()
   123→            ? filter_cells_sorted(full.sorted_way_cells)
   124→            : filter_cells_map(full.cell_to_ways);
   125→    });
   126→    auto f_addrs = std::async(std::launch::async, [&]{
   127→        return !full.sorted_addr_cells.empty()
   128→            ? filter_cells_sorted(full.sorted_addr_cells)
   129→            : filter_cells_map(full.cell_to_addrs);
   130→    });
   131→    auto f_interps = std::async(std::launch::async, [&]{
   132→        return !full.sorted_interp_cells.empty()
   133→            ? filter_cells_sorted(full.sorted_interp_cells)
   134→            : filter_cells_map(full.cell_to_interps);
   135→    });
   136→    auto f_admin = std::async(std::launch::async, [&]{
   137→        return filter_cells_map(full.cell_to_admin, true);
   138→    });
   139→
   140→    auto used_way_ids = f_ways.get();
   141→    auto used_addr_ids = f_addrs.get();
   142→    auto used_interp_ids = f_interps.get();
   143→    auto used_admin_ids = f_admin.get();
   144→    log_phase("      filter: ID collection", _ft, _fc);
   145→
   146→    // Remap all 4 data types in parallel — each builds its own vectors and remap table
   147→    std::unordered_map<uint32_t, uint32_t> way_remap, addr_remap, interp_remap, admin_remap;
   148→
   149→    // Ways
   150→    auto f_remap_ways = std::async(std::launch::async, [&]() {


Bash

grep -r "\.bin" /root/traccar-geocoder/builder/src --include="*.cpp" --include="*.h" | head -30


/root/traccar-geocoder/builder/src/cell_index.cpp:                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
/root/traccar-geocoder/builder/src/cell_index.cpp:                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
/root/traccar-geocoder/builder/src/cell_index.cpp:                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
/root/traccar-geocoder/builder/src/cell_index.cpp:                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
/root/traccar-geocoder/builder/src/cell_index.cpp:                        return write_entries_from_sorted(output_dir + "/interp_entries.bin", sorted_geo_cells, data.sorted_interp_cells);
/root/traccar-geocoder/builder/src/cell_index.cpp:                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
/root/traccar-geocoder/builder/src/cell_index.cpp:            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
/root/traccar-geocoder/builder/src/cell_index.cpp:    log_phase("  Write: geo_cells.bin", _wt);
/root/traccar-geocoder/builder/src/cell_index.cpp:        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
/root/traccar-geocoder/builder/src/cell_index.cpp:            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
/root/traccar-geocoder/builder/src/cell_index.cpp:            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
/root/traccar-geocoder/builder/src/cell_index.cpp:                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
/root/traccar-geocoder/builder/src/cell_index.cpp:                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
/root/traccar-geocoder/builder/src/cell_index.cpp:                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
/root/traccar-geocoder/builder/src/cell_index.cpp:    // before writing admin_polygons and strings.bin.
/root/traccar-geocoder/builder/src/cell_index.cpp:        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
/root/traccar-geocoder/builder/src/cell_index.cpp:        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
/root/traccar-geocoder/builder/src/cell_index.cpp:            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
/root/traccar-geocoder/builder/src/cell_index.cpp:            std::cerr << "strings.bin: " << compact_strings.size() << " bytes (compacted from "
/root/traccar-geocoder/builder/src/cell_index.cpp:            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
/root/traccar-geocoder/builder/src/cell_index.cpp:            std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
/root/traccar-geocoder/builder/src/cell_index.cpp:        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
/root/traccar-geocoder/builder/src/cell_index.cpp:        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
/root/traccar-geocoder/builder/src/cell_index.cpp:    // Shared files (admin_cells.bin, admin_entries.bin, strings.bin)
/root/traccar-geocoder/builder/src/build_index.cpp:        std::string prev_strings = previous_build_path + "/strings.bin";
/root/traccar-geocoder/builder/src/build_index.cpp:            "admin_cells.bin", "admin_entries.bin", "strings.bin"
/root/traccar-geocoder/builder/src/build_index.cpp:            "admin_polygons.bin", "admin_vertices.bin"
/root/traccar-geocoder/builder/src/build_index.cpp:            "street_ways.bin", "street_nodes.bin", "street_entries.bin"
/root/traccar-geocoder/builder/src/build_index.cpp:            "addr_points.bin", "addr_entries.bin",
/root/traccar-geocoder/builder/src/build_index.cpp:            "interp_ways.bin", "interp_nodes.bin", "interp_entries.bin"


     1→#include <algorithm>
     2→#include <atomic>
     3→#include <chrono>
     4→#include <condition_variable>
     5→#include <cstring>
     6→#include <deque>
     7→#include <fstream>
     8→#include <future>
     9→#include <iostream>
    10→#include <mutex>
    11→#include <numeric>
    12→#include <string>
    13→#include <thread>
    14→#include <unordered_set>
    15→#include <vector>
    16→
    17→#include <fcntl.h>
    18→#include <sys/mman.h>
    19→
    20→#include "pbf_reader.h"
    21→
    22→#include <s2/s2cell_id.h>
    23→#include <s2/s2latlng.h>
    24→
    25→#include "types.h"
    26→#include "string_pool.h"
    27→#include "geometry.h"
    28→#include "parsed_data.h"
    29→#include "s2_helpers.h"
    30→#include "ring_assembly.h"
    31→#include "continent_boundaries.h"
    32→#include "interpolation.h"
    33→#include "cache.h"
    34→#include "continent_filter.h"
    35→#include "cell_index.h"
    36→
    37→
    38→// --- Main ---
    39→
    40→int main(int argc, char* argv[]) {
    41→    if (argc < 2) {
    42→        std::cerr << "Usage: build-index <output-dir> <input.osm.pbf> [options]" << std::endl;
    43→        std::cerr << "       build-index <output-dir> --load-cache <path> [options]" << std::endl;
    44→        std::cerr << std::endl;
    45→        std::cerr << "Options:" << std::endl;
    46→        std::cerr << "  --street-level N       S2 cell level for streets (default: 17)" << std::endl;
    47→        std::cerr << "  --admin-level N        S2 cell level for admin (default: 10)" << std::endl;
    48→        std::cerr << "  --max-admin-level N    Only include admin levels <= N" << std::endl;
    49→        std::cerr << "  --save-cache <path>    Save parsed data to cache file" << std::endl;
    50→        std::cerr << "  --load-cache <path>    Load from cache instead of PBF" << std::endl;
    51→        std::cerr << "  --multi-output         Write full, no-addresses, admin-only indexes" << std::endl;
    52→        std::cerr << "  --continents           Also generate per-continent indexes" << std::endl;
    53→        std::cerr << "  --mode <mode>          Index mode: full, no-addresses, admin-only (default: full)" << std::endl;
    54→        std::cerr << "  --admin-only           Shorthand for --mode admin-only" << std::endl;
    55→        std::cerr << "  --no-addresses         Shorthand for --mode no-addresses" << std::endl;
    56→        std::cerr << "  --simplify-epsilon     Use error-bounded simplification (per-level defaults)" << std::endl;
    57→        std::cerr << "  --simplify-epsilon N   Use fixed epsilon of N meters for all levels" << std::endl;
    58→        std::cerr << "  --epsilon-scale F      Multiply all per-level epsilons by F (e.g. 2.0 = 2x coarser)" << std::endl;
    59→        std::cerr << "  --epsilon-levels L2,L3,...,L8  Set epsilon in meters per level (7 values)" << std::endl;
    60→        std::cerr << "  --multi-quality [S,S,...]  Write quality variants at given scales (default: 0,0.2,0.5,1,1.5,2,2.5)" << std::endl;
    61→        std::cerr << "  --previous-build <dir>     Load previous build for stable string offsets (patchability)" << std::endl;
    62→        return 1;
    63→    }
    64→
    65→    std::string output_dir = argv[1];
    66→    std::vector<std::string> input_files;
    67→    std::string save_cache_path;
    68→    std::string load_cache_path;
    69→    std::string previous_build_path;
    70→    bool multi_output = false;
    71→    bool generate_continents = false;
    72→    IndexMode mode = IndexMode::Full;
    73→    SimplifyMode simplify_mode = SimplifyMode::MaxVertices;
    74→    double simplify_epsilon_override = 0; // 0 = use per-level defaults
    75→    bool multi_quality = false;
    76→    std::vector<double> quality_scales;
    77→
    78→    for (int i = 2; i < argc; i++) {
    79→        std::string arg = argv[i];
    80→        if (arg == "--street-level" && i + 1 < argc) {
    81→            kStreetCellLevel = std::atoi(argv[++i]);
    82→        } else if (arg == "--admin-level" && i + 1 < argc) {
    83→            kAdminCellLevel = std::atoi(argv[++i]);
    84→        } else if (arg == "--admin-only") {
    85→            mode = IndexMode::AdminOnly;
    86→        } else if (arg == "--no-addresses") {
    87→            mode = IndexMode::NoAddresses;
    88→        } else if (arg == "--max-admin-level" && i + 1 < argc) {
    89→            kMaxAdminLevel = std::atoi(argv[++i]);
    90→        } else if (arg == "--save-cache" && i + 1 < argc) {
    91→            save_cache_path = argv[++i];
    92→        } else if (arg == "--load-cache" && i + 1 < argc) {
    93→            load_cache_path = argv[++i];
    94→        } else if (arg == "--previous-build" && i + 1 < argc) {
    95→            previous_build_path = argv[++i];
    96→        } else if (arg == "--multi-output") {
    97→            multi_output = true;
    98→        } else if (arg == "--continents") {
    99→            generate_continents = true;
   100→        } else if (arg == "--simplify-epsilon") {
   101→            simplify_mode = SimplifyMode::ErrorBounded;
   102→            // Optional: next arg might be a number (fixed epsilon in meters)
   103→            if (i + 1 < argc) {
   104→                char* end;
   105→                double val = std::strtod(argv[i+1], &end);
   106→                if (*end == '\0' && val > 0) {
   107→                    simplify_epsilon_override = val;
   108→                    i++;
   109→                }
   110→            }
   111→        } else if (arg == "--epsilon-scale" && i + 1 < argc) {
   112→            kEpsilonScale = std::strtod(argv[++i], nullptr);
   113→        } else if (arg == "--multi-quality") {
   114→            multi_quality = true;
   115→            // Optional: next arg might be comma-separated scales
   116→            if (i + 1 < argc && argv[i+1][0] != '-') {
   117→                std::string vals = argv[++i];
   118→                size_t pos = 0;
   119→                while (pos < vals.size()) {
   120→                    size_t next = vals.find(',', pos);
   121→                    if (next == std::string::npos) next = vals.size();
   122→                    double v = std::strtod(vals.substr(pos, next - pos).c_str(), nullptr);
   123→                    quality_scales.push_back(v);
   124→                    pos = next + 1;
   125→                }
   126→            }
   127→            if (quality_scales.empty()) {
   128→                quality_scales = {0, 0.2, 0.5, 1.0, 1.5, 2.0, 2.5};
   129→            }
   130→            // Build with uncapped polygons — quality variants simplify at write time.
   131→            // We need full vertex detail to produce all quality levels accurately.
   132→            simplify_mode = SimplifyMode::ErrorBounded;
   133→            simplify_epsilon_override = 0.1; // effectively uncapped
   134→        } else if (arg == "--epsilon-levels" && i + 1 < argc) {
   135→            // Parse comma-separated values: L2,L3,L4,L5,L6,L7,L8
   136→            std::string vals = argv[++i];
   137→            int level = 2;
   138→            size_t pos = 0;
   139→            while (pos < vals.size() && level <= 8) {
   140→                size_t next = vals.find(',', pos);
   141→                if (next == std::string::npos) next = vals.size();
   142→                double v = std::strtod(vals.substr(pos, next - pos).c_str(), nullptr);
   143→                if (v > 0) kAdminEpsilonMeters[level] = v;
   144→                level++;
   145→                pos = next + 1;
   146→            }
   147→            // Also set levels 9-11 to same as level 8
   148→            for (int l = level; l <= 11; l++) kAdminEpsilonMeters[l] = kAdminEpsilonMeters[8];
   149→        } else if (arg == "--mode" && i + 1 < argc) {
   150→            std::string mode_str = argv[++i];


find /root/traccar-geocoder -type f -name "*.md" | head -20


/root/traccar-geocoder/builder/OPTIMIZATION_LOG.md
/root/traccar-geocoder/builder/test/VALIDATION_REPORT.md
/root/traccar-geocoder/README.md


ls -la /root/traccar-geocoder/builder/ | head -40


total 77
drwxr-xr-x 8 root root   10 Mar 26 20:19 .
drwxr-xr-x 7 root root   14 Mar 21 11:08 ..
-rw-r--r-- 1 root root 1059 Mar 26 01:10 CMakeLists.txt
-rw-r--r-- 1 root root 3827 Mar 26 20:19 OPTIMIZATION_LOG.md
drwxr-xr-x 3 root root   12 Mar 27 03:30 build
drwxr-xr-x 3 root root    7 Mar 23 11:51 build-presplit
drwxr-xr-x 2 root root   10 Mar 26 01:23 data
drwxr-xr-x 2 root root   24 Mar 27 03:30 src
drwxr-xr-x 2 root root   15 Mar 26 11:11 test
drwxr-xr-x 2 root root    6 Mar 24 22:15 tools


     1→#include "cell_index.h"
     2→
     3→#include <algorithm>
     4→#include <atomic>
     5→#include <cstring>
     6→#include <fstream>
     7→#include <future>
     8→#include <iostream>
     9→#include <thread>
    10→#include <unordered_map>
    11→#include <unordered_set>
    12→
    13→#include <unistd.h>
    14→
    15→#include "geometry.h"
    16→#include "s2_helpers.h"
    17→
    18→std::vector<uint32_t> write_entries(
    19→    const std::string& path,
    20→    const std::vector<uint64_t>& sorted_cells,
    21→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
    22→) {
    23→    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
    24→    std::vector<CellRef> sorted_refs;
    25→    sorted_refs.reserve(cell_map.size());
    26→    for (auto& [id, ids] : cell_map) sorted_refs.push_back({id, &ids});
    27→    std::sort(sorted_refs.begin(), sorted_refs.end(),
    28→        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });
    29→
    30→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    31→    size_t total_size = 0;
    32→    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);
    33→
    34→    std::vector<char> buf;
    35→    buf.reserve(total_size);
    36→    uint32_t current = 0;
    37→    size_t ri = 0;
    38→    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
    39→        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
    40→        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
    41→        offsets[si] = current;
    42→        const auto& ids = *sorted_refs[ri].ids;
    43→        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
    44→        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
    45→                   reinterpret_cast<const char*>(&count) + sizeof(count));
    46→        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
    47→                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
    48→        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
    49→        ri++;
    50→    }
    51→    std::ofstream f(path, std::ios::binary);
    52→    f.write(buf.data(), buf.size());
    53→    return offsets;
    54→}
    55→
    56→std::vector<uint32_t> write_entries_from_sorted(
    57→    const std::string& path,
    58→    const std::vector<uint64_t>& sorted_cells,
    59→    const std::vector<CellItemPair>& sorted_pairs
    60→) {
    61→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    62→    if (sorted_pairs.empty()) {
    63→        std::ofstream f(path, std::ios::binary);
    64→        return offsets;
    65→    }
    66→
    67→    unsigned int nthreads = std::thread::hardware_concurrency();
    68→    if (nthreads == 0) nthreads = 4;
    69→    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;
    70→
    71→    struct ChunkResult {
    72→        std::vector<char> buf;
    73→        size_t cell_start, cell_end;
    74→        uint32_t local_size;
    75→    };
    76→    std::vector<ChunkResult> chunks(nthreads);
    77→
    78→    std::vector<std::thread> threads;
    79→    for (unsigned int t = 0; t < nthreads; t++) {
    80→        size_t cs = t * cells_per_chunk;
    81→        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
    82→        if (cs >= sorted_cells.size()) break;
    83→
    84→        threads.emplace_back([&, t, cs, ce]() {
    85→            auto& chunk = chunks[t];
    86→            chunk.cell_start = cs;
    87→            chunk.cell_end = ce;
    88→            chunk.local_size = 0;
    89→
    90→            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
    91→                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
    92→                    return p.cell_id < id;
    93→                }) - sorted_pairs.begin();
    94→
    95→            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
    96→                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
    97→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
    98→                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;
    99→
   100→                offsets[si] = chunk.local_size;
   101→                size_t start = pi;
   102→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
   103→                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(MAX_VERTEX_COUNT)));
   104→                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
   105→                size_t buf_pos = chunk.buf.size();
   106→                chunk.buf.resize(buf_pos + entry_size);
   107→                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
   108→                for (size_t k = start; k < pi; k++) {
   109→                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
   110→                           &sorted_pairs[k].item_id, sizeof(uint32_t));
   111→                }
   112→                chunk.local_size += entry_size;
   113→            }
   114→        });
   115→    }
   116→    for (auto& t : threads) t.join();
   117→
   118→    uint32_t global_offset = 0;
   119→    for (auto& chunk : chunks) {
   120→        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
   121→            if (offsets[si] != NO_DATA) offsets[si] += global_offset;
   122→        }
   123→        global_offset += chunk.local_size;
   124→    }
   125→
   126→    std::vector<char> buf;
   127→    buf.reserve(global_offset);
   128→    for (auto& chunk : chunks) buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());
   129→
   130→    std::ofstream f(path, std::ios::binary);
   131→    f.write(buf.data(), buf.size());
   132→    return offsets;
   133→}
   134→
   135→void write_cell_index(
   136→    const std::string& cells_path,
   137→    const std::string& entries_path,
   138→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
   139→) {
   140→    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(cell_map.begin(), cell_map.end());
   141→    std::sort(sorted.begin(), sorted.end());
   142→
   143→    { std::ofstream f(cells_path, std::ios::binary);
   144→      uint32_t current_offset = 0;
   145→      for (const auto& [cell_id, ids] : sorted) {
   146→          f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
   147→          f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
   148→          current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
   149→      } }
   150→
   151→    { std::ofstream f(entries_path, std::ios::binary);
   152→      for (const auto& [cell_id, ids] : sorted) {
   153→          uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
   154→          f.write(reinterpret_cast<const char*>(&count), sizeof(count));
   155→          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
   156→      } }
   157→}
   158→
   159→void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
   160→    ensure_dir(output_dir);
   161→    auto _wt = std::chrono::steady_clock::now();
   162→
   163→    bool write_streets = (mode != IndexMode::AdminOnly);
   164→    bool write_addresses = (mode == IndexMode::Full);
   165→
   166→    if (write_streets) {
   167→        std::vector<uint64_t> sorted_geo_cells;
   168→        {
   169→            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
   170→                std::vector<uint64_t> cells;
   171→                cells.reserve(pairs.size() / 2);
   172→                for (size_t i = 0; i < pairs.size(); ) {
   173→                    cells.push_back(pairs[i].cell_id);
   174→                    uint64_t cur = pairs[i].cell_id;
   175→                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
   176→                }
   177→                return cells;
   178→            };
   179→            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
   180→                std::vector<uint64_t> cells;
   181→                cells.reserve(m.size());
   182→                for (auto& [id, _] : m) cells.push_back(id);
   183→                std::sort(cells.begin(), cells.end());
   184→                return cells;
   185→            };
   186→
   187→            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
   188→            {
   189→                auto f1 = std::async(std::launch::async, [&] {
   190→                    return !data.sorted_way_cells.empty()
   191→                        ? extract_unique_cells(data.sorted_way_cells)
   192→                        : extract_from_map(data.cell_to_ways);
   193→                });
   194→                if (write_addresses) {
   195→                    auto f2 = std::async(std::launch::async, [&] {
   196→                        return !data.sorted_addr_cells.empty()
   197→                            ? extract_unique_cells(data.sorted_addr_cells)
   198→                            : extract_from_map(data.cell_to_addrs);
   199→                    });
   200→                    auto f3 = std::async(std::launch::async, [&] {


     1→# Traccar Geocoder
     2→
     3→A fast, self-hosted reverse geocoding service built from OpenStreetMap data. Given latitude and longitude coordinates, it returns the nearest street address including house number, street name, city, state, county, postcode, and country.
     4→
     5→Part of the [Traccar](https://www.traccar.org) open source GPS tracking platform. Also available as a [hosted service](https://www.traccar.org/product/geocoder/).
     6→
     7→## Features
     8→
     9→- Street-level reverse geocoding from OSM data
    10→- Address point, street name, and address interpolation lookup
    11→- Administrative boundary resolution (country, state, county, city, postcode)
    12→- Sub-millisecond query latency with memory-mapped index files
    13→- Automatic HTTPS with Let's Encrypt
    14→- Docker support with automatic PBF download and indexing
    15→
    16→## Quick Start
    17→
    18→### Docker Compose
    19→
    20→```yaml
    21→services:
    22→  geocoder:
    23→    image: traccar/traccar-geocoder
    24→    environment:
    25→      - PBF_URLS=https://download.geofabrik.de/europe/monaco-latest.osm.pbf
    26→    ports:
    27→      - "3000:3000"
    28→    volumes:
    29→      - geocoder-data:/data
    30→
    31→volumes:
    32→  geocoder-data:
    33→```
    34→
    35→```bash
    36→docker compose up
    37→```
    38→
    39→### Docker
    40→
    41→```bash
    42→# All-in-one: download, build index, and serve
    43→docker run -e PBF_URLS="https://download.geofabrik.de/europe-latest.osm.pbf" \
    44→  -v geocoder-data:/data -p 3000:3000 traccar/traccar-geocoder
    45→
    46→# Build index only
    47→docker run -e PBF_URLS="https://download.geofabrik.de/europe-latest.osm.pbf" \
    48→  -v geocoder-data:/data traccar/traccar-geocoder build
    49→
    50→# Serve only (from pre-built index)
    51→docker run -v geocoder-data:/data -p 3000:3000 traccar/traccar-geocoder serve
    52→
    53→# Multiple PBF files
    54→docker run -e PBF_URLS="https://download.geofabrik.de/europe/france-latest.osm.pbf https://download.geofabrik.de/europe/germany-latest.osm.pbf" \
    55→  -v geocoder-data:/data -p 3000:3000 traccar/traccar-geocoder
    56→
    57→# With automatic HTTPS
    58→docker run -e PBF_URLS="https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf" \
    59→  -e DOMAIN=geocoder.example.com \
    60→  -v geocoder-data:/data -p 443:443 traccar/traccar-geocoder
    61→```
    62→
    63→PBF files can be downloaded from [Geofabrik](https://download.geofabrik.de/).
    64→
    65→## API
    66→
    67→### GET /reverse
    68→
    69→Query parameters:
    70→- `lat` - latitude (required)
    71→- `lon` - longitude (required)
    72→- `key` - API key (required)
    73→
    74→Example request:
    75→
    76→```
    77→GET /reverse?lat=43.7384&lon=7.4246&key=YOUR_API_KEY
    78→```
    79→
    80→Response follows [Nominatim](https://nominatim.org/release-docs/latest/api/Reverse/) format:
    81→
    82→```json
    83→{
    84→  "display_name": "Avenue de la Costa 42, 98000 Monaco, Monaco",
    85→  "address": {
    86→    "house_number": "42",
    87→    "road": "Avenue de la Costa",
    88→    "city": "Monaco",
    89→    "state": "Monaco",
    90→    "county": "Monaco",
    91→    "postcode": "98000",
    92→    "country": "Monaco",
    93→    "country_code": "MC"
    94→  }
    95→}
    96→```
    97→
    98→Fields are omitted when not available.
    99→
   100→Status codes:
   101→- `200` - success
   102→- `401` - missing or invalid API key
   103→- `429` - rate limit exceeded
   104→
   105→### Authentication
   106→
   107→The server includes a web dashboard for managing API keys. On first launch, navigate to the server URL in a browser to create an admin account. Once logged in, you can generate API keys and create additional users with configurable rate limits.
   108→
   109→## Architecture
   110→
   111→The project consists of two components:
   112→
   113→- **Builder** (C++) - Parses OSM PBF files and creates a compact binary index using S2 geometry cells for spatial lookup.
   114→- **Server** (Rust) - Memory-maps the index files and serves queries via HTTP/HTTPS with sub-millisecond latency.
   115→
   116→### Index Structure
   117→
   118→The builder produces 14 binary files:
   119→
   120→| File | Description |
   121→|------|-------------|
   122→| `geo_cells.bin` | Merged S2 cell index for streets, addresses, and interpolations |
   123→| `street_entries.bin` | Street way IDs per cell |
   124→| `street_ways.bin` | Street way headers (node offset, name) |
   125→| `street_nodes.bin` | Street node coordinates |
   126→| `addr_entries.bin` | Address point IDs per cell |
   127→| `addr_points.bin` | Address point data (coordinates, house number, street) |
   128→| `interp_entries.bin` | Interpolation way IDs per cell |
   129→| `interp_ways.bin` | Interpolation way headers |
   130→| `interp_nodes.bin` | Interpolation node coordinates |
   131→| `admin_cells.bin` | S2 cell index for admin boundaries |
   132→| `admin_entries.bin` | Admin polygon IDs per cell |
   133→| `admin_polygons.bin` | Admin polygon metadata |
   134→| `admin_vertices.bin` | Admin polygon vertices |
   135→| `strings.bin` | Deduplicated string pool |
   136→
   137→## Building from Source
   138→
   139→### Prerequisites
   140→
   141→**Builder (C++):**
   142→- CMake 3.16+
   143→- C++17 compiler
   144→- libosmium, protozero, s2geometry, zlib, bzip2, expat
   145→
   146→**Server (Rust):**
   147→- Rust toolchain
   148→
   149→### Build
   150→
   151→```bash
   152→# Build the indexer
   153→mkdir build && cd build && cmake ../builder && make
   154→
   155→# Build the server
   156→cargo build --release --manifest-path server/Cargo.toml
   157→```
   158→
   159→### Run
   160→
   161→```bash
   162→# Create index from PBF file
   163→./build/build-index output-dir input.osm.pbf [input2.osm.pbf ...]
   164→
   165→# Start the server
   166→./server/target/release/query-server output-dir [bind-address]
   167→
   168→# Start with automatic HTTPS
   169→./server/target/release/query-server output-dir --domain geocoder.example.com
   170→```
   171→
   172→## Environment Variables (Docker)
   173→
   174→| Variable | Description | Default |
   175→|----------|-------------|---------|
   176→| `PBF_URLS` | Space-separated list of PBF download URLs | (required for auto/build) |
   177→| `DOMAIN` | Domain name for automatic HTTPS via Let's Encrypt | (disabled) |
   178→| `BIND_ADDR` | HTTP bind address | `0.0.0.0:3000` |
   179→| `DATA_DIR` | Data directory for PBF files and index | `/data` |
   180→| `CACHE_DIR` | ACME certificate cache directory | `acme-cache` |
   181→
   182→## License
   183→
   184→    Apache License, Version 2.0
   185→
   186→    Licensed under the Apache License, Version 2.0 (the "License");
   187→    you may not use this file except in compliance with the License.
   188→    You may obtain a copy of the License at
   189→
   190→        http://www.apache.org/licenses/LICENSE-2.0
   191→
   192→    Unless required by applicable law or agreed to in writing, software
   193→    distributed under the License is distributed on an "AS IS" BASIS,
   194→    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   195→    See the License for the specific language governing permissions and
   196→    limitations under the License.
   197→


   200→                    auto f3 = std::async(std::launch::async, [&] {
   201→                        return !data.sorted_interp_cells.empty()
   202→                            ? extract_unique_cells(data.sorted_interp_cells)
   203→                            : extract_from_map(data.cell_to_interps);
   204→                    });
   205→                    addr_cells = f2.get();
   206→                    interp_cells = f3.get();
   207→                }
   208→                way_cells = f1.get();
   209→            }
   210→
   211→            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
   212→            std::merge(way_cells.begin(), way_cells.end(), addr_cells.begin(), addr_cells.end(),
   213→                       std::back_inserter(sorted_geo_cells));
   214→            if (!interp_cells.empty()) {
   215→                std::vector<uint64_t> tmp;
   216→                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
   217→                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
   218→                           interp_cells.begin(), interp_cells.end(), std::back_inserter(tmp));
   219→                sorted_geo_cells = std::move(tmp);
   220→            }
   221→            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
   222→                                    sorted_geo_cells.end());
   223→        }
   224→
   225→        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
   226→        {
   227→            auto f1 = std::async(std::launch::async, [&]() {
   228→                if (!data.sorted_way_cells.empty())
   229→                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
   230→                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
   231→            });
   232→            if (write_addresses) {
   233→                auto f2 = std::async(std::launch::async, [&]() {
   234→                    if (!data.sorted_addr_cells.empty())
   235→                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
   236→                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
   237→                });
   238→                auto f3 = std::async(std::launch::async, [&]() {
   239→                    if (!data.sorted_interp_cells.empty())
   240→                        return write_entries_from_sorted(output_dir + "/interp_entries.bin", sorted_geo_cells, data.sorted_interp_cells);
   241→                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
   242→                });
   243→                addr_offsets = f2.get();
   244→                interp_offsets = f3.get();
   245→            }
   246→            street_offsets = f1.get();
   247→        }
   248→
   249→        log_phase("  Write: entry files", _wt);
   250→        {
   251→            size_t n = sorted_geo_cells.size();
   252→            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
   253→            std::vector<char> buf(n * row_size);
   254→            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
   255→            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);
   256→
   257→            unsigned int nthreads = std::thread::hardware_concurrency();
   258→            if (nthreads == 0) nthreads = 4;
   259→            size_t chunk = (n + nthreads - 1) / nthreads;
   260→            std::vector<std::thread> fill_threads;
   261→            for (unsigned int t = 0; t < nthreads; t++) {
   262→                size_t start = t * chunk;
   263→                size_t end = std::min(start + chunk, n);
   264→                if (start >= n) break;
   265→                fill_threads.emplace_back([&, start, end]() {
   266→                    char* ptr = buf.data() + start * row_size;
   267→                    for (size_t i = start; i < end; i++) {
   268→                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
   269→                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   270→                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   271→                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   272→                    }
   273→                });
   274→            }
   275→            for (auto& t : fill_threads) t.join();
   276→
   277→            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
   278→            f.write(buf.data(), buf.size());
   279→        }
   280→
   281→        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
   282→                  << data.ways.size() << " ways, " << data.addr_points.size() << " addrs, "
   283→                  << data.interp_ways.size() << " interps)" << std::endl;
   284→    }
   285→
   286→    log_phase("  Write: geo_cells.bin", _wt);
   287→    auto admin_future = std::async(std::launch::async, [&] {
   288→        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
   289→        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
   290→    });
   291→
   292→    std::vector<std::future<void>> write_futures;
   293→    if (write_streets) {
   294→        write_futures.push_back(std::async(std::launch::async, [&] {
   295→            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
   296→            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
   297→        }));
   298→        write_futures.push_back(std::async(std::launch::async, [&] {
   299→            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
   300→            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
   301→        }));
   302→        if (write_addresses) {
   303→            write_futures.push_back(std::async(std::launch::async, [&] {
   304→                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
   305→                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
   306→            }));
   307→            write_futures.push_back(std::async(std::launch::async, [&] {
   308→                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
   309→                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
   310→            }));
   311→            write_futures.push_back(std::async(std::launch::async, [&] {
   312→                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
   313→                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
   314→            }));
   315→        }
   316→    }
   317→    // If not writing all data types, compact string pool and remap references
   318→    // before writing admin_polygons and strings.bin.
   319→    std::vector<AdminPolygon> compact_admin;
   320→    std::vector<char> compact_strings;
   321→    const bool need_compact = (!write_streets || !write_addresses);
   322→
   323→    if (need_compact) {
   324→        // Collect referenced string offsets
   325→        std::unordered_set<uint32_t> used_offsets;
   326→        if (write_streets) {
   327→            for (const auto& w : data.ways) used_offsets.insert(w.name_id);
   328→            if (write_addresses) {
   329→                for (const auto& a : data.addr_points) {
   330→                    used_offsets.insert(a.housenumber_id);
   331→                    used_offsets.insert(a.street_id);
   332→                }
   333→                for (const auto& iw : data.interp_ways) used_offsets.insert(iw.street_id);
   334→            }
   335→        }
   336→        for (const auto& ap : data.admin_polygons) used_offsets.insert(ap.name_id);
   337→
   338→        // Build compact pool and remap
   339→        const auto& old_sp = data.string_pool.data();
   340→        std::vector<uint32_t> sorted_offs(used_offsets.begin(), used_offsets.end());
   341→        std::sort(sorted_offs.begin(), sorted_offs.end());
   342→        std::unordered_map<uint32_t, uint32_t> remap;
   343→        compact_strings.reserve(used_offsets.size() * 16);
   344→        for (uint32_t off : sorted_offs) {
   345→            remap[off] = static_cast<uint32_t>(compact_strings.size());
   346→            size_t len = std::strlen(old_sp.data() + off);
   347→            compact_strings.insert(compact_strings.end(), old_sp.data() + off,
   348→                old_sp.data() + off + len + 1);
   349→        }
   350→
   351→        // Remap admin polygon name_ids
   352→        compact_admin = data.admin_polygons;
   353→        for (auto& ap : compact_admin) ap.name_id = remap[ap.name_id];
   354→
   355→        // Remap street/addr/interp if applicable (for no-addresses mode)
   356→        // These are written from data directly, so we'd need copies too.
   357→        // For simplicity, only compact when we have fewer types.
   358→    }
   359→
   360→    const auto& admin_to_write = need_compact ? compact_admin : data.admin_polygons;
   361→    write_futures.push_back(std::async(std::launch::async, [&] {
   362→        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
   363→        f.write(reinterpret_cast<const char*>(admin_to_write.data()), admin_to_write.size() * sizeof(AdminPolygon));
   364→    }));
   365→    write_futures.push_back(std::async(std::launch::async, [&] {
   366→        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
   367→        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
   368→    }));
   369→    write_futures.push_back(std::async(std::launch::async, [&] {
   370→        if (need_compact) {
   371→            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
   372→            f.write(compact_strings.data(), compact_strings.size());
   373→            std::cerr << "strings.bin: " << compact_strings.size() << " bytes (compacted from "
   374→                      << data.string_pool.data().size() << ")" << std::endl;
   375→        } else {
   376→            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
   377→            f.write(data.string_pool.data().data(), data.string_pool.data().size());
   378→            std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
   379→        }
   380→    }));
   381→
   382→    log_phase("  Write: parallel data files launched", _wt);
   383→    admin_future.get();
   384→    for (auto& f : write_futures) f.get();
   385→}
   386→
   387→void write_quality_variant(const ParsedData& data, const std::string& source_dir,
   388→                           const std::string& output_dir, double epsilon_scale) {
   389→    ensure_dir(output_dir);
   390→
   391→    // Re-simplify admin polygons at the given epsilon scale
   392→    std::vector<AdminPolygon> new_polys;
   393→    std::vector<NodeCoord> new_verts;
   394→    new_polys.reserve(data.admin_polygons.size());
   395→
   396→    // Parallel simplification
   397→    struct SimplifiedPoly {
   398→        std::vector<std::pair<double,double>> verts;
   399→    };


Found 6 files
src/build_index.cpp
src/pbf_reader.cpp
src/pbf_reader.h
src/continent_filter.h
src/s2_helpers.h
src/cache.cpp


   150→            std::string mode_str = argv[++i];
   151→            if (mode_str == "full") {
   152→                mode = IndexMode::Full;
   153→            } else if (mode_str == "no-addresses") {
   154→                mode = IndexMode::NoAddresses;
   155→            } else if (mode_str == "admin-only") {
   156→                mode = IndexMode::AdminOnly;
   157→            } else {
   158→                std::cerr << "Error: unknown mode '" << mode_str << "'" << std::endl;
   159→                return 1;
   160→            }
   161→        } else {
   162→            input_files.push_back(arg);
   163→        }
   164→    }
   165→
   166→    // Set global simplification mode
   167→    kSimplifyMode = simplify_mode;
   168→    kSimplifyEpsilonOverride = simplify_epsilon_override;
   169→    if (simplify_mode == SimplifyMode::ErrorBounded) {
   170→        if (simplify_epsilon_override > 0) {
   171→            std::cerr << "Simplification: error-bounded, fixed " << simplify_epsilon_override << "m epsilon" << std::endl;
   172→        } else {
   173→            std::cerr << "Simplification: error-bounded, scale=" << kEpsilonScale
   174→                      << ", epsilons: L2=" << admin_epsilon_meters(2)
   175→                      << "m L4=" << admin_epsilon_meters(4)
   176→                      << "m L6=" << admin_epsilon_meters(6)
   177→                      << "m L8=" << admin_epsilon_meters(8) << "m" << std::endl;
   178→        }
   179→    }
   180→
   181→    ParsedData data;
   182→    auto _pt = std::chrono::steady_clock::now();
   183→    auto _cpu = CpuTicks::now();
   184→
   185→    // Load previous build's string pool for stable offsets across builds
   186→    if (!previous_build_path.empty()) {
   187→        std::string prev_strings = previous_build_path + "/strings.bin";
   188→        if (data.string_pool.load_previous(prev_strings)) {
   189→            data.string_pool.mark_previous_loaded();
   190→            std::cerr << "Loaded previous string pool: " << data.string_pool.previous_size()
   191→                      << " bytes from " << prev_strings << std::endl;
   192→        } else {
   193→            std::cerr << "Warning: could not load previous string pool from " << prev_strings << std::endl;
   194→        }
   195→    }
   196→
   197→    if (!load_cache_path.empty()) {
   198→        // Load from cache
   199→        if (!deserialize_cache(data, load_cache_path)) {
   200→            std::cerr << "Error: failed to load cache" << std::endl;
   201→            return 1;
   202→        }
   203→    } else {
   204→        // Parse PBF files (always collect everything)
   205→        if (input_files.empty()) {
   206→            std::cerr << "Error: no input files specified and no --load-cache" << std::endl;
   207→            return 1;
   208→        }
   209→
   210→        // Create thread pool for concurrent admin polygon S2 covering
   211→        unsigned int num_threads = std::max(1u, std::thread::hardware_concurrency() - 1);
   212→        std::cerr << "Using " << num_threads << " worker threads." << std::endl;
   213→        AdminCoverPool admin_pool(num_threads);
   214→        // BuildHandler no longer used — parallel processing handles everything
   215→
   216→        // Dense array node location index — lockless parallel writes
   217→        // Planet OSM node IDs max ~12.5 billion. 8 bytes per entry = 100GB virtual.
   218→        // MAP_NORESERVE means OS only allocates pages on write (~80GB for 10B nodes).
   219→        static const size_t MAX_NODE_ID = MAX_NODE_ID_DEFAULT;
   220→        struct PackedLocation {
   221→            int32_t lat_e7;  // latitude * 10^7
   222→            int32_t lon_e7;  // longitude * 10^7
   223→            bool valid() const { return lat_e7 != 0 || lon_e7 != 0; }
   224→            double lat() const { return lat_e7 / 10000000.0; }
   225→            double lon() const { return lon_e7 / 10000000.0; }
   226→        };
   227→        struct DenseIndex {
   228→            PackedLocation* data;
   229→            size_t capacity;
   230→
   231→            DenseIndex() : capacity(MAX_NODE_ID) {
   232→                size_t byte_size = capacity * sizeof(PackedLocation);
   233→                void* ptr = mmap(nullptr, byte_size,
   234→                    PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE,
   235→                    -1, 0);
   236→                if (ptr == MAP_FAILED) {
   237→                    std::cerr << "Error: failed to mmap dense node index ("
   238→                              << (byte_size / (1024*1024*1024)) << "GB)" << std::endl;
   239→                    std::exit(1);
   240→                }
   241→                madvise(ptr, byte_size, MADV_HUGEPAGE);
   242→                data = static_cast<PackedLocation*>(ptr);
   243→                std::cerr << "Allocated dense node index: " << (byte_size / (1024*1024*1024))
   244→                          << "GB virtual address space" << std::endl;
   245→            }
   246→
   247→            ~DenseIndex() {
   248→                munmap(data, capacity * sizeof(PackedLocation));
   249→            }
   250→
   251→            // Lockless — each node ID maps to a unique array slot
   252→            void set(uint64_t id, double lat, double lng) {
   253→                if (id >= capacity) return;
   254→                data[id] = {static_cast<int32_t>(lat * 10000000.0 + (lat >= 0 ? 0.5 : -0.5)),
   255→                            static_cast<int32_t>(lng * 10000000.0 + (lng >= 0 ? 0.5 : -0.5))};
   256→            }
   257→
   258→            PackedLocation get(uint64_t id) const {
   259→                if (id >= capacity) return {0, 0};
   260→                return data[id];
   261→            }
   262→        } index;
   263→
   264→        for (const auto& input_file : input_files) {
   265→            std::cerr << "Processing " << input_file << "..." << std::endl;
   266→
   267→            // --- Pass 1: collect relation members for parallel admin assembly ---
   268→            std::cerr << "  Pass 1: scanning relations..." << std::endl;
   269→            PbfFile pbf(input_file, num_threads);
   270→
   271→            {
   272→                std::mutex rel_mutex;
   273→                pbf.read_blocks([&](PbfBlock& block, unsigned) {
   274→                    for (auto& rel : block.relations) {
   275→                        const char* boundary = rel.tag("boundary");
   276→                        if (!boundary) continue;
   277→
   278→                        bool is_admin = (std::strcmp(boundary, "administrative") == 0);
   279→                        bool is_postal = (std::strcmp(boundary, "postal_code") == 0);
   280→                        if (!is_admin && !is_postal) continue;
   281→
   282→                        uint8_t admin_level = 0;
   283→                        if (is_admin) {
   284→                            const char* level_str = rel.tag("admin_level");
   285→                            if (!level_str) continue;
   286→                            admin_level = static_cast<uint8_t>(std::atoi(level_str));
   287→                            if (admin_level < 2 || admin_level > 10) continue;
   288→                        } else {
   289→                            admin_level = 11;
   290→                        }
   291→                        if (kMaxAdminLevel > 0 && admin_level > kMaxAdminLevel) continue;
   292→
   293→                        const char* name = rel.tag("name");
   294→                        if (!name && is_admin) continue;
   295→
   296→                        std::string name_str;
   297→                        if (is_postal) {
   298→                            const char* postal_code = rel.tag("postal_code");
   299→                            if (!postal_code) postal_code = name;
   300→                            if (!postal_code) continue;
   301→                            name_str = postal_code;
   302→                        } else {
   303→                            name_str = name;
   304→                        }
   305→
   306→                        std::string country_code;
   307→                        if (admin_level == 2) {
   308→                            const char* cc = rel.tag("ISO3166-1:alpha2");
   309→                            if (cc) country_code = cc;
   310→                        }
   311→
   312→                        CollectedRelation cr;
   313→                        cr.id = rel.id;
   314→                        cr.admin_level = admin_level;
   315→                        cr.name = std::move(name_str);
   316→                        cr.country_code = std::move(country_code);
   317→                        cr.is_postal = is_postal;
   318→
   319→                        for (size_t mi = 0; mi < rel.members.size(); mi++) {
   320→                            if (rel.members[mi].type == 'w') {
   321→                                cr.members.emplace_back(rel.members[mi].ref, rel.member_role(mi));
   322→                            }
   323→                        }
   324→
   325→                        if (!cr.members.empty()) {
   326→                            std::lock_guard<std::mutex> lock(rel_mutex);
   327→                            data.collected_relations.push_back(std::move(cr));
   328→                        }
   329→                    }
   330→                }, "r");
   331→            }
   332→            std::cerr << "  Collected " << data.collected_relations.size()
   333→                      << " admin/postal relations for parallel assembly." << std::endl;
   334→            log_phase("Pass 1: relation scanning", _pt, _cpu);
   335→
   336→            // Build admin_way_ids as a bitset for O(1) lookup
   337→            // Max way ID ~1.3B → ~162 MiB bitset (much faster than unordered_set)
   338→            static constexpr size_t MAX_WAY_ID = 2000000000ULL; // 2B, ~250 MiB
   339→            std::vector<uint8_t> admin_way_bits(MAX_WAY_ID / 8 + 1, 0);
   340→            size_t admin_way_count = 0;
   341→            for (const auto& rel : data.collected_relations) {
   342→                for (const auto& [way_id, role] : rel.members) {
   343→                    if (way_id > 0 && static_cast<size_t>(way_id) < MAX_WAY_ID) {
   344→                        admin_way_bits[way_id / 8] |= (1 << (way_id % 8));
   345→                        admin_way_count++;
   346→                    }
   347→                }
   348→            }
   349→            auto is_admin_way = [&](int64_t id) -> bool {
   350→                if (id <= 0 || static_cast<size_t>(id) >= MAX_WAY_ID) return false;
   351→                return admin_way_bits[id / 8] & (1 << (id % 8));
   352→            };
   353→            std::cerr << "  Admin assembly needs " << admin_way_count << " way geometries." << std::endl;
   354→
   355→            // --- Pass 2: Node processing (streaming — no PbfNode objects) ---
   356→            std::cerr << "  Pass 2: processing nodes with " << num_threads << " threads..." << std::endl;
   357→            {
   358→                struct NodeThreadLocal {
   359→                    std::vector<std::pair<double,double>> addr_coords;
   360→                    std::vector<std::pair<std::string,std::string>> addr_strings;
   361→                    uint64_t count = 0;
   362→                };
   363→                // Use thread_local for streaming callback (no thread index available)
   364→                static thread_local NodeThreadLocal* tl_node_data = nullptr;
   365→                std::vector<NodeThreadLocal> ntld(num_threads);
   366→                std::atomic<unsigned> next_tl{0};
   367→
   368→                pbf.read_nodes_streaming([&](int64_t id, double lat, double lng,
   369→                    const uint32_t* tag_keys, const uint32_t* tag_vals, size_t ntags,
   370→                    const std::vector<std::string>& st) {
   371→
   372→                    // Lazy init thread-local pointer
   373→                    if (!tl_node_data) {
   374→                        unsigned idx = next_tl.fetch_add(1);
   375→                        tl_node_data = &ntld[idx % ntld.size()];
   376→                    }
   377→
   378→                    if (id > 0) {
   379→                        index.set(static_cast<uint64_t>(id), lat, lng);
   380→                    }
   381→
   382→                    // Check for address tags (fast — no string copies for tagless nodes)
   383→                    if (ntags > 0) {
   384→                        const char* housenumber = nullptr;
   385→                        const char* street = nullptr;
   386→                        for (size_t i = 0; i < ntags; i++) {
   387→                            if (tag_keys[i] < st.size()) {
   388→                                if (st[tag_keys[i]] == "addr:housenumber")
   389→                                    housenumber = tag_vals[i] < st.size() ? st[tag_vals[i]].c_str() : nullptr;
   390→                                else if (st[tag_keys[i]] == "addr:street")
   391→                                    street = tag_vals[i] < st.size() ? st[tag_vals[i]].c_str() : nullptr;
   392→                            }
   393→                        }
   394→                        if (housenumber && street) {
   395→                            tl_node_data->addr_coords.push_back({lat, lng});
   396→                            tl_node_data->addr_strings.push_back({housenumber, street});
   397→                            tl_node_data->count++;
   398→                        }
   399→                    }
   400→                });
   401→
   402→                // Reset thread_local for next use
   403→                tl_node_data = nullptr;
   404→
   405→                // Merge address points
   406→                uint64_t total_addrs = 0;
   407→                for (auto& local : ntld) {
   408→                    for (size_t j = 0; j < local.addr_coords.size(); j++) {
   409→                        uint64_t dummy = 0;
   410→                        add_addr_point(data, local.addr_coords[j].first, local.addr_coords[j].second,
   411→                                       local.addr_strings[j].first.c_str(),
   412→                                       local.addr_strings[j].second.c_str(), dummy);
   413→                    }
   414→                    total_addrs += local.count;
   415→                }
   416→                std::cerr << "  Node processing complete: " << total_addrs
   417→                          << " address points collected." << std::endl;
   418→            }
   419→            log_phase("Pass 2: node processing", _pt, _cpu);
   420→            pbf.release_pages(); // free PBF mmap pages, will re-fault for way pass
   421→
   422→            // --- Pass 2b: Way processing (fully parallel) ---
   423→            std::cerr << "  Processing ways with " << num_threads << " threads..." << std::endl;
   424→            {
   425→                // Thread-local data for parallel way processing
   426→                struct ThreadLocalData {
   427→                    std::vector<WayHeader> ways;
   428→                    std::vector<NodeCoord> street_nodes;
   429→                    std::vector<DeferredWay> deferred_ways;
   430→                    std::vector<InterpWay> interp_ways;
   431→                    std::vector<NodeCoord> interp_nodes;
   432→                    std::vector<DeferredInterp> deferred_interps;
   433→                    std::vector<AddrPoint> building_addrs;
   434→                    std::vector<std::pair<double, double>> building_addr_coords; // lat,lng for S2 cell
   435→                    std::vector<std::string> way_strings;      // way name strings
   436→                    std::vector<std::pair<std::string,std::string>> addr_strings; // building addr {hn, street}
   437→                    std::vector<std::string> interp_strings;   // interp street name strings
   438→                    uint64_t way_count = 0;
   439→                    uint64_t building_addr_count = 0;
   440→                    uint64_t interp_count = 0;
   441→                    // Way geometries for parallel admin assembly
   442→                    struct WayGeomEntry {
   443→                        int64_t way_id;
   444→                        std::vector<std::pair<double,double>> coords;
   445→                        int64_t first_node_id;
   446→                        int64_t last_node_id;
   447→                    };
   448→                    std::vector<WayGeomEntry> way_geoms;
   449→                    // Closed-way admin polygons (ways with boundary=administrative)


Error: EISDIR: illegal operation on a directory, read '/root/traccar-geocoder/builder/src'


grep -n "write_index\|continents\|--continents\|generate_continents" /root/traccar-geocoder/builder/src/build_index.cpp | head -40


   150→    auto f_remap_ways = std::async(std::launch::async, [&]() {
   151→        std::vector<uint32_t> sorted_ids(used_way_ids.begin(), used_way_ids.end());
   152→        std::sort(sorted_ids.begin(), sorted_ids.end());
   153→        std::vector<WayHeader> ways;
   154→        std::vector<NodeCoord> nodes;
   155→        ways.reserve(sorted_ids.size());
   156→        nodes.reserve(sorted_ids.size() * 5);
   157→        std::unordered_map<uint32_t, uint32_t> remap;
   158→        remap.reserve(sorted_ids.size());
   159→        for (uint32_t old_id : sorted_ids) {
   160→            remap[old_id] = static_cast<uint32_t>(ways.size());
   161→            const auto& w = full.ways[old_id];
   162→            WayHeader nw = w;
   163→            nw.node_offset = static_cast<uint32_t>(nodes.size());
   164→            ways.push_back(nw);
   165→            for (uint8_t n = 0; n < w.node_count; n++)
   166→                nodes.push_back(full.street_nodes[w.node_offset + n]);
   167→        }
   168→        return std::make_tuple(std::move(remap), std::move(ways), std::move(nodes));
   169→    });
   170→
   171→    // Addrs
   172→    auto f_remap_addrs = std::async(std::launch::async, [&]() {
   173→        std::vector<uint32_t> sorted_ids(used_addr_ids.begin(), used_addr_ids.end());
   174→        std::sort(sorted_ids.begin(), sorted_ids.end());
   175→        std::vector<AddrPoint> addrs;
   176→        addrs.reserve(sorted_ids.size());
   177→        std::unordered_map<uint32_t, uint32_t> remap;
   178→        remap.reserve(sorted_ids.size());
   179→        for (uint32_t old_id : sorted_ids) {
   180→            remap[old_id] = static_cast<uint32_t>(addrs.size());
   181→            addrs.push_back(full.addr_points[old_id]);
   182→        }
   183→        return std::make_tuple(std::move(remap), std::move(addrs));
   184→    });
   185→
   186→    // Interps
   187→    auto f_remap_interps = std::async(std::launch::async, [&]() {
   188→        std::vector<uint32_t> sorted_ids(used_interp_ids.begin(), used_interp_ids.end());
   189→        std::sort(sorted_ids.begin(), sorted_ids.end());
   190→        std::vector<InterpWay> iways;
   191→        std::vector<NodeCoord> inodes;
   192→        iways.reserve(sorted_ids.size());
   193→        std::unordered_map<uint32_t, uint32_t> remap;
   194→        remap.reserve(sorted_ids.size());
   195→        for (uint32_t old_id : sorted_ids) {
   196→            remap[old_id] = static_cast<uint32_t>(iways.size());
   197→            const auto& iw = full.interp_ways[old_id];
   198→            InterpWay niw = iw;
   199→            niw.node_offset = static_cast<uint32_t>(inodes.size());
   200→            iways.push_back(niw);
   201→            for (uint8_t n = 0; n < iw.node_count; n++)
   202→                inodes.push_back(full.interp_nodes[iw.node_offset + n]);
   203→        }
   204→        return std::make_tuple(std::move(remap), std::move(iways), std::move(inodes));
   205→    });
   206→
   207→    // Admins
   208→    auto f_remap_admins = std::async(std::launch::async, [&]() {
   209→        std::vector<uint32_t> sorted_ids(used_admin_ids.begin(), used_admin_ids.end());
   210→        std::sort(sorted_ids.begin(), sorted_ids.end());
   211→        std::vector<AdminPolygon> polys;
   212→        std::vector<NodeCoord> verts;
   213→        polys.reserve(sorted_ids.size());
   214→        std::unordered_map<uint32_t, uint32_t> remap;
   215→        remap.reserve(sorted_ids.size());
   216→        for (uint32_t old_id : sorted_ids) {
   217→            remap[old_id] = static_cast<uint32_t>(polys.size());
   218→            const auto& ap = full.admin_polygons[old_id];
   219→            AdminPolygon nap = ap;
   220→            nap.vertex_offset = static_cast<uint32_t>(verts.size());
   221→            polys.push_back(nap);
   222→            for (uint32_t v = 0; v < ap.vertex_count; v++)
   223→                verts.push_back(full.admin_vertices[ap.vertex_offset + v]);
   224→        }
   225→        return std::make_tuple(std::move(remap), std::move(polys), std::move(verts));
   226→    });
   227→
   228→    // Collect results
   229→    {
   230→        auto [wr, ways, nodes] = f_remap_ways.get();
   231→        way_remap = std::move(wr);
   232→        out.ways = std::move(ways);
   233→        out.street_nodes = std::move(nodes);
   234→    }
   235→    {
   236→        auto [ar, addrs] = f_remap_addrs.get();
   237→        addr_remap = std::move(ar);
   238→        out.addr_points = std::move(addrs);
   239→    }
   240→    {
   241→        auto [ir, iways, inodes] = f_remap_interps.get();
   242→        interp_remap = std::move(ir);
   243→        out.interp_ways = std::move(iways);
   244→        out.interp_nodes = std::move(inodes);
   245→    }
   246→    {
   247→        auto [ar, polys, verts] = f_remap_admins.get();
   248→        admin_remap = std::move(ar);
   249→        out.admin_polygons = std::move(polys);
   250→        out.admin_vertices = std::move(verts);
   251→    }
   252→    log_phase("      filter: data remap", _ft, _fc);
   253→
   254→    // Remap cell maps — use sorted pairs where available (parallel), hash maps otherwise
   255→    auto remap_cells_map = [&](const std::unordered_map<uint64_t, std::vector<uint32_t>>& src,
   256→                               const std::unordered_map<uint32_t, uint32_t>& remap,
   257→                               std::unordered_map<uint64_t, std::vector<uint32_t>>& dst,
   258→                               bool handle_flags = false) {
   259→        for (const auto& [cell_id, ids] : src) {
   260→            if (!cell_in_bbox(cell_id, bbox)) continue;
   261→            std::vector<uint32_t> new_ids;
   262→            for (uint32_t id : ids) {
   263→                uint32_t raw_id = handle_flags ? (id & ID_MASK) : id;
   264→                uint32_t flags = handle_flags ? (id & INTERIOR_FLAG) : 0;
   265→                auto it = remap.find(raw_id);
   266→                if (it != remap.end()) new_ids.push_back(it->second | flags);
   267→            }
   268→            if (!new_ids.empty()) dst[cell_id] = std::move(new_ids);
   269→        }
   270→    };
   271→
   272→    // Remap from sorted pairs — builds output cell map by scanning sorted pairs.
   273→    // Uses same parallel chunked pattern as filter_cells_sorted.
   274→    auto remap_cells_sorted = [&](const std::vector<CellItemPair>& sorted,
   275→                                   const std::unordered_map<uint32_t, uint32_t>& remap,
   276→                                   std::unordered_map<uint64_t, std::vector<uint32_t>>& dst) {
   277→        if (sorted.empty()) return;
   278→        auto _rt = std::chrono::steady_clock::now();
   279→        auto _rc = CpuTicks::now();
   280→
   281→        unsigned nthreads = std::max(1u, std::thread::hardware_concurrency() / 4);
   282→        size_t chunk = (sorted.size() + nthreads - 1) / nthreads;
   283→
   284→        // Find chunk boundaries at cell_id transitions
   285→        std::vector<size_t> bounds = {0};
   286→        for (unsigned t = 1; t < nthreads; t++) {
   287→            size_t target = t * chunk;
   288→            if (target >= sorted.size()) break;
   289→            while (target < sorted.size() && sorted[target].cell_id == sorted[target-1].cell_id)
   290→                target++;
   291→            if (target < sorted.size()) bounds.push_back(target);
   292→        }
   293→        bounds.push_back(sorted.size());
   294→
   295→        std::cerr << "        remap_sorted: " << sorted.size() << " pairs, "
   296→                  << bounds.size()-1 << " chunks, " << nthreads << " threads" << std::endl;
   297→
   298→        // Each thread builds its own partial map
   299→        std::vector<std::unordered_map<uint64_t, std::vector<uint32_t>>> partial(bounds.size() - 1);
   300→        std::vector<std::thread> threads;
   301→        for (size_t t = 0; t + 1 < bounds.size(); t++) {
   302→            threads.emplace_back([&, t]() {
   303→                auto& local = partial[t];
   304→                for (size_t i = bounds[t]; i < bounds[t+1]; ) {
   305→                    uint64_t cell_id = sorted[i].cell_id;
   306→                    bool match = cell_in_bbox(cell_id, bbox);
   307→                    std::vector<uint32_t> new_ids;
   308→                    while (i < bounds[t+1] && sorted[i].cell_id == cell_id) {
   309→                        if (match) {
   310→                            auto it = remap.find(sorted[i].item_id);
   311→                            if (it != remap.end()) new_ids.push_back(it->second);
   312→                        }
   313→                        i++;
   314→                    }
   315→                    if (!new_ids.empty()) local[cell_id] = std::move(new_ids);
   316→                }
   317→            });
   318→        }
   319→        for (auto& t : threads) t.join();
   320→        log_phase("        remap_sorted: scan", _rt, _rc);
   321→
   322→        // Merge partial maps (no conflicts — cell boundaries are clean splits)
   323→        size_t total = 0;
   324→        for (auto& p : partial) total += p.size();
   325→        dst.reserve(total);
   326→        for (auto& p : partial) {
   327→            for (auto& [k, v] : p) dst[k] = std::move(v);
   328→        }
   329→        log_phase("        remap_sorted: merge", _rt, _rc);
   330→    };
   331→
   332→    // Build sorted pairs for continent by filtering + remapping in one pass.
   333→    // Much faster than building hash maps — write_index uses sorted pairs directly.
   334→    auto filter_and_remap_sorted = [&](const std::vector<CellItemPair>& sorted,
   335→                                        const std::unordered_map<uint32_t, uint32_t>& remap,
   336→                                        std::vector<CellItemPair>& dst) {
   337→        if (sorted.empty()) return;
   338→        auto _rt = std::chrono::steady_clock::now();
   339→        auto _rc = CpuTicks::now();
   340→
   341→        unsigned nthreads = std::max(1u, std::thread::hardware_concurrency() / 4);
   342→        size_t chunk = (sorted.size() + nthreads - 1) / nthreads;
   343→
   344→        std::vector<size_t> bounds = {0};
   345→        for (unsigned t = 1; t < nthreads; t++) {
   346→            size_t target = t * chunk;
   347→            if (target >= sorted.size()) break;
   348→            while (target < sorted.size() && sorted[target].cell_id == sorted[target-1].cell_id)
   349→                target++;
   350→            if (target < sorted.size()) bounds.push_back(target);
   351→        }
   352→        bounds.push_back(sorted.size());
   353→
   354→        // Each thread produces a local vector of filtered+remapped pairs
   355→        std::vector<std::vector<CellItemPair>> thread_pairs(bounds.size() - 1);
   356→        std::vector<std::thread> threads;
   357→        for (size_t t = 0; t + 1 < bounds.size(); t++) {
   358→            threads.emplace_back([&, t]() {
   359→                auto& local = thread_pairs[t];
   360→                for (size_t i = bounds[t]; i < bounds[t+1]; ) {
   361→                    uint64_t cell_id = sorted[i].cell_id;
   362→                    bool match = cell_in_bbox(cell_id, bbox);
   363→                    while (i < bounds[t+1] && sorted[i].cell_id == cell_id) {
   364→                        if (match) {
   365→                            auto it = remap.find(sorted[i].item_id);
   366→                            if (it != remap.end())
   367→                                local.push_back({cell_id, it->second});
   368→                        }
   369→                        i++;
   370→                    }
   371→                }
   372→            });
   373→        }
   374→        for (auto& t : threads) t.join();
   375→        log_phase("        sorted remap: scan", _rt, _rc);
   376→
   377→        // Concatenate (already in sorted cell_id order since chunks are at cell boundaries)
   378→        size_t total = 0;
   379→        for (auto& v : thread_pairs) total += v.size();
   380→        dst.reserve(total);
   381→        for (auto& v : thread_pairs) dst.insert(dst.end(), v.begin(), v.end());
   382→        log_phase("        sorted remap: concat", _rt, _rc);
   383→    };
   384→
   385→    {
   386→        // Build sorted pairs for ways/addrs/interps (write_index uses these directly)
   387→        auto f1 = std::async(std::launch::async, [&]{ filter_and_remap_sorted(full.sorted_way_cells, way_remap, out.sorted_way_cells); });
   388→        auto f2 = std::async(std::launch::async, [&]{ filter_and_remap_sorted(full.sorted_addr_cells, addr_remap, out.sorted_addr_cells); });
   389→        auto f3 = std::async(std::launch::async, [&]{ filter_and_remap_sorted(full.sorted_interp_cells, interp_remap, out.sorted_interp_cells); });
   390→        // Admin still uses hash map (has interior flags)
   391→        auto f4 = std::async(std::launch::async, [&]{ remap_cells_map(full.cell_to_admin, admin_remap, out.cell_to_admin, true); });
   392→        f1.get(); f2.get(); f3.get(); f4.get();
   393→    }
   394→    log_phase("      filter: cell map remap", _ft, _fc);
   395→
   396→    // Rebuild compact string pool
   397→    std::unordered_set<uint32_t> used_offsets;
   398→    for (const auto& w : out.ways) used_offsets.insert(w.name_id);
   399→    for (const auto& a : out.addr_points) { used_offsets.insert(a.housenumber_id); used_offsets.insert(a.street_id); }


52:        std::cerr << "  --continents           Also generate per-continent indexes" << std::endl;
71:    bool generate_continents = false;
98:        } else if (arg == "--continents") {
99:            generate_continents = true;
1464:            auto wf1 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/full", IndexMode::Full); });
1465:            auto wf2 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/no-addresses", IndexMode::NoAddresses); });
1466:            auto wf3 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/admin", IndexMode::AdminOnly); });
1469:            write_index(d, base_dir, mode);
1485:    // Process continents with bounded concurrency, largest first.
1486:    if (generate_continents) {
1550:        // Sort by bbox area descending — largest continents first
1562:        std::cerr << "Processing " << kContinentCount << " continents ("
1637:        if (generate_continents) {


     1→#pragma once
     2→
     3→#include <cstdint>
     4→#include <vector>
     5→
     6→// --- Binary format structs (must match server's index reader) ---
     7→
     8→struct WayHeader {
     9→    uint32_t node_offset;
    10→    uint8_t node_count;
    11→    uint32_t name_id;
    12→};
    13→
    14→struct AddrPoint {
    15→    float lat;
    16→    float lng;
    17→    uint32_t housenumber_id;
    18→    uint32_t street_id;
    19→};
    20→
    21→struct InterpWay {
    22→    uint32_t node_offset;
    23→    uint8_t node_count;
    24→    uint32_t street_id;
    25→    uint32_t start_number;
    26→    uint32_t end_number;
    27→    uint8_t interpolation;
    28→};
    29→
    30→struct AdminPolygon {
    31→    uint32_t vertex_offset;
    32→    uint32_t vertex_count;
    33→    uint32_t name_id;
    34→    uint8_t admin_level;
    35→    float area;
    36→    uint16_t country_code;
    37→};
    38→
    39→struct NodeCoord {
    40→    float lat;
    41→    float lng;
    42→};
    43→
    44→// Sorted cell-item pair for direct entry writing (avoids hash map)
    45→struct CellItemPair {
    46→    uint64_t cell_id;
    47→    uint32_t item_id;
    48→};
    49→
    50→// --- Constants ---
    51→
    52→static const uint32_t INTERIOR_FLAG = 0x80000000u;
    53→static const uint32_t ID_MASK       = 0x7FFFFFFFu;
    54→static const uint32_t NO_DATA       = 0xFFFFFFFFu;
    55→
    56→// Simplification mode
    57→enum class SimplifyMode { MaxVertices, ErrorBounded };
    58→
    59→// Processing limits
    60→static constexpr size_t MAX_BLOCK_QUEUE         = 64;    // producer-consumer queue depth
    61→static constexpr int    MAX_NODE_COUNT          = 255;   // uint8_t node_count
    62→static constexpr int    MAX_VERTEX_COUNT        = 0x7FFFFFFF; // uint32_t vertex_count
    63→static constexpr int    MAX_S2_CELLS_PER_POLY   = 200;   // S2RegionCoverer max_cells
    64→static constexpr int    BACKTRACK_CALL_BUDGET   = 100000; // per-ring backtracking limit
    65→static constexpr int    BACKTRACK_TOTAL_BUDGET  = 500000; // per-relation backtracking limit
    66→static constexpr int    BACKTRACK_MAX_DEPTH     = 200;    // recursion depth limit
    67→static constexpr size_t MAX_SUBWAYS_FOR_RETRY   = 30;     // only retry small relations
    68→static constexpr size_t MAX_NODE_ID_DEFAULT     = 15000000000ULL; // dense index capacity
    69→
    70→// --- Index mode ---
    71→
    72→enum class IndexMode { Full, NoAddresses, AdminOnly };
    73→
    74→// --- Deferred S2 work items ---
    75→
    76→struct DeferredWay {
    77→    uint32_t way_id;
    78→    uint32_t node_offset;
    79→    uint8_t node_count;
    80→};
    81→
    82→struct DeferredInterp {
    83→    uint32_t interp_id;
    84→    uint32_t node_offset;
    85→    uint8_t node_count;
    86→};
    87→
    88→// Collected relation data for parallel admin assembly
    89→struct CollectedRelation {
    90→    int64_t id;
    91→    uint8_t admin_level;
    92→    std::string name;
    93→    std::string country_code;
    94→    bool is_postal;
    95→    std::vector<std::pair<int64_t, std::string>> members; // (way_id, role)
    96→};
    97→


1736 /root/traccar-geocoder/builder/src/build_index.cpp


30-}
31-
32:// --- Binary format structs (must match C++ build pipeline) ---
33-
34-#[repr(C)]
--
109-
110-fn mmap_file(path: &str) -> Result<Mmap, String> {
111:    let file = File::open(path).map_err(|e| format!("Failed to open {}: {}", path, e))?;
112:    unsafe { Mmap::map(&file).map_err(|e| format!("Failed to mmap {}: {}", path, e)) }
113-}
114-
--
123-        let max_distance_sq = meters_to_rad * meters_to_rad;
124-
125:        let geo_cells = mmap_file_optional(&format!("{}/geo_cells.bin", dir));
126-        let has_geo = geo_cells.is_some();
127-
128-        let required_geo = |name: &str| -> Result<Option<Mmap>, String> {
129:            let path = format!("{}/{}", dir, name);
130-            let mmap = mmap_file_optional(&path);
131-            if has_geo && mmap.is_none() {
132:                return Err(format!("Partial geo index: {} is missing", name));
133-            }
134-            Ok(mmap)
--
140-            street_ways: required_geo("street_ways.bin")?,
141-            street_nodes: required_geo("street_nodes.bin")?,
142:            addr_entries: mmap_file_optional(&format!("{}/addr_entries.bin", dir)),
143:            addr_points: mmap_file_optional(&format!("{}/addr_points.bin", dir)),
144:            interp_entries: mmap_file_optional(&format!("{}/interp_entries.bin", dir)),
145:            interp_ways: mmap_file_optional(&format!("{}/interp_ways.bin", dir)),
146:            interp_nodes: mmap_file_optional(&format!("{}/interp_nodes.bin", dir)),
147:            admin_cells: mmap_file(&format!("{}/admin_cells.bin", dir))?,
148:            admin_entries: mmap_file(&format!("{}/admin_entries.bin", dir))?,
149:            admin_polygons: mmap_file(&format!("{}/admin_polygons.bin", dir))?,
150:            admin_vertices: mmap_file(&format!("{}/admin_vertices.bin", dir))?,
151:            strings: mmap_file(&format!("{}/strings.bin", dir))?,
152-            street_cell_level,
153-            admin_cell_level,
--
550-            country_code: admin.country_code.map(|c| String::from_utf8_lossy(&c).into_owned()),
551-        };
552:        let display_name = format_address(&address);
553-        Address { display_name, address }
554-    }
--
653-}
654-
655:// Address formatting patterns by country code
656-// Returns (number_after_street, postcode_before_city, include_state)
657:fn format_rules(country_code: Option<&str>) -> (bool, bool, bool) {
658-    match country_code {
659-        // Number before street, postcode after city, include state
--
671-}
672-
673:fn format_address(addr: &AddressDetails<'_>) -> Option<String> {
674-    if addr.road.is_none() && addr.city.is_none() && addr.country.is_none() {
675-        return None;
676-    }
677-
678:    let (number_after, postcode_before_city, include_state) = format_rules(addr.country_code.as_deref());
679-    let mut parts: Vec<String> = Vec::new();
680-
--
683-        if let Some(ref hn) = addr.house_number {
684-            if number_after {
685:                parts.push(format!("{} {}", road, hn));
686-            } else {
687:                parts.push(format!("{} {}", hn, road));
688-            }
689-        } else {
--
764-
765-    let rate_key = if by_ip {
766:        format!("{}:{}", login, connect_info.0.ip())
767-    } else {
768-        login
--
790-    let search_distance = arg_value("--search-distance").and_then(|v| v.parse().ok()).unwrap_or(DEFAULT_SEARCH_DISTANCE);
791-
792:    let db_path = format!("{}/geocoder.json", data_dir);
793-    let db = auth::Db::load(&db_path);
794-


  1600→    log_phase("All index writing (total)", _pt, _cpu);
  1601→
  1602→    // --- Generate manifest.json ---
  1603→    {
  1604→        auto file_size = [](const std::string& path) -> int64_t {
  1605→            struct stat st;
  1606→            if (stat(path.c_str(), &st) == 0) return st.st_size;
  1607→            return -1;
  1608→        };
  1609→
  1610→        auto dir_size = [&](const std::string& dir, const std::vector<std::string>& files) -> int64_t {
  1611→            int64_t total = 0;
  1612→            for (const auto& f : files) {
  1613→                int64_t s = file_size(dir + "/" + f);
  1614→                if (s > 0) total += s;
  1615→            }
  1616→            return total;
  1617→        };
  1618→
  1619→        std::vector<std::string> admin_base_files = {
  1620→            "admin_cells.bin", "admin_entries.bin", "strings.bin"
  1621→        };
  1622→        std::vector<std::string> admin_quality_files = {
  1623→            "admin_polygons.bin", "admin_vertices.bin"
  1624→        };
  1625→        std::vector<std::string> street_files = {
  1626→            "street_ways.bin", "street_nodes.bin", "street_entries.bin"
  1627→        };
  1628→        std::vector<std::string> address_files = {
  1629→            "addr_points.bin", "addr_entries.bin",
  1630→            "interp_ways.bin", "interp_nodes.bin", "interp_entries.bin"
  1631→        };
  1632→        std::vector<std::string> geo_files = {"geo_cells.bin"};
  1633→
  1634→        // Build list of regions
  1635→        std::vector<std::pair<std::string, std::string>> regions; // (name, path)
  1636→        regions.push_back({"planet", output_dir});
  1637→        if (generate_continents) {
  1638→            for (size_t ci = 0; ci < kContinentCount; ci++) {
  1639→                regions.push_back({kContinents[ci].name,
  1640→                    output_dir + "/" + kContinents[ci].name});
  1641→            }
  1642→        }
  1643→
  1644→        // Quality level names
  1645→        auto quality_dir_name = [](double scale) -> std::string {
  1646→            if (scale == 0) return "uncapped";
  1647→            char buf[32]; snprintf(buf, sizeof(buf), "q%.4g", scale);
  1648→            return buf;
  1649→        };
  1650→
  1651→        std::ofstream mf(output_dir + "/manifest.json");
  1652→        mf << "{\n";
  1653→        mf << "  \"version\": 1,\n";
  1654→        mf << "  \"regions\": [\n";
  1655→
  1656→        for (size_t ri = 0; ri < regions.size(); ri++) {
  1657→            const auto& [rname, rpath] = regions[ri];
  1658→            mf << "    {\n";
  1659→            mf << "      \"name\": \"" << rname << "\",\n";
  1660→
  1661→            // Modes
  1662→            mf << "      \"modes\": {\n";
  1663→            if (multi_output) {
  1664→                // Full
  1665→                std::string full_dir = rpath + "/full";
  1666→                int64_t full_sz = dir_size(full_dir, geo_files) +
  1667→                    dir_size(full_dir, street_files) +
  1668→                    dir_size(full_dir, address_files) +
  1669→                    dir_size(full_dir, admin_base_files) +
  1670→                    dir_size(full_dir, admin_quality_files);
  1671→                mf << "        \"full\": {\"size\": " << full_sz << "},\n";
  1672→
  1673→                // Streets (no addresses)
  1674→                std::string na_dir = rpath + "/no-addresses";
  1675→                int64_t na_sz = dir_size(na_dir, geo_files) +
  1676→                    dir_size(na_dir, street_files) +
  1677→                    dir_size(na_dir, admin_base_files) +
  1678→                    dir_size(na_dir, admin_quality_files);
  1679→                mf << "        \"no-addresses\": {\"size\": " << na_sz << "},\n";
  1680→
  1681→                // Admin base (shared files)
  1682→                std::string admin_dir = rpath + "/admin";
  1683→                int64_t admin_base_sz = dir_size(admin_dir, admin_base_files);
  1684→                mf << "        \"admin\": {\"base_size\": " << admin_base_sz << "}\n";
  1685→            } else {
  1686→                int64_t sz = 0;
  1687→                for (const auto& lists : {geo_files, street_files, address_files,
  1688→                                          admin_base_files, admin_quality_files}) {
  1689→                    sz += dir_size(rpath, lists);
  1690→                }
  1691→                mf << "        \"" << (mode == IndexMode::Full ? "full" :
  1692→                    mode == IndexMode::NoAddresses ? "no-addresses" : "admin") << "\": {\"size\": " << sz << "}\n";
  1693→            }
  1694→            mf << "      },\n";
  1695→
  1696→            // Quality variants
  1697→            if (multi_quality) {
  1698→                std::string admin_dir = multi_output ? rpath + "/admin" : rpath;
  1699→                mf << "      \"qualities\": [\n";
  1700→                for (size_t qi = 0; qi < quality_scales.size(); qi++) {
  1701→                    double scale = quality_scales[qi];
  1702→                    std::string qname = quality_dir_name(scale);
  1703→                    std::string qdir = admin_dir + "/" + qname;
  1704→                    int64_t qsz = dir_size(qdir, admin_quality_files);
  1705→
  1706→                    // Compute epsilon values for this scale
  1707→                    double eps_l2 = (scale == 0) ? 0 : 500.0 * scale * kEpsilonScale;
  1708→                    double eps_l8 = (scale == 0) ? 0 : 15.0 * scale * kEpsilonScale;
  1709→
  1710→                    mf << "        {\"name\": \"" << qname << "\", \"scale\": " << scale
  1711→                       << ", \"size\": " << qsz
  1712→                       << ", \"epsilon_l2_m\": " << eps_l2
  1713→                       << ", \"epsilon_l8_m\": " << eps_l8
  1714→                       << "}";
  1715→                    if (qi + 1 < quality_scales.size()) mf << ",";
  1716→                    mf << "\n";
  1717→                }
  1718→                mf << "      ]\n";
  1719→            } else {
  1720→                mf << "      \"qualities\": []\n";
  1721→            }
  1722→
  1723→            mf << "    }";
  1724→            if (ri + 1 < regions.size()) mf << ",";
  1725→            mf << "\n";
  1726→        }
  1727→
  1728→        mf << "  ]\n";
  1729→        mf << "}\n";
  1730→
  1731→        std::cerr << "Wrote " << output_dir << "/manifest.json" << std::endl;
  1732→    }
  1733→
  1734→    std::cerr << "Done." << std::endl;
  1735→    return 0;