  1300→    // concurrency to avoid over-subscribing CPU.
  1301→    auto write_qualities = [&](const ParsedData& d, const std::string& admin_dir) {
  1302→        constexpr unsigned max_concurrent_qualities = 3;
  1303→        std::atomic<unsigned> active{0};
  1304→        std::mutex qmtx;
  1305→        std::condition_variable qcv;
  1306→        std::vector<std::future<void>> qfutures;
  1307→
  1308→        for (double scale : quality_scales) {
  1309→            {
  1310→                std::unique_lock<std::mutex> lock(qmtx);
  1311→                qcv.wait(lock, [&]{ return active.load() < max_concurrent_qualities; });
  1312→            }
  1313→            active.fetch_add(1);
  1314→
  1315→            std::string qname = quality_dir_name(scale);
  1316→            std::string qdir = admin_dir + "/" + qname;
  1317→            qfutures.push_back(std::async(std::launch::async, [&, qdir, scale]() {
  1318→                write_quality_variant(d, admin_dir, qdir, scale);
  1319→                active.fetch_sub(1);
  1320→                qcv.notify_one();
  1321→            }));
  1322→        }
  1323→        for (auto& f : qfutures) f.get();
  1324→    };
  1325→
  1326→    // Write one region: modes + quality variants
  1327→    auto write_region = [&](const ParsedData& d, const std::string& base_dir) {
  1328→        ensure_dir(base_dir);
  1329→
  1330→        if (multi_output) {
  1331→            // Write all 3 modes in parallel (they read shared data, write to separate dirs)
  1332→            auto wf1 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/full", IndexMode::Full); });
  1333→            auto wf2 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/no-addresses", IndexMode::NoAddresses); });
  1334→            auto wf3 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/admin", IndexMode::AdminOnly); });
  1335→            wf1.get(); wf2.get(); wf3.get();
  1336→        } else {
  1337→            write_index(d, base_dir, mode);
  1338→        }
  1339→
  1340→        // Write quality variants under admin/ (or base_dir if not multi_output)
  1341→        if (multi_quality) {
  1342→            std::string admin_dir = multi_output ? base_dir + "/admin" : base_dir;
  1343→            std::cerr << "  Writing quality variants for " << base_dir << "..." << std::endl;
  1344→            write_qualities(d, admin_dir);
  1345→        }
  1346→    };
  1347→
  1348→    // Write planet (async — overlaps with continent filtering start)
  1349→    auto planet_future = std::async(std::launch::async, [&]() {
  1350→        write_region(data, output_dir);
  1351→    });
  1352→
  1353→    // Process continents with bounded concurrency, largest first.
  1354→    if (generate_continents) {
  1355→        // Pre-compute continent membership for each unique cell in sorted pairs.
  1356→        // One parallel scan replaces 8 × cell_in_bbox per cell during filtering.
  1357→        auto _pct = std::chrono::steady_clock::now();
  1358→        auto _pcc = CpuTicks::now();
  1359→
  1360→        // Load continent polygons for boundary-based filtering
  1361→        auto continent_polys = get_continent_polygons();
  1362→        std::cerr << "  Loaded " << continent_polys.size() << " continent boundary polygons" << std::endl;
  1363→
  1364→        // For each sorted pair array, build a parallel array of continent bitmasks.
  1365→        // Since pairs are sorted by cell_id, consecutive entries share the same mask.
  1366→        auto precompute_masks = [&continent_polys](const std::vector<CellItemPair>& sorted) -> std::vector<uint8_t> {
  1367→            if (sorted.empty()) return {};
  1368→            std::vector<uint8_t> masks(sorted.size(), 0);
  1369→
  1370→            unsigned nthreads = std::max(1u, std::thread::hardware_concurrency());
  1371→            size_t chunk = (sorted.size() + nthreads - 1) / nthreads;
  1372→            std::vector<size_t> bounds = {0};
  1373→            for (unsigned t = 1; t < nthreads; t++) {
  1374→                size_t target = t * chunk;
  1375→                if (target >= sorted.size()) break;
  1376→                while (target < sorted.size() && sorted[target].cell_id == sorted[target-1].cell_id)
  1377→                    target++;
  1378→                if (target < sorted.size()) bounds.push_back(target);
  1379→            }
  1380→            bounds.push_back(sorted.size());
  1381→
  1382→            std::vector<std::thread> threads;
  1383→            for (size_t th = 0; th + 1 < bounds.size(); th++) {
  1384→                threads.emplace_back([&, th]() {
  1385→                    for (size_t i = bounds[th]; i < bounds[th+1]; ) {
  1386→                        uint64_t cell_id = sorted[i].cell_id;
  1387→                        S2CellId cell(cell_id);
  1388→                        S2LatLng center = cell.ToLatLng();
  1389→                        double lat = center.lat().degrees();
  1390→                        double lng = center.lng().degrees();
  1391→                        uint8_t mask = 0;
  1392→                        // Test against continent polygons (not bboxes)
  1393→                        for (size_t ci = 0; ci < continent_polys.size() && ci < 8; ci++) {
  1394→                            if (point_in_polygon(lat, lng, continent_polys[ci].vertices))
  1395→                                mask |= (1u << ci);
  1396→                        }
  1397→                        while (i < bounds[th+1] && sorted[i].cell_id == cell_id) {
  1398→                            masks[i] = mask;
  1399→                            i++;


Bash

grep -n "void write_index\|write_index(" /root/traccar-geocoder/builder/src/build_index.cpp | head -5

Find write_index function


1332:            auto wf1 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/full", IndexMode::Full); });
1333:            auto wf2 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/no-addresses", IndexMode::NoAddresses); });
1334:            auto wf3 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/admin", IndexMode::AdminOnly); });
1337:            write_index(d, base_dir, mode);


No files found


grep -r "void write_index" /root/traccar-geocoder/builder/src/ | head -3

Find write_index definition


/root/traccar-geocoder/builder/src/cell_index.cpp:void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
/root/traccar-geocoder/builder/src/cell_index.h:void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode);


     1→#include "cell_index.h"
     2→
     3→#include <algorithm>
     4→#include <atomic>
     5→#include <cstring>
     6→#include <fstream>
     7→#include <future>
     8→#include <iostream>
     9→#include <thread>
    10→#include <unordered_map>
    11→#include <unordered_set>
    12→
    13→#include <unistd.h>
    14→
    15→#include "geometry.h"
    16→#include "s2_helpers.h"
    17→
    18→std::vector<uint32_t> write_entries(
    19→    const std::string& path,
    20→    const std::vector<uint64_t>& sorted_cells,
    21→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
    22→) {
    23→    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
    24→    std::vector<CellRef> sorted_refs;
    25→    sorted_refs.reserve(cell_map.size());
    26→    for (auto& [id, ids] : cell_map) sorted_refs.push_back({id, &ids});
    27→    std::sort(sorted_refs.begin(), sorted_refs.end(),
    28→        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });
    29→
    30→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    31→    size_t total_size = 0;
    32→    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);
    33→
    34→    std::vector<char> buf;
    35→    buf.reserve(total_size);
    36→    uint32_t current = 0;
    37→    size_t ri = 0;
    38→    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
    39→        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
    40→        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
    41→        offsets[si] = current;
    42→        const auto& ids = *sorted_refs[ri].ids;
    43→        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
    44→        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
    45→                   reinterpret_cast<const char*>(&count) + sizeof(count));
    46→        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
    47→                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
    48→        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
    49→        ri++;
    50→    }
    51→    std::ofstream f(path, std::ios::binary);
    52→    f.write(buf.data(), buf.size());
    53→    return offsets;
    54→}
    55→
    56→std::vector<uint32_t> write_entries_from_sorted(
    57→    const std::string& path,
    58→    const std::vector<uint64_t>& sorted_cells,
    59→    const std::vector<CellItemPair>& sorted_pairs
    60→) {
    61→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    62→    if (sorted_pairs.empty()) {
    63→        std::ofstream f(path, std::ios::binary);
    64→        return offsets;
    65→    }
    66→
    67→    unsigned int nthreads = std::thread::hardware_concurrency();
    68→    if (nthreads == 0) nthreads = 4;
    69→    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;
    70→
    71→    struct ChunkResult {
    72→        std::vector<char> buf;
    73→        size_t cell_start, cell_end;
    74→        uint32_t local_size;
    75→    };
    76→    std::vector<ChunkResult> chunks(nthreads);
    77→
    78→    std::vector<std::thread> threads;
    79→    for (unsigned int t = 0; t < nthreads; t++) {
    80→        size_t cs = t * cells_per_chunk;
    81→        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
    82→        if (cs >= sorted_cells.size()) break;
    83→
    84→        threads.emplace_back([&, t, cs, ce]() {
    85→            auto& chunk = chunks[t];
    86→            chunk.cell_start = cs;
    87→            chunk.cell_end = ce;
    88→            chunk.local_size = 0;
    89→
    90→            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
    91→                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
    92→                    return p.cell_id < id;
    93→                }) - sorted_pairs.begin();
    94→
    95→            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
    96→                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
    97→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
    98→                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;
    99→
   100→                offsets[si] = chunk.local_size;


   100→                offsets[si] = chunk.local_size;
   101→                size_t start = pi;
   102→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
   103→                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(MAX_VERTEX_COUNT)));
   104→                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
   105→                size_t buf_pos = chunk.buf.size();
   106→                chunk.buf.resize(buf_pos + entry_size);
   107→                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
   108→                for (size_t k = start; k < pi; k++) {
   109→                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
   110→                           &sorted_pairs[k].item_id, sizeof(uint32_t));
   111→                }
   112→                chunk.local_size += entry_size;
   113→            }
   114→        });
   115→    }
   116→    for (auto& t : threads) t.join();
   117→
   118→    uint32_t global_offset = 0;
   119→    for (auto& chunk : chunks) {
   120→        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
   121→            if (offsets[si] != NO_DATA) offsets[si] += global_offset;
   122→        }
   123→        global_offset += chunk.local_size;
   124→    }
   125→
   126→    std::vector<char> buf;
   127→    buf.reserve(global_offset);
   128→    for (auto& chunk : chunks) buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());
   129→
   130→    std::ofstream f(path, std::ios::binary);
   131→    f.write(buf.data(), buf.size());
   132→    return offsets;
   133→}
   134→
   135→void write_cell_index(
   136→    const std::string& cells_path,
   137→    const std::string& entries_path,
   138→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
   139→) {
   140→    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(cell_map.begin(), cell_map.end());
   141→    std::sort(sorted.begin(), sorted.end());
   142→
   143→    { std::ofstream f(cells_path, std::ios::binary);
   144→      uint32_t current_offset = 0;
   145→      for (const auto& [cell_id, ids] : sorted) {
   146→          f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
   147→          f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
   148→          current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
   149→      } }
   150→
   151→    { std::ofstream f(entries_path, std::ios::binary);
   152→      for (const auto& [cell_id, ids] : sorted) {
   153→          uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
   154→          f.write(reinterpret_cast<const char*>(&count), sizeof(count));
   155→          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
   156→      } }
   157→}
   158→
   159→void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
   160→    ensure_dir(output_dir);
   161→    auto _wt = std::chrono::steady_clock::now();
   162→
   163→    bool write_streets = (mode != IndexMode::AdminOnly);
   164→    bool write_addresses = (mode == IndexMode::Full);
   165→
   166→    if (write_streets) {
   167→        std::vector<uint64_t> sorted_geo_cells;
   168→        {
   169→            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
   170→                std::vector<uint64_t> cells;
   171→                cells.reserve(pairs.size() / 2);
   172→                for (size_t i = 0; i < pairs.size(); ) {
   173→                    cells.push_back(pairs[i].cell_id);
   174→                    uint64_t cur = pairs[i].cell_id;
   175→                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
   176→                }
   177→                return cells;
   178→            };
   179→            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
   180→                std::vector<uint64_t> cells;
   181→                cells.reserve(m.size());
   182→                for (auto& [id, _] : m) cells.push_back(id);
   183→                std::sort(cells.begin(), cells.end());
   184→                return cells;
   185→            };
   186→
   187→            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
   188→            {
   189→                auto f1 = std::async(std::launch::async, [&] {
   190→                    return !data.sorted_way_cells.empty()
   191→                        ? extract_unique_cells(data.sorted_way_cells)
   192→                        : extract_from_map(data.cell_to_ways);
   193→                });
   194→                if (write_addresses) {
   195→                    auto f2 = std::async(std::launch::async, [&] {
   196→                        return !data.sorted_addr_cells.empty()
   197→                            ? extract_unique_cells(data.sorted_addr_cells)
   198→                            : extract_from_map(data.cell_to_addrs);
   199→                    });
   200→                    auto f3 = std::async(std::launch::async, [&] {
   201→                        return !data.sorted_interp_cells.empty()
   202→                            ? extract_unique_cells(data.sorted_interp_cells)
   203→                            : extract_from_map(data.cell_to_interps);
   204→                    });
   205→                    addr_cells = f2.get();
   206→                    interp_cells = f3.get();
   207→                }
   208→                way_cells = f1.get();
   209→            }
   210→
   211→            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
   212→            std::merge(way_cells.begin(), way_cells.end(), addr_cells.begin(), addr_cells.end(),
   213→                       std::back_inserter(sorted_geo_cells));
   214→            if (!interp_cells.empty()) {
   215→                std::vector<uint64_t> tmp;
   216→                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
   217→                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
   218→                           interp_cells.begin(), interp_cells.end(), std::back_inserter(tmp));
   219→                sorted_geo_cells = std::move(tmp);
   220→            }
   221→            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
   222→                                    sorted_geo_cells.end());
   223→        }
   224→
   225→        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
   226→        {
   227→            auto f1 = std::async(std::launch::async, [&]() {
   228→                if (!data.sorted_way_cells.empty())
   229→                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
   230→                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
   231→            });
   232→            if (write_addresses) {
   233→                auto f2 = std::async(std::launch::async, [&]() {
   234→                    if (!data.sorted_addr_cells.empty())
   235→                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
   236→                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
   237→                });
   238→                auto f3 = std::async(std::launch::async, [&]() {
   239→                    if (!data.sorted_interp_cells.empty())
   240→                        return write_entries_from_sorted(output_dir + "/interp_entries.bin", sorted_geo_cells, data.sorted_interp_cells);
   241→                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
   242→                });
   243→                addr_offsets = f2.get();
   244→                interp_offsets = f3.get();
   245→            }
   246→            street_offsets = f1.get();
   247→        }
   248→
   249→        log_phase("  Write: entry files", _wt);


Get line count for cell_index


469 /root/traccar-geocoder/builder/src/cell_index.cpp


   249→        log_phase("  Write: entry files", _wt);
   250→        {
   251→            size_t n = sorted_geo_cells.size();
   252→            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
   253→            std::vector<char> buf(n * row_size);
   254→            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
   255→            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);
   256→
   257→            unsigned int nthreads = std::thread::hardware_concurrency();
   258→            if (nthreads == 0) nthreads = 4;
   259→            size_t chunk = (n + nthreads - 1) / nthreads;
   260→            std::vector<std::thread> fill_threads;
   261→            for (unsigned int t = 0; t < nthreads; t++) {
   262→                size_t start = t * chunk;
   263→                size_t end = std::min(start + chunk, n);
   264→                if (start >= n) break;
   265→                fill_threads.emplace_back([&, start, end]() {
   266→                    char* ptr = buf.data() + start * row_size;
   267→                    for (size_t i = start; i < end; i++) {
   268→                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
   269→                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   270→                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   271→                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
   272→                    }
   273→                });
   274→            }
   275→            for (auto& t : fill_threads) t.join();
   276→
   277→            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
   278→            f.write(buf.data(), buf.size());
   279→        }
   280→
   281→        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
   282→                  << data.ways.size() << " ways, " << data.addr_points.size() << " addrs, "
   283→                  << data.interp_ways.size() << " interps)" << std::endl;
   284→    }
   285→
   286→    log_phase("  Write: geo_cells.bin", _wt);
   287→    auto admin_future = std::async(std::launch::async, [&] {
   288→        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
   289→        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
   290→    });
   291→
   292→    std::vector<std::future<void>> write_futures;
   293→    if (write_streets) {
   294→        write_futures.push_back(std::async(std::launch::async, [&] {
   295→            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
   296→            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
   297→        }));
   298→        write_futures.push_back(std::async(std::launch::async, [&] {
   299→            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
   300→            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
   301→        }));
   302→        if (write_addresses) {
   303→            write_futures.push_back(std::async(std::launch::async, [&] {
   304→                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
   305→                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
   306→            }));
   307→            write_futures.push_back(std::async(std::launch::async, [&] {
   308→                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
   309→                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
   310→            }));
   311→            write_futures.push_back(std::async(std::launch::async, [&] {
   312→                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
   313→                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
   314→            }));
   315→        }
   316→    }
   317→    // If not writing all data types, compact string pool and remap references
   318→    // before writing admin_polygons and strings.bin.
   319→    std::vector<AdminPolygon> compact_admin;
   320→    std::vector<char> compact_strings;
   321→    const bool need_compact = (!write_streets || !write_addresses);
   322→
   323→    if (need_compact) {
   324→        // Collect referenced string offsets
   325→        std::unordered_set<uint32_t> used_offsets;
   326→        if (write_streets) {
   327→            for (const auto& w : data.ways) used_offsets.insert(w.name_id);
   328→            if (write_addresses) {
   329→                for (const auto& a : data.addr_points) {
   330→                    used_offsets.insert(a.housenumber_id);
   331→                    used_offsets.insert(a.street_id);
   332→                }
   333→                for (const auto& iw : data.interp_ways) used_offsets.insert(iw.street_id);
   334→            }
   335→        }
   336→        for (const auto& ap : data.admin_polygons) used_offsets.insert(ap.name_id);
   337→
   338→        // Build compact pool and remap
   339→        const auto& old_sp = data.string_pool.data();
   340→        std::vector<uint32_t> sorted_offs(used_offsets.begin(), used_offsets.end());
   341→        std::sort(sorted_offs.begin(), sorted_offs.end());
   342→        std::unordered_map<uint32_t, uint32_t> remap;
   343→        compact_strings.reserve(used_offsets.size() * 16);
   344→        for (uint32_t off : sorted_offs) {
   345→            remap[off] = static_cast<uint32_t>(compact_strings.size());
   346→            size_t len = std::strlen(old_sp.data() + off);
   347→            compact_strings.insert(compact_strings.end(), old_sp.data() + off,
   348→                old_sp.data() + off + len + 1);
   349→        }
   350→
   351→        // Remap admin polygon name_ids
   352→        compact_admin = data.admin_polygons;
   353→        for (auto& ap : compact_admin) ap.name_id = remap[ap.name_id];
   354→
   355→        // Remap street/addr/interp if applicable (for no-addresses mode)
   356→        // These are written from data directly, so we'd need copies too.
   357→        // For simplicity, only compact when we have fewer types.
   358→    }
   359→
   360→    const auto& admin_to_write = need_compact ? compact_admin : data.admin_polygons;
   361→    write_futures.push_back(std::async(std::launch::async, [&] {
   362→        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
   363→        f.write(reinterpret_cast<const char*>(admin_to_write.data()), admin_to_write.size() * sizeof(AdminPolygon));
   364→    }));
   365→    write_futures.push_back(std::async(std::launch::async, [&] {
   366→        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
   367→        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
   368→    }));
   369→    write_futures.push_back(std::async(std::launch::async, [&] {
   370→        if (need_compact) {
   371→            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
   372→            f.write(compact_strings.data(), compact_strings.size());
   373→            std::cerr << "strings.bin: " << compact_strings.size() << " bytes (compacted from "
   374→                      << data.string_pool.data().size() << ")" << std::endl;
   375→        } else {
   376→            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
   377→            f.write(data.string_pool.data().data(), data.string_pool.data().size());
   378→            std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
   379→        }
   380→    }));
   381→
   382→    log_phase("  Write: parallel data files launched", _wt);
   383→    admin_future.get();
   384→    for (auto& f : write_futures) f.get();
   385→}
   386→
   387→void write_quality_variant(const ParsedData& data, const std::string& source_dir,
   388→                           const std::string& output_dir, double epsilon_scale) {
   389→    ensure_dir(output_dir);
   390→
   391→    // Re-simplify admin polygons at the given epsilon scale
   392→    std::vector<AdminPolygon> new_polys;
   393→    std::vector<NodeCoord> new_verts;
   394→    new_polys.reserve(data.admin_polygons.size());
   395→
   396→    // Parallel simplification
   397→    struct SimplifiedPoly {
   398→        std::vector<std::pair<double,double>> verts;


     1→#pragma once
     2→
     3→#include <algorithm>
     4→#include <chrono>
     5→#include <iostream>
     6→#include <string>
     7→#include <unordered_map>
     8→#include <vector>
     9→
    10→#include <cstdio>
    11→#include <cstring>
    12→#include <dirent.h>
    13→#include <iomanip>
    14→#include <thread>
    15→#include <unistd.h>
    16→#include <sys/stat.h>
    17→
    18→#include "types.h"
    19→#include "string_pool.h"
    20→
    21→// --- Directory creation ---
    22→
    23→inline void ensure_dir(const std::string& path) {
    24→    // Recursive mkdir — create all parent directories
    25→    for (size_t i = 1; i < path.size(); i++) {
    26→        if (path[i] == '/') {
    27→            mkdir(path.substr(0, i).c_str(), 0755);
    28→        }
    29→    }
    30→    mkdir(path.c_str(), 0755);
    31→}
    32→
    33→// --- CPU tick reading (all threads via getrusage) ---
    34→
    35→#include <sys/resource.h>
    36→
    37→struct CpuTicks {
    38→    long long process_us = 0;  // user+system microseconds (all threads)
    39→
    40→    static CpuTicks now() {
    41→        CpuTicks ct;
    42→        struct rusage ru;
    43→        if (getrusage(RUSAGE_SELF, &ru) == 0) {
    44→            ct.process_us = (long long)ru.ru_utime.tv_sec * 1000000 + ru.ru_utime.tv_usec
    45→                          + (long long)ru.ru_stime.tv_sec * 1000000 + ru.ru_stime.tv_usec;
    46→        }
    47→        return ct;
    48→    }
    49→};
    50→
    51→// --- Phase timer with CPU utilization ---
    52→
    53→inline long get_rss_mb() {
    54→    long rss_pages = 0;
    55→    FILE* f = fopen("/proc/self/statm", "r");
    56→    if (f) {
    57→        long size;
    58→        if (fscanf(f, "%ld %ld", &size, &rss_pages) != 2) rss_pages = 0;
    59→        fclose(f);
    60→    }
    61→    return rss_pages * sysconf(_SC_PAGESIZE) / (1024 * 1024);
    62→}
    63→
    64→inline void log_phase(const char* name, std::chrono::steady_clock::time_point& t,
    65→                       CpuTicks& prev_cpu) {
    66→    auto now = std::chrono::steady_clock::now();
    67→    auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(now - t).count();
    68→    auto cpu_now = CpuTicks::now();
    69→    double cpu_sec = (cpu_now.process_us - prev_cpu.process_us) / 1e6;
    70→    double wall_sec = ms / 1000.0;
    71→    double cores_used = (wall_sec > 0.1) ? cpu_sec / wall_sec : 0;
    72→    unsigned ncpu = std::thread::hardware_concurrency();
    73→    int pct = (wall_sec > 0.1 && ncpu > 0) ? (int)(cores_used * 100.0 / ncpu) : 0;
    74→    long rss = get_rss_mb();
    75→    std::cerr << "  [" << ms/1000 << "." << (ms%1000)/100 << "s"
    76→              << " " << std::fixed << std::setprecision(1) << cores_used << "/" << ncpu << "cores"
    77→              << " " << pct << "%"
    78→              << " " << rss << "MiB] " << name << std::endl;
    79→    t = now;
    80→    prev_cpu = cpu_now;
    81→}
    82→
    83→// Backward compat overload (no CPU tracking)
    84→inline void log_phase(const char* name, std::chrono::steady_clock::time_point& t) {
    85→    auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(
    86→        std::chrono::steady_clock::now() - t).count();
    87→    std::cerr << "  [" << ms/1000 << "." << (ms%1000)/100 << "s] " << name << std::endl;
    88→    t = std::chrono::steady_clock::now();
    89→}
    90→
    91→// --- Parsed data container ---
    92→
    93→struct ParsedData {
    94→    StringPool string_pool;
    95→    std::vector<WayHeader> ways;
    96→    std::vector<NodeCoord> street_nodes;
    97→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_ways;
    98→    std::vector<AddrPoint> addr_points;
    99→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_addrs;
   100→    std::vector<InterpWay> interp_ways;
   101→    std::vector<NodeCoord> interp_nodes;
   102→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_interps;
   103→    std::vector<AdminPolygon> admin_polygons;
   104→    std::vector<NodeCoord> admin_vertices;
   105→    std::unordered_map<uint64_t, std::vector<uint32_t>> cell_to_admin;
   106→
   107→    // Deferred work for parallel S2 computation (ways + interps)
   108→    std::vector<DeferredWay> deferred_ways;
   109→    std::vector<DeferredInterp> deferred_interps;
   110→
   111→    // Sorted (cell_id, item_id) pairs — kept for direct entry writing
   112→    std::vector<CellItemPair> sorted_way_cells;
   113→    std::vector<CellItemPair> sorted_addr_cells;
   114→    std::vector<CellItemPair> sorted_interp_cells;
   115→
   116→    // Collected data for parallel admin assembly
   117→    std::vector<CollectedRelation> collected_relations;
   118→    struct WayGeometry {
   119→        std::vector<std::pair<double,double>> coords;
   120→        int64_t first_node_id;
   121→        int64_t last_node_id;
   122→    };
   123→    std::unordered_map<int64_t, WayGeometry> way_geometries;
   124→};
   125→
   126→// --- Deduplicate IDs per cell ---
   127→
   128→template<typename Map>
   129→inline void deduplicate(Map& cell_map) {
   130→    for (auto& [cell_id, ids] : cell_map) {
   131→        std::sort(ids.begin(), ids.end());
   132→        ids.erase(std::unique(ids.begin(), ids.end()), ids.end());
   133→    }
   134→}
   135→


grep -n "void add_addr_point\|add_addr_point(" /root/traccar-geocoder/builder/src/build_index.cpp | head -10

Find add_addr_point function


394:                        add_addr_point(data, local.addr_coords[j].first, local.addr_coords[j].second,
633:                        add_addr_point(data, local.building_addrs[i].lat, local.building_addrs[i].lng,


grep -r "void add_addr_point\|inline.*add_addr_point" /root/traccar-geocoder/builder/src/

Find add_addr_point definition


/root/traccar-geocoder/builder/src/s2_helpers.h:void add_addr_point(ParsedData& data, double lat, double lng,
/root/traccar-geocoder/builder/src/s2_helpers.cpp:void add_addr_point(ParsedData& data, double lat, double lng,


     1→#include "s2_helpers.h"
     2→#include "geometry.h"
     3→
     4→#include <algorithm>
     5→#include <cmath>
     6→#include <iostream>
     7→#include <memory>
     8→
     9→#include <s2/s2latlng.h>
    10→#include <s2/s2region_coverer.h>
    11→#include <s2/s2polyline.h>
    12→#include <s2/s2polygon.h>
    13→#include <s2/s2loop.h>
    14→
    15→// --- S2 cell level globals ---
    16→
    17→int kStreetCellLevel = 17;
    18→int kAdminCellLevel = 10;
    19→int kMaxAdminLevel = 0;  // 0 means no filtering
    20→
    21→SimplifyMode kSimplifyMode = SimplifyMode::MaxVertices;
    22→double kSimplifyEpsilonOverride = 0;
    23→
    24→// --- cover_edge ---
    25→
    26→void cover_edge(double lat1, double lng1, double lat2, double lng2,
    27→                std::vector<S2CellId>& out) {
    28→    out.clear();
    29→
    30→    S2CellId c1 = S2CellId(S2LatLng::FromDegrees(lat1, lng1)).parent(kStreetCellLevel);
    31→    S2CellId c2 = S2CellId(S2LatLng::FromDegrees(lat2, lng2)).parent(kStreetCellLevel);
    32→
    33→    // Same cell — most common case for short edges
    34→    if (c1 == c2) {
    35→        out.push_back(c1);
    36→        return;
    37→    }
    38→
    39→    // Adjacent cells — check midpoint
    40→    S2CellId cm = S2CellId(S2LatLng::FromDegrees((lat1+lat2)/2, (lng1+lng2)/2)).parent(kStreetCellLevel);
    41→    if (cm == c1 || cm == c2) {
    42→        out.push_back(c1);
    43→        out.push_back(c2);
    44→        return;
    45→    }
    46→
    47→    // Longer edge — use S2RegionCoverer for correctness (handles geodesic curves)
    48→    thread_local S2RegionCoverer coverer = []() {
    49→        S2RegionCoverer::Options options;
    50→        options.set_fixed_level(kStreetCellLevel);
    51→        return S2RegionCoverer(options);
    52→    }();
    53→
    54→    S2Point p1 = S2LatLng::FromDegrees(lat1, lng1).ToPoint();
    55→    S2Point p2 = S2LatLng::FromDegrees(lat2, lng2).ToPoint();
    56→    if (p1 == p2) { out.push_back(c1); return; }
    57→
    58→    std::vector<S2Point> points = {p1, p2};
    59→    S2Polyline polyline(points);
    60→    S2CellUnion covering = coverer.GetCovering(polyline);
    61→    out = std::move(covering.cell_ids());
    62→}
    63→
    64→std::vector<S2CellId> cover_edge(double lat1, double lng1, double lat2, double lng2) {
    65→    std::vector<S2CellId> result;
    66→    cover_edge(lat1, lng1, lat2, lng2, result);
    67→    return result;
    68→}
    69→
    70→S2CellId point_to_cell(double lat, double lng) {
    71→    return S2CellId(S2LatLng::FromDegrees(lat, lng)).parent(kStreetCellLevel);
    72→}
    73→
    74→// --- cover_polygon ---
    75→
    76→std::vector<std::pair<S2CellId, bool>> cover_polygon(
    77→    const std::vector<std::pair<double,double>>& vertices) {
    78→    std::vector<S2Point> points;
    79→    points.reserve(vertices.size());
    80→    for (const auto& [lat, lng] : vertices) {
    81→        S2Point p = S2LatLng::FromDegrees(lat, lng).ToPoint();
    82→        if (!points.empty() && points.back() == p) continue;
    83→        points.push_back(p);
    84→    }
    85→    if (points.size() > 1 && points.front() == points.back()) {
    86→        points.pop_back();
    87→    }
    88→    if (points.size() < 3) return {};
    89→
    90→    S2Error error;
    91→    auto loop = std::make_unique<S2Loop>(points, S2Debug::DISABLE);
    92→    loop->Normalize();
    93→    if (loop->FindValidationError(&error)) return {};
    94→
    95→    S2Polygon polygon(std::move(loop));
    96→
    97→    S2RegionCoverer::Options options;
    98→    options.set_max_level(kAdminCellLevel);
    99→    options.set_max_cells(MAX_S2_CELLS_PER_POLY);
   100→


grep -n "void add_addr_point" /root/traccar-geocoder/builder/src/s2_helpers.cpp

Find add_addr_point line number


229:void add_addr_point(ParsedData& data, double lat, double lng,


   229→void add_addr_point(ParsedData& data, double lat, double lng,
   230→                    const char* housenumber, const char* street,
   231→                    uint64_t& addr_count_total) {
   232→    uint32_t addr_id = static_cast<uint32_t>(data.addr_points.size());
   233→    data.addr_points.push_back({
   234→        static_cast<float>(lat),
   235→        static_cast<float>(lng),
   236→        data.string_pool.intern(housenumber),
   237→        data.string_pool.intern(street)
   238→    });
   239→
   240→    S2CellId cell = point_to_cell(lat, lng);
   241→    data.cell_to_addrs[cell.id()].push_back(addr_id);
   242→
   243→    addr_count_total++;
   244→    if (addr_count_total % 1000000 == 0) {
   245→        std::cerr << "Collected " << addr_count_total / 1000000 << "M addresses..." << std::endl;
   246→    }
   247→}
   248→
   249→// --- add_admin_polygon ---
   250→
   251→void add_admin_polygon(ParsedData& data,
   252→                       const std::vector<std::pair<double,double>>& vertices_in,
   253→                       const char* name, uint8_t admin_level,
   254→                       const char* country_code,
   255→                       AdminCoverPool* admin_pool) {
   256→    auto vertices = vertices_in;
   257→    if (vertices.size() >= 4 &&
   258→        std::fabs(vertices.front().first - vertices.back().first) < 1e-7 &&
   259→        std::fabs(vertices.front().second - vertices.back().second) < 1e-7) {
   260→        vertices.pop_back();
   261→        auto min_it = std::min_element(vertices.begin(), vertices.end());
   262→        std::rotate(vertices.begin(), min_it, vertices.end());
   263→        vertices.push_back(vertices.front());
   264→    }
   265→
   266→    auto simplified = simplify_admin_polygon(vertices, admin_level);
   267→    if (simplified.size() < 3) return;
   268→
   269→    uint32_t poly_id = static_cast<uint32_t>(data.admin_polygons.size());
   270→    uint32_t vertex_offset = static_cast<uint32_t>(data.admin_vertices.size());
   271→
   272→    for (const auto& [lat, lng] : simplified) {
   273→        data.admin_vertices.push_back({static_cast<float>(lat), static_cast<float>(lng)});
   274→    }
   275→
   276→    AdminPolygon poly{};
   277→    poly.vertex_offset = vertex_offset;
   278→    poly.vertex_count = static_cast<uint32_t>(simplified.size());


grep -n "read_nodes_streaming\|read_ways_streaming\|thread" /root/traccar-geocoder/builder/src/pbf_reader.h | head -20

Check PBF reader streaming methods


5:// to a thread pool with zero sequential bottleneck.
129:// `num_threads` controls parallelism (0 = hardware_concurrency).
133:                       unsigned num_threads = 0,
139:    explicit PbfFile(const std::string& filename, unsigned num_threads = 0);
147:    // Callback is called from worker threads with (block, thread_index).
148:    // The callback must be thread-safe OR use the thread_index for thread-local storage.
153:    void read_blocks(std::function<void(PbfBlock&, unsigned thread_idx)> callback,
158:    unsigned thread_count() const { return num_threads_; }
161:    void read_nodes_streaming(const NodeCallback& callback);
162:    void read_ways_streaming(const WayCallback& callback);
168:    unsigned num_threads_;


     1→#include "pbf_reader.h"
     2→
     3→#include <algorithm>
     4→#include <atomic>
     5→#include <cerrno>
     6→#include <cstdlib>
     7→#include <condition_variable>
     8→#include <cstring>
     9→#include <fcntl.h>
    10→#include <iostream>
    11→#include <sys/mman.h>
    12→#include <mutex>
    13→#include <stdexcept>
    14→#include <thread>
    15→#include <unistd.h>
    16→#include <zlib.h>
    17→
    18→#include <protozero/pbf_reader.hpp>
    19→#include <protozero/varint.hpp>
    20→
    21→// --- PBF file format constants ---
    22→// BlobHeader max size: 32 MiB (per spec), but typically <100 bytes
    23→// Blob max size: 32 MiB (per spec)
    24→static constexpr size_t MAX_BLOB_HEADER_SIZE = 64 * 1024;
    25→static constexpr size_t MAX_BLOB_SIZE = 64 * 1024 * 1024;
    26→
    27→// Protobuf field tags (from OSM PBF spec)
    28→namespace BlobHeaderTag {
    29→    constexpr int TYPE = 1;      // string
    30→    constexpr int DATASIZE = 3;  // int32
    31→}
    32→namespace BlobTag {
    33→    constexpr int RAW = 1;       // bytes
    34→    constexpr int RAW_SIZE = 2;  // int32
    35→    constexpr int ZLIB = 3;      // bytes
    36→}
    37→namespace PrimitiveBlockTag {
    38→    constexpr int STRINGTABLE = 1;   // StringTable
    39→    constexpr int PRIMITIVEGROUP = 2; // repeated PrimitiveGroup
    40→    constexpr int GRANULARITY = 17;  // int32 (default 100)
    41→    constexpr int LAT_OFFSET = 19;   // int64 (default 0)
    42→    constexpr int LON_OFFSET = 20;   // int64 (default 0)
    43→}
    44→namespace PrimitiveGroupTag {
    45→    constexpr int NODES = 1;
    46→    constexpr int DENSE = 2;
    47→    constexpr int WAYS = 3;
    48→    constexpr int RELATIONS = 4;
    49→}
    50→namespace StringTableTag {
    51→    constexpr int S = 1; // repeated bytes
    52→}
    53→namespace DenseNodesTag {
    54→    constexpr int ID = 1;
    55→    constexpr int LAT = 8;
    56→    constexpr int LON = 9;
    57→    constexpr int KEYS_VALS = 10;
    58→}
    59→namespace WayTag {
    60→    constexpr int ID = 1;
    61→    constexpr int KEYS = 2;
    62→    constexpr int VALS = 3;
    63→    constexpr int REFS = 8;
    64→}
    65→namespace RelationTag {
    66→    constexpr int ID = 1;
    67→    constexpr int KEYS = 2;
    68→    constexpr int VALS = 3;
    69→    constexpr int ROLES_SID = 8;
    70→    constexpr int MEMIDS = 9;
    71→    constexpr int TYPES = 10;
    72→}
    73→
    74→// --- File I/O helpers ---
    75→
    76→static std::string read_bytes(int fd, size_t offset, size_t count) {
    77→    std::string buf(count, '\0');
    78→    ssize_t n = pread(fd, buf.data(), count, offset);
    79→    if (n != (ssize_t)count) {
    80→        throw std::runtime_error("pread failed at offset " + std::to_string(offset) +
    81→                                 ": expected " + std::to_string(count) +
    82→                                 " got " + std::to_string(n));
    83→    }
    84→    return buf;
    85→}
    86→
    87→static uint32_t read_u32_be(int fd, size_t offset) {
    88→    uint8_t buf[4];
    89→    if (pread(fd, buf, 4, offset) != 4) {
    90→        throw std::runtime_error("failed to read 4 bytes at offset " + std::to_string(offset));
    91→    }
    92→    return (uint32_t(buf[0]) << 24) | (uint32_t(buf[1]) << 16) |
    93→           (uint32_t(buf[2]) << 8) | uint32_t(buf[3]);
    94→}
    95→
    96→// --- Blob scanning ---
    97→
    98→std::vector<BlobInfo> scan_pbf_blobs(const std::string& filename) {
    99→    int fd = open(filename.c_str(), O_RDONLY);
   100→    if (fd < 0) throw std::runtime_error("cannot open " + filename + ": " + strerror(errno));
   101→
   102→    // Get file size
   103→    off_t file_size = lseek(fd, 0, SEEK_END);
   104→
   105→    std::vector<BlobInfo> blobs;
   106→    size_t offset = 0;
   107→
   108→    while (offset < (size_t)file_size) {
   109→        // Read 4-byte big-endian BlobHeader size
   110→        uint32_t header_size = read_u32_be(fd, offset);
   111→        if (header_size > MAX_BLOB_HEADER_SIZE) {
   112→            throw std::runtime_error("BlobHeader too large: " + std::to_string(header_size));
   113→        }
   114→
   115→        // Read BlobHeader
   116→        std::string header_data = read_bytes(fd, offset + 4, header_size);
   117→        protozero::pbf_reader header_pbf(header_data);
   118→
   119→        std::string type;
   120→        int32_t data_size = 0;
   121→        while (header_pbf.next()) {
   122→            switch (header_pbf.tag()) {
   123→                case BlobHeaderTag::TYPE:
   124→                    type = header_pbf.get_string();
   125→                    break;
   126→                case BlobHeaderTag::DATASIZE:
   127→                    data_size = header_pbf.get_int32();
   128→                    break;
   129→                default:
   130→                    header_pbf.skip();
   131→            }
   132→        }
   133→
   134→        BlobInfo info;
   135→        info.offset = offset;
   136→        info.header_size = header_size;
   137→        info.data_size = data_size;
   138→        info.type = type;
   139→        blobs.push_back(info);
   140→
   141→        // Advance past: 4 bytes length + header + blob data
   142→        offset += 4 + header_size + data_size;
   143→    }
   144→
   145→    close(fd);
   146→    return blobs;
   147→}
   148→
   149→// --- Blob decompression ---
   150→


grep -n "read_nodes_streaming\|read_ways_streaming" /root/traccar-geocoder/builder/src/pbf_reader.cpp | head -5

Find streaming functions


956:void PbfFile::read_nodes_streaming(const NodeCallback& callback) {
1021:void PbfFile::read_ways_streaming(const WayCallback& callback) {


   956→void PbfFile::read_nodes_streaming(const NodeCallback& callback) {
   957→    std::vector<size_t> data_indices;
   958→    for (size_t i = 0; i < blobs_.size(); i++)
   959→        if (blobs_[i].type == "OSMData") data_indices.push_back(i);
   960→    if (data_indices.empty()) return;
   961→
   962→    // Limit concurrent I/O to avoid saturating slow storage.
   963→    // On fast storage, all threads get through the semaphore quickly.
   964→    // On slow storage, excess threads do decode+callback while waiting.
   965→    unsigned max_io;
   966→    const char* max_io_env = getenv("BUILD_MAX_IO");
   967→    if (max_io_env) {
   968→        max_io = static_cast<unsigned>(std::atoi(max_io_env));
   969→        if (max_io == 0) max_io = num_threads_;
   970→    } else {
   971→        max_io = std::min(num_threads_, std::max(8u, num_threads_ / 2));
   972→    }
   973→    IoSemaphore io_sem(max_io);
   974→
   975→    std::atomic<size_t> next_idx{0};
   976→    std::atomic<size_t> blocks_done{0};
   977→    std::vector<std::thread> threads;
   978→
   979→    struct ThreadStats { double read_us = 0, decode_us = 0; size_t count = 0; };
   980→    std::vector<ThreadStats> stats(num_threads_);
   981→
   982→    for (unsigned t = 0; t < num_threads_; t++) {
   983→        threads.emplace_back([&, t]() {
   984→            int local_fd = open(filename_.c_str(), O_RDONLY);
   985→            if (local_fd < 0) return;
   986→            std::string decomp, blob_buf;
   987→            auto& st = stats[t];
   988→            while (true) {
   989→                size_t j = next_idx.fetch_add(1);
   990→                if (j >= data_indices.size()) break;
   991→
   992→                io_sem.acquire();
   993→                auto t0 = std::chrono::steady_clock::now();
   994→                read_and_decompress_blob_into(local_fd, blobs_[data_indices[j]], blob_buf, decomp);
   995→                auto t1 = std::chrono::steady_clock::now();
   996→                io_sem.release();
   997→
   998→                decode_nodes_streaming(decomp.data(), decomp.size(), callback);
   999→                auto t2 = std::chrono::steady_clock::now();
  1000→                st.read_us += std::chrono::duration<double, std::micro>(t1 - t0).count();
  1001→                st.decode_us += std::chrono::duration<double, std::micro>(t2 - t1).count();
  1002→                st.count++;
  1003→                size_t done = blocks_done.fetch_add(1) + 1;
  1004→                if (done % 5000 == 0)
  1005→                    std::cerr << "  Processed " << done << "/" << data_indices.size() << " blocks..." << std::endl;
  1006→            }
  1007→            close(local_fd);
  1008→        });
  1009→    }
  1010→    for (auto& t : threads) t.join();
  1011→
  1012→    double total_read = 0, total_decode = 0; size_t total_blocks = 0;
  1013→    for (auto& s : stats) { total_read += s.read_us; total_decode += s.decode_us; total_blocks += s.count; }
  1014→    std::cerr << "  Node streaming (" << total_blocks << " blocks, " << num_threads_ << " threads, "
  1015→              << max_io << " max_io): "
  1016→              << "read=" << (total_read/1e6) << "s decode+callback=" << (total_decode/1e6) << "s"
  1017→              << " (per-thread avg: read=" << (total_read/1e6/num_threads_)
  1018→              << "s decode+cb=" << (total_decode/1e6/num_threads_) << "s)" << std::endl;
  1019→}
  1020→
  1021→void PbfFile::read_ways_streaming(const WayCallback& callback) {
  1022→    std::vector<size_t> data_indices;
  1023→    for (size_t i = 0; i < blobs_.size(); i++)
  1024→        if (blobs_[i].type == "OSMData") data_indices.push_back(i);
  1025→    if (data_indices.empty()) return;
  1026→
  1027→    unsigned max_io;
  1028→    const char* max_io_env = getenv("BUILD_MAX_IO");
  1029→    if (max_io_env) {
  1030→        max_io = static_cast<unsigned>(std::atoi(max_io_env));
  1031→        if (max_io == 0) max_io = num_threads_;
  1032→    } else {
  1033→        max_io = std::min(num_threads_, std::max(8u, num_threads_ / 2));
  1034→    }
  1035→    IoSemaphore io_sem(max_io);
  1036→
  1037→    std::atomic<size_t> next_idx{0};
  1038→    std::atomic<size_t> blocks_done{0};
  1039→    std::vector<std::thread> threads;
  1040→
  1041→    struct ThreadStats { double read_us = 0, decode_us = 0; size_t count = 0; };
  1042→    std::vector<ThreadStats> stats(num_threads_);
  1043→
  1044→    for (unsigned t = 0; t < num_threads_; t++) {
  1045→        threads.emplace_back([&, t]() {
  1046→            int local_fd = open(filename_.c_str(), O_RDONLY);
  1047→            if (local_fd < 0) return;
  1048→            std::string decomp, blob_buf;
  1049→            auto& st = stats[t];
  1050→            while (true) {
  1051→                size_t j = next_idx.fetch_add(1);
  1052→                if (j >= data_indices.size()) break;
  1053→
  1054→                io_sem.acquire();
  1055→                auto t0 = std::chrono::steady_clock::now();


grep -n "intern\|string_pool" /root/traccar-geocoder/builder/src/build_index.cpp | head -30

Check string pool usage


26:#include "string_pool.h"
617:                        h.name_id = data.string_pool.intern(local.way_strings[i]);
642:                        iw.street_id = data.string_pool.intern(local.interp_strings[i]);
875:                    // Sequential append + submit to S2 pool (cheap: just vector push + string intern)
889:                        poly.name_id = data.string_pool.intern(pp.name);
1299:    // Each variant does parallel simplification internally, so limit


Excellent. Now I have enough information. Let me create a comprehensive analysis document:

## Comprehensive Non-Determinism Analysis: build_index.cpp

Based on my thorough examination of the codebase, I've identified the critical sources of non-determinism. Here's the complete analysis:

---

## 1. STRING POOL ORDERING - NON-DETERMINISTIC

**File:** `/root/traccar-geocoder/builder/src/string_pool.h` (lines 8-28)

**Problem:**
```cpp
class StringPool {
    uint32_t intern(const std::string& s) {
        auto it = index_.find(s);
        if (it != index_.end()) return it->second;
        uint32_t offset = static_cast<uint32_t>(data_.size());
        index_[s] = offset;  // unordered_map iteration order
        data_.insert(...);
        return offset;
    }
private:
    std::unordered_map<std::string, uint32_t> index_;
    std::vector<char> data_;
};
```

**Root Cause:** Uses `std::unordered_map` without thread-safe interning during parallel processing.

**Non-Determinism Sources:**
1. **Unordered map iteration** (lines 617, 642, 889 in build_index.cpp): When strings are interned from multiple threads during way/interp/admin processing, the HashMap iteration order for `index_` is undefined
2. **Thread-local string interning** (line 617): `data.string_pool.intern(local.way_strings[i])` is called sequentially (in thread merge loop, lines 607-671), but the order depends on thread iteration order
3. **Collision handling**: ASLR + malloc randomization mean even same strings get different insertion timings across runs

**Severity:** **SEVERE - Affects Content**
- Different offset assignments for same string across runs
- Results in different string.bin content
- Breaks byte-identical reproducibility entirely

**What's Needed:**
- Use `std::map` instead of `unordered_map` for deterministic ordering
- Add mutex for thread-safe parallel interning (if strings are interned from multiple threads)
- Or: pre-collect all unique strings, sort, then assign offsets sequentially

---

## 2. THREAD-LOCAL DATA MERGE ORDERING - NON-DETERMINISTIC

**File:** `/root/traccar-geocoder/builder/src/build_index.cpp` (lines 603-671)

**Problem:**
```cpp
// Thread-local way processing (lines 407-600)
std::vector<ThreadLocalData> tld(num_threads);

// Later merge (lines 607-671):
for (auto& local : tld) {  // Iterates tld[0], tld[1], ..., tld[num_threads-1]
    uint32_t way_base = data.ways.size();
    for (size_t i = 0; i < local.ways.size(); i++) {
        data.ways.push_back(h);  // Order depends on thread assignment
    }
    // Also merges deferred_ways, interp_ways, building_addrs, way_geometries
}
```

**Root Cause:** The merge order depends on how PBF blocks are assigned to threads.

**Non-Determinism Sources:**
1. **PBF block scheduling** (pbf_reader.cpp, lines 988-990):
   ```cpp
   while (true) {
       size_t j = next_idx.fetch_add(1);  // Which thread gets which block is non-deterministic
       if (j >= data_indices.size()) break;
   ```
   - Thread scheduling is OS-dependent
   - Block assignment is first-come-first-served via atomic fetch_add
   - Same PBF, different runs = different thread gets block N

2. **Merge loop ordering** (build_index.cpp, line 607):
   ```cpp
   for (auto& local : tld) {  // Depends on which thread_idx processed which blocks
   ```
   - Thread 0 may have processed blocks 0,3,6 in run 1 but blocks 1,4,7 in run 2
   - This changes the order ways appear in `data.ways`

3. **Thread-local accumulation order**:
   - Way IDs from different blocks land in different thread-local buckets
   - Merge order = concatenation order = affects way ordering in final output

**Severity:** **SEVERE - Affects Ordering**
- `data.ways` order differs between runs
- `data.addr_points` order differs between runs (lines 630-636)
- `data.interp_ways` order differs between runs (lines 638-646)
- street_ways.bin and addr_points.bin have different content bytes

**What's Needed:**
- Sort thread-local data by PBF block index before merging
- Or: maintain a "source block order" tracker for each item during processing
- Or: switch to single-threaded pass 2 (slower but deterministic)

---

## 3. ADDRESS POINT MERGING - NON-DETERMINISTIC

**File:** `/root/traccar-geocoder/builder/src/build_index.cpp` (lines 389-402 and 630-636)

**Problem:**
```cpp
// Pass 2: Address points (lines 389-402)
static thread_local NodeThreadLocal* tl_node_data = nullptr;
std::vector<NodeThreadLocal> ntld(num_threads);
std::atomic<unsigned> next_tl{0};

pbf.read_nodes_streaming([&](int64_t id, double lat, ...) {
    if (!tl_node_data) {
        unsigned idx = next_tl.fetch_add(1);  // NON-DETERMINISTIC ASSIGNMENT
        tl_node_data = &ntld[idx % ntld.size()];
    }
    // Accumulate in tl_node_data
});

// Later merge (lines 391-399):
for (auto& local : ntld) {  // Order depends on thread assignment
    for (size_t j = 0; j < local.addr_coords.size(); j++) {
        add_addr_point(data, ...);  // Determines addr_id = size() before push
    }
}
```

**Root Cause:** Thread assignment is random based on which thread first encounters a node callback.

**Non-Determinism Sources:**
1. **Thread-local pointer assignment** (line 358): `next_tl.fetch_add(1)` assigns threads in order of appearance
2. **Callback thread order**: PBF decoder can call back from any worker thread (pbf_reader.cpp, line 998)
3. **Address ID assignment** (s2_helpers.cpp, line 232): `addr_id = data.addr_points.size()` depends on merge order

**Severity:** **SEVERE - Affects Ordering**
- `data.addr_points` has different order
- Different addr_id assignments
- addr_points.bin differs

**What's Needed:**
- Collect all addresses in a temporary sorted structure during streaming
- Sort by node ID before inserting into main data structure
- Or: use a single thread-local accumulator for all addresses

---

## 4. S2 CELL COMPUTATION & SORTING - PARTIALLY DETERMINISTIC

**File:** `/root/traccar-geocoder/builder/src/build_index.cpp` (lines 1056-1145)

**Problem:**
```cpp
// Parallel sort (lines 1072-1085):
std::vector<std::thread> sort_threads;
for (size_t t = 0; t < thread_pairs.size(); t++) {
    sort_threads.emplace_back([&, t]() {
        std::sort(dst.begin(), dst.end(), [](const CellItemPair& a, const CellItemPair& b) {
            return a.cell_id < b.cell_id || (a.cell_id == b.cell_id && a.item_id < b.item_id);
        });
    });
}

// Then parallel tree merge (lines 1109-1129):
while (run_bounds.size() > 2) {
    // Pairwise inplace_merge of adjacent runs
    merge_threads.emplace_back([&, left, mid, right] {
        std::inplace_merge(sorted_out.begin() + left, ...);
    });
}
```

**Analysis:**
- **Within-thread sorting**: DETERMINISTIC (same items, same sort comparator)
- **Parallel tree merge**: DETERMINISTIC (specific pair-wise merge pattern)
- **BUT**: Depends on thread-local data from step 2 & 3

**However**, if input `way_pairs` order differs (due to thread merge non-determinism):
- Thread 0 might have items {1,5,9} in run 1, {2,4,8} in run 2
- Sorting each chunk: still deterministic
- But final merged order differs

**Severity:** **HIGH - Cascading Effect**
- `sorted_way_cells` and `sorted_addr_cells` differ
- Entry file writing (cell_index.cpp, lines 56-133) uses these directly
- street_entries.bin and addr_entries.bin differ

**What's Needed:**
- Fix sources 2 & 3 first (thread-local merge ordering)
- Then S2 computation is deterministic

---

## 5. ADMIN POLYGON ASSEMBLY - PARTIALLY NON-DETERMINISTIC

**File:** `/root/traccar-geocoder/builder/src/build_index.cpp` (lines 694-902)

**Problem:**
```cpp
// Parallel admin assembly (lines 714-815):
std::atomic<size_t> rel_idx{0};
std::vector<std::thread> admin_workers;
for (unsigned int t = 0; t < num_threads; t++) {
    admin_workers.emplace_back([&, t]() {
        auto& local_results = thread_admin_results[t];
        while (true) {
            size_t i = rel_idx.fetch_add(1);  // Work-stealing by thread index
            if (i >= data.collected_relations.size()) break;
            // Process relation i, append to local_results
        }
    });
}

// Sequential merge (lines 819-825):
for (auto& local_results : thread_admin_results) {
    all_admin_results.insert(...);  // Concatenate in thread order
}

// Sequential append to data.admin_polygons (lines 876-898):
for (auto& pp : prepared) {
    uint32_t poly_id = data.admin_polygons.size();
    data.admin_polygons.push_back(poly);  // Order depends on thread execution
}
```

**Analysis:**
- **Relation processing order**: DETERMINISTIC (relations processed in `data.collected_relations` order)
- **Within-thread results**: Thread 0 might process relations [0,5,10], thread 1 [1,6,11], etc.
- **Flattening order** (line 820): Concatenates `thread_admin_results[0]`, `[1]`, ... `[num_threads-1]`
- This depends on which relations each thread processed
- If work distribution differs (due to scheduling variance), order changes

**Severity:** **MEDIUM - Affects Content**
- `admin_polygons.bin` differs
- Polygon IDs reassigned
- But relations themselves are deterministic; it's just the output order

**What's Needed:**
- After parallel assembly, re-sort results by relation ID before appending
- Or: merge directly in deterministic order instead of by thread

---

## 6. ADMIN POLYGON S2 COVERING - NON-DETERMINISTIC

**File:** `/root/traccar-geocoder/builder/src/build_index.cpp` (lines 920-928)

**Problem:**
```cpp
// Drain admin pool results (sequential, but depends on pool's internal state)
auto admin_results = admin_pool.drain();  // Pools use thread-local storage
for (auto& [cell_id, ids] : admin_results) {  // unordered_map iteration
    auto& target = data.cell_to_admin[cell_id];
    target.insert(target.end(), ids.begin(), ids.end());  // Order depends on map iteration
}
```

**Root Cause:** Uses unordered_map for results; iteration order undefined.

**Severity:** **MEDIUM - Affects Ordering**
- `cell_to_admin` hash map order differs
- But when written (cell_index.cpp, line 140-141), gets re-sorted:
  ```cpp
  std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(cell_map.begin(), cell_map.end());
  std::sort(sorted.begin(), sorted.end());
  ```
- So admin_entries.bin is eventually deterministic
- But intermediate form varies

**What's Needed:**
- No fix needed here; write path sorts everything

---

## 7. FINAL OUTPUT WRITING - DETERMINISTIC

**File:** `/root/traccar-geocoder/builder/src/cell_index.cpp` (lines 159-385)

**Analysis:**
- `write_index()` uses `sorted_geo_cells` (explicitly sorted, line 221)
- `write_entries_from_sorted()` (lines 56-133) processes sorted pairs in parallel, but with deterministic output
- Parallel writing uses thread-safe chunk assignment

**However:**
- If `sorted_way_cells`, `sorted_addr_cells`, `sorted_interp_cells` differ (due to non-determinism above), output differs
- String pool differences propagate to strings.bin

**Severity:** **Writing is OK; garbage in = garbage out**

---

## SUMMARY TABLE

| Issue | Component | Severity | Root Cause | Output Diff |
|-------|-----------|----------|-----------|------------|
| String pool ordering | StringPool::intern() | SEVERE | unordered_map | strings.bin |
| Thread-local merge order (ways) | build_index.cpp 607-671 | SEVERE | PBF block scheduling | street_ways.bin, way offsets |
| Thread-local merge order (addr) | build_index.cpp 391-399 | SEVERE | Thread assignment order | addr_points.bin, addr IDs |
| S2 cell ordering (cascading) | build_index.cpp 1056-1145 | HIGH | Depends on 1-3 | street/addr_entries.bin |
| Admin polygon order | build_index.cpp 819-825 | MEDIUM | Thread scheduling | admin_polygons.bin |
| Admin polygon S2 covering | build_index.cpp 920-928 | MEDIUM | unordered_map (sorted on write) | Minimal (reordered at write) |

---

## ANSWER: Byte-Identical Output?

**NO.** Running the same build twice will produce **different binary outputs** due to:

1. **Strings pool offsets**: Different per run (unordered_map)
2. **Way ordering**: Different per run (thread scheduling)
3. **Address point ordering**: Different per run (thread assignment)
4. **String references in index files**: Different per run (cascading from 1)
5. **Admin polygon ordering**: Different per run (scheduling)

Even on the same machine, consecutive runs will differ.

---

## FIXES REQUIRED FOR DETERMINISM

**Priority 1 (Critical):**
1. Replace `std::unordered_map<std::string, uint32_t>` with `std::map` in StringPool
2. Sort thread-local way data by way ID before merging
3. Sort thread-local address data by node ID before merging
4. Fix address point thread assignment (use round-robin instead of atomic)

**Priority 2 (Important):**
5. Sort admin polygon results by relation ID before output
6. Add seeds to ASLR-vulnerable randomization

**Priority 3 (Nice-to-have):**
7. Consider single-threaded pass 2 for byte-reproducibility (if perf allows)
8. Document that output is non-deterministic (add comments)

---

**Files to modify:**
- `/root/traccar-geocoder/builder/src/string_pool.h` - Use std::map
- `/root/traccar-geocoder/builder/src/build_index.cpp` - Sort thread-local data before merge
- Optionally: `/root/traccar-geocoder/builder/src/s2_helpers.cpp` - Thread-safe string interning

Prompt:
I need to understand what makes the geocoder build non-deterministic. Examine /root/traccar-geocoder/builder/src/build_index.cpp focusing on:

1. **String pool ordering**: How does StringPool::intern() assign offsets? Is it first-come-first-served based on thread encounter order?

2. **Record ordering**: How are addr_points, street_ways, interp_ways ordered in the final output? Are they in PBF encounter order merged by thread index?

3. **Thread-local data merging**: Look at the thread-local merge sections (around lines 390-670). How are thread-local buffers merged? Does the merge order depend on thread scheduling?

4. **S2 cell computation**: Is the sorted_way_cells / sorted_addr_cells ordering deterministic given the same input?

5. **Admin polygon assembly**: Is the parallel admin assembly deterministic?

For each source of non-determinism, note:
- What causes it (thread scheduling, hash map iteration, PBF block assignment)
- How severe it is (affects ordering, affects content, affects both)
- What would be needed to make it deterministic

The goal: if we run the same build twice on the same PBF, do we get byte-identical output? If not, what needs to change?

Be thorough - check every parallel section and hash map usage.


Good — now I have a clear picture of non-determinism sources. Let me design the plan.


Write

# Geocoder Incremental Patch System

## Context
We need nightly patch files so users can update their geocoder index without re-downloading the full dataset. A user starting from day 1 should be able to apply patch_1→2, then patch_2→3, and arrive at an output byte-identical to a fresh day 3 build.

This requires:
1. **Deterministic builds** — same PBF input always produces the same bytes
2. **A diff tool** — compares two builds, produces a compact patch
3. **A patch tool** — applies patch to old build, producing output byte-identical to the new build

## Part 1: Deterministic Builds

The current builder produces different bytes on each run due to thread scheduling affecting record ordering and string pool layout. Five fixes are needed in `build_index.cpp`:

### Fix 1: Deterministic string pool
**File:** `src/string_pool.h`
- Currently uses `std::unordered_map` — iteration order is random
- Change: after all strings are interned, sort the pool alphabetically and remap all offsets
- This happens once, after Pass 2b merge but before writing. Same approach as the `geocoder-canonicalize` tool already does

### Fix 2: Sort addr_points before writing
**File:** `src/build_index.cpp` (after thread-local merge, ~line 400)
- Currently: addr_points are in thread-merge order (depends on PBF block scheduling)
- Change: sort `data.addr_points` by canonical key (street_name, housenumber, lat_grid, lng_grid) before S2 cell computation
- Must also rebuild `cell_to_addrs` after sorting since addr IDs change

### Fix 3: Sort ways before writing
**File:** `src/build_index.cpp` (after thread-local merge, ~line 670)
- Currently: ways are in thread-merge order
- Change: sort `data.ways` + `data.street_nodes` by canonical key (name, node_count, first_node coords)
- Must update `deferred_ways` with new IDs after sorting

### Fix 4: Sort interps before writing
Same approach as ways for `data.interp_ways` + `data.interp_nodes`

### Fix 5: Sort admin polygons before writing
**File:** `src/build_index.cpp` (after admin assembly, ~line 900)
- Sort `data.admin_polygons` + `data.admin_vertices` by (name, admin_level, country_code, vertex_count)

### Performance impact
Sorting 86M addr_points: ~10s. Sorting 19M ways: ~20s. Total: ~30-40s added to a 12-minute build. ~5% overhead.

### Validation
Run the same build twice on the same PBF → output must be byte-identical. Use `diff -r` or `md5sum` on all 14 files.

## Part 2: Diff Tool (`geocoder-diff`)

Compares two deterministic builds and produces a `.gcpatch` file.

Because builds are deterministic, the same record gets the same position in any build from the same PBF. Between two PBFs (day N and day N+1), most records are identical and in approximately the same position. The diff tool exploits this.

### Algorithm
For each of the 14 files:
1. Compare old and new files
2. If files differ: produce a COPY/INSERT delta stream
3. Compress the stream with zlib

The COPY/INSERT stream:
- `COPY(offset, length)`: copy bytes from old file at given offset
- `INSERT(length, data)`: insert new literal bytes

Since deterministic builds sort records by content, matching records between builds are at similar positions. Insertions/deletions shift subsequent records, but zstd/COPY-INSERT handles this with sliding-window matching.

### Expected patch sizes
With ~0.3% daily change rate and deterministic ordering:
- Coordinate files (street_nodes, admin_vertices): <1% — coords barely change
- Data files (addr_points, street_ways): 1-5% — sorted by content, insertions cause local shifts
- String pool: <1% — alphabetically sorted, insertions cause small shifts
- Entry/cell files: 1-5% — derived from sorted data, shifts cascade but are localized
- Estimated total: **50-200 MiB** for 7 GiB Europe (much better than 1.9 GiB without determinism, but the exact number depends on how well the sorted order resists insertion shifts)

**Note:** If this isn't small enough, we can add string-offset remapping as a second pass (the technique proven to achieve 14 MiB). But let's measure deterministic-only first — it may be sufficient.

### Patch format (.gcpatch)
```
Header: magic "GCPATCH\0", version u32, flags u32, num_sections u32
Per section:
  file_id: u32 (which of the 14 files)
  encoding: u32 (0=full_replace, 1=copy_insert)
  old_size: u64, new_size: u64
  compressed_data_size: u64
  data: zlib-compressed COPY/INSERT ops (or full file)
End marker: file_id = 0xFFFFFFFF
```

## Part 3: Patch Tool (`geocoder-patch`)

Applies a `.gcpatch` to an existing build directory.

### Algorithm
1. Validate header + old file sizes match
2. For each section:
   - `full_replace`: decompress → write new file
   - `copy_insert`: mmap old file, replay COPY/INSERT ops → write new file
3. Output goes to a new directory (atomic — old dir untouched until verified)

The tool is simple: it replays byte-level operations. No understanding of record semantics needed.

## Part 4: Validation Plan

### Test 1: Determinism
```bash
build-index out-A/ europe-old.pbf --simplify-epsilon --epsilon-scale 1
build-index out-B/ europe-old.pbf --simplify-epsilon --epsilon-scale 1
diff -r out-A/ out-B/  # Must show no differences
```

### Test 2: Single patch round-trip
```bash
build-index build-old/ europe-old.pbf ...
build-index build-new/ europe-new.pbf ...
geocoder-diff build-old/ build-new/ -o patch_1to2.gcpatch
geocoder-patch build-old/ patch_1to2.gcpatch -o patched/
diff -r patched/ build-new/  # Must match
```

### Test 3: Sequential patches
```bash
build-index build-1/ europe-1.pbf ...
build-index build-2/ europe-2.pbf ...
build-index build-3/ europe-3.pbf ...
geocoder-diff build-1/ build-2/ -o patch_1to2.gcpatch
geocoder-diff build-2/ build-3/ -o patch_2to3.gcpatch
geocoder-patch build-1/ patch_1to2.gcpatch -o result-2/
geocoder-patch result-2/ patch_2to3.gcpatch -o result-3/
diff -r result-3/ build-3/  # Must match
```

For the third PBF: download the latest europe from Geofabrik (different date than the two we already have on node 3).

## Files to Create/Modify

### Modify
- `builder/src/string_pool.h` — no changes needed (sort happens in build_index)
- `builder/src/build_index.cpp` — add sorting after each merge phase

### Create
- `builder/tools/geocoder_diff.cpp` — diff tool (~400 lines)
- `builder/tools/geocoder_patch.cpp` — patch tool (~200 lines)
- `builder/tools/patch_format.h` — shared format definitions (already exists, update)

### Reuse
- `builder/tools/patch_format.h` — PatchFileId enum, file helpers (from current work)
- `builder/tools/geocoder_canonicalize.cpp` — sorting logic can be adapted for build_index.cpp

## Implementation Order

1. Make builds deterministic (modify build_index.cpp)
2. Validate determinism (Test 1)
3. Implement geocoder-diff with simple COPY/INSERT encoding
4. Implement geocoder-patch
5. Validate single-patch round-trip (Test 2)
6. Measure patch sizes — if >100 MiB, add string remap optimization
7. Download third PBF, validate sequential patches (Test 3)


The file /root/.claude/plans/concurrent-humming-sonnet.md has been updated successfully.

# Geocoder Incremental Patch System

## Context
We need nightly patch files so users can update their geocoder index without re-downloading the full dataset. A user starting from day 1 should be able to apply patch_1→2, then patch_2→3, and arrive at an output byte-identical to a fresh day 3 build.

This requires:
1. **Deterministic builds** — same PBF input always produces the same bytes
2. **A diff tool** — compares two builds, produces a compact patch
3. **A patch tool** — applies patch to old build, producing output byte-identical to the new build

## Part 1: Deterministic Builds

The current builder produces different bytes on each run due to thread scheduling affecting record ordering and string pool layout. Five fixes are needed in `build_index.cpp`:

### Fix 1: Deterministic string pool
**File:** `src/string_pool.h`
- Currently uses `std::unordered_map` — iteration order is random
- Change: after all strings are interned, sort the pool alphabetically and remap all offsets
- This happens once, after Pass 2b merge but before writing. Same approach as the `geocoder-canonicalize` tool already does

### Fix 2: Sort addr_points before writing
**File:** `src/build_index.cpp` (after thread-local merge, ~line 400)
- Currently: addr_points are in thread-merge order (depends on PBF block scheduling)
- Change: sort `data.addr_points` by canonical key (street_name, housenumber, lat_grid, lng_grid) before S2 cell computation
- Must also rebuild `cell_to_addrs` after sorting since addr IDs change

### Fix 3: Sort ways before writing
**File:** `src/build_index.cpp` (after thread-local merge, ~line 670)
- Currently: ways are in thread-merge order
- Change: sort `data.ways` + `data.street_nodes` by canonical key (name, node_count, first_node coords)
- Must update `deferred_ways` with new IDs after sorting

### Fix 4: Sort interps before writing
Same approach as ways for `data.interp_ways` + `data.interp_nodes`

### Fix 5: Sort admin polygons before writing
**File:** `src/build_index.cpp` (after admin assembly, ~line 900)
- Sort `data.admin_polygons` + `data.admin_vertices` by (name, admin_level, country_code, vertex_count)

### Performance impact
Sorting 86M addr_points: ~10s. Sorting 19M ways: ~20s. Total: ~30-40s added to a 12-minute build. ~5% overhead.

### Validation
Run the same build twice on the same PBF → output must be byte-identical. Use `diff -r` or `md5sum` on all 14 files.

## Part 2: Diff Tool (`geocoder-diff`)

Compares two deterministic builds and produces a `.gcpatch` file.

Because builds are deterministic, the same record gets the same position in any build from the same PBF. Between two PBFs (day N and day N+1), most records are identical and in approximately the same position. The diff tool exploits this.

### Algorithm
For each of the 14 files:
1. Compare old and new files
2. If files differ: produce a COPY/INSERT delta stream
3. Compress the stream with zlib

The COPY/INSERT stream:
- `COPY(offset, length)`: copy bytes from old file at given offset
- `INSERT(length, data)`: insert new literal bytes

Since deterministic builds sort records by content, matching records between builds are at similar positions. Insertions/deletions shift subsequent records, but zstd/COPY-INSERT handles this with sliding-window matching.

### Expected patch sizes
With ~0.3% daily change rate and deterministic ordering:
- Coordinate files (street_nodes, admin_vertices): <1% — coords barely change
- Data files (addr_points, street_ways): 1-5% — sorted by content, insertions cause local shifts
- String pool: <1% — alphabetically sorted, insertions cause small shifts
- Entry/cell files: 1-5% — derived from sorted data, shifts cascade but are localized
- Estimated total: **50-200 MiB** for 7 GiB Europe (much better than 1.9 GiB without determinism, but the exact number depends on how well the sorted order resists insertion shifts)

**Note:** If this isn't small enough, we can add string-offset remapping as a second pass (the technique proven to achieve 14 MiB). But let's measure deterministic-only first — it may be sufficient.

### Patch format (.gcpatch)
```
Header: magic "GCPATCH\0", version u32, flags u32, num_sections u32
Per section:
  file_id: u32 (which of the 14 files)
  encoding: u32 (0=full_replace, 1=copy_insert)
  old_size: u64, new_size: u64
  compressed_data_size: u64
  data: zlib-compressed COPY/INSERT ops (or full file)
End marker: file_id = 0xFFFFFFFF
```

## Part 3: Patch Tool (`geocoder-patch`)

Applies a `.gcpatch` to an existing build directory.

### Algorithm
1. Validate header + old file sizes match
2. For each section:
   - `full_replace`: decompress → write new file
   - `copy_insert`: mmap old file, replay COPY/INSERT ops → write new file
3. Output goes to a new directory (atomic — old dir untouched until verified)

The tool is simple: it replays byte-level operations. No understanding of record semantics needed.

## Part 4: Validation Plan

### Test 1: Determinism
```bash
build-index out-A/ europe-old.pbf --simplify-epsilon --epsilon-scale 1
build-index out-B/ europe-old.pbf --simplify-epsilon --epsilon-scale 1
diff -r out-A/ out-B/  # Must show no differences
```

### Test 2: Single patch round-trip
```bash
build-index build-old/ europe-old.pbf ...
build-index build-new/ europe-new.pbf ...
geocoder-diff build-old/ build-new/ -o patch_1to2.gcpatch
geocoder-patch build-old/ patch_1to2.gcpatch -o patched/
diff -r patched/ build-new/  # Must match
```

### Test 3: Sequential patches
```bash
build-index build-1/ europe-1.pbf ...
build-index build-2/ europe-2.pbf ...
build-index build-3/ europe-3.pbf ...
geocoder-diff build-1/ build-2/ -o patch_1to2.gcpatch
geocoder-diff build-2/ build-3/ -o patch_2to3.gcpatch
geocoder-patch build-1/ patch_1to2.gcpatch -o result-2/
geocoder-patch result-2/ patch_2to3.gcpatch -o result-3/
diff -r result-3/ build-3/  # Must match
```

For the third PBF: download the latest europe from Geofabrik (different date than the two we already have on node 3).

## Files to Create/Modify

### Modify
- `builder/src/string_pool.h` — no changes needed (sort happens in build_index)
- `builder/src/build_index.cpp` — add sorting after each merge phase

### Create
- `builder/tools/geocoder_diff.cpp` — diff tool (~400 lines)
- `builder/tools/geocoder_patch.cpp` — patch tool (~200 lines)
- `builder/tools/patch_format.h` — shared format definitions (already exists, update)

### Reuse
- `builder/tools/patch_format.h` — PatchFileId enum, file helpers (from current work)
- `builder/tools/geocoder_canonicalize.cpp` — sorting logic can be adapted for build_index.cpp

## Implementation Order

1. Make builds deterministic (modify build_index.cpp)
2. Validate determinism (Test 1)
3. Implement geocoder-diff with simple COPY/INSERT encoding
4. Implement geocoder-patch
5. Validate single-patch round-trip (Test 2)
6. Measure patch sizes — if >100 MiB, add string remap optimization
7. Download third PBF, validate sequential patches (Test 3)

-Binary delta patching (zstd --patch-from) produces patches that are 25-67% of the original file sizes because independent builds have different string offsets, record orderings, and entry layouts. The user wants a custom tool that understands the format and produces small (~5-15 MiB) patches for a 7 GiB Europe dataset with ~0.3% daily changes.

+We need nightly patch files so users can update their geocoder index without re-downloading the full dataset. A user starting from day 1 should be able to apply patch_1→2, then patch_2→3, and arrive at an output byte-identical to a fresh day 3 build.

+1. **Deterministic builds** — same PBF input always produces the same bytes

+2. **A diff tool** — compares two builds, produces a compact patch

+3. **A patch tool** — applies patch to old build, producing output byte-identical to the new build

-1. **geocoder-canonicalize** — normalizes a fresh build into deterministic byte-identical canonical format

-2. **geocoder-diff** — compares two canonical directories, produces `.gcpatch` file (COPY/INSERT byte stream)

-3. **geocoder-patch** — applies `.gcpatch` to a canonical directory, producing new canonical directory

+The current builder produces different bytes on each run due to thread scheduling affecting record ordering and string pool layout. Five fixes are needed in `build_index.cpp`:

+- Currently uses `std::unordered_map` — iteration order is random

+- Change: after all strings are interned, sort the pool alphabetically and remap all offsets

+- This happens once, after Pass 2b merge but before writing. Same approach as the `geocoder-canonicalize` tool already does

+### Fix 2: Sort addr_points before writing

+**File:** `src/build_index.cpp` (after thread-local merge, ~line 400)

+- Currently: addr_points are in thread-merge order (depends on PBF block scheduling)

+- Change: sort `data.addr_points` by canonical key (street_name, housenumber, lat_grid, lng_grid) before S2 cell computation

+- Must also rebuild `cell_to_addrs` after sorting since addr IDs change

+**File:** `src/build_index.cpp` (after thread-local merge, ~line 670)

+- Currently: ways are in thread-merge order

+- Change: sort `data.ways` + `data.street_nodes` by canonical key (name, node_count, first_node coords)

+- Must update `deferred_ways` with new IDs after sorting

+Same approach as ways for `data.interp_ways` + `data.interp_nodes`

+### Fix 5: Sort admin polygons before writing

+**File:** `src/build_index.cpp` (after admin assembly, ~line 900)

+- Sort `data.admin_polygons` + `data.admin_vertices` by (name, admin_level, country_code, vertex_count)

+Sorting 86M addr_points: ~10s. Sorting 19M ways: ~20s. Total: ~30-40s added to a 12-minute build. ~5% overhead.

+Run the same build twice on the same PBF → output must be byte-identical. Use `diff -r` or `md5sum` on all 14 files.

+Compares two deterministic builds and produces a `.gcpatch` file.

+Because builds are deterministic, the same record gets the same position in any build from the same PBF. Between two PBFs (day N and day N+1), most records are identical and in approximately the same position. The diff tool exploits this.

+2. If files differ: produce a COPY/INSERT delta stream

+- `COPY(offset, length)`: copy bytes from old file at given offset

+- `INSERT(length, data)`: insert new literal bytes

+Since deterministic builds sort records by content, matching records between builds are at similar positions. Insertions/deletions shift subsequent records, but zstd/COPY-INSERT handles this with sliding-window matching.

+With ~0.3% daily change rate and deterministic ordering:

+- Coordinate files (street_nodes, admin_vertices): <1% — coords barely change

+- Data files (addr_points, street_ways): 1-5% — sorted by content, insertions cause local shifts

+- String pool: <1% — alphabetically sorted, insertions cause small shifts

+- Entry/cell files: 1-5% — derived from sorted data, shifts cascade but are localized

+- Estimated total: **50-200 MiB** for 7 GiB Europe (much better than 1.9 GiB without determinism, but the exact number depends on how well the sorted order resists insertion shifts)

+**Note:** If this isn't small enough, we can add string-offset remapping as a second pass (the technique proven to achieve 14 MiB). But let's measure deterministic-only first — it may be sufficient.

-fresh build → geocoder-canonicalize → canonical/

-diff(prev_canonical/, canonical/) → patch.gcpatch

+Header: magic "GCPATCH\0", version u32, flags u32, num_sections u32

+  encoding: u32 (0=full_replace, 1=copy_insert)

+  data: zlib-compressed COPY/INSERT ops (or full file)

+## Part 3: Patch Tool (`geocoder-patch`)

+Applies a `.gcpatch` to an existing build directory.

+1. Validate header + old file sizes match

+   - `full_replace`: decompress → write new file

+   - `copy_insert`: mmap old file, replay COPY/INSERT ops → write new file

+3. Output goes to a new directory (atomic — old dir untouched until verified)

+The tool is simple: it replays byte-level operations. No understanding of record semantics needed.

+build-index out-A/ europe-old.pbf --simplify-epsilon --epsilon-scale 1

+build-index out-B/ europe-old.pbf --simplify-epsilon --epsilon-scale 1

+diff -r out-A/ out-B/  # Must show no differences

-canonical_day1 + patch_1→2 → canonical_day2

-canonical_day2 + patch_2→3 → canonical_day3

+build-index build-old/ europe-old.pbf ...

+build-index build-new/ europe-new.pbf ...

+geocoder-diff build-old/ build-new/ -o patch_1to2.gcpatch

+geocoder-patch build-old/ patch_1to2.gcpatch -o patched/

+diff -r patched/ build-new/  # Must match

-- **strings.bin**: strings sorted alphabetically

-- **Data files** (addr_points, street_ways, interp_ways, admin_polygons): records sorted by content fingerprint (reusing logic from compare_indexes.cpp)

-- **Node/vertex files**: grouped by parent record in canonical order

-- **Entries/cells**: derived from canonical data ordering + cell assignments carried from original build (no S2 recomputation)

+geocoder-diff build-1/ build-2/ -o patch_1to2.gcpatch

+geocoder-diff build-2/ build-3/ -o patch_2to3.gcpatch

+geocoder-patch build-1/ patch_1to2.gcpatch -o result-2/

+geocoder-patch result-2/ patch_2to3.gcpatch -o result-3/

+diff -r result-3/ build-3/  # Must match

-Binary COPY/INSERT stream per file, zstd compressed. For each of the 14 files:

-- COPY(old_offset, length) — copy bytes from old canonical file

-- INSERT(length, data) — insert new bytes

-Since canonical format is stable, ~99.7% of content is COPYed.

+For the third PBF: download the latest europe from Geofabrik (different date than the two we already have on node 3).

-- `builder/tools/patch_format.h` — shared format definitions

-- `builder/tools/canonical_order.h` — fingerprint/sort functions (extracted from compare_indexes.cpp)

-- `builder/src/types.h` — binary record structs

-- `builder/test/compare_indexes.cpp` — existing fingerprint logic

-- `builder/src/cell_index.cpp` — entry/cell write functions

-- `builder/src/string_pool.h` — string pool implementation

+- `builder/src/string_pool.h` — no changes needed (sort happens in build_index)

+- `builder/src/build_index.cpp` — add sorting after each merge phase

+- `builder/tools/geocoder_diff.cpp` — diff tool (~400 lines)

+- `builder/tools/geocoder_patch.cpp` — patch tool (~200 lines)

+- `builder/tools/patch_format.h` — shared format definitions (already exists, update)

+- `builder/tools/patch_format.h` — PatchFileId enum, file helpers (from current work)

+- `builder/tools/geocoder_canonicalize.cpp` — sorting logic can be adapted for build_index.cpp

+1. Make builds deterministic (modify build_index.cpp)

+3. Implement geocoder-diff with simple COPY/INSERT encoding

+5. Validate single-patch round-trip (Test 2)

+6. Measure patch sizes — if >100 MiB, add string remap optimization

+7. Download third PBF, validate sequential patches (Test 3)

# Custom Geocoder Patch System

## Context
Binary delta patching (zstd --patch-from) produces patches that are 25-67% of the original file sizes because independent builds have different string offsets, record orderings, and entry layouts. The user wants a custom tool that understands the format and produces small (~5-15 MiB) patches for a 7 GiB Europe dataset with ~0.3% daily changes.

## Architecture

Three tools:
1. **geocoder-canonicalize** — normalizes a fresh build into deterministic byte-identical canonical format
2. **geocoder-diff** — compares two canonical directories, produces `.gcpatch` file (COPY/INSERT byte stream)
3. **geocoder-patch** — applies `.gcpatch` to a canonical directory, producing new canonical directory

### Build Server Pipeline
```
fresh build → geocoder-canonicalize → canonical/
diff(prev_canonical/, canonical/) → patch.gcpatch
```

### Client Pipeline
```
canonical_day1 + patch_1→2 → canonical_day2
canonical_day2 + patch_2→3 → canonical_day3
```

## Canonical Format
- **strings.bin**: strings sorted alphabetically
- **Data files** (addr_points, street_ways, interp_ways, admin_polygons): records sorted by content fingerprint (reusing logic from compare_indexes.cpp)
- **Node/vertex files**: grouped by parent record in canonical order
- **Entries/cells**: derived from canonical data ordering + cell assignments carried from original build (no S2 recomputation)

## Patch Format (.gcpatch)
Binary COPY/INSERT stream per file, zstd compressed. For each of the 14 files:
- COPY(old_offset, length) — copy bytes from old canonical file
- INSERT(length, data) — insert new bytes
Since canonical format is stable, ~99.7% of content is COPYed.

## Files to Create
- `builder/tools/patch_format.h` — shared format definitions
- `builder/tools/canonical_order.h` — fingerprint/sort functions (extracted from compare_indexes.cpp)
- `builder/tools/geocoder_canonicalize.cpp`
- `builder/tools/geocoder_diff.cpp`
- `builder/tools/geocoder_patch.cpp`

## Key Files to Read
- `builder/src/types.h` — binary record structs
- `builder/test/compare_indexes.cpp` — existing fingerprint logic
- `builder/src/cell_index.cpp` — entry/cell write functions
- `builder/src/string_pool.h` — string pool implementation


default-src 'none'; frame-ancestors 'none'

_cfuvid=7VCypxMGntcKCxEFw9B4tllJRcQ1KvLNbp5pM45x3Zc-1774604560.7298-1.0.1.1-jLNrtxcjpZvhdBkVpWXOkwN3p0jYRGmx6E4P66PD4gw; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com

Overloaded. https://docs.claude.com/en/api/errors


ExitPlanMode

Prompt: build the project with cmake

Prompt: deploy binary to node 3 via scp

Prompt: run build/diff/patch tools on node 3 via ssh

Prompt: download PBF files on node 3

Prompt: compare files on node 3

# Geocoder Incremental Patch System

## Context
We need nightly patch files so users can update their geocoder index without re-downloading the full dataset. A user starting from day 1 should be able to apply patch_1→2, then patch_2→3, and arrive at an output byte-identical to a fresh day 3 build.

This requires:
1. **Deterministic builds** — same PBF input always produces the same bytes
2. **A diff tool** — compares two builds, produces a compact patch
3. **A patch tool** — applies patch to old build, producing output byte-identical to the new build

## Part 1: Deterministic Builds

The current builder produces different bytes on each run due to thread scheduling affecting record ordering and string pool layout. Five fixes are needed in `build_index.cpp`:

### Fix 1: Deterministic string pool
**File:** `src/string_pool.h`
- Currently uses `std::unordered_map` — iteration order is random
- Change: after all strings are interned, sort the pool alphabetically and remap all offsets
- This happens once, after Pass 2b merge but before writing. Same approach as the `geocoder-canonicalize` tool already does

### Fix 2: Sort addr_points before writing
**File:** `src/build_index.cpp` (after thread-local merge, ~line 400)
- Currently: addr_points are in thread-merge order (depends on PBF block scheduling)
- Change: sort `data.addr_points` by canonical key (street_name, housenumber, lat_grid, lng_grid) before S2 cell computation
- Must also rebuild `cell_to_addrs` after sorting since addr IDs change

### Fix 3: Sort ways before writing
**File:** `src/build_index.cpp` (after thread-local merge, ~line 670)
- Currently: ways are in thread-merge order
- Change: sort `data.ways` + `data.street_nodes` by canonical key (name, node_count, first_node coords)
- Must update `deferred_ways` with new IDs after sorting

### Fix 4: Sort interps before writing
Same approach as ways for `data.interp_ways` + `data.interp_nodes`

### Fix 5: Sort admin polygons before writing
**File:** `src/build_index.cpp` (after admin assembly, ~line 900)
- Sort `data.admin_polygons` + `data.admin_vertices` by (name, admin_level, country_code, vertex_count)

### Performance impact
Sorting 86M addr_points: ~10s. Sorting 19M ways: ~20s. Total: ~30-40s added to a 12-minute build. ~5% overhead.

### Validation
Run the same build twice on the same PBF → output must be byte-identical. Use `diff -r` or `md5sum` on all 14 files.

## Part 2: Diff Tool (`geocoder-diff`)

Compares two deterministic builds and produces a `.gcpatch` file.

Because builds are deterministic, the same record gets the same position in any build from the same PBF. Between two PBFs (day N and day N+1), most records are identical and in approximately the same position. The diff tool exploits this.

### Algorithm
For each of the 14 files:
1. Compare old and new files
2. If files differ: produce a COPY/INSERT delta stream
3. Compress the stream with zlib

The COPY/INSERT stream:
- `COPY(offset, length)`: copy bytes from old file at given offset
- `INSERT(length, data)`: insert new literal bytes

Since deterministic builds sort records by content, matching records between builds are at similar positions. Insertions/deletions shift subsequent records, but zstd/COPY-INSERT handles this with sliding-window matching.

### Expected patch sizes
With ~0.3% daily change rate and deterministic ordering:
- Coordinate files (street_nodes, admin_vertices): <1% — coords barely change
- Data files (addr_points, street_ways): 1-5% — sorted by content, insertions cause local shifts
- String pool: <1% — alphabetically sorted, insertions cause small shifts
- Entry/cell files: 1-5% — derived from sorted data, shifts cascade but are localized
- Estimated total: **50-200 MiB** for 7 GiB Europe (much better than 1.9 GiB without determinism, but the exact number depends on how well the sorted order resists insertion shifts)

**Note:** If this isn't small enough, we can add string-offset remapping as a second pass (the technique proven to achieve 14 MiB). But let's measure deterministic-only first — it may be sufficient.

### Patch format (.gcpatch)
```
Header: magic "GCPATCH\0", version u32, flags u32, num_sections u32
Per section:
  file_id: u32 (which of the 14 files)
  encoding: u32 (0=full_replace, 1=copy_insert)
  old_size: u64, new_size: u64
  compressed_data_size: u64
  data: zlib-compressed COPY/INSERT ops (or full file)
End marker: file_id = 0xFFFFFFFF
```

## Part 3: Patch Tool (`geocoder-patch`)

Applies a `.gcpatch` to an existing build directory.

### Algorithm
1. Validate header + old file sizes match
2. For each section:
   - `full_replace`: decompress → write new file
   - `copy_insert`: mmap old file, replay COPY/INSERT ops → write new file
3. Output goes to a new directory (atomic — old dir untouched until verified)

The tool is simple: it replays byte-level operations. No understanding of record semantics needed.

## Part 4: Validation Plan

### Test 1: Determinism
```bash
build-index out-A/ europe-old.pbf --simplify-epsilon --epsilon-scale 1
build-index out-B/ europe-old.pbf --simplify-epsilon --epsilon-scale 1
diff -r out-A/ out-B/  # Must show no differences
```

### Test 2: Single patch round-trip
```bash
build-index build-old/ europe-old.pbf ...
build-index build-new/ europe-new.pbf ...
geocoder-diff build-old/ build-new/ -o patch_1to2.gcpatch
geocoder-patch build-old/ patch_1to2.gcpatch -o patched/
diff -r patched/ build-new/  # Must match
```

### Test 3: Sequential patches
```bash
build-index build-1/ europe-1.pbf ...
build-index build-2/ europe-2.pbf ...
build-index build-3/ europe-3.pbf ...
geocoder-diff build-1/ build-2/ -o patch_1to2.gcpatch
geocoder-diff build-2/ build-3/ -o patch_2to3.gcpatch
geocoder-patch build-1/ patch_1to2.gcpatch -o result-2/
geocoder-patch result-2/ patch_2to3.gcpatch -o result-3/
diff -r result-3/ build-3/  # Must match
```

For the third PBF: download the latest europe from Geofabrik (different date than the two we already have on node 3).

## Files to Create/Modify

### Modify
- `builder/src/string_pool.h` — no changes needed (sort happens in build_index)
- `builder/src/build_index.cpp` — add sorting after each merge phase

### Create
- `builder/tools/geocoder_diff.cpp` — diff tool (~400 lines)
- `builder/tools/geocoder_patch.cpp` — patch tool (~200 lines)
- `builder/tools/patch_format.h` — shared format definitions (already exists, update)

### Reuse
- `builder/tools/patch_format.h` — PatchFileId enum, file helpers (from current work)
- `builder/tools/geocoder_canonicalize.cpp` — sorting logic can be adapted for build_index.cpp

## Implementation Order

1. Make builds deterministic (modify build_index.cpp)
2. Validate determinism (Test 1)
3. Implement geocoder-diff with simple COPY/INSERT encoding
4. Implement geocoder-patch
5. Validate single-patch round-trip (Test 2)
6. Measure patch sizes — if >100 MiB, add string remap optimization
7. Download third PBF, validate sequential patches (Test 3)