The file /root/traccar-geocoder/builder/tools/geocoder_diff.cpp has been updated successfully.

// geocoder-diff: Compare two canonical geocoder indexes and produce a compact patch.
//
// Strategy:
// 1. Build string remap (old string offsets → new string offsets)
// 2. For data files with string refs: apply remap to old file in memory, then binary diff
// 3. For coordinate files: direct binary diff (no string refs)
// 4. For small derived files: full replacement (compressed)
//
// The binary diff uses a simple block-based COPY/INSERT encoder:
// - Compare 64-byte blocks between (remapped) old and new
// - COPY for matching blocks, INSERT for different blocks
// - Handles size differences (new may be larger)
//
// Usage: geocoder-diff <old-canonical-dir> <new-canonical-dir> -o <patch-file>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>
#include <zlib.h>

#include "patch_format.h"

// --- zlib helpers ---

static std::vector<char> zlib_compress(const char* data, size_t size) {
    if (size == 0) return {};
    uLongf bound = compressBound(size);
    std::vector<char> out(bound);
    int ret = compress2(reinterpret_cast<Bytef*>(out.data()), &bound,
                        reinterpret_cast<const Bytef*>(data), size, 9);
    if (ret != Z_OK) return {};
    out.resize(bound);
    return out;
}

// --- String remap ---

static std::unordered_map<uint32_t, uint32_t> build_string_remap(
    const std::vector<char>& old_pool, const std::vector<char>& new_pool)
{
    // Index new pool
    std::unordered_map<std::string, uint32_t> new_index;
    size_t pos = 0;
    while (pos < new_pool.size()) {
        const char* s = new_pool.data() + pos;
        size_t len = strlen(s);
        new_index[std::string(s, len)] = static_cast<uint32_t>(pos);
        pos += len + 1;
    }

    // Map old offsets → new offsets
    std::unordered_map<uint32_t, uint32_t> remap;
    pos = 0;
    while (pos < old_pool.size()) {
        const char* s = old_pool.data() + pos;
        size_t len = strlen(s);
        auto it = new_index.find(std::string(s, len));
        if (it != new_index.end()) {
            remap[static_cast<uint32_t>(pos)] = it->second;
        }
        pos += len + 1;
    }
    return remap;
}

// Apply string remap to specific fields in a data buffer
static void remap_addr_points(std::vector<char>& data,
                                const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / 16;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * 16;
        // housenumber_id at offset 8, street_id at offset 12
        for (size_t off : {8, 12}) {
            uint32_t old_id;
            memcpy(&old_id, p + off, 4);
            auto it = remap.find(old_id);
            if (it != remap.end()) {
                memcpy(p + off, &it->second, 4);
            }
        }
    }
}

static void remap_ways(std::vector<char>& data, size_t stride,
                        const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / stride;
    size_t name_off = (stride == 12) ? 8 : 5;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * stride;
        uint32_t old_id;
        memcpy(&old_id, p + name_off, 4);
        auto it = remap.find(old_id);
        if (it != remap.end()) {
            memcpy(p + name_off, &it->second, 4);
        }
    }
}

static void remap_interps(std::vector<char>& data, size_t stride,
                           const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / stride;
    size_t street_off = (stride == 20) ? 8 : 5;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * stride;
        uint32_t old_id;
        memcpy(&old_id, p + street_off, 4);
        auto it = remap.find(old_id);
        if (it != remap.end()) {
            memcpy(p + street_off, &it->second, 4);
        }
    }
}

static void remap_admin_polygons(std::vector<char>& data, size_t stride,
                                   const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / stride;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * stride;
        uint32_t old_id;
        memcpy(&old_id, p + 8, 4); // name_id at offset 8
        auto it = remap.find(old_id);
        if (it != remap.end()) {
            memcpy(p + 8, &it->second, 4);
        }
    }
}

// --- Block-based COPY/INSERT binary diff ---
// Compares (remapped) old vs new, produces a compact delta.
// Block size of 64 bytes balances granularity vs overhead.

static constexpr size_t DIFF_BLOCK = 64;

static std::vector<char> compute_block_diff(const std::vector<char>& old_data,
                                              const std::vector<char>& new_data) {
    std::vector<char> ops;
    size_t old_pos = 0, new_pos = 0;

    while (new_pos < new_data.size()) {
        // Check if current block matches
        size_t remaining_new = new_data.size() - new_pos;
        size_t remaining_old = (old_pos < old_data.size()) ? old_data.size() - old_pos : 0;
        size_t block = std::min(DIFF_BLOCK, remaining_new);

        if (remaining_old >= block &&
            memcmp(old_data.data() + old_pos, new_data.data() + new_pos, block) == 0) {
            // Find full extent of match
            size_t match_len = block;
            while (old_pos + match_len < old_data.size() &&
                   new_pos + match_len < new_data.size() &&
                   old_data[old_pos + match_len] == new_data[new_pos + match_len]) {
                match_len++;
            }
            // Emit COPY: tag(1) + offset(8) + length(4) = 13 bytes
            ops.push_back(OP_COPY);
            uint64_t off = old_pos;
            uint32_t len = static_cast<uint32_t>(match_len);
            ops.insert(ops.end(), reinterpret_cast<const char*>(&off),
                      reinterpret_cast<const char*>(&off) + 8);
            ops.insert(ops.end(), reinterpret_cast<const char*>(&len),
                      reinterpret_cast<const char*>(&len) + 4);
            old_pos += match_len;
            new_pos += match_len;
        } else {
            // Find extent of non-match (up to next matching block)
            size_t insert_start = new_pos;
            size_t old_scan = old_pos;

            // Advance by one block
            new_pos += block;
            old_scan += std::min(block, remaining_old);

            // Keep scanning until we find a matching block again
            while (new_pos < new_data.size()) {
                size_t rem_new = new_data.size() - new_pos;
                size_t rem_old = (old_scan < old_data.size()) ? old_data.size() - old_scan : 0;
                size_t b = std::min(DIFF_BLOCK, rem_new);
                if (rem_old >= b &&
                    memcmp(old_data.data() + old_scan, new_data.data() + new_pos, b) == 0) {
                    break; // Found match
                }
                new_pos += b;
                old_scan += std::min(b, rem_old);
            }

            // Emit INSERT: tag(1) + length(4) + data
            size_t insert_len = new_pos - insert_start;
            ops.push_back(OP_INSERT);
            uint32_t len = static_cast<uint32_t>(insert_len);
            ops.insert(ops.end(), reinterpret_cast<const char*>(&len),
                      reinterpret_cast<const char*>(&len) + 4);
            ops.insert(ops.end(), new_data.data() + insert_start,
                      new_data.data() + insert_start + insert_len);
            old_pos = old_scan;
        }
    }
    return ops;
}

// --- Write helpers ---
static void write_u32(std::ofstream& f, uint32_t v) { f.write(reinterpret_cast<const char*>(&v), 4); }
static void write_u64(std::ofstream& f, uint64_t v) { f.write(reinterpret_cast<const char*>(&v), 8); }

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>" << std::endl;
        return 1;
    }

    std::string old_dir = argv[1];
    std::string new_dir = argv[2];
    std::string patch_path = argv[4];

    std::cerr << "Loading string pools..." << std::endl;
    auto old_strings = read_file(old_dir + "/strings.bin");
    auto new_strings = read_file(new_dir + "/strings.bin");
    auto remap = build_string_remap(old_strings, new_strings);
    std::cerr << "  Remap: " << remap.size() << " strings mapped" << std::endl;

    std::ofstream pf(patch_path, std::ios::binary);
    pf.write(GCPATCH_MAGIC, 8);
    write_u32(pf, GCPATCH_VERSION);
    write_u32(pf, 0);

    // Section: strings.bin (full replacement)
    {
        std::cerr << "  strings.bin: full replacement..." << std::endl;
        auto compressed = zlib_compress(new_strings.data(), new_strings.size());
        write_u32(pf, static_cast<uint32_t>(PatchFileId::STRINGS));
        write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));
        write_u64(pf, new_strings.size());
        write_u64(pf, compressed.size());
        pf.write(compressed.data(), compressed.size());
        std::cerr << "    " << new_strings.size() << " -> " << compressed.size() << std::endl;
    }

    // Helper: diff a file with optional string remapping
    auto diff_file = [&](PatchFileId id, const std::string& name,
                          std::function<void(std::vector<char>&)> remap_fn) {
        std::cerr << "  " << name << ": ";
        auto old_data = read_file(old_dir + "/" + name);
        auto new_data = read_file(new_dir + "/" + name);

        // Apply string remap to old data (makes it comparable to new)
        if (remap_fn) {
            remap_fn(old_data);
        }

        // Compute block diff
        auto ops = compute_block_diff(old_data, new_data);

        // Compress the ops stream
        auto compressed = zlib_compress(ops.data(), ops.size());

        // Also compute full replacement size for comparison
        auto full_compressed = zlib_compress(new_data.data(), new_data.size());

        // Use whichever is smaller
        bool use_delta = compressed.size() < full_compressed.size();

        write_u32(pf, static_cast<uint32_t>(id));
        if (use_delta) {
            write_u32(pf, static_cast<uint32_t>(PatchEncoding::COPY_INSERT));
            write_u64(pf, old_data.size()); // old size
            write_u64(pf, new_data.size()); // new size
            write_u64(pf, compressed.size());
            pf.write(compressed.data(), compressed.size());
            std::cerr << "delta " << compressed.size() << " bytes";
        } else {
            write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));
            write_u64(pf, new_data.size());
            write_u64(pf, full_compressed.size());
            pf.write(full_compressed.data(), full_compressed.size());
            std::cerr << "full " << full_compressed.size() << " bytes";
        }
        double pct = (new_data.size() > 0) ?
            (use_delta ? compressed.size() : full_compressed.size()) * 100.0 / new_data.size() : 0;
        std::cerr << " (" << std::fixed << std::setprecision(1) << pct << "% of "
                  << new_data.size() / 1024 / 1024 << " MiB)" << std::endl;
    };

    // Detect strides for struct files
    size_t way_stride = detect_way_stride(old_dir + "/street_ways.bin");
    auto old_interp_data = read_file(old_dir + "/interp_ways.bin");
    size_t interp_stride = (old_interp_data.size() % 20 == 0 && old_interp_data.size() > 0) ? 20 : 18;
    auto old_admin_data = read_file(old_dir + "/admin_polygons.bin");
    size_t admin_stride = 24;
    for (size_t s : {24, 20, 19}) {
        if (old_admin_data.size() % s == 0 && old_admin_data.size() > 0) { admin_stride = s; break; }
    }

    // Data files with string remapping
    diff_file(PatchFileId::ADDR_POINTS, "addr_points.bin",
        [&](std::vector<char>& d) { remap_addr_points(d, remap); });

    diff_file(PatchFileId::STREET_WAYS, "street_ways.bin",
        [&, way_stride](std::vector<char>& d) { remap_ways(d, way_stride, remap); });

    diff_file(PatchFileId::INTERP_WAYS, "interp_ways.bin",
        [&, interp_stride](std::vector<char>& d) { remap_interps(d, interp_stride, remap); });

    diff_file(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin",
        [&, admin_stride](std::vector<char>& d) { remap_admin_polygons(d, admin_stride, remap); });

    // Coordinate files (no string refs, direct diff)
    diff_file(PatchFileId::STREET_NODES, "street_nodes.bin", nullptr);
    diff_file(PatchFileId::INTERP_NODES, "interp_nodes.bin", nullptr);
    diff_file(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", nullptr);

    // Derived/index files (no remap needed, direct diff)
    diff_file(PatchFileId::GEO_CELLS, "geo_cells.bin", nullptr);
    diff_file(PatchFileId::STREET_ENTRIES, "street_entries.bin", nullptr);
    diff_file(PatchFileId::ADDR_ENTRIES, "addr_entries.bin", nullptr);
    diff_file(PatchFileId::INTERP_ENTRIES, "interp_entries.bin", nullptr);
    diff_file(PatchFileId::ADMIN_CELLS, "admin_cells.bin", nullptr);
    diff_file(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin", nullptr);

    write_u32(pf, 0xFFFFFFFF); // end marker
    auto total = pf.tellp();
    std::cerr << "\nTotal patch: " << total << " bytes (" << total / 1024 / 1024 << " MiB)" << std::endl;
    return 0;
}

-// 1. New strings.bin (full replacement, zlib compressed)

-// 2. String remap table (old_offset → new_offset for every string)

-// 3. Per data file: record-level diff (matched records copied with string fixup,

-//    new records inserted, removed records skipped)

+// 1. Build string remap (old string offsets → new string offsets)

+// 2. For data files with string refs: apply remap to old file in memory, then binary diff

+// 3. For coordinate files: direct binary diff (no string refs)

+// 4. For small derived files: full replacement (compressed)

+// The binary diff uses a simple block-based COPY/INSERT encoder:

+// - Compare 64-byte blocks between (remapped) old and new

+// - COPY for matching blocks, INSERT for different blocks

+// - Handles size differences (new may be larger)

int ret = compress2(reinterpret_cast<Bytef*>(out.data()), &bound,

reinterpret_cast<const Bytef*>(data), size, 9);

-    if (ret != Z_OK) { std::cerr << "zlib compress failed" << std::endl; return {}; }

-// Maps every string offset in old pool to its offset in new pool.

-    std::vector<std::pair<uint32_t, uint32_t>> entries; // (old_off, new_off)

-    std::unordered_map<uint32_t, uint32_t> map; // old_off → new_off

+static std::unordered_map<uint32_t, uint32_t> build_string_remap(

+    const std::vector<char>& old_pool, const std::vector<char>& new_pool)

+    std::unordered_map<std::string, uint32_t> new_index;

+        const char* s = new_pool.data() + pos;

+        new_index[std::string(s, len)] = static_cast<uint32_t>(pos);

-    void build(const std::vector<char>& old_pool, const std::vector<char>& new_pool) {

-        // Index new pool: string content → offset

-        std::unordered_map<std::string, uint32_t> new_index;

-            while (pos < new_pool.size()) {

-                const char* s = new_pool.data() + pos;

-                size_t len = strlen(s);

-                new_index[std::string(s, len)] = static_cast<uint32_t>(pos);

+    std::unordered_map<uint32_t, uint32_t> remap;

+        const char* s = old_pool.data() + pos;

+        auto it = new_index.find(std::string(s, len));

+            remap[static_cast<uint32_t>(pos)] = it->second;

+// Apply string remap to specific fields in a data buffer

+static void remap_addr_points(std::vector<char>& data,

+                                const std::unordered_map<uint32_t, uint32_t>& remap) {

+        char* p = data.data() + i * 16;

+        // housenumber_id at offset 8, street_id at offset 12

+            memcpy(&old_id, p + off, 4);

+            auto it = remap.find(old_id);

+                memcpy(p + off, &it->second, 4);

-        // Map each old string to new offset

-        while (pos < old_pool.size()) {

-            const char* s = old_pool.data() + pos;

-            uint32_t old_off = static_cast<uint32_t>(pos);

-            auto it = new_index.find(std::string(s, len));

-            uint32_t new_off = (it != new_index.end()) ? it->second : 0xFFFFFFFF;

-            entries.push_back({old_off, new_off});

+static void remap_ways(std::vector<char>& data, size_t stride,

+                        const std::unordered_map<uint32_t, uint32_t>& remap) {

+    size_t name_off = (stride == 12) ? 8 : 5;

+        char* p = data.data() + i * stride;

+        memcpy(&old_id, p + name_off, 4);

+            memcpy(p + name_off, &it->second, 4);

-    uint32_t remap(uint32_t old_off) const {

-        return (it != map.end()) ? it->second : 0;

+static void remap_interps(std::vector<char>& data, size_t stride,

+                           const std::unordered_map<uint32_t, uint32_t>& remap) {

+    size_t street_off = (stride == 20) ? 8 : 5;

+        memcpy(&old_id, p + street_off, 4);

+            memcpy(p + street_off, &it->second, 4);

-// --- Record-level diff for fixed-stride files ---

-// Walks both files record by record. For matching records (same fingerprint),

-// emits COPY. For new/removed records, emits INSERT/skip.

+static void remap_admin_polygons(std::vector<char>& data, size_t stride,

+                                   const std::unordered_map<uint32_t, uint32_t>& remap) {

+        memcpy(&old_id, p + 8, 4); // name_id at offset 8

+            memcpy(p + 8, &it->second, 4);

-    // Operations to reconstruct new file from old + patch data

-        enum Type { COPY_REMAP, INSERT };

-        uint32_t old_index;  // for COPY_REMAP: index in old file

-        // For INSERT: data is in insert_data at insert_offset

-    std::vector<char> insert_data; // concatenated new record bytes

+// --- Block-based COPY/INSERT binary diff ---

+// Compares (remapped) old vs new, produces a compact delta.

+// Block size of 64 bytes balances granularity vs overhead.

-// Fingerprint an addr_point using resolved strings

-static uint64_t addr_hash(const PatchAddrPoint& p, const std::vector<char>& pool) {

-    // Simple hash of grid coords + string content

-    const char* hn = get_string(pool, p.housenumber_id);

-    const char* st = get_string(pool, p.street_id);

+static constexpr size_t DIFF_BLOCK = 64;

-static uint64_t way_hash(const PatchWayHeader& w, const PatchNodeCoord* nodes,

-                          const std::vector<char>& pool) {

-    const char* name = get_string(pool, w.name_id);

-    while (*name) { mix(*name); name++; }

-    for (uint8_t i = 0; i < w.node_count; i++) {

-        mix(to_grid(nodes[w.node_offset + i].lat));

-        mix(to_grid(nodes[w.node_offset + i].lng));

+static std::vector<char> compute_block_diff(const std::vector<char>& old_data,

+                                              const std::vector<char>& new_data) {

+    while (new_pos < new_data.size()) {

+        // Check if current block matches

+        size_t remaining_new = new_data.size() - new_pos;

+        size_t remaining_old = (old_pos < old_data.size()) ? old_data.size() - old_pos : 0;

+        size_t block = std::min(DIFF_BLOCK, remaining_new);

+            memcmp(old_data.data() + old_pos, new_data.data() + new_pos, block) == 0) {

+            // Find full extent of match

+            while (old_pos + match_len < old_data.size() &&

+                   new_pos + match_len < new_data.size() &&

+                   old_data[old_pos + match_len] == new_data[new_pos + match_len]) {

+            // Emit COPY: tag(1) + offset(8) + length(4) = 13 bytes

+            uint32_t len = static_cast<uint32_t>(match_len);

+            ops.insert(ops.end(), reinterpret_cast<const char*>(&off),

+                      reinterpret_cast<const char*>(&off) + 8);

+            ops.insert(ops.end(), reinterpret_cast<const char*>(&len),

+                      reinterpret_cast<const char*>(&len) + 4);

+            // Find extent of non-match (up to next matching block)

+            size_t insert_start = new_pos;

+            old_scan += std::min(block, remaining_old);

+            // Keep scanning until we find a matching block again

+            while (new_pos < new_data.size()) {

+                size_t rem_new = new_data.size() - new_pos;

+                size_t rem_old = (old_scan < old_data.size()) ? old_data.size() - old_scan : 0;

+                size_t b = std::min(DIFF_BLOCK, rem_new);

+                    memcmp(old_data.data() + old_scan, new_data.data() + new_pos, b) == 0) {

+                    break; // Found match

+                old_scan += std::min(b, rem_old);

+            // Emit INSERT: tag(1) + length(4) + data

+            size_t insert_len = new_pos - insert_start;

+            uint32_t len = static_cast<uint32_t>(insert_len);

+            ops.insert(ops.end(), new_data.data() + insert_start,

+                      new_data.data() + insert_start + insert_len);

-    std::cerr << "Loading indexes..." << std::endl;

-    // Load string pools and build remap

+    std::cerr << "Loading string pools..." << std::endl;

auto old_strings = read_file(old_dir + "/strings.bin");

auto new_strings = read_file(new_dir + "/strings.bin");

-    remap.build(old_strings, new_strings);

-    std::cerr << "  String remap: " << remap.entries.size() << " entries" << std::endl;

+    auto remap = build_string_remap(old_strings, new_strings);

+    std::cerr << "  Remap: " << remap.size() << " strings mapped" << std::endl;

-    if (!pf) { std::cerr << "Cannot open " << patch_path << std::endl; return 1; }

-    // Section 1: New strings.bin (full replacement, compressed)

-    std::cerr << "  Writing strings section..." << std::endl;

+    // Section: strings.bin (full replacement)

+        std::cerr << "  strings.bin: full replacement..." << std::endl;

-        write_u64(pf, new_strings.size()); // uncompressed size

-        write_u64(pf, compressed.size());  // compressed size

+        write_u64(pf, new_strings.size());

+        write_u64(pf, compressed.size());

-        std::cerr << "    " << new_strings.size() << " -> " << compressed.size() << " bytes" << std::endl;

+        std::cerr << "    " << new_strings.size() << " -> " << compressed.size() << std::endl;

-    std::cerr << "  Writing string remap..." << std::endl;

-        // Only write remaps where old_off != new_off

-        std::vector<std::pair<uint32_t, uint32_t>> changed;

-        for (auto& [old_off, new_off] : remap.entries) {

-            if (old_off != new_off && new_off != 0xFFFFFFFF) {

-                changed.push_back({old_off, new_off});

+    // Helper: diff a file with optional string remapping

+    auto diff_file = [&](PatchFileId id, const std::string& name,

+                          std::function<void(std::vector<char>&)> remap_fn) {

+        auto old_data = read_file(old_dir + "/" + name);

+        auto new_data = read_file(new_dir + "/" + name);

+        // Apply string remap to old data (makes it comparable to new)

-        write_u32(pf, 0xFFFFFFFE); // special marker for remap section

-        write_u32(pf, static_cast<uint32_t>(changed.size()));

-        std::cerr << "    " << changed.size() << " changed offsets (of " << remap.entries.size() << " total)" << std::endl;

-    // Helper: diff a data file where records have fixed stride

-    // Strategy: for each file, send the full new file compressed.

-    // The applicator just decompresses and writes it.

-    // This is simple and correct. For files where zlib compresses well

-    // against their size, this is good enough.

-    auto write_file_section = [&](PatchFileId id, const std::string& name) {

-        std::cerr << "  Writing " << name << "..." << std::endl;

-        auto new_data = read_file(new_dir + "/" + name);

-        auto compressed = zlib_compress(new_data.data(), new_data.size());

+        auto ops = compute_block_diff(old_data, new_data);

+        auto compressed = zlib_compress(ops.data(), ops.size());

+        // Also compute full replacement size for comparison

+        auto full_compressed = zlib_compress(new_data.data(), new_data.size());

+        bool use_delta = compressed.size() < full_compressed.size();

-        write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));

-        write_u64(pf, new_data.size());

-        write_u64(pf, compressed.size());

-        pf.write(compressed.data(), compressed.size());

-        std::cerr << "    " << new_data.size() << " -> " << compressed.size()

-                  << " (" << (new_data.size() > 0 ? compressed.size() * 100 / new_data.size() : 0) << "%)" << std::endl;

+            write_u32(pf, static_cast<uint32_t>(PatchEncoding::COPY_INSERT));

+            write_u64(pf, old_data.size()); // old size

+            write_u64(pf, new_data.size()); // new size

+            write_u64(pf, compressed.size());

+            pf.write(compressed.data(), compressed.size());

+            std::cerr << "delta " << compressed.size() << " bytes";

+            write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));

+            write_u64(pf, new_data.size());

+            write_u64(pf, full_compressed.size());

+            pf.write(full_compressed.data(), full_compressed.size());

+            std::cerr << "full " << full_compressed.size() << " bytes";

+        double pct = (new_data.size() > 0) ?

+            (use_delta ? compressed.size() : full_compressed.size()) * 100.0 / new_data.size() : 0;

+        std::cerr << " (" << std::fixed << std::setprecision(1) << pct << "% of "

-    // For now: send ALL files as full replacement (compressed).

-    // This gives us correct patches that we can test end-to-end.

-    // Optimization: use COPY_INSERT encoding for large files later.

-    write_file_section(PatchFileId::STREET_WAYS, "street_ways.bin");

-    write_file_section(PatchFileId::STREET_NODES, "street_nodes.bin");

-    write_file_section(PatchFileId::ADDR_POINTS, "addr_points.bin");

-    write_file_section(PatchFileId::INTERP_WAYS, "interp_ways.bin");

-    write_file_section(PatchFileId::INTERP_NODES, "interp_nodes.bin");

-    write_file_section(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin");

-    write_file_section(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin");

-    write_file_section(PatchFileId::GEO_CELLS, "geo_cells.bin");

-    write_file_section(PatchFileId::STREET_ENTRIES, "street_entries.bin");

-    write_file_section(PatchFileId::ADDR_ENTRIES, "addr_entries.bin");

-    write_file_section(PatchFileId::INTERP_ENTRIES, "interp_entries.bin");

-    write_file_section(PatchFileId::ADMIN_CELLS, "admin_cells.bin");

-    write_file_section(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin");

+    size_t way_stride = detect_way_stride(old_dir + "/street_ways.bin");

+    auto old_interp_data = read_file(old_dir + "/interp_ways.bin");

+    size_t interp_stride = (old_interp_data.size() % 20 == 0 && old_interp_data.size() > 0) ? 20 : 18;

+    auto old_admin_data = read_file(old_dir + "/admin_polygons.bin");

+        if (old_admin_data.size() % s == 0 && old_admin_data.size() > 0) { admin_stride = s; break; }

+    // Data files with string remapping

+    diff_file(PatchFileId::ADDR_POINTS, "addr_points.bin",

+        [&](std::vector<char>& d) { remap_addr_points(d, remap); });

-    std::cerr << "Patch written to " << patch_path << " (" << pf.tellp() << " bytes)" << std::endl;

+    diff_file(PatchFileId::STREET_WAYS, "street_ways.bin",

+        [&, way_stride](std::vector<char>& d) { remap_ways(d, way_stride, remap); });

+    diff_file(PatchFileId::INTERP_WAYS, "interp_ways.bin",

+        [&, interp_stride](std::vector<char>& d) { remap_interps(d, interp_stride, remap); });

+    diff_file(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin",

+        [&, admin_stride](std::vector<char>& d) { remap_admin_polygons(d, admin_stride, remap); });

+    // Coordinate files (no string refs, direct diff)

+    diff_file(PatchFileId::STREET_NODES, "street_nodes.bin", nullptr);

+    diff_file(PatchFileId::INTERP_NODES, "interp_nodes.bin", nullptr);

+    diff_file(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", nullptr);

+    // Derived/index files (no remap needed, direct diff)

+    diff_file(PatchFileId::GEO_CELLS, "geo_cells.bin", nullptr);

+    diff_file(PatchFileId::STREET_ENTRIES, "street_entries.bin", nullptr);

+    diff_file(PatchFileId::ADDR_ENTRIES, "addr_entries.bin", nullptr);

+    diff_file(PatchFileId::INTERP_ENTRIES, "interp_entries.bin", nullptr);

+    diff_file(PatchFileId::ADMIN_CELLS, "admin_cells.bin", nullptr);

+    diff_file(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin", nullptr);

+    write_u32(pf, 0xFFFFFFFF); // end marker

+    std::cerr << "\nTotal patch: " << total << " bytes (" << total / 1024 / 1024 << " MiB)" << std::endl;

// geocoder-diff: Compare two canonical geocoder indexes and produce a compact patch.
//
// The patch encodes:
// 1. New strings.bin (full replacement, zlib compressed)
// 2. String remap table (old_offset → new_offset for every string)
// 3. Per data file: record-level diff (matched records copied with string fixup,
//    new records inserted, removed records skipped)
//
// Usage: geocoder-diff <old-canonical-dir> <new-canonical-dir> -o <patch-file>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>
#include <zlib.h>

#include "patch_format.h"

// --- zlib compression helpers ---

static std::vector<char> zlib_compress(const char* data, size_t size) {
    uLongf bound = compressBound(size);
    std::vector<char> out(bound);
    int ret = compress2(reinterpret_cast<Bytef*>(out.data()), &bound,
                        reinterpret_cast<const Bytef*>(data), size, 9);
    if (ret != Z_OK) { std::cerr << "zlib compress failed" << std::endl; return {}; }
    out.resize(bound);
    return out;
}

// --- Build string remap ---
// Maps every string offset in old pool to its offset in new pool.

struct StringRemap {
    std::vector<std::pair<uint32_t, uint32_t>> entries; // (old_off, new_off)
    std::unordered_map<uint32_t, uint32_t> map; // old_off → new_off

    void build(const std::vector<char>& old_pool, const std::vector<char>& new_pool) {
        // Index new pool: string content → offset
        std::unordered_map<std::string, uint32_t> new_index;
        {
            size_t pos = 0;
            while (pos < new_pool.size()) {
                const char* s = new_pool.data() + pos;
                size_t len = strlen(s);
                new_index[std::string(s, len)] = static_cast<uint32_t>(pos);
                pos += len + 1;
            }
        }

        // Map each old string to new offset
        size_t pos = 0;
        while (pos < old_pool.size()) {
            const char* s = old_pool.data() + pos;
            size_t len = strlen(s);
            uint32_t old_off = static_cast<uint32_t>(pos);
            auto it = new_index.find(std::string(s, len));
            uint32_t new_off = (it != new_index.end()) ? it->second : 0xFFFFFFFF;
            entries.push_back({old_off, new_off});
            map[old_off] = new_off;
            pos += len + 1;
        }
    }

    uint32_t remap(uint32_t old_off) const {
        auto it = map.find(old_off);
        return (it != map.end()) ? it->second : 0;
    }
};

// --- Record-level diff for fixed-stride files ---
// Walks both files record by record. For matching records (same fingerprint),
// emits COPY. For new/removed records, emits INSERT/skip.

struct RecordDiff {
    // Operations to reconstruct new file from old + patch data
    struct Op {
        enum Type { COPY_REMAP, INSERT };
        Type type;
        uint32_t old_index;  // for COPY_REMAP: index in old file
        // For INSERT: data is in insert_data at insert_offset
        uint32_t insert_offset;
        uint32_t insert_length;
    };
    std::vector<Op> ops;
    std::vector<char> insert_data; // concatenated new record bytes
};

// Fingerprint an addr_point using resolved strings
static uint64_t addr_hash(const PatchAddrPoint& p, const std::vector<char>& pool) {
    // Simple hash of grid coords + string content
    uint64_t h = 14695981039346656037ULL;
    auto mix = [&h](uint64_t v) { h ^= v; h *= 1099511628211ULL; };
    mix(to_grid(p.lat));
    mix(to_grid(p.lng));
    const char* hn = get_string(pool, p.housenumber_id);
    while (*hn) { mix(*hn); hn++; }
    mix(0xFF);
    const char* st = get_string(pool, p.street_id);
    while (*st) { mix(*st); st++; }
    return h;
}

static uint64_t way_hash(const PatchWayHeader& w, const PatchNodeCoord* nodes,
                          const std::vector<char>& pool) {
    uint64_t h = 14695981039346656037ULL;
    auto mix = [&h](uint64_t v) { h ^= v; h *= 1099511628211ULL; };
    const char* name = get_string(pool, w.name_id);
    while (*name) { mix(*name); name++; }
    mix(0xFF);
    mix(w.node_count);
    for (uint8_t i = 0; i < w.node_count; i++) {
        mix(to_grid(nodes[w.node_offset + i].lat));
        mix(to_grid(nodes[w.node_offset + i].lng));
    }
    return h;
}

// --- Write patch file ---

static void write_u32(std::ofstream& f, uint32_t v) { f.write(reinterpret_cast<const char*>(&v), 4); }
static void write_u64(std::ofstream& f, uint64_t v) { f.write(reinterpret_cast<const char*>(&v), 8); }

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>" << std::endl;
        return 1;
    }

    std::string old_dir = argv[1];
    std::string new_dir = argv[2];
    std::string patch_path = argv[4];

    std::cerr << "Loading indexes..." << std::endl;

    // Load string pools and build remap
    auto old_strings = read_file(old_dir + "/strings.bin");
    auto new_strings = read_file(new_dir + "/strings.bin");
    StringRemap remap;
    remap.build(old_strings, new_strings);
    std::cerr << "  String remap: " << remap.entries.size() << " entries" << std::endl;

    // Open patch file
    std::ofstream pf(patch_path, std::ios::binary);
    if (!pf) { std::cerr << "Cannot open " << patch_path << std::endl; return 1; }

    // Write header
    pf.write(GCPATCH_MAGIC, 8);
    write_u32(pf, GCPATCH_VERSION);
    write_u32(pf, 0); // flags

    // Section 1: New strings.bin (full replacement, compressed)
    std::cerr << "  Writing strings section..." << std::endl;
    {
        auto compressed = zlib_compress(new_strings.data(), new_strings.size());
        write_u32(pf, static_cast<uint32_t>(PatchFileId::STRINGS));
        write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));
        write_u64(pf, new_strings.size()); // uncompressed size
        write_u64(pf, compressed.size());  // compressed size
        pf.write(compressed.data(), compressed.size());
        std::cerr << "    " << new_strings.size() << " -> " << compressed.size() << " bytes" << std::endl;
    }

    // Section 2: String remap table
    std::cerr << "  Writing string remap..." << std::endl;
    {
        // Only write remaps where old_off != new_off
        std::vector<std::pair<uint32_t, uint32_t>> changed;
        for (auto& [old_off, new_off] : remap.entries) {
            if (old_off != new_off && new_off != 0xFFFFFFFF) {
                changed.push_back({old_off, new_off});
            }
        }
        write_u32(pf, 0xFFFFFFFE); // special marker for remap section
        write_u32(pf, static_cast<uint32_t>(changed.size()));
        for (auto& [o, n] : changed) {
            write_u32(pf, o);
            write_u32(pf, n);
        }
        std::cerr << "    " << changed.size() << " changed offsets (of " << remap.entries.size() << " total)" << std::endl;
    }

    // Helper: diff a data file where records have fixed stride
    // Strategy: for each file, send the full new file compressed.
    // The applicator just decompresses and writes it.
    // This is simple and correct. For files where zlib compresses well
    // against their size, this is good enough.
    auto write_file_section = [&](PatchFileId id, const std::string& name) {
        std::cerr << "  Writing " << name << "..." << std::endl;
        auto new_data = read_file(new_dir + "/" + name);
        auto compressed = zlib_compress(new_data.data(), new_data.size());
        write_u32(pf, static_cast<uint32_t>(id));
        write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));
        write_u64(pf, new_data.size());
        write_u64(pf, compressed.size());
        pf.write(compressed.data(), compressed.size());
        std::cerr << "    " << new_data.size() << " -> " << compressed.size()
                  << " (" << (new_data.size() > 0 ? compressed.size() * 100 / new_data.size() : 0) << "%)" << std::endl;
    };

    // For now: send ALL files as full replacement (compressed).
    // This gives us correct patches that we can test end-to-end.
    // Optimization: use COPY_INSERT encoding for large files later.
    write_file_section(PatchFileId::STREET_WAYS, "street_ways.bin");
    write_file_section(PatchFileId::STREET_NODES, "street_nodes.bin");
    write_file_section(PatchFileId::ADDR_POINTS, "addr_points.bin");
    write_file_section(PatchFileId::INTERP_WAYS, "interp_ways.bin");
    write_file_section(PatchFileId::INTERP_NODES, "interp_nodes.bin");
    write_file_section(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin");
    write_file_section(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin");
    write_file_section(PatchFileId::GEO_CELLS, "geo_cells.bin");
    write_file_section(PatchFileId::STREET_ENTRIES, "street_entries.bin");
    write_file_section(PatchFileId::ADDR_ENTRIES, "addr_entries.bin");
    write_file_section(PatchFileId::INTERP_ENTRIES, "interp_entries.bin");
    write_file_section(PatchFileId::ADMIN_CELLS, "admin_cells.bin");
    write_file_section(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin");

    // Write end marker
    write_u32(pf, 0xFFFFFFFF);

    std::cerr << "Patch written to " << patch_path << " (" << pf.tellp() << " bytes)" << std::endl;
    return 0;
}


Now update the patch tool to handle COPY_INSERT encoding:


Edit

        if (encoding == static_cast<uint32_t>(PatchEncoding::RAW_REPLACE)) {
            // Read compressed data
            std::vector<char> compressed(compressed_size);
            pf.read(compressed.data(), compressed_size);

            // Decompress
            auto data = zlib_decompress(compressed.data(), compressed_size, uncompressed_size);
            if (data.empty() && uncompressed_size > 0) {
                std::cerr << "  FAILED to decompress " << filename << std::endl;
                return 1;
            }

            // Write output file
            std::string out_path = output_dir + "/" + filename;
            write_file(out_path, data);
            std::cerr << "  " << filename << ": " << compressed_size << " -> " << data.size() << " bytes" << std::endl;
        } else {
            std::cerr << "  Unknown encoding " << encoding << " for " << filename << std::endl;
            pf.seekg(compressed_size, std::ios::cur);
        }

        if (encoding == static_cast<uint32_t>(PatchEncoding::RAW_REPLACE)) {
            uint64_t compressed_size_local = read_u64(pf);
            std::vector<char> compressed(compressed_size_local);
            pf.read(compressed.data(), compressed_size_local);
            auto data = zlib_decompress(compressed.data(), compressed_size_local, uncompressed_size);
            if (data.empty() && uncompressed_size > 0) {
                std::cerr << "  FAILED to decompress " << filename << std::endl;
                return 1;
            }
            write_file(output_dir + "/" + filename, data);
            std::cerr << "  " << filename << ": replaced (" << data.size() << " bytes)" << std::endl;

        } else if (encoding == static_cast<uint32_t>(PatchEncoding::COPY_INSERT)) {
            uint64_t old_size = uncompressed_size; // repurposed
            uint64_t new_size = read_u64(pf);
            uint64_t compressed_size_local = read_u64(pf);

            // Read and decompress ops
            std::vector<char> compressed(compressed_size_local);
            pf.read(compressed.data(), compressed_size_local);

            // Estimate ops uncompressed size (generous: 4x compressed)
            size_t est_size = compressed_size_local * 8;
            if (est_size < new_size * 2) est_size = new_size * 2;
            auto ops = zlib_decompress(compressed.data(), compressed_size_local, est_size);
            if (ops.empty() && compressed_size_local > 0) {
                std::cerr << "  FAILED to decompress ops for " << filename << std::endl;
                return 1;
            }

            // Load old file
            auto old_data = read_file(current_dir + "/" + filename);
            if (old_data.size() != old_size) {
                std::cerr << "  WARNING: " << filename << " size mismatch (expected "
                          << old_size << ", got " << old_data.size() << ")" << std::endl;
            }

            // Replay COPY/INSERT ops
            std::vector<char> new_data;
            new_data.reserve(new_size);
            size_t pos = 0;
            while (pos < ops.size()) {
                uint8_t tag = static_cast<uint8_t>(ops[pos]); pos++;
                if (tag == OP_COPY) {
                    uint64_t off; memcpy(&off, ops.data() + pos, 8); pos += 8;
                    uint32_t len; memcpy(&len, ops.data() + pos, 4); pos += 4;
                    if (off + len <= old_data.size()) {
                        new_data.insert(new_data.end(), old_data.data() + off,
                                       old_data.data() + off + len);
                    }
                } else if (tag == OP_INSERT) {
                    uint32_t len; memcpy(&len, ops.data() + pos, 4); pos += 4;
                    new_data.insert(new_data.end(), ops.data() + pos, ops.data() + pos + len);
                    pos += len;
                } else {
                    std::cerr << "  Unknown op " << (int)tag << " in " << filename << std::endl;
                    break;
                }
            }

            write_file(output_dir + "/" + filename, new_data);
            std::cerr << "  " << filename << ": delta applied (" << new_data.size() << " bytes)" << std::endl;

        } else {
            std::cerr << "  Unknown encoding " << encoding << " for " << filename << std::endl;
        }


The file /root/traccar-geocoder/builder/tools/geocoder_patch.cpp has been updated successfully.

        if (encoding == static_cast<uint32_t>(PatchEncoding::RAW_REPLACE)) {
            uint64_t compressed_size_local = read_u64(pf);
            std::vector<char> compressed(compressed_size_local);
            pf.read(compressed.data(), compressed_size_local);
            auto data = zlib_decompress(compressed.data(), compressed_size_local, uncompressed_size);
            if (data.empty() && uncompressed_size > 0) {
                std::cerr << "  FAILED to decompress " << filename << std::endl;
                return 1;
            }
            write_file(output_dir + "/" + filename, data);
            std::cerr << "  " << filename << ": replaced (" << data.size() << " bytes)" << std::endl;

        } else if (encoding == static_cast<uint32_t>(PatchEncoding::COPY_INSERT)) {
            uint64_t old_size = uncompressed_size; // repurposed
            uint64_t new_size = read_u64(pf);
            uint64_t compressed_size_local = read_u64(pf);

            // Read and decompress ops
            std::vector<char> compressed(compressed_size_local);
            pf.read(compressed.data(), compressed_size_local);

            // Estimate ops uncompressed size (generous: 4x compressed)
            size_t est_size = compressed_size_local * 8;
            if (est_size < new_size * 2) est_size = new_size * 2;
            auto ops = zlib_decompress(compressed.data(), compressed_size_local, est_size);
            if (ops.empty() && compressed_size_local > 0) {
                std::cerr << "  FAILED to decompress ops for " << filename << std::endl;
                return 1;
            }

            // Load old file
            auto old_data = read_file(current_dir + "/" + filename);
            if (old_data.size() != old_size) {
                std::cerr << "  WARNING: " << filename << " size mismatch (expected "
                          << old_size << ", got " << old_data.size() << ")" << std::endl;
            }

            // Replay COPY/INSERT ops
            std::vector<char> new_data;
            new_data.reserve(new_size);
            size_t pos = 0;
            while (pos < ops.size()) {
                uint8_t tag = static_cast<uint8_t>(ops[pos]); pos++;
                if (tag == OP_COPY) {
                    uint64_t off; memcpy(&off, ops.data() + pos, 8); pos += 8;
                    uint32_t len; memcpy(&len, ops.data() + pos, 4); pos += 4;
                    if (off + len <= old_data.size()) {
                        new_data.insert(new_data.end(), old_data.data() + off,
                                       old_data.data() + off + len);
                    }
                } else if (tag == OP_INSERT) {
                    uint32_t len; memcpy(&len, ops.data() + pos, 4); pos += 4;
                    new_data.insert(new_data.end(), ops.data() + pos, ops.data() + pos + len);
                    pos += len;
                } else {
                    std::cerr << "  Unknown op " << (int)tag << " in " << filename << std::endl;
                    break;
                }
            }

            write_file(output_dir + "/" + filename, new_data);
            std::cerr << "  " << filename << ": delta applied (" << new_data.size() << " bytes)" << std::endl;

        } else {
            std::cerr << "  Unknown encoding " << encoding << " for " << filename << std::endl;
        }

// geocoder-patch: Apply a .gcpatch file to a canonical geocoder index.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
#include <zlib.h>

#include "patch_format.h"

static uint32_t read_u32(std::ifstream& f) {
    uint32_t v; f.read(reinterpret_cast<char*>(&v), 4); return v;
}
static uint64_t read_u64(std::ifstream& f) {
    uint64_t v; f.read(reinterpret_cast<char*>(&v), 8); return v;
}

static std::vector<char> zlib_decompress(const char* data, size_t compressed_size, size_t uncompressed_size) {
    std::vector<char> out(uncompressed_size);
    uLongf dest_len = uncompressed_size;
    int ret = uncompress(reinterpret_cast<Bytef*>(out.data()), &dest_len,
                         reinterpret_cast<const Bytef*>(data), compressed_size);
    if (ret != Z_OK) {
        std::cerr << "zlib decompress failed: " << ret << std::endl;
        return {};
    }
    out.resize(dest_len);
    return out;
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }

    std::string current_dir = argv[1];
    std::string patch_path = argv[2];
    std::string output_dir = argv[4];
    ensure_dir(output_dir);

    // Open patch file
    std::ifstream pf(patch_path, std::ios::binary);
    if (!pf) { std::cerr << "Cannot open " << patch_path << std::endl; return 1; }

    // Read and verify header
    char magic[8];
    pf.read(magic, 8);
    if (memcmp(magic, GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch file (bad magic)" << std::endl;
        return 1;
    }
    uint32_t version = read_u32(pf);
    if (version != GCPATCH_VERSION) {
        std::cerr << "Unsupported patch version: " << version << std::endl;
        return 1;
    }
    uint32_t flags = read_u32(pf);
    (void)flags;

    std::cerr << "Applying patch from " << patch_path << "..." << std::endl;

    // Process sections
    while (pf) {
        uint32_t section_id = read_u32(pf);
        if (!pf || section_id == 0xFFFFFFFF) break; // end marker

        // String remap section (special)
        if (section_id == 0xFFFFFFFE) {
            uint32_t count = read_u32(pf);
            // Skip remap entries (not needed for RAW_REPLACE mode)
            pf.seekg(count * 8, std::ios::cur);
            std::cerr << "  Skipping string remap (" << count << " entries)" << std::endl;
            continue;
        }

        uint32_t encoding = read_u32(pf);
        uint64_t uncompressed_size = read_u64(pf);
        uint64_t compressed_size = read_u64(pf);

        if (section_id >= static_cast<uint32_t>(PatchFileId::COUNT)) {
            std::cerr << "  Unknown section " << section_id << ", skipping " << compressed_size << " bytes" << std::endl;
            pf.seekg(compressed_size, std::ios::cur);
            continue;
        }

        const char* filename = patch_file_names[section_id];

        if (encoding == static_cast<uint32_t>(PatchEncoding::RAW_REPLACE)) {
            // Read compressed data
            std::vector<char> compressed(compressed_size);
            pf.read(compressed.data(), compressed_size);

            // Decompress
            auto data = zlib_decompress(compressed.data(), compressed_size, uncompressed_size);
            if (data.empty() && uncompressed_size > 0) {
                std::cerr << "  FAILED to decompress " << filename << std::endl;
                return 1;
            }

            // Write output file
            std::string out_path = output_dir + "/" + filename;
            write_file(out_path, data);
            std::cerr << "  " << filename << ": " << compressed_size << " -> " << data.size() << " bytes" << std::endl;
        } else {
            std::cerr << "  Unknown encoding " << encoding << " for " << filename << std::endl;
            pf.seekg(compressed_size, std::ios::cur);
        }
    }

    std::cerr << "Patch applied. Output in " << output_dir << std::endl;
    return 0;
}

-            std::vector<char> compressed(compressed_size);

-            pf.read(compressed.data(), compressed_size);

-            auto data = zlib_decompress(compressed.data(), compressed_size, uncompressed_size);

+            uint64_t compressed_size_local = read_u64(pf);

+            std::vector<char> compressed(compressed_size_local);

+            pf.read(compressed.data(), compressed_size_local);

+            auto data = zlib_decompress(compressed.data(), compressed_size_local, uncompressed_size);

+            write_file(output_dir + "/" + filename, data);

+            std::cerr << "  " << filename << ": replaced (" << data.size() << " bytes)" << std::endl;

-            std::string out_path = output_dir + "/" + filename;

-            write_file(out_path, data);

-            std::cerr << "  " << filename << ": " << compressed_size << " -> " << data.size() << " bytes" << std::endl;

+        } else if (encoding == static_cast<uint32_t>(PatchEncoding::COPY_INSERT)) {

+            uint64_t old_size = uncompressed_size; // repurposed

+            uint64_t new_size = read_u64(pf);

+            // Estimate ops uncompressed size (generous: 4x compressed)

+            size_t est_size = compressed_size_local * 8;

+            if (est_size < new_size * 2) est_size = new_size * 2;

+            auto ops = zlib_decompress(compressed.data(), compressed_size_local, est_size);

+            if (ops.empty() && compressed_size_local > 0) {

+                std::cerr << "  FAILED to decompress ops for " << filename << std::endl;

+            auto old_data = read_file(current_dir + "/" + filename);

+            if (old_data.size() != old_size) {

+                std::cerr << "  WARNING: " << filename << " size mismatch (expected "

+                          << old_size << ", got " << old_data.size() << ")" << std::endl;

+            std::vector<char> new_data;

+            new_data.reserve(new_size);

+                uint8_t tag = static_cast<uint8_t>(ops[pos]); pos++;

+                    uint64_t off; memcpy(&off, ops.data() + pos, 8); pos += 8;

+                    uint32_t len; memcpy(&len, ops.data() + pos, 4); pos += 4;

+                    if (off + len <= old_data.size()) {

+                        new_data.insert(new_data.end(), old_data.data() + off,

+                                       old_data.data() + off + len);

+                } else if (tag == OP_INSERT) {

+                    new_data.insert(new_data.end(), ops.data() + pos, ops.data() + pos + len);

+                    std::cerr << "  Unknown op " << (int)tag << " in " << filename << std::endl;

+            write_file(output_dir + "/" + filename, new_data);

+            std::cerr << "  " << filename << ": delta applied (" << new_data.size() << " bytes)" << std::endl;

-            pf.seekg(compressed_size, std::ios::cur);


I also need to fix the RAW_REPLACE reading in the patch tool — the section format has old_size/new_size/compressed_size fields for COPY_INSERT but only uncompressed_size/compressed_size for RAW_REPLACE. Let me fix the section reading to be consistent:


        uint32_t encoding = read_u32(pf);
        uint64_t uncompressed_size = read_u64(pf);
        uint64_t compressed_size = read_u64(pf);

        if (section_id >= static_cast<uint32_t>(PatchFileId::COUNT)) {
            std::cerr << "  Unknown section " << section_id << ", skipping " << compressed_size << " bytes" << std::endl;
            pf.seekg(compressed_size, std::ios::cur);
            continue;
        }

        const char* filename = patch_file_names[section_id];

        if (encoding == static_cast<uint32_t>(PatchEncoding::RAW_REPLACE)) {
            uint64_t compressed_size_local = read_u64(pf);
            std::vector<char> compressed(compressed_size_local);
            pf.read(compressed.data(), compressed_size_local);
            auto data = zlib_decompress(compressed.data(), compressed_size_local, uncompressed_size);

        uint32_t encoding = read_u32(pf);

        if (section_id >= static_cast<uint32_t>(PatchFileId::COUNT)) {
            std::cerr << "  Unknown section " << section_id << std::endl;
            // Can't skip unknown sections reliably, abort
            break;
        }

        const char* filename = patch_file_names[section_id];

        if (encoding == static_cast<uint32_t>(PatchEncoding::RAW_REPLACE)) {
            uint64_t uncompressed_size = read_u64(pf);
            uint64_t compressed_size = read_u64(pf);
            std::vector<char> compressed(compressed_size);
            pf.read(compressed.data(), compressed_size);
            auto data = zlib_decompress(compressed.data(), compressed_size, uncompressed_size);


        uint32_t encoding = read_u32(pf);

        if (section_id >= static_cast<uint32_t>(PatchFileId::COUNT)) {
            std::cerr << "  Unknown section " << section_id << std::endl;
            // Can't skip unknown sections reliably, abort
            break;
        }

        const char* filename = patch_file_names[section_id];

        if (encoding == static_cast<uint32_t>(PatchEncoding::RAW_REPLACE)) {
            uint64_t uncompressed_size = read_u64(pf);
            uint64_t compressed_size = read_u64(pf);
            std::vector<char> compressed(compressed_size);
            pf.read(compressed.data(), compressed_size);
            auto data = zlib_decompress(compressed.data(), compressed_size, uncompressed_size);

// geocoder-patch: Apply a .gcpatch file to a canonical geocoder index.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
#include <zlib.h>

#include "patch_format.h"

static uint32_t read_u32(std::ifstream& f) {
    uint32_t v; f.read(reinterpret_cast<char*>(&v), 4); return v;
}
static uint64_t read_u64(std::ifstream& f) {
    uint64_t v; f.read(reinterpret_cast<char*>(&v), 8); return v;
}

static std::vector<char> zlib_decompress(const char* data, size_t compressed_size, size_t uncompressed_size) {
    std::vector<char> out(uncompressed_size);
    uLongf dest_len = uncompressed_size;
    int ret = uncompress(reinterpret_cast<Bytef*>(out.data()), &dest_len,
                         reinterpret_cast<const Bytef*>(data), compressed_size);
    if (ret != Z_OK) {
        std::cerr << "zlib decompress failed: " << ret << std::endl;
        return {};
    }
    out.resize(dest_len);
    return out;
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }

    std::string current_dir = argv[1];
    std::string patch_path = argv[2];
    std::string output_dir = argv[4];
    ensure_dir(output_dir);

    // Open patch file
    std::ifstream pf(patch_path, std::ios::binary);
    if (!pf) { std::cerr << "Cannot open " << patch_path << std::endl; return 1; }

    // Read and verify header
    char magic[8];
    pf.read(magic, 8);
    if (memcmp(magic, GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch file (bad magic)" << std::endl;
        return 1;
    }
    uint32_t version = read_u32(pf);
    if (version != GCPATCH_VERSION) {
        std::cerr << "Unsupported patch version: " << version << std::endl;
        return 1;
    }
    uint32_t flags = read_u32(pf);
    (void)flags;

    std::cerr << "Applying patch from " << patch_path << "..." << std::endl;

    // Process sections
    while (pf) {
        uint32_t section_id = read_u32(pf);
        if (!pf || section_id == 0xFFFFFFFF) break; // end marker

        // String remap section (special)
        if (section_id == 0xFFFFFFFE) {
            uint32_t count = read_u32(pf);
            // Skip remap entries (not needed for RAW_REPLACE mode)
            pf.seekg(count * 8, std::ios::cur);
            std::cerr << "  Skipping string remap (" << count << " entries)" << std::endl;
            continue;
        }

        uint32_t encoding = read_u32(pf);
        uint64_t uncompressed_size = read_u64(pf);
        uint64_t compressed_size = read_u64(pf);

        if (section_id >= static_cast<uint32_t>(PatchFileId::COUNT)) {
            std::cerr << "  Unknown section " << section_id << ", skipping " << compressed_size << " bytes" << std::endl;
            pf.seekg(compressed_size, std::ios::cur);
            continue;
        }

        const char* filename = patch_file_names[section_id];

        if (encoding == static_cast<uint32_t>(PatchEncoding::RAW_REPLACE)) {
            uint64_t compressed_size_local = read_u64(pf);
            std::vector<char> compressed(compressed_size_local);
            pf.read(compressed.data(), compressed_size_local);
            auto data = zlib_decompress(compressed.data(), compressed_size_local, uncompressed_size);
            if (data.empty() && uncompressed_size > 0) {
                std::cerr << "  FAILED to decompress " << filename << std::endl;
                return 1;
            }
            write_file(output_dir + "/" + filename, data);
            std::cerr << "  " << filename << ": replaced (" << data.size() << " bytes)" << std::endl;

        } else if (encoding == static_cast<uint32_t>(PatchEncoding::COPY_INSERT)) {
            uint64_t old_size = uncompressed_size; // repurposed
            uint64_t new_size = read_u64(pf);
            uint64_t compressed_size_local = read_u64(pf);

            // Read and decompress ops
            std::vector<char> compressed(compressed_size_local);
            pf.read(compressed.data(), compressed_size_local);

            // Estimate ops uncompressed size (generous: 4x compressed)
            size_t est_size = compressed_size_local * 8;
            if (est_size < new_size * 2) est_size = new_size * 2;
            auto ops = zlib_decompress(compressed.data(), compressed_size_local, est_size);
            if (ops.empty() && compressed_size_local > 0) {
                std::cerr << "  FAILED to decompress ops for " << filename << std::endl;
                return 1;
            }

            // Load old file
            auto old_data = read_file(current_dir + "/" + filename);
            if (old_data.size() != old_size) {
                std::cerr << "  WARNING: " << filename << " size mismatch (expected "
                          << old_size << ", got " << old_data.size() << ")" << std::endl;
            }

            // Replay COPY/INSERT ops
            std::vector<char> new_data;
            new_data.reserve(new_size);
            size_t pos = 0;
            while (pos < ops.size()) {
                uint8_t tag = static_cast<uint8_t>(ops[pos]); pos++;
                if (tag == OP_COPY) {
                    uint64_t off; memcpy(&off, ops.data() + pos, 8); pos += 8;
                    uint32_t len; memcpy(&len, ops.data() + pos, 4); pos += 4;
                    if (off + len <= old_data.size()) {
                        new_data.insert(new_data.end(), old_data.data() + off,
                                       old_data.data() + off + len);
                    }
                } else if (tag == OP_INSERT) {
                    uint32_t len; memcpy(&len, ops.data() + pos, 4); pos += 4;
                    new_data.insert(new_data.end(), ops.data() + pos, ops.data() + pos + len);
                    pos += len;
                } else {
                    std::cerr << "  Unknown op " << (int)tag << " in " << filename << std::endl;
                    break;
                }
            }

            write_file(output_dir + "/" + filename, new_data);
            std::cerr << "  " << filename << ": delta applied (" << new_data.size() << " bytes)" << std::endl;

        } else {
            std::cerr << "  Unknown encoding " << encoding << " for " << filename << std::endl;
        }
    }

    std::cerr << "Patch applied. Output in " << output_dir << std::endl;
    return 0;
}

-        uint64_t uncompressed_size = read_u64(pf);

-        uint64_t compressed_size = read_u64(pf);

-            std::cerr << "  Unknown section " << section_id << ", skipping " << compressed_size << " bytes" << std::endl;

+            std::cerr << "  Unknown section " << section_id << std::endl;

+            // Can't skip unknown sections reliably, abort

-            uint64_t compressed_size_local = read_u64(pf);

-            std::vector<char> compressed(compressed_size_local);

-            pf.read(compressed.data(), compressed_size_local);

-            auto data = zlib_decompress(compressed.data(), compressed_size_local, uncompressed_size);

+            uint64_t uncompressed_size = read_u64(pf);

+            uint64_t compressed_size = read_u64(pf);

+            std::vector<char> compressed(compressed_size);

+            pf.read(compressed.data(), compressed_size);

+            auto data = zlib_decompress(compressed.data(), compressed_size, uncompressed_size);


Also fix the COPY_INSERT branch to read the right fields:


        } else if (encoding == static_cast<uint32_t>(PatchEncoding::COPY_INSERT)) {
            uint64_t old_size = uncompressed_size; // repurposed
            uint64_t new_size = read_u64(pf);
            uint64_t compressed_size_local = read_u64(pf);

        } else if (encoding == static_cast<uint32_t>(PatchEncoding::COPY_INSERT)) {
            uint64_t old_size = read_u64(pf);
            uint64_t new_size = read_u64(pf);
            uint64_t compressed_size_local = read_u64(pf);


        } else if (encoding == static_cast<uint32_t>(PatchEncoding::COPY_INSERT)) {
            uint64_t old_size = read_u64(pf);
            uint64_t new_size = read_u64(pf);
            uint64_t compressed_size_local = read_u64(pf);

// geocoder-patch: Apply a .gcpatch file to a canonical geocoder index.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
#include <zlib.h>

#include "patch_format.h"

static uint32_t read_u32(std::ifstream& f) {
    uint32_t v; f.read(reinterpret_cast<char*>(&v), 4); return v;
}
static uint64_t read_u64(std::ifstream& f) {
    uint64_t v; f.read(reinterpret_cast<char*>(&v), 8); return v;
}

static std::vector<char> zlib_decompress(const char* data, size_t compressed_size, size_t uncompressed_size) {
    std::vector<char> out(uncompressed_size);
    uLongf dest_len = uncompressed_size;
    int ret = uncompress(reinterpret_cast<Bytef*>(out.data()), &dest_len,
                         reinterpret_cast<const Bytef*>(data), compressed_size);
    if (ret != Z_OK) {
        std::cerr << "zlib decompress failed: " << ret << std::endl;
        return {};
    }
    out.resize(dest_len);
    return out;
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }

    std::string current_dir = argv[1];
    std::string patch_path = argv[2];
    std::string output_dir = argv[4];
    ensure_dir(output_dir);

    // Open patch file
    std::ifstream pf(patch_path, std::ios::binary);
    if (!pf) { std::cerr << "Cannot open " << patch_path << std::endl; return 1; }

    // Read and verify header
    char magic[8];
    pf.read(magic, 8);
    if (memcmp(magic, GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch file (bad magic)" << std::endl;
        return 1;
    }
    uint32_t version = read_u32(pf);
    if (version != GCPATCH_VERSION) {
        std::cerr << "Unsupported patch version: " << version << std::endl;
        return 1;
    }
    uint32_t flags = read_u32(pf);
    (void)flags;

    std::cerr << "Applying patch from " << patch_path << "..." << std::endl;

    // Process sections
    while (pf) {
        uint32_t section_id = read_u32(pf);
        if (!pf || section_id == 0xFFFFFFFF) break; // end marker

        // String remap section (special)
        if (section_id == 0xFFFFFFFE) {
            uint32_t count = read_u32(pf);
            // Skip remap entries (not needed for RAW_REPLACE mode)
            pf.seekg(count * 8, std::ios::cur);
            std::cerr << "  Skipping string remap (" << count << " entries)" << std::endl;
            continue;
        }

        uint32_t encoding = read_u32(pf);

        if (section_id >= static_cast<uint32_t>(PatchFileId::COUNT)) {
            std::cerr << "  Unknown section " << section_id << std::endl;
            // Can't skip unknown sections reliably, abort
            break;
        }

        const char* filename = patch_file_names[section_id];

        if (encoding == static_cast<uint32_t>(PatchEncoding::RAW_REPLACE)) {
            uint64_t uncompressed_size = read_u64(pf);
            uint64_t compressed_size = read_u64(pf);
            std::vector<char> compressed(compressed_size);
            pf.read(compressed.data(), compressed_size);
            auto data = zlib_decompress(compressed.data(), compressed_size, uncompressed_size);
            if (data.empty() && uncompressed_size > 0) {
                std::cerr << "  FAILED to decompress " << filename << std::endl;
                return 1;
            }
            write_file(output_dir + "/" + filename, data);
            std::cerr << "  " << filename << ": replaced (" << data.size() << " bytes)" << std::endl;

        } else if (encoding == static_cast<uint32_t>(PatchEncoding::COPY_INSERT)) {
            uint64_t old_size = uncompressed_size; // repurposed
            uint64_t new_size = read_u64(pf);
            uint64_t compressed_size_local = read_u64(pf);

            // Read and decompress ops
            std::vector<char> compressed(compressed_size_local);
            pf.read(compressed.data(), compressed_size_local);

            // Estimate ops uncompressed size (generous: 4x compressed)
            size_t est_size = compressed_size_local * 8;
            if (est_size < new_size * 2) est_size = new_size * 2;
            auto ops = zlib_decompress(compressed.data(), compressed_size_local, est_size);
            if (ops.empty() && compressed_size_local > 0) {
                std::cerr << "  FAILED to decompress ops for " << filename << std::endl;
                return 1;
            }

            // Load old file
            auto old_data = read_file(current_dir + "/" + filename);
            if (old_data.size() != old_size) {
                std::cerr << "  WARNING: " << filename << " size mismatch (expected "
                          << old_size << ", got " << old_data.size() << ")" << std::endl;
            }

            // Replay COPY/INSERT ops
            std::vector<char> new_data;
            new_data.reserve(new_size);
            size_t pos = 0;
            while (pos < ops.size()) {
                uint8_t tag = static_cast<uint8_t>(ops[pos]); pos++;
                if (tag == OP_COPY) {
                    uint64_t off; memcpy(&off, ops.data() + pos, 8); pos += 8;
                    uint32_t len; memcpy(&len, ops.data() + pos, 4); pos += 4;
                    if (off + len <= old_data.size()) {
                        new_data.insert(new_data.end(), old_data.data() + off,
                                       old_data.data() + off + len);
                    }
                } else if (tag == OP_INSERT) {
                    uint32_t len; memcpy(&len, ops.data() + pos, 4); pos += 4;
                    new_data.insert(new_data.end(), ops.data() + pos, ops.data() + pos + len);
                    pos += len;
                } else {
                    std::cerr << "  Unknown op " << (int)tag << " in " << filename << std::endl;
                    break;
                }
            }

            write_file(output_dir + "/" + filename, new_data);
            std::cerr << "  " << filename << ": delta applied (" << new_data.size() << " bytes)" << std::endl;

        } else {
            std::cerr << "  Unknown encoding " << encoding << " for " << filename << std::endl;
        }
    }

    std::cerr << "Patch applied. Output in " << output_dir << std::endl;
    return 0;
}

-            uint64_t old_size = uncompressed_size; // repurposed

+            uint64_t old_size = read_u64(pf);


Bash

cmake --build build -j$(nproc) 2>&1 | tail -10

Build all tools


/root/traccar-geocoder/builder/tools/geocoder_diff.cpp:243:22: note: candidate: 'main(int, char**)::<lambda(PatchFileId, const std::string&, int)>'
  243 |     auto diff_file = [&](PatchFileId id, const std::string& name,
      |                      ^
/root/traccar-geocoder/builder/tools/geocoder_diff.cpp:243:22: note:   no known conversion for argument 3 from 'std::nullptr_t' to 'int'
gmake[2]: *** [CMakeFiles/geocoder-diff.dir/build.make:79: CMakeFiles/geocoder-diff.dir/tools/geocoder_diff.cpp.o] Error 1
gmake[1]: *** [CMakeFiles/Makefile2:160: CMakeFiles/geocoder-diff.dir/all] Error 2
gmake[1]: *** Waiting for unfinished jobs....
[ 93%] Linking CXX executable geocoder-patch
[ 93%] Built target geocoder-patch
gmake: *** [Makefile:91: all] Error 2


Need to fix the lambda type for the remap function parameter:


   240→    }
   241→
   242→    // Helper: diff a file with optional string remapping
   243→    auto diff_file = [&](PatchFileId id, const std::string& name,
   244→                          std::function<void(std::vector<char>&)> remap_fn) {
   245→        std::cerr << "  " << name << ": ";
   246→        auto old_data = read_file(old_dir + "/" + name);
   247→        auto new_data = read_file(new_dir + "/" + name);
   248→
   249→        // Apply string remap to old data (makes it comparable to new)

    }

    // Helper: diff a file with optional string remapping
    auto diff_file = [&](PatchFileId id, const std::string& name,
                          std::function<void(std::vector<char>&)> remap_fn) {
        std::cerr << "  " << name << ": ";
        auto old_data = read_file(old_dir + "/" + name);
        auto new_data = read_file(new_dir + "/" + name);

        // Apply string remap to old data (makes it comparable to new)


The issue is passing `nullptr` where `std::function<void(...)>` is expected. Let me fix the calls:


Edit

    diff_file(PatchFileId::STREET_NODES, "street_nodes.bin", nullptr);
    diff_file(PatchFileId::INTERP_NODES, "interp_nodes.bin", nullptr);
    diff_file(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", nullptr);

    // Derived/index files (no remap needed, direct diff)
    diff_file(PatchFileId::GEO_CELLS, "geo_cells.bin", nullptr);
    diff_file(PatchFileId::STREET_ENTRIES, "street_entries.bin", nullptr);
    diff_file(PatchFileId::ADDR_ENTRIES, "addr_entries.bin", nullptr);
    diff_file(PatchFileId::INTERP_ENTRIES, "interp_entries.bin", nullptr);
    diff_file(PatchFileId::ADMIN_CELLS, "admin_cells.bin", nullptr);
    diff_file(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin", nullptr);

    std::function<void(std::vector<char>&)> no_remap;
    diff_file(PatchFileId::STREET_NODES, "street_nodes.bin", no_remap);
    diff_file(PatchFileId::INTERP_NODES, "interp_nodes.bin", no_remap);
    diff_file(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", no_remap);

    // Derived/index files (no remap needed, direct diff)
    diff_file(PatchFileId::GEO_CELLS, "geo_cells.bin", no_remap);
    diff_file(PatchFileId::STREET_ENTRIES, "street_entries.bin", no_remap);
    diff_file(PatchFileId::ADDR_ENTRIES, "addr_entries.bin", no_remap);
    diff_file(PatchFileId::INTERP_ENTRIES, "interp_entries.bin", no_remap);
    diff_file(PatchFileId::ADMIN_CELLS, "admin_cells.bin", no_remap);
    diff_file(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin", no_remap);


The file /root/traccar-geocoder/builder/tools/geocoder_diff.cpp has been updated. All occurrences were successfully replaced.

    std::function<void(std::vector<char>&)> no_remap;
    diff_file(PatchFileId::STREET_NODES, "street_nodes.bin", no_remap);
    diff_file(PatchFileId::INTERP_NODES, "interp_nodes.bin", no_remap);
    diff_file(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", no_remap);

    // Derived/index files (no remap needed, direct diff)
    diff_file(PatchFileId::GEO_CELLS, "geo_cells.bin", no_remap);
    diff_file(PatchFileId::STREET_ENTRIES, "street_entries.bin", no_remap);
    diff_file(PatchFileId::ADDR_ENTRIES, "addr_entries.bin", no_remap);
    diff_file(PatchFileId::INTERP_ENTRIES, "interp_entries.bin", no_remap);
    diff_file(PatchFileId::ADMIN_CELLS, "admin_cells.bin", no_remap);
    diff_file(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin", no_remap);

// geocoder-diff: Compare two canonical geocoder indexes and produce a compact patch.
//
// Strategy:
// 1. Build string remap (old string offsets → new string offsets)
// 2. For data files with string refs: apply remap to old file in memory, then binary diff
// 3. For coordinate files: direct binary diff (no string refs)
// 4. For small derived files: full replacement (compressed)
//
// The binary diff uses a simple block-based COPY/INSERT encoder:
// - Compare 64-byte blocks between (remapped) old and new
// - COPY for matching blocks, INSERT for different blocks
// - Handles size differences (new may be larger)
//
// Usage: geocoder-diff <old-canonical-dir> <new-canonical-dir> -o <patch-file>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>
#include <zlib.h>

#include "patch_format.h"

// --- zlib helpers ---

static std::vector<char> zlib_compress(const char* data, size_t size) {
    if (size == 0) return {};
    uLongf bound = compressBound(size);
    std::vector<char> out(bound);
    int ret = compress2(reinterpret_cast<Bytef*>(out.data()), &bound,
                        reinterpret_cast<const Bytef*>(data), size, 9);
    if (ret != Z_OK) return {};
    out.resize(bound);
    return out;
}

// --- String remap ---

static std::unordered_map<uint32_t, uint32_t> build_string_remap(
    const std::vector<char>& old_pool, const std::vector<char>& new_pool)
{
    // Index new pool
    std::unordered_map<std::string, uint32_t> new_index;
    size_t pos = 0;
    while (pos < new_pool.size()) {
        const char* s = new_pool.data() + pos;
        size_t len = strlen(s);
        new_index[std::string(s, len)] = static_cast<uint32_t>(pos);
        pos += len + 1;
    }

    // Map old offsets → new offsets
    std::unordered_map<uint32_t, uint32_t> remap;
    pos = 0;
    while (pos < old_pool.size()) {
        const char* s = old_pool.data() + pos;
        size_t len = strlen(s);
        auto it = new_index.find(std::string(s, len));
        if (it != new_index.end()) {
            remap[static_cast<uint32_t>(pos)] = it->second;
        }
        pos += len + 1;
    }
    return remap;
}

// Apply string remap to specific fields in a data buffer
static void remap_addr_points(std::vector<char>& data,
                                const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / 16;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * 16;
        // housenumber_id at offset 8, street_id at offset 12
        for (size_t off : {8, 12}) {
            uint32_t old_id;
            memcpy(&old_id, p + off, 4);
            auto it = remap.find(old_id);
            if (it != remap.end()) {
                memcpy(p + off, &it->second, 4);
            }
        }
    }
}

static void remap_ways(std::vector<char>& data, size_t stride,
                        const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / stride;
    size_t name_off = (stride == 12) ? 8 : 5;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * stride;
        uint32_t old_id;
        memcpy(&old_id, p + name_off, 4);
        auto it = remap.find(old_id);
        if (it != remap.end()) {
            memcpy(p + name_off, &it->second, 4);
        }
    }
}

static void remap_interps(std::vector<char>& data, size_t stride,
                           const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / stride;
    size_t street_off = (stride == 20) ? 8 : 5;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * stride;
        uint32_t old_id;
        memcpy(&old_id, p + street_off, 4);
        auto it = remap.find(old_id);
        if (it != remap.end()) {
            memcpy(p + street_off, &it->second, 4);
        }
    }
}

static void remap_admin_polygons(std::vector<char>& data, size_t stride,
                                   const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / stride;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * stride;
        uint32_t old_id;
        memcpy(&old_id, p + 8, 4); // name_id at offset 8
        auto it = remap.find(old_id);
        if (it != remap.end()) {
            memcpy(p + 8, &it->second, 4);
        }
    }
}

// --- Block-based COPY/INSERT binary diff ---
// Compares (remapped) old vs new, produces a compact delta.
// Block size of 64 bytes balances granularity vs overhead.

static constexpr size_t DIFF_BLOCK = 64;

static std::vector<char> compute_block_diff(const std::vector<char>& old_data,
                                              const std::vector<char>& new_data) {
    std::vector<char> ops;
    size_t old_pos = 0, new_pos = 0;

    while (new_pos < new_data.size()) {
        // Check if current block matches
        size_t remaining_new = new_data.size() - new_pos;
        size_t remaining_old = (old_pos < old_data.size()) ? old_data.size() - old_pos : 0;
        size_t block = std::min(DIFF_BLOCK, remaining_new);

        if (remaining_old >= block &&
            memcmp(old_data.data() + old_pos, new_data.data() + new_pos, block) == 0) {
            // Find full extent of match
            size_t match_len = block;
            while (old_pos + match_len < old_data.size() &&
                   new_pos + match_len < new_data.size() &&
                   old_data[old_pos + match_len] == new_data[new_pos + match_len]) {
                match_len++;
            }
            // Emit COPY: tag(1) + offset(8) + length(4) = 13 bytes
            ops.push_back(OP_COPY);
            uint64_t off = old_pos;
            uint32_t len = static_cast<uint32_t>(match_len);
            ops.insert(ops.end(), reinterpret_cast<const char*>(&off),
                      reinterpret_cast<const char*>(&off) + 8);
            ops.insert(ops.end(), reinterpret_cast<const char*>(&len),
                      reinterpret_cast<const char*>(&len) + 4);
            old_pos += match_len;
            new_pos += match_len;
        } else {
            // Find extent of non-match (up to next matching block)
            size_t insert_start = new_pos;
            size_t old_scan = old_pos;

            // Advance by one block
            new_pos += block;
            old_scan += std::min(block, remaining_old);

            // Keep scanning until we find a matching block again
            while (new_pos < new_data.size()) {
                size_t rem_new = new_data.size() - new_pos;
                size_t rem_old = (old_scan < old_data.size()) ? old_data.size() - old_scan : 0;
                size_t b = std::min(DIFF_BLOCK, rem_new);
                if (rem_old >= b &&
                    memcmp(old_data.data() + old_scan, new_data.data() + new_pos, b) == 0) {
                    break; // Found match
                }
                new_pos += b;
                old_scan += std::min(b, rem_old);
            }

            // Emit INSERT: tag(1) + length(4) + data
            size_t insert_len = new_pos - insert_start;
            ops.push_back(OP_INSERT);
            uint32_t len = static_cast<uint32_t>(insert_len);
            ops.insert(ops.end(), reinterpret_cast<const char*>(&len),
                      reinterpret_cast<const char*>(&len) + 4);
            ops.insert(ops.end(), new_data.data() + insert_start,
                      new_data.data() + insert_start + insert_len);
            old_pos = old_scan;
        }
    }
    return ops;
}

// --- Write helpers ---
static void write_u32(std::ofstream& f, uint32_t v) { f.write(reinterpret_cast<const char*>(&v), 4); }
static void write_u64(std::ofstream& f, uint64_t v) { f.write(reinterpret_cast<const char*>(&v), 8); }

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>" << std::endl;
        return 1;
    }

    std::string old_dir = argv[1];
    std::string new_dir = argv[2];
    std::string patch_path = argv[4];

    std::cerr << "Loading string pools..." << std::endl;
    auto old_strings = read_file(old_dir + "/strings.bin");
    auto new_strings = read_file(new_dir + "/strings.bin");
    auto remap = build_string_remap(old_strings, new_strings);
    std::cerr << "  Remap: " << remap.size() << " strings mapped" << std::endl;

    std::ofstream pf(patch_path, std::ios::binary);
    pf.write(GCPATCH_MAGIC, 8);
    write_u32(pf, GCPATCH_VERSION);
    write_u32(pf, 0);

    // Section: strings.bin (full replacement)
    {
        std::cerr << "  strings.bin: full replacement..." << std::endl;
        auto compressed = zlib_compress(new_strings.data(), new_strings.size());
        write_u32(pf, static_cast<uint32_t>(PatchFileId::STRINGS));
        write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));
        write_u64(pf, new_strings.size());
        write_u64(pf, compressed.size());
        pf.write(compressed.data(), compressed.size());
        std::cerr << "    " << new_strings.size() << " -> " << compressed.size() << std::endl;
    }

    // Helper: diff a file with optional string remapping
    auto diff_file = [&](PatchFileId id, const std::string& name,
                          std::function<void(std::vector<char>&)> remap_fn) {
        std::cerr << "  " << name << ": ";
        auto old_data = read_file(old_dir + "/" + name);
        auto new_data = read_file(new_dir + "/" + name);

        // Apply string remap to old data (makes it comparable to new)
        if (remap_fn) {
            remap_fn(old_data);
        }

        // Compute block diff
        auto ops = compute_block_diff(old_data, new_data);

        // Compress the ops stream
        auto compressed = zlib_compress(ops.data(), ops.size());

        // Also compute full replacement size for comparison
        auto full_compressed = zlib_compress(new_data.data(), new_data.size());

        // Use whichever is smaller
        bool use_delta = compressed.size() < full_compressed.size();

        write_u32(pf, static_cast<uint32_t>(id));
        if (use_delta) {
            write_u32(pf, static_cast<uint32_t>(PatchEncoding::COPY_INSERT));
            write_u64(pf, old_data.size()); // old size
            write_u64(pf, new_data.size()); // new size
            write_u64(pf, compressed.size());
            pf.write(compressed.data(), compressed.size());
            std::cerr << "delta " << compressed.size() << " bytes";
        } else {
            write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));
            write_u64(pf, new_data.size());
            write_u64(pf, full_compressed.size());
            pf.write(full_compressed.data(), full_compressed.size());
            std::cerr << "full " << full_compressed.size() << " bytes";
        }
        double pct = (new_data.size() > 0) ?
            (use_delta ? compressed.size() : full_compressed.size()) * 100.0 / new_data.size() : 0;
        std::cerr << " (" << std::fixed << std::setprecision(1) << pct << "% of "
                  << new_data.size() / 1024 / 1024 << " MiB)" << std::endl;
    };

    // Detect strides for struct files
    size_t way_stride = detect_way_stride(old_dir + "/street_ways.bin");
    auto old_interp_data = read_file(old_dir + "/interp_ways.bin");
    size_t interp_stride = (old_interp_data.size() % 20 == 0 && old_interp_data.size() > 0) ? 20 : 18;
    auto old_admin_data = read_file(old_dir + "/admin_polygons.bin");
    size_t admin_stride = 24;
    for (size_t s : {24, 20, 19}) {
        if (old_admin_data.size() % s == 0 && old_admin_data.size() > 0) { admin_stride = s; break; }
    }

    // Data files with string remapping
    diff_file(PatchFileId::ADDR_POINTS, "addr_points.bin",
        [&](std::vector<char>& d) { remap_addr_points(d, remap); });

    diff_file(PatchFileId::STREET_WAYS, "street_ways.bin",
        [&, way_stride](std::vector<char>& d) { remap_ways(d, way_stride, remap); });

    diff_file(PatchFileId::INTERP_WAYS, "interp_ways.bin",
        [&, interp_stride](std::vector<char>& d) { remap_interps(d, interp_stride, remap); });

    diff_file(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin",
        [&, admin_stride](std::vector<char>& d) { remap_admin_polygons(d, admin_stride, remap); });

    // Coordinate files (no string refs, direct diff)
    diff_file(PatchFileId::STREET_NODES, "street_nodes.bin", nullptr);
    diff_file(PatchFileId::INTERP_NODES, "interp_nodes.bin", nullptr);
    diff_file(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", nullptr);

    // Derived/index files (no remap needed, direct diff)
    diff_file(PatchFileId::GEO_CELLS, "geo_cells.bin", nullptr);
    diff_file(PatchFileId::STREET_ENTRIES, "street_entries.bin", nullptr);
    diff_file(PatchFileId::ADDR_ENTRIES, "addr_entries.bin", nullptr);
    diff_file(PatchFileId::INTERP_ENTRIES, "interp_entries.bin", nullptr);
    diff_file(PatchFileId::ADMIN_CELLS, "admin_cells.bin", nullptr);
    diff_file(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin", nullptr);

    write_u32(pf, 0xFFFFFFFF); // end marker
    auto total = pf.tellp();
    std::cerr << "\nTotal patch: " << total << " bytes (" << total / 1024 / 1024 << " MiB)" << std::endl;
    return 0;
}

-    diff_file(PatchFileId::STREET_NODES, "street_nodes.bin", nullptr);

-    diff_file(PatchFileId::INTERP_NODES, "interp_nodes.bin", nullptr);

-    diff_file(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", nullptr);

+    std::function<void(std::vector<char>&)> no_remap;

+    diff_file(PatchFileId::STREET_NODES, "street_nodes.bin", no_remap);

+    diff_file(PatchFileId::INTERP_NODES, "interp_nodes.bin", no_remap);

+    diff_file(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", no_remap);

-    diff_file(PatchFileId::GEO_CELLS, "geo_cells.bin", nullptr);

-    diff_file(PatchFileId::STREET_ENTRIES, "street_entries.bin", nullptr);

-    diff_file(PatchFileId::ADDR_ENTRIES, "addr_entries.bin", nullptr);

-    diff_file(PatchFileId::INTERP_ENTRIES, "interp_entries.bin", nullptr);

-    diff_file(PatchFileId::ADMIN_CELLS, "admin_cells.bin", nullptr);

-    diff_file(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin", nullptr);

+    diff_file(PatchFileId::GEO_CELLS, "geo_cells.bin", no_remap);

+    diff_file(PatchFileId::STREET_ENTRIES, "street_entries.bin", no_remap);

+    diff_file(PatchFileId::ADDR_ENTRIES, "addr_entries.bin", no_remap);

+    diff_file(PatchFileId::INTERP_ENTRIES, "interp_entries.bin", no_remap);

+    diff_file(PatchFileId::ADMIN_CELLS, "admin_cells.bin", no_remap);

+    diff_file(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin", no_remap);


Bash

cmake --build build -j$(nproc) 2>&1 | tail -5

Build all tools


      |                                                              ^~~~~~~~
      |                                                              remap
gmake[2]: *** [CMakeFiles/geocoder-diff.dir/build.make:79: CMakeFiles/geocoder-diff.dir/tools/geocoder_diff.cpp.o] Error 1
gmake[1]: *** [CMakeFiles/Makefile2:160: CMakeFiles/geocoder-diff.dir/all] Error 2
gmake: *** [Makefile:91: all] Error 2


cmake --build build -j$(nproc) --target geocoder-diff 2>&1 | grep "error:" | head -5

Show compilation errors


/root/traccar-geocoder/builder/tools/geocoder_diff.cpp:244:27: error: 'std::function' has not been declared
/root/traccar-geocoder/builder/tools/geocoder_diff.cpp:244:40: error: expected ',' or '...' before '<' token
/root/traccar-geocoder/builder/tools/geocoder_diff.cpp:250:13: error: 'remap_fn' was not declared in this scope; did you mean 'remap'?
/root/traccar-geocoder/builder/tools/geocoder_diff.cpp:283:49: error: 'setprecision' is not a member of 'std'
/root/traccar-geocoder/builder/tools/geocoder_diff.cpp:298:14: error: no match for call to '(main(int, char**)::<lambda(PatchFileId, const std::string&, int)>) (PatchFileId, const char [16], main(int, char**)::<lambda(std::vector<char>&)>)'


Missing `<functional>` and `<iomanip>` includes:


Edit

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>
#include <zlib.h>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <functional>
#include <iomanip>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>
#include <zlib.h>