    // Entry/cell files: NOT included.
    // The patch tool reconstructs them from the derived ID remaps
    // (extracted implicitly from the merge sequences above).
    // geo_cells.bin, admin_cells.bin, and all entry files are rebuilt.
    std::cerr << "  (entry/cell files: omitted, rebuilt by patch tool)" << std::endl;

    // Placeholder: we could include cell_id changes here for perfect reconstruction.
    // For now, the patch tool uses the old cell set with remapped IDs.

    if (false) // disabled — entry files not included
    for (auto& [id, name] : std::vector<std::pair<PatchFileId, std::string>>{
        {PatchFileId::GEO_CELLS, "geo_cells.bin"}})

// geocoder-diff v3: Fully custom patch format using merge-sequence encoding.
//
// For each data file: walk old (string-remapped) and new in parallel,
// match records by content, emit MATCH/INSERT/DELETE operations.
// For entry/cell files: include changed cell entry data directly.
// geo_cells rebuilt by patch tool from entries (not delta-patched).
//
// Patch format: custom binary, zstd-compressed as a whole for transport.
//
// Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>

#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <functional>
#include <iomanip>
#include <iostream>
#include <string>
#include <unistd.h>
#include <unordered_map>
#include <unordered_set>
#include <vector>

#include "patch_format.h"

// --- Merge sequence encoding ---
// Walk two sorted arrays (old remapped, new), matching by record bytes.
// Emit: MATCH(n) = copy n records from old, INSERT(n,data) = add n new records,
//       DELETE(n) = skip n old records.

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

struct MergeSequence {
    std::vector<char> data; // serialized ops

    void add_match(uint32_t count) {
        uint8_t op = OP_MATCH_RUN;
        data.insert(data.end(), (char*)&op, (char*)&op + 1);
        data.insert(data.end(), (char*)&count, (char*)&count + 4);
    }
    void add_delete(uint32_t count) {
        uint8_t op = OP_DELETE_RUN;
        data.insert(data.end(), (char*)&op, (char*)&op + 1);
        data.insert(data.end(), (char*)&count, (char*)&count + 4);
    }
    void add_insert(const char* records, uint32_t count, size_t stride) {
        uint8_t op = OP_INSERT_RUN;
        data.insert(data.end(), (char*)&op, (char*)&op + 1);
        data.insert(data.end(), (char*)&count, (char*)&count + 4);
        data.insert(data.end(), records, records + count * stride);
    }
};

// Build merge sequence for a data file.
// old_data has string remap already applied.
// Records are compared by byte equality (stride bytes).
static MergeSequence build_merge_seq(
    const std::vector<char>& old_data, const std::vector<char>& new_data,
    size_t stride)
{
    size_t old_n = old_data.size() / stride;
    size_t new_n = new_data.size() / stride;
    size_t oi = 0, ni = 0;
    MergeSequence seq;
    uint32_t match_run = 0, del_run = 0;

    auto flush_match = [&]() { if (match_run > 0) { seq.add_match(match_run); match_run = 0; } };
    auto flush_del = [&]() { if (del_run > 0) { seq.add_delete(del_run); del_run = 0; } };

    // For small strides (<=8), records aren't unique enough for hash matching.
    // Use simple sequential scan instead.
    bool use_hash = (stride > 8);

    // Pre-build hash index of new records for fast mismatch resolution
    auto record_hash = [&](const char* p, size_t s) -> uint64_t {
        uint64_t h = 14695981039346656037ULL;
        for (size_t i = 0; i < s; i++) { h ^= (uint8_t)p[i]; h *= 1099511628211ULL; }
        return h;
    };
    std::unordered_multimap<uint64_t, uint32_t> new_hash;
    if (use_hash) {
        new_hash.reserve(new_n);
        for (uint32_t i = 0; i < new_n; i++)
            new_hash.emplace(record_hash(new_data.data() + i * stride, stride), i);
    }

    while (oi < old_n && ni < new_n) {
        const char* op = old_data.data() + oi * stride;
        const char* np = new_data.data() + ni * stride;

        if (memcmp(op, np, stride) == 0) {
            flush_del();
            match_run++;
            oi++; ni++;
        } else if (use_hash) {
            // Hash-based mismatch resolution (for records with unique content)
            uint64_t oh = record_hash(op, stride);
            auto range = new_hash.equal_range(oh);
            uint32_t best_ni = UINT32_MAX;
            for (auto it = range.first; it != range.second; ++it) {
                if (it->second >= ni && it->second < ni + 10000 &&
                    memcmp(op, new_data.data() + it->second * stride, stride) == 0) {
                    if (it->second < best_ni) best_ni = it->second;
                }
            }
            if (best_ni != UINT32_MAX && best_ni > ni) {
                flush_match(); flush_del();
                seq.add_insert(new_data.data() + ni * stride, best_ni - ni, stride);
                ni = best_ni;
            } else if (best_ni == ni) {
                flush_del(); match_run++; oi++; ni++;
            } else {
                flush_match(); del_run++; oi++;
            }
        } else {
            // Sequential scan (for coordinate files where records aren't unique)
            size_t lookahead = std::min((size_t)200, std::min(old_n - oi, new_n - ni));
            bool found = false;
            for (size_t k = 1; k <= lookahead; k++) {
                if (ni + k < new_n && memcmp(op, new_data.data() + (ni + k) * stride, stride) == 0) {
                    flush_match(); flush_del();
                    seq.add_insert(new_data.data() + ni * stride, k, stride);
                    ni += k; found = true; break;
                }
                if (oi + k < old_n && memcmp(np, old_data.data() + (oi + k) * stride, stride) == 0) {
                    flush_match(); del_run += k; oi += k; found = true; break;
                }
            }
            if (!found) { flush_match(); del_run++; oi++; flush_del(); seq.add_insert(np, 1, stride); ni++; }
        }
    }

    flush_match(); flush_del();
    // Remaining old records: deletions
    if (oi < old_n) seq.add_delete(old_n - oi);
    // Remaining new records: insertions
    if (ni < new_n) seq.add_insert(new_data.data() + ni * stride, new_n - ni, stride);

    return seq;
}

// --- String remap ---

static std::unordered_map<uint32_t, uint32_t> build_string_remap(
    const std::vector<char>& old_pool, const std::vector<char>& new_pool)
{
    std::unordered_map<std::string, uint32_t> new_idx;
    size_t pos = 0;
    while (pos < new_pool.size()) {
        const char* s = new_pool.data() + pos;
        size_t len = strlen(s);
        new_idx[std::string(s, len)] = static_cast<uint32_t>(pos);
        pos += len + 1;
    }
    std::unordered_map<uint32_t, uint32_t> remap;
    pos = 0;
    while (pos < old_pool.size()) {
        const char* s = old_pool.data() + pos;
        size_t len = strlen(s);
        auto it = new_idx.find(std::string(s, len));
        if (it != new_idx.end())
            remap[static_cast<uint32_t>(pos)] = it->second;
        pos += len + 1;
    }
    return remap;
}

// --- Remap + fixup helpers ---

static void remap_addr_points(std::vector<char>& data, const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + 16 <= data.size(); i += 16)
        for (size_t off : {8, 12}) {
            uint32_t v; memcpy(&v, data.data() + i + off, 4);
            auto it = rm.find(v); if (it != rm.end()) memcpy(data.data() + i + off, &it->second, 4);
        }
}
static void remap_field(std::vector<char>& data, size_t stride, size_t field_off,
                         const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + stride <= data.size(); i += stride) {
        uint32_t v; memcpy(&v, data.data() + i + field_off, 4);
        auto it = rm.find(v); if (it != rm.end()) memcpy(data.data() + i + field_off, &it->second, 4);
    }
}

// Content matching for ways (by name + nodes, ignoring node_offset)
static uint64_t fnv_mix(uint64_t h, uint64_t v) { h ^= v; h *= 1099511628211ULL; return h; }

static void fixup_way_offsets(std::vector<char>& old_ways, const std::vector<char>& old_nodes,
                                const std::vector<char>& new_ways, const std::vector<char>& new_nodes,
                                size_t stride) {
    size_t name_off = (stride == 12) ? 8 : 5;
    size_t old_n = old_ways.size() / stride, new_n = new_ways.size() / stride;
    size_t old_nc = old_nodes.size() / 8, new_nc = new_nodes.size() / 8;

    auto way_hash = [&](const char* w, const char* nodes, size_t max_n) -> uint64_t {
        uint32_t node_offset, name_id; uint8_t node_count;
        memcpy(&node_offset, w, 4); node_count = (uint8_t)w[4]; memcpy(&name_id, w + name_off, 4);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, node_count);
        for (uint8_t j = 0; j < node_count && (node_offset + j) < max_n; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4);
            memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(way_hash(new_ways.data() + i * stride, new_nodes.data(), new_nc), i);

    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = way_hash(old_ways.data() + i * stride, old_nodes.data(), old_nc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            // Fix node_offset to match new way
            uint32_t new_node_off;
            memcpy(&new_node_off, new_ways.data() + it->second * stride, 4);
            memcpy(old_ways.data() + i * stride, &new_node_off, 4);
            new_map.erase(it);
        }
    }
}

// Same for admin polygons (vertex_offset)
static void fixup_admin_offsets(std::vector<char>& old_polys, const std::vector<char>& old_verts,
                                  const std::vector<char>& new_polys, const std::vector<char>& new_verts,
                                  size_t stride) {
    size_t old_n = old_polys.size() / stride, new_n = new_polys.size() / stride;
    size_t old_vc = old_verts.size() / 8, new_vc = new_verts.size() / 8;

    auto poly_hash = [&](const char* p, const char* verts, size_t max_v) -> uint64_t {
        uint32_t vert_offset, vert_count, name_id;
        memcpy(&vert_offset, p, 4); memcpy(&vert_count, p + 4, 4); memcpy(&name_id, p + 8, 4);
        uint8_t level = (uint8_t)p[12]; uint16_t cc; memcpy(&cc, p + stride - 2, 2);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, level); h = fnv_mix(h, cc); h = fnv_mix(h, vert_count);
        for (uint32_t j = 0; j < std::min(vert_count, 10u) && (vert_offset + j) < max_v; j++) {
            float lat, lng;
            memcpy(&lat, verts + (vert_offset + j) * 8, 4); memcpy(&lng, verts + (vert_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(poly_hash(new_polys.data() + i * stride, new_verts.data(), new_vc), i);

    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = poly_hash(old_polys.data() + i * stride, old_verts.data(), old_vc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            uint32_t new_vert_off;
            memcpy(&new_vert_off, new_polys.data() + it->second * stride, 4);
            memcpy(old_polys.data() + i * stride, &new_vert_off, 4);
            new_map.erase(it);
        }
    }
}

// Same for interp ways
static void fixup_interp_offsets(std::vector<char>& old_data, const std::vector<char>& old_nodes,
                                   const std::vector<char>& new_data, const std::vector<char>& new_nodes,
                                   size_t stride) {
    size_t street_off = (stride >= 20) ? 8 : 5;
    size_t old_n = old_data.size() / stride, new_n = new_data.size() / stride;
    size_t old_nc = old_nodes.size() / 8, new_nc = new_nodes.size() / 8;

    auto ihash = [&](const char* p, const char* nodes, size_t max_n) -> uint64_t {
        uint32_t node_offset, street_id, start, end; uint8_t count, itype;
        memcpy(&node_offset, p, 4); count = (uint8_t)p[4];
        memcpy(&street_id, p + street_off, 4); memcpy(&start, p + street_off + 4, 4);
        memcpy(&end, p + street_off + 8, 4);
        itype = (street_off + 12 < stride) ? (uint8_t)p[street_off + 12] : 0;
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, street_id); h = fnv_mix(h, start); h = fnv_mix(h, end);
        h = fnv_mix(h, itype); h = fnv_mix(h, count);
        for (uint8_t j = 0; j < count && (node_offset + j) < max_n; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4); memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(ihash(new_data.data() + i * stride, new_nodes.data(), new_nc), i);
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = ihash(old_data.data() + i * stride, old_nodes.data(), old_nc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            uint32_t new_off; memcpy(&new_off, new_data.data() + it->second * stride, 4);
            memcpy(old_data.data() + i * stride, &new_off, 4);
            new_map.erase(it);
        }
    }
}

// --- Write helpers ---
static void wval(std::vector<char>& buf, const void* data, size_t size) {
    buf.insert(buf.end(), (const char*)data, (const char*)data + size);
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>" << std::endl;
        return 1;
    }
    std::string old_dir = argv[1], new_dir = argv[2], patch_path = argv[4];
    std::string tmpdir = "/tmp/geocoder-diff-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    // Build string remap
    std::cerr << "Building string remap..." << std::endl;
    auto old_strings = read_file(old_dir + "/strings.bin");
    auto new_strings = read_file(new_dir + "/strings.bin");
    auto str_remap = build_string_remap(old_strings, new_strings);

    // Detect strides
    auto detect = [](const std::string& path, std::vector<size_t> cs) -> size_t {
        auto d = read_file(path);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect(old_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect(old_dir + "/interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect(old_dir + "/admin_polygons.bin", {24, 20, 19});

    // Build patch data (uncompressed, will be zstd-compressed at the end)
    std::vector<char> patch;
    patch.insert(patch.end(), GCPATCH_MAGIC, GCPATCH_MAGIC + 8);
    uint32_t ver = 2, flags = 0; // version 2 = custom format
    wval(patch, &ver, 4); wval(patch, &flags, 4);

    // --- Section: String remap ---
    {
        std::vector<std::pair<uint32_t,uint32_t>> entries;
        size_t pos = 0;
        while (pos < old_strings.size()) {
            uint32_t old_off = static_cast<uint32_t>(pos);
            auto it = str_remap.find(old_off);
            entries.push_back({old_off, (it != str_remap.end()) ? it->second : old_off});
            pos += strlen(old_strings.data() + pos) + 1;
        }
        uint32_t marker = 0xFFFFFFFE, count = static_cast<uint32_t>(entries.size());
        wval(patch, &marker, 4); wval(patch, &count, 4);
        for (auto& [o, n] : entries) { wval(patch, &o, 4); wval(patch, &n, 4); }
        std::cerr << "  String remap: " << count << " entries (" << count * 8 << " bytes)" << std::endl;
    }

    // --- Section: Per-file merge sequences ---
    // Track fixups per file (applied before merge, included in patch for patch tool)
    std::unordered_map<uint32_t, std::vector<std::pair<uint32_t,uint32_t>>> file_fixups;

    auto write_merge = [&](PatchFileId id, const std::string& name,
                             const std::vector<char>& old_data, const std::vector<char>& new_data,
                             size_t stride) {
        auto seq = build_merge_seq(old_data, new_data, stride);
        uint32_t fid = static_cast<uint32_t>(id);
        uint64_t os = old_data.size(), ns = new_data.size(), ss = seq.data.size();

        // Check for fixups
        auto fit = file_fixups.find(fid);
        uint32_t n_fixups = fit != file_fixups.end() ? static_cast<uint32_t>(fit->second.size()) : 0;

        wval(patch, &fid, 4);
        uint32_t st = static_cast<uint32_t>(stride);
        wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8);
        // Fixup count + data before merge sequence
        wval(patch, &n_fixups, 4);
        if (n_fixups > 0) {
            for (auto& [idx, val] : fit->second) { wval(patch, &idx, 4); wval(patch, &val, 4); }
        }
        wval(patch, &ss, 8);
        patch.insert(patch.end(), seq.data.begin(), seq.data.end());
        std::cerr << "  " << name << ": seq=" << ss << " fixups=" << n_fixups
                  << " (" << std::fixed << std::setprecision(2)
                  << (ns > 0 ? ss * 100.0 / ns : 0) << "%)" << std::endl;
    };

    // strings.bin: full replacement (byte-level merge is inefficient for string pools)
    {
        uint32_t fid = static_cast<uint32_t>(PatchFileId::STRINGS);
        uint32_t st = 0; // full replacement
        uint64_t os = 0, ns = new_strings.size();
        uint32_t nfix = 0;
        uint64_t ss = new_strings.size();
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8);
        wval(patch, &nfix, 4); wval(patch, &ss, 8);
        patch.insert(patch.end(), new_strings.begin(), new_strings.end());
        std::cerr << "  strings.bin: full " << new_strings.size() << " bytes" << std::endl;
    }

    // addr_points.bin
    {
        auto old_data = read_file(old_dir + "/addr_points.bin");
        remap_addr_points(old_data, str_remap);
        auto new_data = read_file(new_dir + "/addr_points.bin");
        write_merge(PatchFileId::ADDR_POINTS, "addr_points.bin", old_data, new_data, 16);
    }

    // street_ways.bin (string remap + offset fixup for node_offset)
    {
        auto old_w = read_file(old_dir + "/street_ways.bin");
        remap_field(old_w, way_stride, way_stride == 12 ? 8 : 5, str_remap);
        auto new_w = read_file(new_dir + "/street_ways.bin");
        auto old_n = read_file(old_dir + "/street_nodes.bin");
        auto new_n = read_file(new_dir + "/street_nodes.bin");
        // Record fixups: (record_index, new_node_offset)
        auto& fixups = file_fixups[static_cast<uint32_t>(PatchFileId::STREET_WAYS)];
        // Save old offsets, apply fixups, then record changes
        size_t wn = old_w.size() / way_stride;
        std::vector<uint32_t> old_offsets(wn);
        for (size_t i = 0; i < wn; i++) memcpy(&old_offsets[i], old_w.data() + i * way_stride, 4);
        fixup_way_offsets(old_w, old_n, new_w, new_n, way_stride);
        for (size_t i = 0; i < wn; i++) {
            uint32_t new_off; memcpy(&new_off, old_w.data() + i * way_stride, 4);
            if (new_off != old_offsets[i]) fixups.push_back({static_cast<uint32_t>(i), new_off});
        }
        write_merge(PatchFileId::STREET_WAYS, "street_ways.bin", old_w, new_w, way_stride);
    }

    // street_nodes.bin (no remap)
    {
        auto old_data = read_file(old_dir + "/street_nodes.bin");
        auto new_data = read_file(new_dir + "/street_nodes.bin");
        write_merge(PatchFileId::STREET_NODES, "street_nodes.bin", old_data, new_data, 8);
    }

    // interp_ways.bin
    {
        auto old_data = read_file(old_dir + "/interp_ways.bin");
        remap_field(old_data, interp_stride, interp_stride >= 20 ? 8 : 5, str_remap);
        auto old_n = read_file(old_dir + "/interp_nodes.bin");
        auto new_data = read_file(new_dir + "/interp_ways.bin");
        auto new_n = read_file(new_dir + "/interp_nodes.bin");
        auto& fixups = file_fixups[static_cast<uint32_t>(PatchFileId::INTERP_WAYS)];
        size_t in_count = old_data.size() / interp_stride;
        std::vector<uint32_t> old_offsets(in_count);
        for (size_t i = 0; i < in_count; i++) memcpy(&old_offsets[i], old_data.data() + i * interp_stride, 4);
        fixup_interp_offsets(old_data, old_n, new_data, new_n, interp_stride);
        for (size_t i = 0; i < in_count; i++) {
            uint32_t new_off; memcpy(&new_off, old_data.data() + i * interp_stride, 4);
            if (new_off != old_offsets[i]) fixups.push_back({static_cast<uint32_t>(i), new_off});
        }
        write_merge(PatchFileId::INTERP_WAYS, "interp_ways.bin", old_data, new_data, interp_stride);
    }

    // interp_nodes.bin
    {
        auto old_data = read_file(old_dir + "/interp_nodes.bin");
        auto new_data = read_file(new_dir + "/interp_nodes.bin");
        write_merge(PatchFileId::INTERP_NODES, "interp_nodes.bin", old_data, new_data, 8);
    }

    // admin_polygons.bin
    {
        auto old_data = read_file(old_dir + "/admin_polygons.bin");
        remap_field(old_data, admin_stride, 8, str_remap);
        auto old_v = read_file(old_dir + "/admin_vertices.bin");
        auto new_data = read_file(new_dir + "/admin_polygons.bin");
        auto new_v = read_file(new_dir + "/admin_vertices.bin");
        auto& fixups = file_fixups[static_cast<uint32_t>(PatchFileId::ADMIN_POLYGONS)];
        size_t an = old_data.size() / admin_stride;
        std::vector<uint32_t> old_offsets(an);
        for (size_t i = 0; i < an; i++) memcpy(&old_offsets[i], old_data.data() + i * admin_stride, 4);
        fixup_admin_offsets(old_data, old_v, new_data, new_v, admin_stride);
        for (size_t i = 0; i < an; i++) {
            uint32_t new_off; memcpy(&new_off, old_data.data() + i * admin_stride, 4);
            if (new_off != old_offsets[i]) fixups.push_back({static_cast<uint32_t>(i), new_off});
        }
        write_merge(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin", old_data, new_data, admin_stride);
    }

    // admin_vertices.bin
    {
        auto old_data = read_file(old_dir + "/admin_vertices.bin");
        auto new_data = read_file(new_dir + "/admin_vertices.bin");
        write_merge(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", old_data, new_data, 8);
    }

    // --- Section: Entry/cell files as full new data ---
    // These are small relative to data files and rebuilt by patch tool isn't reliable.
    // Include them as full replacement. zstd transport compression handles the rest.
    // Entry/cell files: full replacement (zstd transport handles compression)
    for (auto& [id, name] : std::vector<std::pair<PatchFileId, std::string>>{
        {PatchFileId::GEO_CELLS, "geo_cells.bin"},
        {PatchFileId::STREET_ENTRIES, "street_entries.bin"},
        {PatchFileId::ADDR_ENTRIES, "addr_entries.bin"},
        {PatchFileId::INTERP_ENTRIES, "interp_entries.bin"},
        {PatchFileId::ADMIN_CELLS, "admin_cells.bin"},
        {PatchFileId::ADMIN_ENTRIES, "admin_entries.bin"}}) {
        auto data = read_file(new_dir + "/" + name);
        uint32_t fid = static_cast<uint32_t>(id);
        uint32_t st = 0; // stride 0 = full replacement
        uint64_t os = 0, ns = data.size();
        uint32_t nfix = 0;
        uint64_t ss = data.size();
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8);
        wval(patch, &nfix, 4); // no fixups
        wval(patch, &ss, 8);   // data size
        patch.insert(patch.end(), data.begin(), data.end());
        std::cerr << "  " << name << ": full " << data.size() << " bytes" << std::endl;
    }

    // End marker
    uint32_t end_marker = 0xFFFFFFFF;
    wval(patch, &end_marker, 4);

    std::cerr << "\nUncompressed patch: " << patch.size() << " bytes ("
              << patch.size() / 1024 / 1024 << " MiB)" << std::endl;

    // Compress whole patch with zstd for transport
    {
        std::string raw_path = tmpdir + "/patch.raw";
        write_file(raw_path, patch);
        std::string cmd = "zstd -19 '" + raw_path + "' -o '" + patch_path + "' -f --quiet 2>/dev/null";
        system(cmd.c_str());
        auto compressed = read_file(patch_path);
        std::cerr << "Compressed patch: " << compressed.size() << " bytes ("
                  << compressed.size() / 1024 / 1024 << " MiB)" << std::endl;
        remove(raw_path.c_str());
    }

    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());
    return 0;
}

-    // Entry/cell files: full replacement (zstd transport handles compression)

+    // The patch tool reconstructs them from the derived ID remaps

+    // (extracted implicitly from the merge sequences above).

+    // geo_cells.bin, admin_cells.bin, and all entry files are rebuilt.

+    std::cerr << "  (entry/cell files: omitted, rebuilt by patch tool)" << std::endl;

+    // Placeholder: we could include cell_id changes here for perfect reconstruction.

+    // For now, the patch tool uses the old cell set with remapped IDs.

+    if (false) // disabled — entry files not included

-        {PatchFileId::GEO_CELLS, "geo_cells.bin"},

-        {PatchFileId::STREET_ENTRIES, "street_entries.bin"},

-        {PatchFileId::ADDR_ENTRIES, "addr_entries.bin"},

-        {PatchFileId::INTERP_ENTRIES, "interp_entries.bin"},

-        {PatchFileId::ADMIN_CELLS, "admin_cells.bin"},

-        {PatchFileId::ADMIN_ENTRIES, "admin_entries.bin"}}) {

+        {PatchFileId::GEO_CELLS, "geo_cells.bin"}}) {


Bash

cmake --build build -j$(nproc) --target geocoder-diff --target geocoder-patch 2>&1 | grep -E "(error:|Built)" | head -5

Build


[100%] Built target geocoder-diff
[100%] Built target geocoder-patch


sshpass -p 'elon-is-insane' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" /root/traccar-geocoder/builder/build/geocoder-diff /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:~/ 2>&1 && sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
rm -f /home/michtest/test-nofull.gcpatch
time /home/michtest/geocoder-diff /home/michtest/det-A /home/michtest/det-new -o /home/michtest/test-nofull.gcpatch 2>&1
echo "Patch: $(ls -lh /home/michtest/test-nofull.gcpatch | awk "{print \$5}")"
echo ""
rm -rf /home/michtest/patched-nofull
time /home/michtest/geocoder-patch /home/michtest/det-A /home/michtest/test-nofull.gcpatch -o /home/michtest/patched-nofull 2>&1
echo ""
all_match=true
for f in /home/michtest/det-new/*.bin; do
    b=$(basename $f); p=/home/michtest/patched-nofull/$b
    [ -f "$p" ] && cmp -s "$f" "$p" && echo "  $b: MATCH" || { echo "  $b: FAIL"; all_match=false; }
done
$all_match && echo "ALL FILES MATCH" || echo "SOME DIFFER"
' 2>&1

Test without entry/cell files in patch


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Building string remap...
  String remap: 4982052 entries (39856416 bytes)
  strings.bin: full 82573397 bytes
  addr_points.bin: seq=3903933 fixups=0 (0.28%)
  street_ways.bin: seq=2385521 fixups=19372497 (1.02%)
  street_nodes.bin: seq=10898537 fixups=0 (0.75%)
  interp_ways.bin: seq=545 fixups=13395 (0.16%)
  interp_nodes.bin: seq=2085 fixups=0 (0.29%)
  admin_polygons.bin: seq=81199 fixups=449077 (0.75%)
  admin_vertices.bin: seq=82545588 fixups=0 (28.36%)
  (entry/cell files: omitted, rebuilt by patch tool)

Uncompressed patch: 380927289 bytes (363 MiB)
Compressed patch: 101735205 bytes (97 MiB)

real	1m42.371s
user	3m12.199s
sys	0m5.949s
Patch: 98M

Decompressed patch: 380927289 bytes
Loaded 4980054 string remap entries (pos=39856440)
  strings.bin: full replacement 82573397 bytes
  addr_points.bin: 1375841776 bytes (mapped 85825938/85891830 IDs)
    Applied 19372497 fixups
  street_ways.bin: 233626176 bytes (mapped 19372840/19452480 IDs)
  street_nodes.bin: 1445600360 bytes
    Applied 13395 fixups
  interp_ways.bin: 335424 bytes (mapped 13961/13988 IDs)
  interp_nodes.bin: 725184 bytes
    Applied 449077 fixups
  admin_polygons.bin: 10820736 bytes (mapped 448753/450907 IDs)
  admin_vertices.bin: 291061192 bytes
Reconstructing entry/cell files from derived ID remaps...
  Rebuilt geo_cells + 3 entry files
  Rebuilt admin_cells + admin_entries
Patch applied. Output in /home/michtest/patched-nofull

real	2m13.334s
user	1m19.761s
sys	0m53.278s
  addr_entries.bin: FAIL
  addr_points.bin: MATCH
  admin_cells.bin: FAIL
  admin_entries.bin: FAIL
  admin_polygons.bin: MATCH
  admin_vertices.bin: MATCH
  geo_cells.bin: FAIL
  interp_entries.bin: FAIL
  interp_nodes.bin: MATCH
  interp_ways.bin: MATCH
  street_entries.bin: FAIL
  street_nodes.bin: MATCH
  street_ways.bin: MATCH
  strings.bin: MATCH
SOME DIFFER


**Patch is 97 MiB!** Down from 691 MiB! But entry/cell files don't match because the rebuilt entries use the OLD cell set (no cell additions/removals applied).

The data files all MATCH (addr_points, street_ways, street_nodes, interp_ways, interp_nodes, admin_polygons, admin_vertices, strings). The entry/cell files are rebuilt but with the wrong cell set.

The fix: include cell_id changes (added/removed) in the patch. Only 475 KB of data needed. Let me add that — the diff tool already computes them, just need to include them and have the patch tool apply them:

The diff tool already has `geo_added` and `geo_removed` computed. I just need to write them as a section. And the admin cells need the same treatment.


   490→    }
   491→
   492→    // --- Section: Entry/cell files as full new data ---
   493→    // These are small relative to data files and rebuilt by patch tool isn't reliable.
   494→    // Include them as full replacement. zstd transport compression handles the rest.
   495→    // Entry/cell files: NOT included.
   496→    // The patch tool reconstructs them from the derived ID remaps
   497→    // (extracted implicitly from the merge sequences above).
   498→    // geo_cells.bin, admin_cells.bin, and all entry files are rebuilt.
   499→    std::cerr << "  (entry/cell files: omitted, rebuilt by patch tool)" << std::endl;
   500→
   501→    // Placeholder: we could include cell_id changes here for perfect reconstruction.
   502→    // For now, the patch tool uses the old cell set with remapped IDs.
   503→
   504→    if (false) // disabled — entry files not included

    }

    // --- Section: Entry/cell files as full new data ---
    // These are small relative to data files and rebuilt by patch tool isn't reliable.
    // Include them as full replacement. zstd transport compression handles the rest.
    // Entry/cell files: NOT included.
    // The patch tool reconstructs them from the derived ID remaps
    // (extracted implicitly from the merge sequences above).
    // geo_cells.bin, admin_cells.bin, and all entry files are rebuilt.
    std::cerr << "  (entry/cell files: omitted, rebuilt by patch tool)" << std::endl;

    // Placeholder: we could include cell_id changes here for perfect reconstruction.
    // For now, the patch tool uses the old cell set with remapped IDs.

    if (false) // disabled — entry files not included


Edit

    // Placeholder: we could include cell_id changes here for perfect reconstruction.
    // For now, the patch tool uses the old cell set with remapped IDs.

    if (false) // disabled — entry files not included

    // Include cell_id changes for entry reconstruction
    {
        // Geo cell changes
        auto new_geo = read_file(new_dir + "/geo_cells.bin");
        auto old_geo = read_file(old_dir + "/geo_cells.bin");
        std::unordered_set<uint64_t> old_set, new_set;
        for (size_t i = 0; i < old_geo.size() / 20; i++) {
            uint64_t c; memcpy(&c, old_geo.data() + i * 20, 8); old_set.insert(c);
        }
        for (size_t i = 0; i < new_geo.size() / 20; i++) {
            uint64_t c; memcpy(&c, new_geo.data() + i * 20, 8); new_set.insert(c);
        }
        std::vector<uint64_t> g_added, g_removed;
        for (auto c : new_set) if (!old_set.count(c)) g_added.push_back(c);
        for (auto c : old_set) if (!new_set.count(c)) g_removed.push_back(c);
        std::sort(g_added.begin(), g_added.end());
        std::sort(g_removed.begin(), g_removed.end());

        uint32_t marker = CELL_CHANGES_GEO_MARKER;
        uint32_t na = g_added.size(), nr = g_removed.size();
        wval(patch, &marker, 4); wval(patch, &na, 4); wval(patch, &nr, 4);
        for (auto c : g_added) wval(patch, &c, 8);
        for (auto c : g_removed) wval(patch, &c, 8);
        std::cerr << "  Geo cell changes: +" << na << " -" << nr << " ("
                  << (na + nr) * 8 << " bytes)" << std::endl;

        // Admin cell changes
        auto new_ac = read_file(new_dir + "/admin_cells.bin");
        auto old_ac = read_file(old_dir + "/admin_cells.bin");
        old_set.clear(); new_set.clear();
        for (size_t i = 0; i < old_ac.size() / 12; i++) {
            uint64_t c; memcpy(&c, old_ac.data() + i * 12, 8); old_set.insert(c);
        }
        for (size_t i = 0; i < new_ac.size() / 12; i++) {
            uint64_t c; memcpy(&c, new_ac.data() + i * 12, 8); new_set.insert(c);
        }
        std::vector<uint64_t> a_added, a_removed;
        for (auto c : new_set) if (!old_set.count(c)) a_added.push_back(c);
        for (auto c : old_set) if (!new_set.count(c)) a_removed.push_back(c);
        std::sort(a_added.begin(), a_added.end());
        std::sort(a_removed.begin(), a_removed.end());

        marker = CELL_CHANGES_ADMIN_MARKER;
        na = a_added.size(); nr = a_removed.size();
        wval(patch, &marker, 4); wval(patch, &na, 4); wval(patch, &nr, 4);
        for (auto c : a_added) wval(patch, &c, 8);
        for (auto c : a_removed) wval(patch, &c, 8);
        std::cerr << "  Admin cell changes: +" << na << " -" << nr << std::endl;
    }

    if (false) // disabled — entry files not included


The file /root/traccar-geocoder/builder/tools/geocoder_diff.cpp has been updated successfully.

    // Include cell_id changes for entry reconstruction
    {
        // Geo cell changes
        auto new_geo = read_file(new_dir + "/geo_cells.bin");
        auto old_geo = read_file(old_dir + "/geo_cells.bin");
        std::unordered_set<uint64_t> old_set, new_set;
        for (size_t i = 0; i < old_geo.size() / 20; i++) {
            uint64_t c; memcpy(&c, old_geo.data() + i * 20, 8); old_set.insert(c);
        }
        for (size_t i = 0; i < new_geo.size() / 20; i++) {
            uint64_t c; memcpy(&c, new_geo.data() + i * 20, 8); new_set.insert(c);
        }
        std::vector<uint64_t> g_added, g_removed;
        for (auto c : new_set) if (!old_set.count(c)) g_added.push_back(c);
        for (auto c : old_set) if (!new_set.count(c)) g_removed.push_back(c);
        std::sort(g_added.begin(), g_added.end());
        std::sort(g_removed.begin(), g_removed.end());

        uint32_t marker = CELL_CHANGES_GEO_MARKER;
        uint32_t na = g_added.size(), nr = g_removed.size();
        wval(patch, &marker, 4); wval(patch, &na, 4); wval(patch, &nr, 4);
        for (auto c : g_added) wval(patch, &c, 8);
        for (auto c : g_removed) wval(patch, &c, 8);
        std::cerr << "  Geo cell changes: +" << na << " -" << nr << " ("
                  << (na + nr) * 8 << " bytes)" << std::endl;

        // Admin cell changes
        auto new_ac = read_file(new_dir + "/admin_cells.bin");
        auto old_ac = read_file(old_dir + "/admin_cells.bin");
        old_set.clear(); new_set.clear();
        for (size_t i = 0; i < old_ac.size() / 12; i++) {
            uint64_t c; memcpy(&c, old_ac.data() + i * 12, 8); old_set.insert(c);
        }
        for (size_t i = 0; i < new_ac.size() / 12; i++) {
            uint64_t c; memcpy(&c, new_ac.data() + i * 12, 8); new_set.insert(c);
        }
        std::vector<uint64_t> a_added, a_removed;
        for (auto c : new_set) if (!old_set.count(c)) a_added.push_back(c);
        for (auto c : old_set) if (!new_set.count(c)) a_removed.push_back(c);
        std::sort(a_added.begin(), a_added.end());
        std::sort(a_removed.begin(), a_removed.end());

        marker = CELL_CHANGES_ADMIN_MARKER;
        na = a_added.size(); nr = a_removed.size();
        wval(patch, &marker, 4); wval(patch, &na, 4); wval(patch, &nr, 4);
        for (auto c : a_added) wval(patch, &c, 8);
        for (auto c : a_removed) wval(patch, &c, 8);
        std::cerr << "  Admin cell changes: +" << na << " -" << nr << std::endl;
    }

    if (false) // disabled — entry files not included

// geocoder-diff v3: Fully custom patch format using merge-sequence encoding.
//
// For each data file: walk old (string-remapped) and new in parallel,
// match records by content, emit MATCH/INSERT/DELETE operations.
// For entry/cell files: include changed cell entry data directly.
// geo_cells rebuilt by patch tool from entries (not delta-patched).
//
// Patch format: custom binary, zstd-compressed as a whole for transport.
//
// Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>

#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <functional>
#include <iomanip>
#include <iostream>
#include <string>
#include <unistd.h>
#include <unordered_map>
#include <unordered_set>
#include <vector>

#include "patch_format.h"

// --- Merge sequence encoding ---
// Walk two sorted arrays (old remapped, new), matching by record bytes.
// Emit: MATCH(n) = copy n records from old, INSERT(n,data) = add n new records,
//       DELETE(n) = skip n old records.

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

struct MergeSequence {
    std::vector<char> data; // serialized ops

    void add_match(uint32_t count) {
        uint8_t op = OP_MATCH_RUN;
        data.insert(data.end(), (char*)&op, (char*)&op + 1);
        data.insert(data.end(), (char*)&count, (char*)&count + 4);
    }
    void add_delete(uint32_t count) {
        uint8_t op = OP_DELETE_RUN;
        data.insert(data.end(), (char*)&op, (char*)&op + 1);
        data.insert(data.end(), (char*)&count, (char*)&count + 4);
    }
    void add_insert(const char* records, uint32_t count, size_t stride) {
        uint8_t op = OP_INSERT_RUN;
        data.insert(data.end(), (char*)&op, (char*)&op + 1);
        data.insert(data.end(), (char*)&count, (char*)&count + 4);
        data.insert(data.end(), records, records + count * stride);
    }
};

// Build merge sequence for a data file.
// old_data has string remap already applied.
// Records are compared by byte equality (stride bytes).
static MergeSequence build_merge_seq(
    const std::vector<char>& old_data, const std::vector<char>& new_data,
    size_t stride)
{
    size_t old_n = old_data.size() / stride;
    size_t new_n = new_data.size() / stride;
    size_t oi = 0, ni = 0;
    MergeSequence seq;
    uint32_t match_run = 0, del_run = 0;

    auto flush_match = [&]() { if (match_run > 0) { seq.add_match(match_run); match_run = 0; } };
    auto flush_del = [&]() { if (del_run > 0) { seq.add_delete(del_run); del_run = 0; } };

    // For small strides (<=8), records aren't unique enough for hash matching.
    // Use simple sequential scan instead.
    bool use_hash = (stride > 8);

    // Pre-build hash index of new records for fast mismatch resolution
    auto record_hash = [&](const char* p, size_t s) -> uint64_t {
        uint64_t h = 14695981039346656037ULL;
        for (size_t i = 0; i < s; i++) { h ^= (uint8_t)p[i]; h *= 1099511628211ULL; }
        return h;
    };
    std::unordered_multimap<uint64_t, uint32_t> new_hash;
    if (use_hash) {
        new_hash.reserve(new_n);
        for (uint32_t i = 0; i < new_n; i++)
            new_hash.emplace(record_hash(new_data.data() + i * stride, stride), i);
    }

    while (oi < old_n && ni < new_n) {
        const char* op = old_data.data() + oi * stride;
        const char* np = new_data.data() + ni * stride;

        if (memcmp(op, np, stride) == 0) {
            flush_del();
            match_run++;
            oi++; ni++;
        } else if (use_hash) {
            // Hash-based mismatch resolution (for records with unique content)
            uint64_t oh = record_hash(op, stride);
            auto range = new_hash.equal_range(oh);
            uint32_t best_ni = UINT32_MAX;
            for (auto it = range.first; it != range.second; ++it) {
                if (it->second >= ni && it->second < ni + 10000 &&
                    memcmp(op, new_data.data() + it->second * stride, stride) == 0) {
                    if (it->second < best_ni) best_ni = it->second;
                }
            }
            if (best_ni != UINT32_MAX && best_ni > ni) {
                flush_match(); flush_del();
                seq.add_insert(new_data.data() + ni * stride, best_ni - ni, stride);
                ni = best_ni;
            } else if (best_ni == ni) {
                flush_del(); match_run++; oi++; ni++;
            } else {
                flush_match(); del_run++; oi++;
            }
        } else {
            // Sequential scan (for coordinate files where records aren't unique)
            size_t lookahead = std::min((size_t)200, std::min(old_n - oi, new_n - ni));
            bool found = false;
            for (size_t k = 1; k <= lookahead; k++) {
                if (ni + k < new_n && memcmp(op, new_data.data() + (ni + k) * stride, stride) == 0) {
                    flush_match(); flush_del();
                    seq.add_insert(new_data.data() + ni * stride, k, stride);
                    ni += k; found = true; break;
                }
                if (oi + k < old_n && memcmp(np, old_data.data() + (oi + k) * stride, stride) == 0) {
                    flush_match(); del_run += k; oi += k; found = true; break;
                }
            }
            if (!found) { flush_match(); del_run++; oi++; flush_del(); seq.add_insert(np, 1, stride); ni++; }
        }
    }

    flush_match(); flush_del();
    // Remaining old records: deletions
    if (oi < old_n) seq.add_delete(old_n - oi);
    // Remaining new records: insertions
    if (ni < new_n) seq.add_insert(new_data.data() + ni * stride, new_n - ni, stride);

    return seq;
}

// --- String remap ---

static std::unordered_map<uint32_t, uint32_t> build_string_remap(
    const std::vector<char>& old_pool, const std::vector<char>& new_pool)
{
    std::unordered_map<std::string, uint32_t> new_idx;
    size_t pos = 0;
    while (pos < new_pool.size()) {
        const char* s = new_pool.data() + pos;
        size_t len = strlen(s);
        new_idx[std::string(s, len)] = static_cast<uint32_t>(pos);
        pos += len + 1;
    }
    std::unordered_map<uint32_t, uint32_t> remap;
    pos = 0;
    while (pos < old_pool.size()) {
        const char* s = old_pool.data() + pos;
        size_t len = strlen(s);
        auto it = new_idx.find(std::string(s, len));
        if (it != new_idx.end())
            remap[static_cast<uint32_t>(pos)] = it->second;
        pos += len + 1;
    }
    return remap;
}

// --- Remap + fixup helpers ---

static void remap_addr_points(std::vector<char>& data, const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + 16 <= data.size(); i += 16)
        for (size_t off : {8, 12}) {
            uint32_t v; memcpy(&v, data.data() + i + off, 4);
            auto it = rm.find(v); if (it != rm.end()) memcpy(data.data() + i + off, &it->second, 4);
        }
}
static void remap_field(std::vector<char>& data, size_t stride, size_t field_off,
                         const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + stride <= data.size(); i += stride) {
        uint32_t v; memcpy(&v, data.data() + i + field_off, 4);
        auto it = rm.find(v); if (it != rm.end()) memcpy(data.data() + i + field_off, &it->second, 4);
    }
}

// Content matching for ways (by name + nodes, ignoring node_offset)
static uint64_t fnv_mix(uint64_t h, uint64_t v) { h ^= v; h *= 1099511628211ULL; return h; }

static void fixup_way_offsets(std::vector<char>& old_ways, const std::vector<char>& old_nodes,
                                const std::vector<char>& new_ways, const std::vector<char>& new_nodes,
                                size_t stride) {
    size_t name_off = (stride == 12) ? 8 : 5;
    size_t old_n = old_ways.size() / stride, new_n = new_ways.size() / stride;
    size_t old_nc = old_nodes.size() / 8, new_nc = new_nodes.size() / 8;

    auto way_hash = [&](const char* w, const char* nodes, size_t max_n) -> uint64_t {
        uint32_t node_offset, name_id; uint8_t node_count;
        memcpy(&node_offset, w, 4); node_count = (uint8_t)w[4]; memcpy(&name_id, w + name_off, 4);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, node_count);
        for (uint8_t j = 0; j < node_count && (node_offset + j) < max_n; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4);
            memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(way_hash(new_ways.data() + i * stride, new_nodes.data(), new_nc), i);

    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = way_hash(old_ways.data() + i * stride, old_nodes.data(), old_nc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            // Fix node_offset to match new way
            uint32_t new_node_off;
            memcpy(&new_node_off, new_ways.data() + it->second * stride, 4);
            memcpy(old_ways.data() + i * stride, &new_node_off, 4);
            new_map.erase(it);
        }
    }
}

// Same for admin polygons (vertex_offset)
static void fixup_admin_offsets(std::vector<char>& old_polys, const std::vector<char>& old_verts,
                                  const std::vector<char>& new_polys, const std::vector<char>& new_verts,
                                  size_t stride) {
    size_t old_n = old_polys.size() / stride, new_n = new_polys.size() / stride;
    size_t old_vc = old_verts.size() / 8, new_vc = new_verts.size() / 8;

    auto poly_hash = [&](const char* p, const char* verts, size_t max_v) -> uint64_t {
        uint32_t vert_offset, vert_count, name_id;
        memcpy(&vert_offset, p, 4); memcpy(&vert_count, p + 4, 4); memcpy(&name_id, p + 8, 4);
        uint8_t level = (uint8_t)p[12]; uint16_t cc; memcpy(&cc, p + stride - 2, 2);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, level); h = fnv_mix(h, cc); h = fnv_mix(h, vert_count);
        for (uint32_t j = 0; j < std::min(vert_count, 10u) && (vert_offset + j) < max_v; j++) {
            float lat, lng;
            memcpy(&lat, verts + (vert_offset + j) * 8, 4); memcpy(&lng, verts + (vert_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(poly_hash(new_polys.data() + i * stride, new_verts.data(), new_vc), i);

    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = poly_hash(old_polys.data() + i * stride, old_verts.data(), old_vc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            uint32_t new_vert_off;
            memcpy(&new_vert_off, new_polys.data() + it->second * stride, 4);
            memcpy(old_polys.data() + i * stride, &new_vert_off, 4);
            new_map.erase(it);
        }
    }
}

// Same for interp ways
static void fixup_interp_offsets(std::vector<char>& old_data, const std::vector<char>& old_nodes,
                                   const std::vector<char>& new_data, const std::vector<char>& new_nodes,
                                   size_t stride) {
    size_t street_off = (stride >= 20) ? 8 : 5;
    size_t old_n = old_data.size() / stride, new_n = new_data.size() / stride;
    size_t old_nc = old_nodes.size() / 8, new_nc = new_nodes.size() / 8;

    auto ihash = [&](const char* p, const char* nodes, size_t max_n) -> uint64_t {
        uint32_t node_offset, street_id, start, end; uint8_t count, itype;
        memcpy(&node_offset, p, 4); count = (uint8_t)p[4];
        memcpy(&street_id, p + street_off, 4); memcpy(&start, p + street_off + 4, 4);
        memcpy(&end, p + street_off + 8, 4);
        itype = (street_off + 12 < stride) ? (uint8_t)p[street_off + 12] : 0;
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, street_id); h = fnv_mix(h, start); h = fnv_mix(h, end);
        h = fnv_mix(h, itype); h = fnv_mix(h, count);
        for (uint8_t j = 0; j < count && (node_offset + j) < max_n; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4); memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(ihash(new_data.data() + i * stride, new_nodes.data(), new_nc), i);
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = ihash(old_data.data() + i * stride, old_nodes.data(), old_nc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            uint32_t new_off; memcpy(&new_off, new_data.data() + it->second * stride, 4);
            memcpy(old_data.data() + i * stride, &new_off, 4);
            new_map.erase(it);
        }
    }
}

// --- Write helpers ---
static void wval(std::vector<char>& buf, const void* data, size_t size) {
    buf.insert(buf.end(), (const char*)data, (const char*)data + size);
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>" << std::endl;
        return 1;
    }
    std::string old_dir = argv[1], new_dir = argv[2], patch_path = argv[4];
    std::string tmpdir = "/tmp/geocoder-diff-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    // Build string remap
    std::cerr << "Building string remap..." << std::endl;
    auto old_strings = read_file(old_dir + "/strings.bin");
    auto new_strings = read_file(new_dir + "/strings.bin");
    auto str_remap = build_string_remap(old_strings, new_strings);

    // Detect strides
    auto detect = [](const std::string& path, std::vector<size_t> cs) -> size_t {
        auto d = read_file(path);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect(old_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect(old_dir + "/interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect(old_dir + "/admin_polygons.bin", {24, 20, 19});

    // Build patch data (uncompressed, will be zstd-compressed at the end)
    std::vector<char> patch;
    patch.insert(patch.end(), GCPATCH_MAGIC, GCPATCH_MAGIC + 8);
    uint32_t ver = 2, flags = 0; // version 2 = custom format
    wval(patch, &ver, 4); wval(patch, &flags, 4);

    // --- Section: String remap ---
    {
        std::vector<std::pair<uint32_t,uint32_t>> entries;
        size_t pos = 0;
        while (pos < old_strings.size()) {
            uint32_t old_off = static_cast<uint32_t>(pos);
            auto it = str_remap.find(old_off);
            entries.push_back({old_off, (it != str_remap.end()) ? it->second : old_off});
            pos += strlen(old_strings.data() + pos) + 1;
        }
        uint32_t marker = 0xFFFFFFFE, count = static_cast<uint32_t>(entries.size());
        wval(patch, &marker, 4); wval(patch, &count, 4);
        for (auto& [o, n] : entries) { wval(patch, &o, 4); wval(patch, &n, 4); }
        std::cerr << "  String remap: " << count << " entries (" << count * 8 << " bytes)" << std::endl;
    }

    // --- Section: Per-file merge sequences ---
    // Track fixups per file (applied before merge, included in patch for patch tool)
    std::unordered_map<uint32_t, std::vector<std::pair<uint32_t,uint32_t>>> file_fixups;

    auto write_merge = [&](PatchFileId id, const std::string& name,
                             const std::vector<char>& old_data, const std::vector<char>& new_data,
                             size_t stride) {
        auto seq = build_merge_seq(old_data, new_data, stride);
        uint32_t fid = static_cast<uint32_t>(id);
        uint64_t os = old_data.size(), ns = new_data.size(), ss = seq.data.size();

        // Check for fixups
        auto fit = file_fixups.find(fid);
        uint32_t n_fixups = fit != file_fixups.end() ? static_cast<uint32_t>(fit->second.size()) : 0;

        wval(patch, &fid, 4);
        uint32_t st = static_cast<uint32_t>(stride);
        wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8);
        // Fixup count + data before merge sequence
        wval(patch, &n_fixups, 4);
        if (n_fixups > 0) {
            for (auto& [idx, val] : fit->second) { wval(patch, &idx, 4); wval(patch, &val, 4); }
        }
        wval(patch, &ss, 8);
        patch.insert(patch.end(), seq.data.begin(), seq.data.end());
        std::cerr << "  " << name << ": seq=" << ss << " fixups=" << n_fixups
                  << " (" << std::fixed << std::setprecision(2)
                  << (ns > 0 ? ss * 100.0 / ns : 0) << "%)" << std::endl;
    };

    // strings.bin: full replacement (byte-level merge is inefficient for string pools)
    {
        uint32_t fid = static_cast<uint32_t>(PatchFileId::STRINGS);
        uint32_t st = 0; // full replacement
        uint64_t os = 0, ns = new_strings.size();
        uint32_t nfix = 0;
        uint64_t ss = new_strings.size();
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8);
        wval(patch, &nfix, 4); wval(patch, &ss, 8);
        patch.insert(patch.end(), new_strings.begin(), new_strings.end());
        std::cerr << "  strings.bin: full " << new_strings.size() << " bytes" << std::endl;
    }

    // addr_points.bin
    {
        auto old_data = read_file(old_dir + "/addr_points.bin");
        remap_addr_points(old_data, str_remap);
        auto new_data = read_file(new_dir + "/addr_points.bin");
        write_merge(PatchFileId::ADDR_POINTS, "addr_points.bin", old_data, new_data, 16);
    }

    // street_ways.bin (string remap + offset fixup for node_offset)
    {
        auto old_w = read_file(old_dir + "/street_ways.bin");
        remap_field(old_w, way_stride, way_stride == 12 ? 8 : 5, str_remap);
        auto new_w = read_file(new_dir + "/street_ways.bin");
        auto old_n = read_file(old_dir + "/street_nodes.bin");
        auto new_n = read_file(new_dir + "/street_nodes.bin");
        // Record fixups: (record_index, new_node_offset)
        auto& fixups = file_fixups[static_cast<uint32_t>(PatchFileId::STREET_WAYS)];
        // Save old offsets, apply fixups, then record changes
        size_t wn = old_w.size() / way_stride;
        std::vector<uint32_t> old_offsets(wn);
        for (size_t i = 0; i < wn; i++) memcpy(&old_offsets[i], old_w.data() + i * way_stride, 4);
        fixup_way_offsets(old_w, old_n, new_w, new_n, way_stride);
        for (size_t i = 0; i < wn; i++) {
            uint32_t new_off; memcpy(&new_off, old_w.data() + i * way_stride, 4);
            if (new_off != old_offsets[i]) fixups.push_back({static_cast<uint32_t>(i), new_off});
        }
        write_merge(PatchFileId::STREET_WAYS, "street_ways.bin", old_w, new_w, way_stride);
    }

    // street_nodes.bin (no remap)
    {
        auto old_data = read_file(old_dir + "/street_nodes.bin");
        auto new_data = read_file(new_dir + "/street_nodes.bin");
        write_merge(PatchFileId::STREET_NODES, "street_nodes.bin", old_data, new_data, 8);
    }

    // interp_ways.bin
    {
        auto old_data = read_file(old_dir + "/interp_ways.bin");
        remap_field(old_data, interp_stride, interp_stride >= 20 ? 8 : 5, str_remap);
        auto old_n = read_file(old_dir + "/interp_nodes.bin");
        auto new_data = read_file(new_dir + "/interp_ways.bin");
        auto new_n = read_file(new_dir + "/interp_nodes.bin");
        auto& fixups = file_fixups[static_cast<uint32_t>(PatchFileId::INTERP_WAYS)];
        size_t in_count = old_data.size() / interp_stride;
        std::vector<uint32_t> old_offsets(in_count);
        for (size_t i = 0; i < in_count; i++) memcpy(&old_offsets[i], old_data.data() + i * interp_stride, 4);
        fixup_interp_offsets(old_data, old_n, new_data, new_n, interp_stride);
        for (size_t i = 0; i < in_count; i++) {
            uint32_t new_off; memcpy(&new_off, old_data.data() + i * interp_stride, 4);
            if (new_off != old_offsets[i]) fixups.push_back({static_cast<uint32_t>(i), new_off});
        }
        write_merge(PatchFileId::INTERP_WAYS, "interp_ways.bin", old_data, new_data, interp_stride);
    }

    // interp_nodes.bin
    {
        auto old_data = read_file(old_dir + "/interp_nodes.bin");
        auto new_data = read_file(new_dir + "/interp_nodes.bin");
        write_merge(PatchFileId::INTERP_NODES, "interp_nodes.bin", old_data, new_data, 8);
    }

    // admin_polygons.bin
    {
        auto old_data = read_file(old_dir + "/admin_polygons.bin");
        remap_field(old_data, admin_stride, 8, str_remap);
        auto old_v = read_file(old_dir + "/admin_vertices.bin");
        auto new_data = read_file(new_dir + "/admin_polygons.bin");
        auto new_v = read_file(new_dir + "/admin_vertices.bin");
        auto& fixups = file_fixups[static_cast<uint32_t>(PatchFileId::ADMIN_POLYGONS)];
        size_t an = old_data.size() / admin_stride;
        std::vector<uint32_t> old_offsets(an);
        for (size_t i = 0; i < an; i++) memcpy(&old_offsets[i], old_data.data() + i * admin_stride, 4);
        fixup_admin_offsets(old_data, old_v, new_data, new_v, admin_stride);
        for (size_t i = 0; i < an; i++) {
            uint32_t new_off; memcpy(&new_off, old_data.data() + i * admin_stride, 4);
            if (new_off != old_offsets[i]) fixups.push_back({static_cast<uint32_t>(i), new_off});
        }
        write_merge(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin", old_data, new_data, admin_stride);
    }

    // admin_vertices.bin
    {
        auto old_data = read_file(old_dir + "/admin_vertices.bin");
        auto new_data = read_file(new_dir + "/admin_vertices.bin");
        write_merge(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", old_data, new_data, 8);
    }

    // --- Section: Entry/cell files as full new data ---
    // These are small relative to data files and rebuilt by patch tool isn't reliable.
    // Include them as full replacement. zstd transport compression handles the rest.
    // Entry/cell files: NOT included.
    // The patch tool reconstructs them from the derived ID remaps
    // (extracted implicitly from the merge sequences above).
    // geo_cells.bin, admin_cells.bin, and all entry files are rebuilt.
    std::cerr << "  (entry/cell files: omitted, rebuilt by patch tool)" << std::endl;

    // Placeholder: we could include cell_id changes here for perfect reconstruction.
    // For now, the patch tool uses the old cell set with remapped IDs.

    if (false) // disabled — entry files not included
    for (auto& [id, name] : std::vector<std::pair<PatchFileId, std::string>>{
        {PatchFileId::GEO_CELLS, "geo_cells.bin"}}) {
        auto data = read_file(new_dir + "/" + name);
        uint32_t fid = static_cast<uint32_t>(id);
        uint32_t st = 0; // stride 0 = full replacement
        uint64_t os = 0, ns = data.size();
        uint32_t nfix = 0;
        uint64_t ss = data.size();
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8);
        wval(patch, &nfix, 4); // no fixups
        wval(patch, &ss, 8);   // data size
        patch.insert(patch.end(), data.begin(), data.end());
        std::cerr << "  " << name << ": full " << data.size() << " bytes" << std::endl;
    }

    // End marker
    uint32_t end_marker = 0xFFFFFFFF;
    wval(patch, &end_marker, 4);

    std::cerr << "\nUncompressed patch: " << patch.size() << " bytes ("
              << patch.size() / 1024 / 1024 << " MiB)" << std::endl;

    // Compress whole patch with zstd for transport
    {
        std::string raw_path = tmpdir + "/patch.raw";
        write_file(raw_path, patch);
        std::string cmd = "zstd -19 '" + raw_path + "' -o '" + patch_path + "' -f --quiet 2>/dev/null";
        system(cmd.c_str());
        auto compressed = read_file(patch_path);
        std::cerr << "Compressed patch: " << compressed.size() << " bytes ("
                  << compressed.size() / 1024 / 1024 << " MiB)" << std::endl;
        remove(raw_path.c_str());
    }

    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());
    return 0;
}

-    // Placeholder: we could include cell_id changes here for perfect reconstruction.

-    // For now, the patch tool uses the old cell set with remapped IDs.

+    // Include cell_id changes for entry reconstruction

+        auto new_geo = read_file(new_dir + "/geo_cells.bin");

+        auto old_geo = read_file(old_dir + "/geo_cells.bin");

+        std::unordered_set<uint64_t> old_set, new_set;

+        for (size_t i = 0; i < old_geo.size() / 20; i++) {

+            uint64_t c; memcpy(&c, old_geo.data() + i * 20, 8); old_set.insert(c);

+        for (size_t i = 0; i < new_geo.size() / 20; i++) {

+            uint64_t c; memcpy(&c, new_geo.data() + i * 20, 8); new_set.insert(c);

+        std::vector<uint64_t> g_added, g_removed;

+        for (auto c : new_set) if (!old_set.count(c)) g_added.push_back(c);

+        for (auto c : old_set) if (!new_set.count(c)) g_removed.push_back(c);

+        std::sort(g_added.begin(), g_added.end());

+        std::sort(g_removed.begin(), g_removed.end());

+        uint32_t marker = CELL_CHANGES_GEO_MARKER;

+        uint32_t na = g_added.size(), nr = g_removed.size();

+        wval(patch, &marker, 4); wval(patch, &na, 4); wval(patch, &nr, 4);

+        for (auto c : g_added) wval(patch, &c, 8);

+        for (auto c : g_removed) wval(patch, &c, 8);

+        std::cerr << "  Geo cell changes: +" << na << " -" << nr << " ("

+        auto new_ac = read_file(new_dir + "/admin_cells.bin");

+        auto old_ac = read_file(old_dir + "/admin_cells.bin");

+        old_set.clear(); new_set.clear();

+        for (size_t i = 0; i < old_ac.size() / 12; i++) {

+            uint64_t c; memcpy(&c, old_ac.data() + i * 12, 8); old_set.insert(c);

+        for (size_t i = 0; i < new_ac.size() / 12; i++) {

+            uint64_t c; memcpy(&c, new_ac.data() + i * 12, 8); new_set.insert(c);

+        std::vector<uint64_t> a_added, a_removed;

+        for (auto c : new_set) if (!old_set.count(c)) a_added.push_back(c);

+        for (auto c : old_set) if (!new_set.count(c)) a_removed.push_back(c);

+        std::sort(a_added.begin(), a_added.end());

+        std::sort(a_removed.begin(), a_removed.end());

+        marker = CELL_CHANGES_ADMIN_MARKER;

+        na = a_added.size(); nr = a_removed.size();

+        for (auto c : a_added) wval(patch, &c, 8);

+        for (auto c : a_removed) wval(patch, &c, 8);

+        std::cerr << "  Admin cell changes: +" << na << " -" << nr << std::endl;


Now update the patch tool to read cell changes and pass them to the rebuild function:


        if (!have_geo && !derived_id_remaps.empty()) {
            std::cerr << "Reconstructing entry/cell files from derived ID remaps..." << std::endl;
            // Convert vector remaps to unordered_maps for the shared rebuild function
            std::unordered_map<uint32_t,uint32_t> w_rm, a_rm, i_rm;
            auto convert = [](const std::vector<uint32_t>& vec) {
                std::unordered_map<uint32_t,uint32_t> m;
                for (uint32_t i = 0; i < vec.size(); i++)
                    if (vec[i] != 0xFFFFFFFF) m[i] = vec[i];
                return m;
            };
            if (derived_id_remaps.count((uint32_t)PatchFileId::STREET_WAYS))
                w_rm = convert(derived_id_remaps[(uint32_t)PatchFileId::STREET_WAYS]);
            if (derived_id_remaps.count((uint32_t)PatchFileId::ADDR_POINTS))
                a_rm = convert(derived_id_remaps[(uint32_t)PatchFileId::ADDR_POINTS]);
            if (derived_id_remaps.count((uint32_t)PatchFileId::INTERP_WAYS))
                i_rm = convert(derived_id_remaps[(uint32_t)PatchFileId::INTERP_WAYS]);

            // Read cell_id list from patch (if included) or from old geo_cells
            auto old_geo = read_file(cur_dir + "/geo_cells.bin");
            auto old_se = read_file(cur_dir + "/street_entries.bin");
            auto old_ae = read_file(cur_dir + "/addr_entries.bin");
            auto old_ie = read_file(cur_dir + "/interp_entries.bin");

            auto geo = rebuild_geo_from_remap(old_geo, old_se, old_ae, old_ie, w_rm, a_rm, i_rm);
            write_file(out_dir + "/geo_cells.bin", geo.geo_cells_data);
            write_file(out_dir + "/street_entries.bin", geo.street_entries_data);
            write_file(out_dir + "/addr_entries.bin", geo.addr_entries_data);
            write_file(out_dir + "/interp_entries.bin", geo.interp_entries_data);
            std::cerr << "  Rebuilt geo_cells + 3 entry files" << std::endl;
        }

        if (!have_admin && !derived_id_remaps.empty()) {
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            if (derived_id_remaps.count((uint32_t)PatchFileId::ADMIN_POLYGONS)) {
                auto& vec = derived_id_remaps[(uint32_t)PatchFileId::ADMIN_POLYGONS];
                for (uint32_t i = 0; i < vec.size(); i++)
                    if (vec[i] != 0xFFFFFFFF) ad_rm[i] = vec[i];
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_ae = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_ae, ad_rm);
            write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
            write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            std::cerr << "  Rebuilt admin_cells + admin_entries" << std::endl;
        }

        // Read cell changes from remaining patch data
        std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;
        while (pos < patch.size()) {
            uint32_t marker = read_u32();
            if (marker == CELL_CHANGES_GEO_MARKER) {
                uint32_t na = read_u32(), nr = read_u32();
                geo_added.resize(na); geo_removed.resize(nr);
                for (uint32_t i = 0; i < na; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; geo_added[i]=c; }
                for (uint32_t i = 0; i < nr; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; geo_removed[i]=c; }
                std::cerr << "  Geo cell changes: +" << na << " -" << nr << std::endl;
            } else if (marker == CELL_CHANGES_ADMIN_MARKER) {
                uint32_t na = read_u32(), nr = read_u32();
                admin_added.resize(na); admin_removed.resize(nr);
                for (uint32_t i = 0; i < na; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; admin_added[i]=c; }
                for (uint32_t i = 0; i < nr; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; admin_removed[i]=c; }
                std::cerr << "  Admin cell changes: +" << na << " -" << nr << std::endl;
            } else {
                pos -= 4; break;
            }
        }

        if (!have_geo && !derived_id_remaps.empty()) {
            std::cerr << "Reconstructing entry/cell files from derived ID remaps..." << std::endl;
            auto convert = [](const std::vector<uint32_t>& vec) {
                std::unordered_map<uint32_t,uint32_t> m;
                for (uint32_t i = 0; i < vec.size(); i++)
                    if (vec[i] != 0xFFFFFFFF) m[i] = vec[i];
                return m;
            };
            std::unordered_map<uint32_t,uint32_t> w_rm, a_rm, i_rm;
            if (derived_id_remaps.count((uint32_t)PatchFileId::STREET_WAYS))
                w_rm = convert(derived_id_remaps[(uint32_t)PatchFileId::STREET_WAYS]);
            if (derived_id_remaps.count((uint32_t)PatchFileId::ADDR_POINTS))
                a_rm = convert(derived_id_remaps[(uint32_t)PatchFileId::ADDR_POINTS]);
            if (derived_id_remaps.count((uint32_t)PatchFileId::INTERP_WAYS))
                i_rm = convert(derived_id_remaps[(uint32_t)PatchFileId::INTERP_WAYS]);

            auto old_geo = read_file(cur_dir + "/geo_cells.bin");
            auto old_se = read_file(cur_dir + "/street_entries.bin");
            auto old_ae = read_file(cur_dir + "/addr_entries.bin");
            auto old_ie = read_file(cur_dir + "/interp_entries.bin");

            auto geo = rebuild_geo_from_remap(old_geo, old_se, old_ae, old_ie, w_rm, a_rm, i_rm,
                                               geo_added, geo_removed);
            write_file(out_dir + "/geo_cells.bin", geo.geo_cells_data);
            write_file(out_dir + "/street_entries.bin", geo.street_entries_data);
            write_file(out_dir + "/addr_entries.bin", geo.addr_entries_data);
            write_file(out_dir + "/interp_entries.bin", geo.interp_entries_data);
            std::cerr << "  Rebuilt geo_cells + 3 entry files" << std::endl;
        }

        if (!have_admin && !derived_id_remaps.empty()) {
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            if (derived_id_remaps.count((uint32_t)PatchFileId::ADMIN_POLYGONS)) {
                auto& vec = derived_id_remaps[(uint32_t)PatchFileId::ADMIN_POLYGONS];
                for (uint32_t i = 0; i < vec.size(); i++)
                    if (vec[i] != 0xFFFFFFFF) ad_rm[i] = vec[i];
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_ae = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_ae, ad_rm);
            write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
            write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            std::cerr << "  Rebuilt admin_cells + admin_entries" << std::endl;
        }


The file /root/traccar-geocoder/builder/tools/geocoder_patch.cpp has been updated successfully.

        // Read cell changes from remaining patch data
        std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;
        while (pos < patch.size()) {
            uint32_t marker = read_u32();
            if (marker == CELL_CHANGES_GEO_MARKER) {
                uint32_t na = read_u32(), nr = read_u32();
                geo_added.resize(na); geo_removed.resize(nr);
                for (uint32_t i = 0; i < na; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; geo_added[i]=c; }
                for (uint32_t i = 0; i < nr; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; geo_removed[i]=c; }
                std::cerr << "  Geo cell changes: +" << na << " -" << nr << std::endl;
            } else if (marker == CELL_CHANGES_ADMIN_MARKER) {
                uint32_t na = read_u32(), nr = read_u32();
                admin_added.resize(na); admin_removed.resize(nr);
                for (uint32_t i = 0; i < na; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; admin_added[i]=c; }
                for (uint32_t i = 0; i < nr; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; admin_removed[i]=c; }
                std::cerr << "  Admin cell changes: +" << na << " -" << nr << std::endl;
            } else {
                pos -= 4; break;
            }
        }

        if (!have_geo && !derived_id_remaps.empty()) {
            std::cerr << "Reconstructing entry/cell files from derived ID remaps..." << std::endl;
            auto convert = [](const std::vector<uint32_t>& vec) {
                std::unordered_map<uint32_t,uint32_t> m;
                for (uint32_t i = 0; i < vec.size(); i++)
                    if (vec[i] != 0xFFFFFFFF) m[i] = vec[i];
                return m;
            };
            std::unordered_map<uint32_t,uint32_t> w_rm, a_rm, i_rm;
            if (derived_id_remaps.count((uint32_t)PatchFileId::STREET_WAYS))
                w_rm = convert(derived_id_remaps[(uint32_t)PatchFileId::STREET_WAYS]);
            if (derived_id_remaps.count((uint32_t)PatchFileId::ADDR_POINTS))
                a_rm = convert(derived_id_remaps[(uint32_t)PatchFileId::ADDR_POINTS]);
            if (derived_id_remaps.count((uint32_t)PatchFileId::INTERP_WAYS))
                i_rm = convert(derived_id_remaps[(uint32_t)PatchFileId::INTERP_WAYS]);

            auto old_geo = read_file(cur_dir + "/geo_cells.bin");
            auto old_se = read_file(cur_dir + "/street_entries.bin");
            auto old_ae = read_file(cur_dir + "/addr_entries.bin");
            auto old_ie = read_file(cur_dir + "/interp_entries.bin");

            auto geo = rebuild_geo_from_remap(old_geo, old_se, old_ae, old_ie, w_rm, a_rm, i_rm,
                                               geo_added, geo_removed);
            write_file(out_dir + "/geo_cells.bin", geo.geo_cells_data);
            write_file(out_dir + "/street_entries.bin", geo.street_entries_data);
            write_file(out_dir + "/addr_entries.bin", geo.addr_entries_data);
            write_file(out_dir + "/interp_entries.bin", geo.interp_entries_data);
            std::cerr << "  Rebuilt geo_cells + 3 entry files" << std::endl;
        }

        if (!have_admin && !derived_id_remaps.empty()) {
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            if (derived_id_remaps.count((uint32_t)PatchFileId::ADMIN_POLYGONS)) {
                auto& vec = derived_id_remaps[(uint32_t)PatchFileId::ADMIN_POLYGONS];
                for (uint32_t i = 0; i < vec.size(); i++)
                    if (vec[i] != 0xFFFFFFFF) ad_rm[i] = vec[i];
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_ae = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_ae, ad_rm);
            write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
            write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            std::cerr << "  Rebuilt admin_cells + admin_entries" << std::endl;
        }

// geocoder-patch v3: Apply custom patch format.
//
// Decompresses zstd transport layer, then:
// 1. Reads string remap, applies to old data files
// 2. Replays merge sequences to reconstruct new data files
// 3. Writes full-replacement entry/cell files
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

// Merge ops (must match diff tool)
enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    // Decompress zstd transport layer
    std::string raw_path = tmpdir + "/patch.raw";
    {
        std::string cmd = "zstd -d '" + patch_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0) {
            std::cerr << "Failed to decompress patch" << std::endl; return 1;
        }
    }
    auto patch = read_file(raw_path);
    remove(raw_path.c_str());
    std::cerr << "Decompressed patch: " << patch.size() << " bytes" << std::endl;

    size_t pos = 0;
    auto read_u32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, patch.data() + pos, 4); pos += 4; return v; };
    auto read_u64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, patch.data() + pos, 8); pos += 8; return v; };

    // Check header
    if (memcmp(patch.data(), GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch magic" << std::endl; return 1;
    }
    pos = 8;
    uint32_t version = read_u32();
    if (version != 2) { std::cerr << "Expected patch v2, got " << version << std::endl; return 1; }
    uint32_t flags = read_u32();
    (void)flags;

    // Read string remap
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t marker = read_u32();
        if (marker != 0xFFFFFFFE) { std::cerr << "Expected remap section" << std::endl; return 1; }
        uint32_t count = read_u32();
        for (uint32_t i = 0; i < count; i++) {
            uint32_t old_off = read_u32(), new_off = read_u32();
            if (old_off != new_off) str_remap[old_off] = new_off;
        }
        std::cerr << "Loaded " << str_remap.size() << " string remap entries (pos=" << pos << ")" << std::endl;
    }

    // Detect strides
    auto detect = [&](const std::string& name, std::vector<size_t> cs) -> size_t {
        auto d = read_file(cur_dir + "/" + name);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect("street_ways.bin", {12, 9});
    size_t interp_stride = detect("interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect("admin_polygons.bin", {24, 20, 19});

    // String remap functions
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t file_id, size_t stride) {
        if (str_remap.empty()) return;
        std::vector<size_t> field_offsets;
        if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) field_offsets = {8, 12};
        else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) field_offsets = {(stride == 12) ? (size_t)8 : (size_t)5};
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) field_offsets = {(stride >= 20) ? (size_t)8 : (size_t)5};
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) field_offsets = {8};
        else return;

        for (size_t i = 0; i + stride <= data.size(); i += stride)
            for (size_t off : field_offsets) {
                uint32_t v; memcpy(&v, data.data() + i + off, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + off, &it->second, 4);
            }
    };

    // ID remaps derived from merge sequences (populated during merge replay)
    std::unordered_map<uint32_t, std::vector<uint32_t>> derived_id_remaps;

    // Process sections
    while (pos < patch.size()) {
        uint32_t file_id = read_u32();
        if (file_id == 0xFFFFFFFF) break;

        uint32_t stride = read_u32();
        uint64_t old_size = read_u64(), new_size = read_u64();

        if (file_id >= (uint32_t)PatchFileId::COUNT) {
            std::cerr << "  Unknown file " << file_id << std::endl;
            break;
        }
        const char* filename = patch_file_names[file_id];

        if (stride == 0) {
            // Full replacement: n_fixups=0, then seq_size = data size
            uint32_t n_fix = read_u32(); (void)n_fix; // always 0 for full replacement
            uint64_t data_size = read_u64();
            std::vector<char> data(patch.data() + pos, patch.data() + pos + data_size);
            write_file(out_dir + "/" + std::string(filename), data);
            std::cerr << "  " << filename << ": full replacement " << data.size() << " bytes" << std::endl;
            pos += data_size;
            continue;
        }

        // Merge sequence: load old file, apply remap + fixups, replay sequence
        auto old_data = read_file(cur_dir + "/" + std::string(filename));
        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        // Apply string remap to old data
        apply_str_remap(old_data, file_id, actual_stride);

        // Read and apply fixups (node_offset/vertex_offset fixes)
        // Read and apply fixups (offset field patches for matched records)
        uint32_t n_fixups = read_u32();
        for (uint32_t i = 0; i < n_fixups; i++) {
            uint32_t idx = read_u32(), val = read_u32();
            size_t byte_pos = (size_t)idx * actual_stride;
            if (byte_pos + 4 <= old_data.size())
                memcpy(old_data.data() + byte_pos, &val, 4);
        }
        if (n_fixups > 0) std::cerr << "    Applied " << n_fixups << " fixups" << std::endl;

        // Read merge sequence size and replay, tracking old→new ID mapping
        uint64_t seq_size = read_u64();
        std::vector<char> output;
        output.reserve(new_size);
        size_t old_rec = 0, new_rec = 0; // record indices
        size_t old_pos_bytes = 0;
        size_t seq_end = pos + seq_size;

        // ID remap for this file (old_record_index → new_record_index)
        std::vector<uint32_t> id_map;
        bool track_ids = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
                          file_id == (uint32_t)PatchFileId::STREET_WAYS ||
                          file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
                          file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
        if (track_ids) id_map.assign(old_data.size() / actual_stride, 0xFFFFFFFF);

        while (pos < seq_end) {
            uint8_t op = static_cast<uint8_t>(patch[pos]); pos++;
            uint32_t count; memcpy(&count, patch.data() + pos, 4); pos += 4;

            if (op == OP_MATCH_RUN) {
                size_t bytes = count * actual_stride;
                if (old_pos_bytes + bytes <= old_data.size())
                    output.insert(output.end(), old_data.data() + old_pos_bytes,
                                 old_data.data() + old_pos_bytes + bytes);
                if (track_ids)
                    for (uint32_t k = 0; k < count; k++)
                        if (old_rec + k < id_map.size())
                            id_map[old_rec + k] = static_cast<uint32_t>(new_rec + k);
                old_rec += count; new_rec += count;
                old_pos_bytes += bytes;
            } else if (op == OP_INSERT_RUN) {
                size_t bytes = count * actual_stride;
                output.insert(output.end(), patch.data() + pos, patch.data() + pos + bytes);
                pos += bytes;
                new_rec += count;
            } else if (op == OP_DELETE_RUN) {
                old_rec += count;
                old_pos_bytes += count * actual_stride;
            }
        }

        write_file(out_dir + "/" + std::string(filename), output);

        // Store ID remap for entry reconstruction
        if (track_ids) {
            derived_id_remaps[file_id] = std::move(id_map);
            size_t matched = 0;
            for (auto v : derived_id_remaps[file_id]) if (v != 0xFFFFFFFF) matched++;
            std::cerr << "  " << filename << ": " << output.size() << " bytes (mapped "
                      << matched << "/" << derived_id_remaps[file_id].size() << " IDs)" << std::endl;
        } else {
            std::cerr << "  " << filename << ": " << output.size() << " bytes" << std::endl;
        }
    }

    // Reconstruct entry/cell files from derived ID remaps (if not provided as full replacement)
    {
        bool have_geo = false, have_admin = false;
        // Check if geo_cells was already written (full replacement)
        auto check = [&](const std::string& name) -> bool {
            auto d = read_file(out_dir + "/" + name);
            return !d.empty();
        };
        have_geo = check("geo_cells.bin");
        have_admin = check("admin_cells.bin");

        if (!have_geo && !derived_id_remaps.empty()) {
            std::cerr << "Reconstructing entry/cell files from derived ID remaps..." << std::endl;
            // Convert vector remaps to unordered_maps for the shared rebuild function
            std::unordered_map<uint32_t,uint32_t> w_rm, a_rm, i_rm;
            auto convert = [](const std::vector<uint32_t>& vec) {
                std::unordered_map<uint32_t,uint32_t> m;
                for (uint32_t i = 0; i < vec.size(); i++)
                    if (vec[i] != 0xFFFFFFFF) m[i] = vec[i];
                return m;
            };
            if (derived_id_remaps.count((uint32_t)PatchFileId::STREET_WAYS))
                w_rm = convert(derived_id_remaps[(uint32_t)PatchFileId::STREET_WAYS]);
            if (derived_id_remaps.count((uint32_t)PatchFileId::ADDR_POINTS))
                a_rm = convert(derived_id_remaps[(uint32_t)PatchFileId::ADDR_POINTS]);
            if (derived_id_remaps.count((uint32_t)PatchFileId::INTERP_WAYS))
                i_rm = convert(derived_id_remaps[(uint32_t)PatchFileId::INTERP_WAYS]);

            // Read cell_id list from patch (if included) or from old geo_cells
            auto old_geo = read_file(cur_dir + "/geo_cells.bin");
            auto old_se = read_file(cur_dir + "/street_entries.bin");
            auto old_ae = read_file(cur_dir + "/addr_entries.bin");
            auto old_ie = read_file(cur_dir + "/interp_entries.bin");

            auto geo = rebuild_geo_from_remap(old_geo, old_se, old_ae, old_ie, w_rm, a_rm, i_rm);
            write_file(out_dir + "/geo_cells.bin", geo.geo_cells_data);
            write_file(out_dir + "/street_entries.bin", geo.street_entries_data);
            write_file(out_dir + "/addr_entries.bin", geo.addr_entries_data);
            write_file(out_dir + "/interp_entries.bin", geo.interp_entries_data);
            std::cerr << "  Rebuilt geo_cells + 3 entry files" << std::endl;
        }

        if (!have_admin && !derived_id_remaps.empty()) {
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            if (derived_id_remaps.count((uint32_t)PatchFileId::ADMIN_POLYGONS)) {
                auto& vec = derived_id_remaps[(uint32_t)PatchFileId::ADMIN_POLYGONS];
                for (uint32_t i = 0; i < vec.size(); i++)
                    if (vec[i] != 0xFFFFFFFF) ad_rm[i] = vec[i];
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_ae = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_ae, ad_rm);
            write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
            write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            std::cerr << "  Rebuilt admin_cells + admin_entries" << std::endl;
        }
    }

    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

+        // Read cell changes from remaining patch data

+        std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;

+            uint32_t marker = read_u32();

+            if (marker == CELL_CHANGES_GEO_MARKER) {

+                uint32_t na = read_u32(), nr = read_u32();

+                geo_added.resize(na); geo_removed.resize(nr);

+                for (uint32_t i = 0; i < na; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; geo_added[i]=c; }

+                for (uint32_t i = 0; i < nr; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; geo_removed[i]=c; }

+                std::cerr << "  Geo cell changes: +" << na << " -" << nr << std::endl;

+            } else if (marker == CELL_CHANGES_ADMIN_MARKER) {

+                admin_added.resize(na); admin_removed.resize(nr);

+                for (uint32_t i = 0; i < na; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; admin_added[i]=c; }

+                for (uint32_t i = 0; i < nr; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; admin_removed[i]=c; }

+                std::cerr << "  Admin cell changes: +" << na << " -" << nr << std::endl;

-            // Convert vector remaps to unordered_maps for the shared rebuild function

-            std::unordered_map<uint32_t,uint32_t> w_rm, a_rm, i_rm;

+            std::unordered_map<uint32_t,uint32_t> w_rm, a_rm, i_rm;

-            // Read cell_id list from patch (if included) or from old geo_cells

-            auto geo = rebuild_geo_from_remap(old_geo, old_se, old_ae, old_ie, w_rm, a_rm, i_rm);

+            auto geo = rebuild_geo_from_remap(old_geo, old_se, old_ae, old_ie, w_rm, a_rm, i_rm,

+                                               geo_added, geo_removed);


Bash

cmake --build build -j$(nproc) --target geocoder-diff --target geocoder-patch 2>&1 | grep -E "(error:|Built)" | head -5

Build


sshpass -p 'elon-is-insane' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" /root/traccar-geocoder/builder/build/geocoder-diff /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:~/ 2>&1 && sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
rm -f /home/michtest/test-opt.gcpatch
time /home/michtest/geocoder-diff /home/michtest/det-A /home/michtest/det-new -o /home/michtest/test-opt.gcpatch 2>&1
echo "Patch: $(ls -lh /home/michtest/test-opt.gcpatch | awk "{print \$5}")"
echo ""
rm -rf /home/michtest/patched-opt
time /home/michtest/geocoder-patch /home/michtest/det-A /home/michtest/test-opt.gcpatch -o /home/michtest/patched-opt 2>&1
echo ""
all_match=true
for f in /home/michtest/det-new/*.bin; do
    b=$(basename $f); p=/home/michtest/patched-opt/$b
    [ -f "$p" ] && cmp -s "$f" "$p" && echo "  $b: MATCH" || { echo "  $b: FAIL"; all_match=false; }
done
$all_match && echo "ALL FILES MATCH" || echo "SOME DIFFER"
' 2>&1

Test with cell changes for entry reconstruction


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Building string remap...
  String remap: 4982052 entries (39856416 bytes)
  strings.bin: full 82573397 bytes
  addr_points.bin: seq=3903933 fixups=0 (0.28%)
  street_ways.bin: seq=2385521 fixups=19372497 (1.02%)
  street_nodes.bin: seq=10898537 fixups=0 (0.75%)
  interp_ways.bin: seq=545 fixups=13395 (0.16%)
  interp_nodes.bin: seq=2085 fixups=0 (0.29%)
  admin_polygons.bin: seq=81199 fixups=449077 (0.75%)
  admin_vertices.bin: seq=82545588 fixups=0 (28.36%)
  (entry/cell files: omitted, rebuilt by patch tool)
  Geo cell changes: +49840 -11038 (487024 bytes)
  Admin cell changes: +0 -0

Uncompressed patch: 381414337 bytes (363 MiB)
Compressed patch: 101846661 bytes (97 MiB)
Patch: 98M

real	2m48.869s
user	4m17.178s
sys	0m3.732s
Decompressed patch: 381414337 bytes
Loaded 4980054 string remap entries (pos=39856440)
  strings.bin: full replacement 82573397 bytes
  addr_points.bin: 1375841776 bytes (mapped 85825938/85891830 IDs)
    Applied 19372497 fixups
  street_ways.bin: 233626176 bytes (mapped 19372840/19452480 IDs)
  street_nodes.bin: 1445600360 bytes
    Applied 13395 fixups
  interp_ways.bin: 335424 bytes (mapped 13961/13988 IDs)
  interp_nodes.bin: 725184 bytes
    Applied 449077 fixups
  admin_polygons.bin: 10820736 bytes (mapped 448753/450907 IDs)
  admin_vertices.bin: 291061192 bytes
  Unknown file 4294967291
Reconstructing entry/cell files from derived ID remaps...
  Rebuilt geo_cells + 3 entry files
  Rebuilt admin_cells + admin_entries
Patch applied. Output in /home/michtest/patched-opt

real	1m25.637s
user	1m18.361s
sys	0m7.332s
  addr_entries.bin: FAIL
  addr_points.bin: MATCH
  admin_cells.bin: FAIL
  admin_entries.bin: FAIL
  admin_polygons.bin: MATCH
  admin_vertices.bin: MATCH
  geo_cells.bin: FAIL
  interp_entries.bin: FAIL
  interp_nodes.bin: MATCH
  interp_ways.bin: MATCH
  street_entries.bin: FAIL
  street_nodes.bin: MATCH
  street_ways.bin: MATCH
  strings.bin: MATCH
SOME DIFFER


**97 MiB patch!** Getting close to the target. But entry/cell files still don't match — same issue as before, the rebuilt entries differ from new entries.

"Unknown file 4294967291" = 0xFFFFFFFB (CELL_CHANGES_GEO_MARKER) — the patch tool reads it as a file section. I need to read cell changes BEFORE the section loop, not after. Let me fix the cell changes reading to happen before the main section loop:

Actually, the cell changes are read correctly (inside the reconstruction block), but the "Unknown file" message appears because the section loop encounters the cell change marker first and doesn't recognize it. Let me add cell change handling to the section loop:


Edit

    // Process sections
    while (pos < patch.size()) {
        uint32_t file_id = read_u32();
        if (file_id == 0xFFFFFFFF) break;

    // Read cell changes (may appear before end marker)
    std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;

    // Process sections
    while (pos < patch.size()) {
        uint32_t file_id = read_u32();
        if (file_id == 0xFFFFFFFF) break;

        // Cell change sections
        if (file_id == CELL_CHANGES_GEO_MARKER) {
            uint32_t na = read_u32(), nr = read_u32();
            geo_added.resize(na); geo_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; geo_added[i]=c; }
            for (uint32_t i = 0; i < nr; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; geo_removed[i]=c; }
            std::cerr << "  Geo cell changes: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_ADMIN_MARKER) {
            uint32_t na = read_u32(), nr = read_u32();
            admin_added.resize(na); admin_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; admin_added[i]=c; }
            for (uint32_t i = 0; i < nr; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; admin_removed[i]=c; }
            std::cerr << "  Admin cell changes: +" << na << " -" << nr << std::endl;
            continue;
        }