The file /root/traccar-geocoder/builder/tools/geocoder_diff.cpp has been updated successfully.

        auto ad_rm_d = derive_remap(PatchFileId::ADMIN_POLYGONS,
            read_file(old_dir + "/admin_polygons.bin").size(), admin_stride);

        // Secondary matching: recover additional ID mappings for modified records
        // that didn't byte-match but represent the same entity (e.g. geometry edits)
        {
            size_t way_name_off = (way_stride == 12) ? 8 : 5;

            // Street ways: match by name_id + node_count
            auto old_w2 = read_file(old_dir + "/street_ways.bin");
            remap_field(old_w2, way_stride, way_name_off, str_remap);
            auto new_w2 = read_file(new_dir + "/street_ways.bin");
            auto soft_w = secondary_match_from_merge(
                stored_merges[static_cast<uint32_t>(PatchFileId::STREET_WAYS)],
                old_w2, new_w2, way_stride,
                [way_name_off](const char* rec) -> uint64_t {
                    uint32_t name_id; memcpy(&name_id, rec + way_name_off, 4);
                    uint8_t nc = static_cast<uint8_t>(rec[4]);
                    return ((uint64_t)name_id << 8) | nc;
                });
            for (auto& [o, n] : soft_w) w_rm_d[o] = n;
            std::cerr << "  Secondary way matches: " << soft_w.size() << std::endl;

            // Addr points: match by (street_id, housenumber_id)
            auto old_a2 = read_file(old_dir + "/addr_points.bin");
            remap_addr_points(old_a2, str_remap);
            auto new_a2 = read_file(new_dir + "/addr_points.bin");
            auto soft_a = secondary_match_from_merge(
                stored_merges[static_cast<uint32_t>(PatchFileId::ADDR_POINTS)],
                old_a2, new_a2, 16,
                [](const char* rec) -> uint64_t {
                    uint32_t hn_id, st_id;
                    memcpy(&hn_id, rec + 8, 4); memcpy(&st_id, rec + 12, 4);
                    return ((uint64_t)st_id << 32) | hn_id;
                });
            for (auto& [o, n] : soft_a) a_rm_d[o] = n;
            std::cerr << "  Secondary addr matches: " << soft_a.size() << std::endl;

            // Interp ways: match by (street_id, start_number, end_number)
            size_t ist_off = (interp_stride >= 20) ? 8 : 5;
            auto old_i2 = read_file(old_dir + "/interp_ways.bin");
            remap_field(old_i2, interp_stride, ist_off, str_remap);
            auto new_i2 = read_file(new_dir + "/interp_ways.bin");
            auto soft_i = secondary_match_from_merge(
                stored_merges[static_cast<uint32_t>(PatchFileId::INTERP_WAYS)],
                old_i2, new_i2, interp_stride,
                [ist_off](const char* rec) -> uint64_t {
                    uint32_t street_id, start;
                    memcpy(&street_id, rec + ist_off, 4);
                    memcpy(&start, rec + ist_off + 4, 4);
                    return ((uint64_t)street_id << 32) | start;
                });
            for (auto& [o, n] : soft_i) i_rm_d[o] = n;
            std::cerr << "  Secondary interp matches: " << soft_i.size() << std::endl;

            // Admin polygons: match by (name_id, admin_level, country_code)
            auto old_ap2 = read_file(old_dir + "/admin_polygons.bin");
            remap_field(old_ap2, admin_stride, 8, str_remap);
            auto new_ap2 = read_file(new_dir + "/admin_polygons.bin");
            auto soft_ad = secondary_match_from_merge(
                stored_merges[static_cast<uint32_t>(PatchFileId::ADMIN_POLYGONS)],
                old_ap2, new_ap2, admin_stride,
                [](const char* rec) -> uint64_t {
                    uint32_t name_id; memcpy(&name_id, rec + 8, 4);
                    uint8_t level = static_cast<uint8_t>(rec[12]);
                    uint16_t cc; memcpy(&cc, rec + 20, 2);
                    return ((uint64_t)name_id << 24) | ((uint64_t)level << 16) | cc;
                });
            for (auto& [o, n] : soft_ad) ad_rm_d[o] = n;
            std::cerr << "  Secondary admin matches: " << soft_ad.size() << std::endl;
        }

        // Rebuild entries using derived remaps (same as patch tool)

// geocoder-diff v3: Fully custom patch format using merge-sequence encoding.
//
// For each data file: walk old (string-remapped) and new in parallel,
// match records by content, emit MATCH/INSERT/DELETE operations.
// For entry/cell files: include changed cell entry data directly.
// geo_cells rebuilt by patch tool from entries (not delta-patched).
//
// Patch format: custom binary, zstd-compressed as a whole for transport.
//
// Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>

#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <functional>
#include <iomanip>
#include <iostream>
#include <string>
#include <unistd.h>
#include <unordered_map>
#include <unordered_set>
#include <vector>

#include "patch_format.h"

// --- Merge sequence encoding ---
// Walk two sorted arrays (old remapped, new), matching by record bytes.
// Emit: MATCH(n) = copy n records from old, INSERT(n,data) = add n new records,
//       DELETE(n) = skip n old records.

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

struct MergeSequence {
    std::vector<char> data; // serialized ops

    void add_match(uint32_t count) {
        uint8_t op = OP_MATCH_RUN;
        data.insert(data.end(), (char*)&op, (char*)&op + 1);
        data.insert(data.end(), (char*)&count, (char*)&count + 4);
    }
    void add_delete(uint32_t count) {
        uint8_t op = OP_DELETE_RUN;
        data.insert(data.end(), (char*)&op, (char*)&op + 1);
        data.insert(data.end(), (char*)&count, (char*)&count + 4);
    }
    void add_insert(const char* records, uint32_t count, size_t stride) {
        uint8_t op = OP_INSERT_RUN;
        data.insert(data.end(), (char*)&op, (char*)&op + 1);
        data.insert(data.end(), (char*)&count, (char*)&count + 4);
        data.insert(data.end(), records, records + count * stride);
    }
};

// Build merge sequence for a data file.
// old_data has string remap already applied.
// Records are compared by byte equality (stride bytes).
static MergeSequence build_merge_seq(
    const std::vector<char>& old_data, const std::vector<char>& new_data,
    size_t stride)
{
    size_t old_n = old_data.size() / stride;
    size_t new_n = new_data.size() / stride;
    size_t oi = 0, ni = 0;
    MergeSequence seq;
    uint32_t match_run = 0, del_run = 0;

    auto flush_match = [&]() { if (match_run > 0) { seq.add_match(match_run); match_run = 0; } };
    auto flush_del = [&]() { if (del_run > 0) { seq.add_delete(del_run); del_run = 0; } };

    // For small strides (<=8), records aren't unique enough for hash matching.
    // Use simple sequential scan instead.
    bool use_hash = (stride > 8);

    // Pre-build hash index of new records for fast mismatch resolution
    auto record_hash = [&](const char* p, size_t s) -> uint64_t {
        uint64_t h = 14695981039346656037ULL;
        for (size_t i = 0; i < s; i++) { h ^= (uint8_t)p[i]; h *= 1099511628211ULL; }
        return h;
    };
    std::unordered_multimap<uint64_t, uint32_t> new_hash;
    if (use_hash) {
        new_hash.reserve(new_n);
        for (uint32_t i = 0; i < new_n; i++)
            new_hash.emplace(record_hash(new_data.data() + i * stride, stride), i);
    }

    while (oi < old_n && ni < new_n) {
        const char* op = old_data.data() + oi * stride;
        const char* np = new_data.data() + ni * stride;

        if (memcmp(op, np, stride) == 0) {
            flush_del();
            match_run++;
            oi++; ni++;
        } else if (use_hash) {
            // Hash-based mismatch resolution (for records with unique content)
            uint64_t oh = record_hash(op, stride);
            auto range = new_hash.equal_range(oh);
            uint32_t best_ni = UINT32_MAX;
            for (auto it = range.first; it != range.second; ++it) {
                if (it->second >= ni && it->second < ni + 10000 &&
                    memcmp(op, new_data.data() + it->second * stride, stride) == 0) {
                    if (it->second < best_ni) best_ni = it->second;
                }
            }
            if (best_ni != UINT32_MAX && best_ni > ni) {
                flush_match(); flush_del();
                seq.add_insert(new_data.data() + ni * stride, best_ni - ni, stride);
                ni = best_ni;
            } else if (best_ni == ni) {
                flush_del(); match_run++; oi++; ni++;
            } else {
                flush_match(); del_run++; oi++;
            }
        } else {
            // Sequential scan (for coordinate files where records aren't unique)
            size_t lookahead = std::min((size_t)200, std::min(old_n - oi, new_n - ni));
            bool found = false;
            for (size_t k = 1; k <= lookahead; k++) {
                if (ni + k < new_n && memcmp(op, new_data.data() + (ni + k) * stride, stride) == 0) {
                    flush_match(); flush_del();
                    seq.add_insert(new_data.data() + ni * stride, k, stride);
                    ni += k; found = true; break;
                }
                if (oi + k < old_n && memcmp(np, old_data.data() + (oi + k) * stride, stride) == 0) {
                    flush_match(); del_run += k; oi += k; found = true; break;
                }
            }
            if (!found) { flush_match(); del_run++; oi++; flush_del(); seq.add_insert(np, 1, stride); ni++; }
        }
    }

    flush_match(); flush_del();
    // Remaining old records: deletions
    if (oi < old_n) seq.add_delete(old_n - oi);
    // Remaining new records: insertions
    if (ni < new_n) seq.add_insert(new_data.data() + ni * stride, new_n - ni, stride);

    return seq;
}

// --- String remap ---

static std::unordered_map<uint32_t, uint32_t> build_string_remap(
    const std::vector<char>& old_pool, const std::vector<char>& new_pool)
{
    std::unordered_map<std::string, uint32_t> new_idx;
    size_t pos = 0;
    while (pos < new_pool.size()) {
        const char* s = new_pool.data() + pos;
        size_t len = strlen(s);
        new_idx[std::string(s, len)] = static_cast<uint32_t>(pos);
        pos += len + 1;
    }
    std::unordered_map<uint32_t, uint32_t> remap;
    pos = 0;
    while (pos < old_pool.size()) {
        const char* s = old_pool.data() + pos;
        size_t len = strlen(s);
        auto it = new_idx.find(std::string(s, len));
        if (it != new_idx.end())
            remap[static_cast<uint32_t>(pos)] = it->second;
        pos += len + 1;
    }
    return remap;
}

// --- Remap + fixup helpers ---

static void remap_addr_points(std::vector<char>& data, const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + 16 <= data.size(); i += 16)
        for (size_t off : {8, 12}) {
            uint32_t v; memcpy(&v, data.data() + i + off, 4);
            auto it = rm.find(v); if (it != rm.end()) memcpy(data.data() + i + off, &it->second, 4);
        }
}
static void remap_field(std::vector<char>& data, size_t stride, size_t field_off,
                         const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + stride <= data.size(); i += stride) {
        uint32_t v; memcpy(&v, data.data() + i + field_off, 4);
        auto it = rm.find(v); if (it != rm.end()) memcpy(data.data() + i + field_off, &it->second, 4);
    }
}

// Content matching for ways (by name + nodes, ignoring node_offset)
static uint64_t fnv_mix(uint64_t h, uint64_t v) { h ^= v; h *= 1099511628211ULL; return h; }

static void fixup_way_offsets(std::vector<char>& old_ways, const std::vector<char>& old_nodes,
                                const std::vector<char>& new_ways, const std::vector<char>& new_nodes,
                                size_t stride) {
    size_t name_off = (stride == 12) ? 8 : 5;
    size_t old_n = old_ways.size() / stride, new_n = new_ways.size() / stride;
    size_t old_nc = old_nodes.size() / 8, new_nc = new_nodes.size() / 8;

    auto way_hash = [&](const char* w, const char* nodes, size_t max_n) -> uint64_t {
        uint32_t node_offset, name_id; uint8_t node_count;
        memcpy(&node_offset, w, 4); node_count = (uint8_t)w[4]; memcpy(&name_id, w + name_off, 4);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, node_count);
        for (uint8_t j = 0; j < node_count && (node_offset + j) < max_n; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4);
            memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(way_hash(new_ways.data() + i * stride, new_nodes.data(), new_nc), i);

    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = way_hash(old_ways.data() + i * stride, old_nodes.data(), old_nc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            // Fix node_offset to match new way
            uint32_t new_node_off;
            memcpy(&new_node_off, new_ways.data() + it->second * stride, 4);
            memcpy(old_ways.data() + i * stride, &new_node_off, 4);
            new_map.erase(it);
        }
    }
}

// Same for admin polygons (vertex_offset)
static void fixup_admin_offsets(std::vector<char>& old_polys, const std::vector<char>& old_verts,
                                  const std::vector<char>& new_polys, const std::vector<char>& new_verts,
                                  size_t stride) {
    size_t old_n = old_polys.size() / stride, new_n = new_polys.size() / stride;
    size_t old_vc = old_verts.size() / 8, new_vc = new_verts.size() / 8;

    auto poly_hash = [&](const char* p, const char* verts, size_t max_v) -> uint64_t {
        uint32_t vert_offset, vert_count, name_id;
        memcpy(&vert_offset, p, 4); memcpy(&vert_count, p + 4, 4); memcpy(&name_id, p + 8, 4);
        uint8_t level = (uint8_t)p[12]; uint16_t cc; memcpy(&cc, p + 20, 2);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, level); h = fnv_mix(h, cc); h = fnv_mix(h, vert_count);
        for (uint32_t j = 0; j < std::min(vert_count, 10u) && (vert_offset + j) < max_v; j++) {
            float lat, lng;
            memcpy(&lat, verts + (vert_offset + j) * 8, 4); memcpy(&lng, verts + (vert_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(poly_hash(new_polys.data() + i * stride, new_verts.data(), new_vc), i);

    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = poly_hash(old_polys.data() + i * stride, old_verts.data(), old_vc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            uint32_t new_vert_off;
            memcpy(&new_vert_off, new_polys.data() + it->second * stride, 4);
            memcpy(old_polys.data() + i * stride, &new_vert_off, 4);
            new_map.erase(it);
        }
    }
}

// Same for interp ways
static void fixup_interp_offsets(std::vector<char>& old_data, const std::vector<char>& old_nodes,
                                   const std::vector<char>& new_data, const std::vector<char>& new_nodes,
                                   size_t stride) {
    size_t street_off = (stride >= 20) ? 8 : 5;
    size_t old_n = old_data.size() / stride, new_n = new_data.size() / stride;
    size_t old_nc = old_nodes.size() / 8, new_nc = new_nodes.size() / 8;

    auto ihash = [&](const char* p, const char* nodes, size_t max_n) -> uint64_t {
        uint32_t node_offset, street_id, start, end; uint8_t count, itype;
        memcpy(&node_offset, p, 4); count = (uint8_t)p[4];
        memcpy(&street_id, p + street_off, 4); memcpy(&start, p + street_off + 4, 4);
        memcpy(&end, p + street_off + 8, 4);
        itype = (street_off + 12 < stride) ? (uint8_t)p[street_off + 12] : 0;
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, street_id); h = fnv_mix(h, start); h = fnv_mix(h, end);
        h = fnv_mix(h, itype); h = fnv_mix(h, count);
        for (uint8_t j = 0; j < count && (node_offset + j) < max_n; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4); memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(ihash(new_data.data() + i * stride, new_nodes.data(), new_nc), i);
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = ihash(old_data.data() + i * stride, old_nodes.data(), old_nc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            uint32_t new_off; memcpy(&new_off, new_data.data() + it->second * stride, 4);
            memcpy(old_data.data() + i * stride, &new_off, 4);
            new_map.erase(it);
        }
    }
}

// --- Secondary matching for modified records ---
// After the merge sequence is built, match DELETE'd and INSERT'd records by a
// relaxed key to recover ID mappings for records that changed (e.g. geometry edit)
// but represent the same logical entity. Only matches when the key group has
// equal counts on both sides to avoid mismatches.
template<typename KeyFn>
static std::unordered_map<uint32_t, uint32_t> secondary_match_from_merge(
    const MergeSequence& seq,
    const std::vector<char>& old_data,
    const std::vector<char>& new_data,
    size_t stride,
    KeyFn key_fn)
{
    std::vector<uint32_t> del_indices, ins_indices;
    size_t pos = 0, old_rec = 0, new_rec = 0;
    while (pos < seq.data.size()) {
        uint8_t op = static_cast<uint8_t>(seq.data[pos]); pos++;
        uint32_t count; memcpy(&count, seq.data.data() + pos, 4); pos += 4;
        if (op == OP_MATCH_RUN) {
            old_rec += count; new_rec += count;
        } else if (op == OP_INSERT_RUN) {
            for (uint32_t k = 0; k < count; k++)
                ins_indices.push_back(static_cast<uint32_t>(new_rec + k));
            pos += count * stride;
            new_rec += count;
        } else if (op == OP_DELETE_RUN) {
            for (uint32_t k = 0; k < count; k++)
                del_indices.push_back(static_cast<uint32_t>(old_rec + k));
            old_rec += count;
        }
    }

    // Group by key
    std::unordered_map<uint64_t, std::vector<uint32_t>> del_by_key, ins_by_key;
    for (uint32_t di : del_indices) {
        if ((size_t)(di + 1) * stride > old_data.size()) continue;
        del_by_key[key_fn(old_data.data() + di * stride)].push_back(di);
    }
    for (uint32_t ii : ins_indices) {
        if ((size_t)(ii + 1) * stride > new_data.size()) continue;
        ins_by_key[key_fn(new_data.data() + ii * stride)].push_back(ii);
    }

    // Match only when equal counts per key (safe pairwise match in index order)
    std::unordered_map<uint32_t, uint32_t> remap;
    for (auto& [key, del_vec] : del_by_key) {
        auto it = ins_by_key.find(key);
        if (it == ins_by_key.end()) continue;
        auto& ins_vec = it->second;
        if (del_vec.size() == ins_vec.size()) {
            std::sort(del_vec.begin(), del_vec.end());
            std::sort(ins_vec.begin(), ins_vec.end());
            for (size_t i = 0; i < del_vec.size(); i++)
                remap[del_vec[i]] = ins_vec[i];
        }
    }
    return remap;
}

// --- Derive ID remap from merge sequence (same logic as patch tool) ---
static std::vector<uint32_t> derive_id_remap_from_merge(
    const MergeSequence& seq, size_t old_count, size_t stride)
{
    std::vector<uint32_t> id_map(old_count, 0xFFFFFFFF);
    size_t pos = 0, old_rec = 0, new_rec = 0;
    while (pos < seq.data.size()) {
        uint8_t op = static_cast<uint8_t>(seq.data[pos]); pos++;
        uint32_t count; memcpy(&count, seq.data.data() + pos, 4); pos += 4;
        if (op == OP_MATCH_RUN) {
            for (uint32_t k = 0; k < count; k++)
                if (old_rec + k < id_map.size())
                    id_map[old_rec + k] = static_cast<uint32_t>(new_rec + k);
            old_rec += count; new_rec += count;
        } else if (op == OP_INSERT_RUN) {
            pos += count * stride;
            new_rec += count;
        } else if (op == OP_DELETE_RUN) {
            old_rec += count;
        }
    }
    return id_map;
}

// --- Write helpers ---
static void wval(std::vector<char>& buf, const void* data, size_t size) {
    buf.insert(buf.end(), (const char*)data, (const char*)data + size);
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>" << std::endl;
        return 1;
    }
    std::string old_dir = argv[1], new_dir = argv[2], patch_path = argv[4];
    std::string tmpdir = "/tmp/geocoder-diff-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    // Build string remap
    std::cerr << "Building string remap..." << std::endl;
    auto old_strings = read_file(old_dir + "/strings.bin");
    auto new_strings = read_file(new_dir + "/strings.bin");
    auto str_remap = build_string_remap(old_strings, new_strings);

    // Detect strides
    auto detect = [](const std::string& path, std::vector<size_t> cs) -> size_t {
        auto d = read_file(path);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect(old_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect(old_dir + "/interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect(old_dir + "/admin_polygons.bin", {24, 20, 19});

    // Build patch data (uncompressed, will be zstd-compressed at the end)
    std::vector<char> patch;
    patch.insert(patch.end(), GCPATCH_MAGIC, GCPATCH_MAGIC + 8);
    uint32_t ver = 2, flags = 0; // version 2 = custom format
    wval(patch, &ver, 4); wval(patch, &flags, 4);

    // --- Section: String remap ---
    // When string-level diff is present, the remap is derived by the patch tool.
    // Only include explicit remap if string diff is NOT used.
    // For now, always include empty remap (string diff handles everything).
    {
        uint32_t marker = 0xFFFFFFFE, count = 0;
        wval(patch, &marker, 4); wval(patch, &count, 4);
        std::cerr << "  String remap: derived from string diff (0 explicit entries)" << std::endl;
    }

    // --- Section: Per-file merge sequences ---
    // Track fixups per file (applied before merge, included in patch for patch tool)
    std::unordered_map<uint32_t, std::vector<std::pair<uint32_t,uint32_t>>> file_fixups;

    // Stored merge sequences for ID remap derivation
    std::unordered_map<uint32_t, MergeSequence> stored_merges;

    auto write_merge = [&](PatchFileId id, const std::string& name,
                             const std::vector<char>& old_data, const std::vector<char>& new_data,
                             size_t stride) {
        auto seq = build_merge_seq(old_data, new_data, stride);
        stored_merges[static_cast<uint32_t>(id)] = seq; // store for entry rebuild
        uint32_t fid = static_cast<uint32_t>(id);
        uint64_t os = old_data.size(), ns = new_data.size(), ss = seq.data.size();

        // Check for fixups
        auto fit = file_fixups.find(fid);
        uint32_t n_fixups = fit != file_fixups.end() ? static_cast<uint32_t>(fit->second.size()) : 0;

        wval(patch, &fid, 4);
        uint32_t st = static_cast<uint32_t>(stride);
        wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8);
        // Fixup count + delta-encoded data before merge sequence
        wval(patch, &n_fixups, 4);
        if (n_fixups > 0) {
            std::vector<char> delta_buf;
            uint32_t prev_idx = 0, prev_val = 0;
            for (auto& [idx, val] : fit->second) {
                write_varint(delta_buf, idx - prev_idx);
                write_varint(delta_buf, val - prev_val);
                prev_idx = idx;
                prev_val = val;
            }
            uint32_t delta_size = static_cast<uint32_t>(delta_buf.size());
            wval(patch, &delta_size, 4);
            patch.insert(patch.end(), delta_buf.begin(), delta_buf.end());
        }
        wval(patch, &ss, 8);
        patch.insert(patch.end(), seq.data.begin(), seq.data.end());
        std::cerr << "  " << name << ": seq=" << ss << " fixups=" << n_fixups
                  << " (" << std::fixed << std::setprecision(2)
                  << (ns > 0 ? ss * 100.0 / ns : 0) << "%)" << std::endl;
    };

    // strings.bin: string-level diff (both pools are sorted alphabetically)
    // Walk both pools, emit new strings to insert and indices of deleted strings.
    // The patch tool merges new strings into old pool alphabetically.
    {
        // Parse both pools into string lists
        std::vector<std::string> old_strs, new_strs;
        {
            size_t p = 0;
            while (p < old_strings.size()) {
                const char* s = old_strings.data() + p;
                old_strs.push_back(s);
                p += strlen(s) + 1;
            }
        }
        {
            size_t p = 0;
            while (p < new_strings.size()) {
                const char* s = new_strings.data() + p;
                new_strs.push_back(s);
                p += strlen(s) + 1;
            }
        }

        // Merge-walk to find insertions and deletions
        std::vector<std::string> added_strings;
        std::vector<uint32_t> deleted_indices;
        size_t oi = 0, ni = 0;
        while (oi < old_strs.size() && ni < new_strs.size()) {
            int c = old_strs[oi].compare(new_strs[ni]);
            if (c == 0) { oi++; ni++; } // same string
            else if (c < 0) { deleted_indices.push_back(oi); oi++; } // old only → deleted
            else { added_strings.push_back(new_strs[ni]); ni++; } // new only → added
        }
        while (oi < old_strs.size()) { deleted_indices.push_back(oi); oi++; }
        while (ni < new_strs.size()) { added_strings.push_back(new_strs[ni]); ni++; }

        // Serialize: marker, n_added, n_deleted, added_string_data, deleted_indices
        uint32_t str_diff_marker = 0xFFFFFFF7;
        uint32_t n_added = static_cast<uint32_t>(added_strings.size());
        uint32_t n_deleted = static_cast<uint32_t>(deleted_indices.size());
        wval(patch, &str_diff_marker, 4);
        wval(patch, &n_added, 4);
        wval(patch, &n_deleted, 4);
        for (auto& s : added_strings) {
            patch.insert(patch.end(), s.begin(), s.end());
            patch.push_back('\0');
        }
        for (auto idx : deleted_indices) wval(patch, &idx, 4);

        size_t str_diff_size = 12 + (n_added > 0 ? new_strings.size() - old_strings.size() + n_deleted * 16 : 0);
        std::cerr << "  strings.bin: +" << n_added << " -" << n_deleted << " strings" << std::endl;
    }

    // addr_points.bin
    {
        auto old_data = read_file(old_dir + "/addr_points.bin");
        remap_addr_points(old_data, str_remap);
        auto new_data = read_file(new_dir + "/addr_points.bin");
        write_merge(PatchFileId::ADDR_POINTS, "addr_points.bin", old_data, new_data, 16);
    }

    // street_ways.bin (string remap + offset fixup for node_offset)
    {
        auto old_w = read_file(old_dir + "/street_ways.bin");
        remap_field(old_w, way_stride, way_stride == 12 ? 8 : 5, str_remap);
        auto new_w = read_file(new_dir + "/street_ways.bin");
        auto old_n = read_file(old_dir + "/street_nodes.bin");
        auto new_n = read_file(new_dir + "/street_nodes.bin");
        // Record fixups: (record_index, new_node_offset)
        auto& fixups = file_fixups[static_cast<uint32_t>(PatchFileId::STREET_WAYS)];
        // Save old offsets, apply fixups, then record changes
        size_t wn = old_w.size() / way_stride;
        std::vector<uint32_t> old_offsets(wn);
        for (size_t i = 0; i < wn; i++) memcpy(&old_offsets[i], old_w.data() + i * way_stride, 4);
        fixup_way_offsets(old_w, old_n, new_w, new_n, way_stride);
        for (size_t i = 0; i < wn; i++) {
            uint32_t new_off; memcpy(&new_off, old_w.data() + i * way_stride, 4);
            if (new_off != old_offsets[i]) fixups.push_back({static_cast<uint32_t>(i), new_off});
        }
        write_merge(PatchFileId::STREET_WAYS, "street_ways.bin", old_w, new_w, way_stride);
    }

    // street_nodes.bin — derive from street_ways merge (parent-aware)
    // Instead of merging 8-byte records independently (fails at scale),
    // use the way merge to determine which node blocks to COPY vs INSERT.
    {
        auto& way_seq = stored_merges[static_cast<uint32_t>(PatchFileId::STREET_WAYS)];
        // Read ORIGINAL old ways (before fixup) to get correct old node offsets
        auto old_w_orig = read_file(old_dir + "/street_ways.bin");
        auto new_w = read_file(new_dir + "/street_ways.bin");
        auto old_n = read_file(old_dir + "/street_nodes.bin");
        auto new_n = read_file(new_dir + "/street_nodes.bin");
        // For MATCH: use original old node_offsets (not the fixup'd ones)
        // For INSERT: read from new node file using new way's node_offset
        auto& old_w = old_w_orig; // original offsets

        // Walk the way merge to build a node merge
        MergeSequence node_seq;
        size_t way_oi = 0, way_ni = 0;
        size_t wpos = 0;
        while (wpos < way_seq.data.size()) {
            uint8_t op = static_cast<uint8_t>(way_seq.data[wpos]); wpos++;
            uint32_t count; memcpy(&count, way_seq.data.data() + wpos, 4); wpos += 4;

            if (op == OP_MATCH_RUN) {
                // Matched ways: verify node blocks are actually identical
                for (uint32_t k = 0; k < count; k++) {
                    uint32_t old_no; memcpy(&old_no, old_w.data() + (way_oi + k) * way_stride, 4);
                    uint8_t nc = static_cast<uint8_t>(old_w[(way_oi + k) * way_stride + 4]);
                    // Get new way's node_offset (from new_w at way_ni + k)
                    uint32_t new_no; memcpy(&new_no, new_w.data() + (way_ni + k) * way_stride, 4);
                    if (nc > 0) {
                        // Check if node data is actually identical
                        bool nodes_match = (old_no + nc) * 8 <= old_n.size() &&
                                          (new_no + nc) * 8 <= new_n.size() &&
                                          memcmp(old_n.data() + old_no * 8,
                                                 new_n.data() + new_no * 8, nc * 8) == 0;
                        if (nodes_match) {
                            node_seq.add_match(nc);
                        } else {
                            // Way matched but nodes differ → INSERT new nodes
                            node_seq.add_delete(nc);
                            node_seq.add_insert(new_n.data() + new_no * 8, nc, 8);
                        }
                    }
                }
                way_oi += count; way_ni += count;
            } else if (op == OP_INSERT_RUN) {
                // New ways: their nodes are new data → INSERT from new
                for (uint32_t k = 0; k < count; k++) {
                    const char* new_way = way_seq.data.data() + wpos + k * way_stride;
                    uint32_t new_node_off; memcpy(&new_node_off, new_way, 4);
                    uint8_t nc = static_cast<uint8_t>(new_way[4]);
                    if (nc > 0 && (new_node_off + nc) * 8 <= new_n.size())
                        node_seq.add_insert(new_n.data() + new_node_off * 8, nc, 8);
                }
                wpos += count * way_stride;
                way_ni += count;
            } else if (op == OP_DELETE_RUN) {
                // Deleted ways: skip their nodes
                for (uint32_t k = 0; k < count; k++) {
                    uint8_t nc = static_cast<uint8_t>(old_w[(way_oi + k) * way_stride + 4]);
                    if (nc > 0) node_seq.add_delete(nc);
                }
                way_oi += count;
            }
        }

        // Write the derived node merge
        uint32_t fid = static_cast<uint32_t>(PatchFileId::STREET_NODES);
        uint64_t os = old_n.size(), ns = new_n.size(), ss = node_seq.data.size();
        uint32_t st = 8, n_fixups = 0;
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8);
        wval(patch, &n_fixups, 4); wval(patch, &ss, 8);
        patch.insert(patch.end(), node_seq.data.begin(), node_seq.data.end());
        std::cerr << "  street_nodes.bin: parent-derived seq=" << ss << " bytes ("
                  << std::fixed << std::setprecision(2) << (ns > 0 ? ss * 100.0 / ns : 0) << "%)" << std::endl;
    }

    // interp_ways.bin
    {
        auto old_data = read_file(old_dir + "/interp_ways.bin");
        auto new_idata = read_file(new_dir + "/interp_ways.bin");
        // Zero padding bytes for deterministic comparison
        if (interp_stride == 24) {
            for (size_t i = 0; i + interp_stride <= old_data.size(); i += interp_stride) {
                memset(old_data.data() + i + 5, 0, 3);  // after node_count
                memset(old_data.data() + i + 21, 0, 3);  // after interpolation
            }
            for (size_t i = 0; i + interp_stride <= new_idata.size(); i += interp_stride) {
                memset(new_idata.data() + i + 5, 0, 3);
                memset(new_idata.data() + i + 21, 0, 3);
            }
        }
        remap_field(old_data, interp_stride, interp_stride >= 20 ? 8 : 5, str_remap);
        auto old_n = read_file(old_dir + "/interp_nodes.bin");
        auto& new_data = new_idata; // use padding-zeroed version
        auto new_n = read_file(new_dir + "/interp_nodes.bin");
        auto& fixups = file_fixups[static_cast<uint32_t>(PatchFileId::INTERP_WAYS)];
        size_t in_count = old_data.size() / interp_stride;
        std::vector<uint32_t> old_offsets(in_count);
        for (size_t i = 0; i < in_count; i++) memcpy(&old_offsets[i], old_data.data() + i * interp_stride, 4);
        fixup_interp_offsets(old_data, old_n, new_data, new_n, interp_stride);
        for (size_t i = 0; i < in_count; i++) {
            uint32_t new_off; memcpy(&new_off, old_data.data() + i * interp_stride, 4);
            if (new_off != old_offsets[i]) fixups.push_back({static_cast<uint32_t>(i), new_off});
        }
        write_merge(PatchFileId::INTERP_WAYS, "interp_ways.bin", old_data, new_data, interp_stride);
    }

    // interp_nodes.bin — derive from interp_ways merge (parent-aware)
    {
        auto& iw_seq = stored_merges[static_cast<uint32_t>(PatchFileId::INTERP_WAYS)];
        auto old_iw_data = read_file(old_dir + "/interp_ways.bin");
        auto new_iw_data = read_file(new_dir + "/interp_ways.bin");
        auto& old_iw = old_iw_data;
        auto& new_iw = new_iw_data;
        auto old_in = read_file(old_dir + "/interp_nodes.bin");
        auto new_in = read_file(new_dir + "/interp_nodes.bin");
        MergeSequence in_seq;
        size_t iw_oi = 0, iw_ni = 0;
        size_t iwpos = 0;
        while (iwpos < iw_seq.data.size()) {
            uint8_t op = static_cast<uint8_t>(iw_seq.data[iwpos]); iwpos++;
            uint32_t count; memcpy(&count, iw_seq.data.data() + iwpos, 4); iwpos += 4;
            if (op == OP_MATCH_RUN) {
                for (uint32_t k = 0; k < count; k++) {
                    uint32_t old_no; memcpy(&old_no, old_iw.data() + (iw_oi + k) * interp_stride, 4);
                    uint8_t nc = static_cast<uint8_t>(old_iw[(iw_oi + k) * interp_stride + 4]);
                    uint32_t new_no; memcpy(&new_no, new_iw.data() + (iw_ni + k) * interp_stride, 4);
                    if (nc > 0) {
                        bool match = (old_no + nc) * 8 <= old_in.size() &&
                                    (new_no + nc) * 8 <= new_in.size() &&
                                    memcmp(old_in.data() + old_no * 8, new_in.data() + new_no * 8, nc * 8) == 0;
                        if (match) in_seq.add_match(nc);
                        else { in_seq.add_delete(nc); in_seq.add_insert(new_in.data() + new_no * 8, nc, 8); }
                    }
                }
                iw_oi += count; iw_ni += count;
            } else if (op == OP_INSERT_RUN) {
                for (uint32_t k = 0; k < count; k++) {
                    const char* rec = iw_seq.data.data() + iwpos + k * interp_stride;
                    uint32_t no; memcpy(&no, rec, 4);
                    uint8_t nc = static_cast<uint8_t>(rec[4]);
                    if (nc > 0 && (no + nc) * 8 <= new_in.size())
                        in_seq.add_insert(new_in.data() + no * 8, nc, 8);
                }
                iwpos += count * interp_stride;
                iw_ni += count;
            } else if (op == OP_DELETE_RUN) {
                for (uint32_t k = 0; k < count; k++) {
                    uint8_t nc = static_cast<uint8_t>(old_iw[(iw_oi + k) * interp_stride + 4]);
                    if (nc > 0) in_seq.add_delete(nc);
                }
                iw_oi += count;
            }
        }
        uint32_t fid = static_cast<uint32_t>(PatchFileId::INTERP_NODES);
        uint64_t os = old_in.size(), ns = new_in.size(), ss = in_seq.data.size();
        uint32_t st = 8, n_fixups = 0;
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8);
        wval(patch, &n_fixups, 4); wval(patch, &ss, 8);
        patch.insert(patch.end(), in_seq.data.begin(), in_seq.data.end());
        std::cerr << "  interp_nodes.bin: parent-derived seq=" << ss << " bytes" << std::endl;
    }

    // admin_polygons.bin
    {
        auto old_data = read_file(old_dir + "/admin_polygons.bin");
        auto new_data = read_file(new_dir + "/admin_polygons.bin");
        // Zero padding bytes (offset 13-15 between admin_level and area) for deterministic comparison
        if (admin_stride == 24) {
            for (size_t i = 0; i + admin_stride <= old_data.size(); i += admin_stride)
                memset(old_data.data() + i + 13, 0, 3);
            for (size_t i = 0; i + admin_stride <= new_data.size(); i += admin_stride)
                memset(new_data.data() + i + 13, 0, 3);
        }
        remap_field(old_data, admin_stride, 8, str_remap);
        auto old_v = read_file(old_dir + "/admin_vertices.bin");
        auto new_v = read_file(new_dir + "/admin_vertices.bin");
        auto& fixups = file_fixups[static_cast<uint32_t>(PatchFileId::ADMIN_POLYGONS)];
        size_t an = old_data.size() / admin_stride;
        std::vector<uint32_t> old_offsets(an);
        for (size_t i = 0; i < an; i++) memcpy(&old_offsets[i], old_data.data() + i * admin_stride, 4);
        fixup_admin_offsets(old_data, old_v, new_data, new_v, admin_stride);
        for (size_t i = 0; i < an; i++) {
            uint32_t new_off; memcpy(&new_off, old_data.data() + i * admin_stride, 4);
            if (new_off != old_offsets[i]) fixups.push_back({static_cast<uint32_t>(i), new_off});
        }
        write_merge(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin", old_data, new_data, admin_stride);
    }

    // admin_vertices.bin — derive from admin_polygons merge (parent-aware)
    {
        auto& ap_seq = stored_merges[static_cast<uint32_t>(PatchFileId::ADMIN_POLYGONS)];
        auto old_ap = read_file(old_dir + "/admin_polygons.bin");
        auto new_ap = read_file(new_dir + "/admin_polygons.bin");
        auto old_av = read_file(old_dir + "/admin_vertices.bin");
        auto new_av = read_file(new_dir + "/admin_vertices.bin");
        MergeSequence av_seq;
        size_t ap_oi = 0, ap_ni = 0;
        size_t appos = 0;
        while (appos < ap_seq.data.size()) {
            uint8_t op = static_cast<uint8_t>(ap_seq.data[appos]); appos++;
            uint32_t count; memcpy(&count, ap_seq.data.data() + appos, 4); appos += 4;
            if (op == OP_MATCH_RUN) {
                for (uint32_t k = 0; k < count; k++) {
                    uint32_t old_vo, vc;
                    memcpy(&old_vo, old_ap.data() + (ap_oi + k) * admin_stride, 4);
                    memcpy(&vc, old_ap.data() + (ap_oi + k) * admin_stride + 4, 4);
                    uint32_t new_vo;
                    memcpy(&new_vo, new_ap.data() + (ap_ni + k) * admin_stride, 4);
                    if (vc > 0) {
                        bool verts_match = (old_vo + vc) * 8 <= old_av.size() &&
                                          (new_vo + vc) * 8 <= new_av.size() &&
                                          memcmp(old_av.data() + old_vo * 8,
                                                 new_av.data() + new_vo * 8, vc * 8) == 0;
                        if (verts_match) {
                            av_seq.add_match(vc);
                        } else {
                            av_seq.add_delete(vc);
                            av_seq.add_insert(new_av.data() + new_vo * 8, vc, 8);
                        }
                    }
                }
                ap_oi += count; ap_ni += count;
            } else if (op == OP_INSERT_RUN) {
                for (uint32_t k = 0; k < count; k++) {
                    const char* rec = ap_seq.data.data() + appos + k * admin_stride;
                    uint32_t vo, vc; memcpy(&vo, rec, 4); memcpy(&vc, rec + 4, 4);
                    if (vc > 0 && (vo + vc) * 8 <= new_av.size())
                        av_seq.add_insert(new_av.data() + vo * 8, vc, 8);
                }
                appos += count * admin_stride;
                ap_ni += count;
            } else if (op == OP_DELETE_RUN) {
                for (uint32_t k = 0; k < count; k++) {
                    uint32_t vc; memcpy(&vc, old_ap.data() + (ap_oi + k) * admin_stride + 4, 4);
                    if (vc > 0) av_seq.add_delete(vc);
                }
                ap_oi += count;
            }
        }
        uint32_t fid = static_cast<uint32_t>(PatchFileId::ADMIN_VERTICES);
        uint64_t os = old_av.size(), ns = new_av.size(), ss = av_seq.data.size();
        uint32_t st = 8, n_fixups = 0;
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8);
        wval(patch, &n_fixups, 4); wval(patch, &ss, 8);
        patch.insert(patch.end(), av_seq.data.begin(), av_seq.data.end());
        std::cerr << "  admin_vertices.bin: parent-derived seq=" << ss << " bytes ("
                  << std::fixed << std::setprecision(2) << (ns > 0 ? ss * 100.0 / ns : 0) << "%)" << std::endl;
    }

    // --- Section: Entry/cell files as full new data ---
    // These are small relative to data files and rebuilt by patch tool isn't reliable.
    // Include them as full replacement. zstd transport compression handles the rest.
    // Entry/cell files: NOT included.
    // The patch tool reconstructs them from the derived ID remaps
    // (extracted implicitly from the merge sequences above).
    // geo_cells.bin, admin_cells.bin, and all entry files are rebuilt.
    std::cerr << "  (entry/cell files: omitted, rebuilt by patch tool)" << std::endl;

    // Include cell_id changes for entry reconstruction
    {
        // Geo cell changes
        auto new_geo = read_file(new_dir + "/geo_cells.bin");
        auto old_geo = read_file(old_dir + "/geo_cells.bin");
        std::unordered_set<uint64_t> old_set, new_set;
        for (size_t i = 0; i < old_geo.size() / 20; i++) {
            uint64_t c; memcpy(&c, old_geo.data() + i * 20, 8); old_set.insert(c);
        }
        for (size_t i = 0; i < new_geo.size() / 20; i++) {
            uint64_t c; memcpy(&c, new_geo.data() + i * 20, 8); new_set.insert(c);
        }
        std::vector<uint64_t> g_added, g_removed;
        for (auto c : new_set) if (!old_set.count(c)) g_added.push_back(c);
        for (auto c : old_set) if (!new_set.count(c)) g_removed.push_back(c);
        std::sort(g_added.begin(), g_added.end());
        std::sort(g_removed.begin(), g_removed.end());

        uint32_t marker = CELL_CHANGES_GEO_MARKER;
        uint32_t na = g_added.size(), nr = g_removed.size();
        wval(patch, &marker, 4); wval(patch, &na, 4); wval(patch, &nr, 4);
        for (auto c : g_added) wval(patch, &c, 8);
        for (auto c : g_removed) wval(patch, &c, 8);
        std::cerr << "  Geo cell changes: +" << na << " -" << nr << " ("
                  << (na + nr) * 8 << " bytes)" << std::endl;

        // Admin cell changes
        auto new_ac = read_file(new_dir + "/admin_cells.bin");
        auto old_ac = read_file(old_dir + "/admin_cells.bin");
        old_set.clear(); new_set.clear();
        for (size_t i = 0; i < old_ac.size() / 12; i++) {
            uint64_t c; memcpy(&c, old_ac.data() + i * 12, 8); old_set.insert(c);
        }
        for (size_t i = 0; i < new_ac.size() / 12; i++) {
            uint64_t c; memcpy(&c, new_ac.data() + i * 12, 8); new_set.insert(c);
        }
        std::vector<uint64_t> a_added, a_removed;
        for (auto c : new_set) if (!old_set.count(c)) a_added.push_back(c);
        for (auto c : old_set) if (!new_set.count(c)) a_removed.push_back(c);
        std::sort(a_added.begin(), a_added.end());
        std::sort(a_removed.begin(), a_removed.end());

        marker = CELL_CHANGES_ADMIN_MARKER;
        na = a_added.size(); nr = a_removed.size();
        wval(patch, &marker, 4); wval(patch, &na, 4); wval(patch, &nr, 4);
        for (auto c : a_added) wval(patch, &c, 8);
        for (auto c : a_removed) wval(patch, &c, 8);
        std::cerr << "  Admin cell changes: +" << na << " -" << nr << std::endl;

        // Derive ID remaps from stored merge sequences (same as patch tool does)
        auto derive_remap = [&](PatchFileId id, size_t old_size, size_t stride) -> std::unordered_map<uint32_t,uint32_t> {
            auto it = stored_merges.find(static_cast<uint32_t>(id));
            if (it == stored_merges.end()) return {};
            auto vec = derive_id_remap_from_merge(it->second, old_size / stride, stride);
            std::unordered_map<uint32_t,uint32_t> m;
            for (uint32_t i = 0; i < vec.size(); i++)
                if (vec[i] != 0xFFFFFFFF) m[i] = vec[i];
            return m;
        };

        auto w_rm_d = derive_remap(PatchFileId::STREET_WAYS,
            read_file(old_dir + "/street_ways.bin").size(), way_stride);
        auto a_rm_d = derive_remap(PatchFileId::ADDR_POINTS,
            read_file(old_dir + "/addr_points.bin").size(), 16);
        auto i_rm_d = derive_remap(PatchFileId::INTERP_WAYS,
            read_file(old_dir + "/interp_ways.bin").size(), interp_stride);
        auto ad_rm_d = derive_remap(PatchFileId::ADMIN_POLYGONS,
            read_file(old_dir + "/admin_polygons.bin").size(), admin_stride);

        // Rebuild entries using derived remaps (same as patch tool)
        auto old_se2 = read_file(old_dir + "/street_entries.bin");
        auto old_ae2 = read_file(old_dir + "/addr_entries.bin");
        auto old_ie2 = read_file(old_dir + "/interp_entries.bin");
        auto derived = rebuild_geo_from_remap(old_geo, old_se2, old_ae2, old_ie2,
                                               w_rm_d, a_rm_d, i_rm_d, g_added, g_removed);
        auto old_admc = read_file(old_dir + "/admin_cells.bin");
        auto old_adme = read_file(old_dir + "/admin_entries.bin");
        auto admin_derived = rebuild_admin_from_remap(old_admc, old_adme, ad_rm_d, a_added, a_removed);

        // Cell-level entry corrections: compare derived vs new entries cell by cell.
        // Include only cells whose entry data differs.
        // This is fully custom — no zstd, no temp files.
        auto write_entry_correction = [&](PatchFileId fid, const std::string& fname,
                                           const std::vector<char>& derived_entries,
                                           const std::vector<char>& derived_cells, size_t cell_stride,
                                           size_t offset_pos) {
            auto new_entries = read_file(new_dir + "/" + fname);
            auto new_cells_data = read_file(new_dir + "/" +
                (fid == PatchFileId::ADMIN_ENTRIES ? "admin_cells.bin" : "geo_cells.bin"));

            // Parse both derived and new entries by cell
            auto parse_all = [](const std::vector<char>& cells, size_t stride, size_t off_pos,
                                 const std::vector<char>& entries)
                -> std::vector<std::pair<uint64_t, std::vector<uint32_t>>>
            {
                size_t n = cells.size() / stride;
                std::vector<std::pair<uint64_t, std::vector<uint32_t>>> result;
                result.reserve(n);
                for (size_t i = 0; i < n; i++) {
                    uint64_t cid; memcpy(&cid, cells.data() + i * stride, 8);
                    uint32_t off; memcpy(&off, cells.data() + i * stride + off_pos, 4);
                    std::vector<uint32_t> ids;
                    if (off != 0xFFFFFFFF && off + 2 <= entries.size()) {
                        uint16_t count; memcpy(&count, entries.data() + off, 2);
                        if (off + 2 + count * 4 <= entries.size()) {
                            ids.resize(count);
                            memcpy(ids.data(), entries.data() + off + 2, count * 4);
                        }
                    }
                    result.push_back({cid, std::move(ids)});
                }
                return result;
            };

            auto derived_parsed = parse_all(derived_cells, cell_stride, offset_pos, derived_entries);
            auto new_parsed = parse_all(new_cells_data,
                cell_stride == 12 ? 12 : 20,
                cell_stride == 12 ? 8 : offset_pos,
                new_entries);

            // Build new cell map for lookup
            std::unordered_map<uint64_t, const std::vector<uint32_t>*> new_map;
            for (auto& [cid, ids] : new_parsed) new_map[cid] = &ids;

            // Find differing cells
            std::vector<char> corr_data;
            uint32_t diff_count = 0;
            for (auto& [cid, d_ids] : derived_parsed) {
                auto it = new_map.find(cid);
                const std::vector<uint32_t>* n_ids = nullptr;
                if (it != new_map.end()) n_ids = it->second;

                bool differs = false;
                if (!n_ids) { differs = !d_ids.empty(); }
                else if (d_ids.size() != n_ids->size()) { differs = true; }
                else if (!d_ids.empty() && memcmp(d_ids.data(), n_ids->data(), d_ids.size() * 4) != 0) { differs = true; }

                if (differs) {
                    wval(corr_data, &cid, 8);
                    uint16_t count = n_ids ? static_cast<uint16_t>(n_ids->size()) : 0;
                    wval(corr_data, &count, 2);
                    if (n_ids && !n_ids->empty())
                        corr_data.insert(corr_data.end(), (const char*)n_ids->data(),
                                        (const char*)n_ids->data() + n_ids->size() * 4);
                    diff_count++;
                }
            }
            // Also check for cells only in new (not in derived)
            std::unordered_set<uint64_t> derived_set;
            for (auto& [cid, _] : derived_parsed) derived_set.insert(cid);
            for (auto& [cid, ids] : new_parsed) {
                if (!derived_set.count(cid) && !ids.empty()) {
                    wval(corr_data, &cid, 8);
                    uint16_t count = static_cast<uint16_t>(ids.size());
                    wval(corr_data, &count, 2);
                    corr_data.insert(corr_data.end(), (const char*)ids.data(),
                                    (const char*)ids.data() + ids.size() * 4);
                    diff_count++;
                }
            }

            uint32_t marker = ENTRY_CORRECTION_MARKER;
            uint32_t file = static_cast<uint32_t>(fid);
            wval(patch, &marker, 4);
            wval(patch, &file, 4);
            wval(patch, &diff_count, 4);
            patch.insert(patch.end(), corr_data.begin(), corr_data.end());
            std::cerr << "  " << fname << ": " << diff_count << " cell corrections ("
                      << corr_data.size() << " bytes)" << std::endl;
        };

        // Geo entries: street, addr, interp
        write_entry_correction(PatchFileId::STREET_ENTRIES, "street_entries.bin",
            derived.street_entries_data, derived.geo_cells_data, 20, 8);
        write_entry_correction(PatchFileId::ADDR_ENTRIES, "addr_entries.bin",
            derived.addr_entries_data, derived.geo_cells_data, 20, 12);
        write_entry_correction(PatchFileId::INTERP_ENTRIES, "interp_entries.bin",
            derived.interp_entries_data, derived.geo_cells_data, 20, 16);
        // Admin entries
        write_entry_correction(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin",
            admin_derived.admin_entries_data, admin_derived.admin_cells_data, 12, 8);

        // Cell flag corrections: identify cells whose has_street/has_addr/has_interp
        // flags differ between old and new. Only ~3K cells typically.
        // The patch tool uses these to correctly assign entry offsets during geo_cells rebuild.
        {
            auto new_geo_data = read_file(new_dir + "/geo_cells.bin");
            std::unordered_map<uint64_t, uint8_t> old_cell_flags;
            for (size_t i = 0; i < old_geo.size() / 20; i++) {
                uint64_t cid; memcpy(&cid, old_geo.data() + i * 20, 8);
                uint32_t s, a, ip;
                memcpy(&s, old_geo.data()+i*20+8, 4); memcpy(&a, old_geo.data()+i*20+12, 4);
                memcpy(&ip, old_geo.data()+i*20+16, 4);
                uint8_t flags = (s != 0xFFFFFFFF ? 1 : 0) | (a != 0xFFFFFFFF ? 2 : 0) | (ip != 0xFFFFFFFF ? 4 : 0);
                old_cell_flags[cid] = flags;
            }
            std::vector<std::pair<uint64_t, uint8_t>> flag_corrections;
            for (size_t i = 0; i < new_geo_data.size() / 20; i++) {
                uint64_t cid; memcpy(&cid, new_geo_data.data() + i * 20, 8);
                uint32_t s, a, ip;
                memcpy(&s, new_geo_data.data()+i*20+8, 4); memcpy(&a, new_geo_data.data()+i*20+12, 4);
                memcpy(&ip, new_geo_data.data()+i*20+16, 4);
                uint8_t new_flags = (s != 0xFFFFFFFF ? 1 : 0) | (a != 0xFFFFFFFF ? 2 : 0) | (ip != 0xFFFFFFFF ? 4 : 0);
                auto it = old_cell_flags.find(cid);
                uint8_t old_f = (it != old_cell_flags.end()) ? it->second : 0;
                if (new_flags != old_f)
                    flag_corrections.push_back({cid, new_flags});
            }
            uint32_t fm = CELL_FLAGS_MARKER;
            uint32_t fc = static_cast<uint32_t>(flag_corrections.size());
            wval(patch, &fm, 4); wval(patch, &fc, 4);
            for (auto& [cid, flags] : flag_corrections) {
                wval(patch, &cid, 8); wval(patch, &flags, 1);
            }
            std::cerr << "  Cell flag corrections: " << fc << " cells ("
                      << fc * 9 << " bytes)" << std::endl;
        }
    }

    // End marker
    uint32_t end_marker = 0xFFFFFFFF;
    wval(patch, &end_marker, 4);

    std::cerr << "\nUncompressed patch: " << patch.size() << " bytes ("
              << patch.size() / 1024 / 1024 << " MiB)" << std::endl;

    // Compress whole patch with zstd for transport
    {
        std::string raw_path = tmpdir + "/patch.raw";
        write_file(raw_path, patch);
        std::string cmd = "zstd -19 '" + raw_path + "' -o '" + patch_path + "' -f --quiet 2>/dev/null";
        system(cmd.c_str());
        auto compressed = read_file(patch_path);
        std::cerr << "Compressed patch: " << compressed.size() << " bytes ("
                  << compressed.size() / 1024 / 1024 << " MiB)" << std::endl;
        remove(raw_path.c_str());
    }

    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());
    return 0;
}

+        // Secondary matching: recover additional ID mappings for modified records

+        // that didn't byte-match but represent the same entity (e.g. geometry edits)

+            size_t way_name_off = (way_stride == 12) ? 8 : 5;

+            // Street ways: match by name_id + node_count

+            auto old_w2 = read_file(old_dir + "/street_ways.bin");

+            remap_field(old_w2, way_stride, way_name_off, str_remap);

+            auto new_w2 = read_file(new_dir + "/street_ways.bin");

+            auto soft_w = secondary_match_from_merge(

+                stored_merges[static_cast<uint32_t>(PatchFileId::STREET_WAYS)],

+                old_w2, new_w2, way_stride,

+                [way_name_off](const char* rec) -> uint64_t {

+                    uint32_t name_id; memcpy(&name_id, rec + way_name_off, 4);

+                    uint8_t nc = static_cast<uint8_t>(rec[4]);

+                    return ((uint64_t)name_id << 8) | nc;

+            for (auto& [o, n] : soft_w) w_rm_d[o] = n;

+            std::cerr << "  Secondary way matches: " << soft_w.size() << std::endl;

+            // Addr points: match by (street_id, housenumber_id)

+            auto old_a2 = read_file(old_dir + "/addr_points.bin");

+            remap_addr_points(old_a2, str_remap);

+            auto new_a2 = read_file(new_dir + "/addr_points.bin");

+            auto soft_a = secondary_match_from_merge(

+                stored_merges[static_cast<uint32_t>(PatchFileId::ADDR_POINTS)],

+                [](const char* rec) -> uint64_t {

+                    uint32_t hn_id, st_id;

+                    memcpy(&hn_id, rec + 8, 4); memcpy(&st_id, rec + 12, 4);

+                    return ((uint64_t)st_id << 32) | hn_id;

+            for (auto& [o, n] : soft_a) a_rm_d[o] = n;

+            std::cerr << "  Secondary addr matches: " << soft_a.size() << std::endl;

+            // Interp ways: match by (street_id, start_number, end_number)

+            size_t ist_off = (interp_stride >= 20) ? 8 : 5;

+            auto old_i2 = read_file(old_dir + "/interp_ways.bin");

+            remap_field(old_i2, interp_stride, ist_off, str_remap);

+            auto new_i2 = read_file(new_dir + "/interp_ways.bin");

+            auto soft_i = secondary_match_from_merge(

+                stored_merges[static_cast<uint32_t>(PatchFileId::INTERP_WAYS)],

+                old_i2, new_i2, interp_stride,

+                [ist_off](const char* rec) -> uint64_t {

+                    uint32_t street_id, start;

+                    memcpy(&street_id, rec + ist_off, 4);

+                    memcpy(&start, rec + ist_off + 4, 4);

+                    return ((uint64_t)street_id << 32) | start;

+            for (auto& [o, n] : soft_i) i_rm_d[o] = n;

+            std::cerr << "  Secondary interp matches: " << soft_i.size() << std::endl;

+            // Admin polygons: match by (name_id, admin_level, country_code)

+            auto old_ap2 = read_file(old_dir + "/admin_polygons.bin");

+            remap_field(old_ap2, admin_stride, 8, str_remap);

+            auto new_ap2 = read_file(new_dir + "/admin_polygons.bin");

+            auto soft_ad = secondary_match_from_merge(

+                stored_merges[static_cast<uint32_t>(PatchFileId::ADMIN_POLYGONS)],

+                old_ap2, new_ap2, admin_stride,

+                [](const char* rec) -> uint64_t {

+                    uint32_t name_id; memcpy(&name_id, rec + 8, 4);

+                    uint8_t level = static_cast<uint8_t>(rec[12]);

+                    uint16_t cc; memcpy(&cc, rec + 20, 2);

+                    return ((uint64_t)name_id << 24) | ((uint64_t)level << 16) | cc;

+            for (auto& [o, n] : soft_ad) ad_rm_d[o] = n;

+            std::cerr << "  Secondary admin matches: " << soft_ad.size() << std::endl;


Updated task #14 status


Updated task #16 status


Now let me build and test on Europe. First, check if node 3 is accessible and what's available:


Bash

ssh node3 "ls -la /home/michtest/det-A/ /home/michtest/det-today/ 2>&1 | head -20"

Check Europe test data on node 3


mkdir -p build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release 2>&1 | tail -5

Configure build


Error: Exit code 255
Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Permission denied, please try again.
Permission denied, please try again.
michtest@100.113.88.121: Permission denied (publickey,password).


<tool_use_error>Cancelled: parallel tool call Bash(ssh node3 "ls -la /home/michtest/det-A/ …) errored</tool_use_error>


<system-reminder>This memory is 7 days old. Memories are point-in-time observations, not live state — claims about code behavior or file:line citations may be outdated. Verify against current code before asserting as fact.</system-reminder>
     1→---
     2→name: node_access
     3→description: SSH access details for build nodes via Tailscale (immich tailnet)
     4→type: reference
     5→---
     6→
     7→All nodes accessed via Tailscale (userspace networking mode, `tailscale nc` proxy).
     8→SSH multiplexing configured in ~/.ssh/config with ControlPath /tmp/ssh-mux/.
     9→
    10→- **Node 3** (michtest-ng): 100.113.88.121, user michtest, password `elon-is-insane`
    11→  - 32 cores, 174GB RAM + 50GB swap, no cgroup
    12→  - /tmp is 79GB tmpfs (RAM-backed) — use ~/ for large files
    13→  - Has PBFs at ~/pbf/ (planet, germany, monaco), old indexes at ~/indexes-old-code/
    14→  - Master connection: `sshpass -p 'elon-is-insane' ssh -M -o ServerAliveInterval=30 node3 "sleep 99999" &`
    15→
    16→- **Node 2a**: 10.10.10.12:30022, user root, OIDC auth via zitadel.internal.immich.cloud
    17→  - 88 physical cores, 251GB physical — cgroup limited to 44 CPUs, 192GB RAM
    18→  - Ephemeral pods — session must be held open or pod is wiped
    19→  - Each new SSH connection requires fresh OIDC device code approval
    20→
    21→- **Node 2b**: 10.10.10.11:30022, user root, same OIDC setup as 2a
    22→  - Same specs as 2a (44c/192GB cgroup)
    23→
    24→**Important**: Nodes 2a/2b can only have ONE open SSH connection at a time. Use SSH multiplexing.
    25→Tailscale requires `--accept-routes` for 10.10.10.x subnet access.
    26→

---
name: node_access
description: SSH access details for build nodes via Tailscale (immich tailnet)
type: reference
---

All nodes accessed via Tailscale (userspace networking mode, `tailscale nc` proxy).
SSH multiplexing configured in ~/.ssh/config with ControlPath /tmp/ssh-mux/.

- **Node 3** (michtest-ng): 100.113.88.121, user michtest, password `elon-is-insane`
  - 32 cores, 174GB RAM + 50GB swap, no cgroup
  - /tmp is 79GB tmpfs (RAM-backed) — use ~/ for large files
  - Has PBFs at ~/pbf/ (planet, germany, monaco), old indexes at ~/indexes-old-code/
  - Master connection: `sshpass -p 'elon-is-insane' ssh -M -o ServerAliveInterval=30 node3 "sleep 99999" &`

- **Node 2a**: 10.10.10.12:30022, user root, OIDC auth via zitadel.internal.immich.cloud
  - 88 physical cores, 251GB physical — cgroup limited to 44 CPUs, 192GB RAM
  - Ephemeral pods — session must be held open or pod is wiped
  - Each new SSH connection requires fresh OIDC device code approval

- **Node 2b**: 10.10.10.11:30022, user root, same OIDC setup as 2a
  - Same specs as 2a (44c/192GB cgroup)

**Important**: Nodes 2a/2b can only have ONE open SSH connection at a time. Use SSH multiplexing.
Tailscale requires `--accept-routes` for 10.10.10.x subnet access.


sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no michtest@100.113.88.121 "ls /home/michtest/det-A/ /home/michtest/det-today/ 2>&1 | head -30"

Check Europe test data on node 3


Error: Exit code 255
ssh: connect to host 100.113.88.121 port 22: Network is unreachable


which tailscale && tailscale status 2>&1 | head -20

Check Tailscale status


/usr/bin/tailscale
100.68.107.4     cc3fcc93b585          zackpollard@    linux    -
100.115.161.111  bigmac                nutgood@        macOS    offline, last seen 8d ago
100.79.151.116   brandons-macbook-pro  bwees@          macOS    offline, last seen 3d ago
100.107.156.50   devins-macbook-pro    onedr0p@        macOS    offline, last seen 2d ago
100.107.247.98   eric                  zackpollard@    linux    offline, last seen 2d ago
100.115.203.128  framework             nirokato@       linux    -
100.123.80.73    kvm-bulbasaur         bwees@          linux    offline, last seen 4d ago
100.81.60.48     kvm-charmander        bwees@          linux    -
100.102.115.99   kvm-eevee             bwees@          linux    -
100.74.141.95    kvm-flareon           bwees@          linux    -
100.100.219.82   kvm-futomacoss        bwees@          linux    -
100.124.129.61   kvm-geodude           bwees@          linux    -
100.122.135.111  kvm-gpu-a             bwees@          linux    -
100.80.36.102    kvm-gpu-b             bwees@          linux    offline, last seen 78d ago
100.64.74.47     kvm-magikarp          bwees@          linux    -
100.68.216.47    kvm-pikachu           bwees@          linux    -
100.116.44.45    kvm-red               bwees@          linux    -
100.71.253.6     kvm-squirtle          bwees@          linux    -
100.69.47.21     kvm-sylveon           bwees@          linux    -
100.125.211.45   kvm-vaporeon          bwees@          linux    -


tailscale status 2>&1 | grep -i 'michtest\|node3\|node 3'

Find node 3 in Tailscale


100.113.88.121   michtest-ng           nutgood@        linux    active; direct 174.95.229.200:55513, tx 5658120 rx 1908720


sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 michtest@100.113.88.121 "echo connected && ls /home/michtest/det-A/" 2>&1

Test SSH to node 3


Ping node 3 via Tailscale


pong from michtest-ng (100.113.88.121) via 174.95.229.200:55513 in 156ms


ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no -o ConnectTimeout=10 michtest@100.113.88.121 "echo connected" 2>&1

SSH via tailscale nc proxy


Error: Exit code 255
Permission denied, please try again.
Permission denied, please try again.
michtest@100.113.88.121: Permission denied (publickey,password).


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "echo connected && ls /home/michtest/det-A/" 2>&1

SSH via tailscale nc with password


connected
addr_entries.bin
addr_points.bin
admin_cells.bin
admin_entries.bin
admin_polygons.bin
admin_vertices.bin
geo_cells.bin
interp_entries.bin
interp_nodes.bin
interp_ways.bin
manifest.json
street_entries.bin
street_nodes.bin
street_ways.bin
strings.bin


Good, node 3 is accessible. Let me build the updated diff tool and send it over.


ls build/CMakeCache.txt 2>/dev/null && echo "build dir exists" || echo "need to configure"

Check if build dir exists


build/CMakeCache.txt
build dir exists


cd /root/traccar-geocoder/builder/build && cmake .. -DCMAKE_BUILD_TYPE=Release 2>&1 | tail -3 && make -j$(nproc) geocoder-diff 2>&1

Build geocoder-diff with secondary matching


-- Configuring done (0.1s)
-- Generating done (0.0s)
-- Build files have been written to: /root/traccar-geocoder/builder/build
[ 50%] Building CXX object CMakeFiles/geocoder-diff.dir/tools/geocoder_diff.cpp.o
[100%] Linking CXX executable geocoder-diff
[100%] Built target geocoder-diff


Compiled cleanly. Now send it to node 3 and run the Europe diff:


sshpass -p 'elon-is-insane' scp -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no /root/traccar-geocoder/builder/build/geocoder-diff michtest@100.113.88.121:/home/michtest/geocoder-diff 2>&1

Copy geocoder-diff to node 3


(Bash completed with no output)


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "cd /home/michtest && time ./geocoder-diff det-A/ det-today/ -o patch-europe-secondary.gcpatch" 2>&1

Run Europe diff with secondary matching


Building string remap...
  String remap: derived from string diff (0 explicit entries)
  strings.bin: +4152 -2396 strings
  addr_points.bin: seq=4405789 fixups=0 (0.32%)
  street_ways.bin: seq=2714467 fixups=19361289 (1.16%)
  street_nodes.bin: parent-derived seq=108326877 bytes (7.49%)
  interp_ways.bin: seq=545 fixups=13395 (0.16%)
  interp_nodes.bin: parent-derived seq=71135 bytes
  admin_polygons.bin: seq=95018 fixups=449047 (0.88%)
  admin_vertices.bin: parent-derived seq=7106042 bytes (2.44%)
  (entry/cell files: omitted, rebuilt by patch tool)
  Geo cell changes: +58772 -12627 (571192 bytes)
  Admin cell changes: +0 -0
  Secondary way matches: 29127
  Secondary addr matches: 55444
  Secondary interp matches: 8
  Secondary admin matches: 2320
  street_entries.bin: 549325 cell corrections (9065358 bytes)
  addr_entries.bin: 64334 cell corrections (1668720 bytes)
  interp_entries.bin: 37 cell corrections (466 bytes)
  admin_entries.bin: 1764 cell corrections (55132 bytes)
  Cell flag corrections: 83178 cells (748602 bytes)

Uncompressed patch: 174691965 bytes (166 MiB)
Compressed patch: 33832987 bytes (32 MiB)

real	8m8.768s
user	8m17.256s
sys	0m27.113s


Good news — secondary matching found 29K way matches, 55K addr matches, and 2.3K admin matches. But I realize there's a critical issue: the patch tool derives its own ID remap from the merge sequence (without secondary matching), so the corrections need to be computed against the same remap the patch tool uses, OR the secondary remap must be included in the patch file.

Let me include the secondary remap in the patch file so the patch tool can apply it too.


   260→// cell_index is the position in the geo_cells/admin_cells array.
   261→static constexpr uint32_t ENTRY_CORRECTION_MARKER = 0xFFFFFFF8;
   262→
   263→// Cell flag corrections marker: 0xFFFFFFF9
   264→// Format: marker, count(u32), [(cell_id:u64, flags:u8)] × count
   265→// flags: bit 0 = has_street, bit 1 = has_addr, bit 2 = has_interp
   266→static constexpr uint32_t CELL_FLAGS_MARKER = 0xFFFFFFF9;
   267→
   268→// --- Shared entry rebuild logic ---
   269→// Used by both diff and patch tools to produce identical rebuilt entries.
   270→// Takes old geo_cells + old entries + ID remap → produces rebuilt entries + geo_cells.
   271→
   272→struct RebuiltGeo {
   273→    std::vector<char> geo_cells_data;
   274→    std::vector<char> street_entries_data;

// cell_index is the position in the geo_cells/admin_cells array.
static constexpr uint32_t ENTRY_CORRECTION_MARKER = 0xFFFFFFF8;

// Cell flag corrections marker: 0xFFFFFFF9
// Format: marker, count(u32), [(cell_id:u64, flags:u8)] × count
// flags: bit 0 = has_street, bit 1 = has_addr, bit 2 = has_interp
static constexpr uint32_t CELL_FLAGS_MARKER = 0xFFFFFFF9;

// --- Shared entry rebuild logic ---
// Used by both diff and patch tools to produce identical rebuilt entries.
// Takes old geo_cells + old entries + ID remap → produces rebuilt entries + geo_cells.

struct RebuiltGeo {
    std::vector<char> geo_cells_data;
    std::vector<char> street_entries_data;


Add the secondary remap marker to patch_format.h:


Edit

// Cell flag corrections marker: 0xFFFFFFF9
// Format: marker, count(u32), [(cell_id:u64, flags:u8)] × count
// flags: bit 0 = has_street, bit 1 = has_addr, bit 2 = has_interp
static constexpr uint32_t CELL_FLAGS_MARKER = 0xFFFFFFF9;

// Cell flag corrections marker: 0xFFFFFFF9
// Format: marker, count(u32), [(cell_id:u64, flags:u8)] × count
// flags: bit 0 = has_street, bit 1 = has_addr, bit 2 = has_interp
static constexpr uint32_t CELL_FLAGS_MARKER = 0xFFFFFFF9;

// Secondary ID remap marker: 0xFFFFFFF6
// Additional old→new ID mappings for modified records (recovered by relaxed key matching).
// Both diff and patch tools must apply these to their derived remaps for consistent results.
// Format: marker(4), n_files(4),
//   for each: file_id(4), n_pairs(4), [(old_id:u32, new_id:u32)] × n_pairs
static constexpr uint32_t SECONDARY_REMAP_MARKER = 0xFFFFFFF6;


The file /root/traccar-geocoder/builder/tools/patch_format.h has been updated successfully.

// Cell flag corrections marker: 0xFFFFFFF9
// Format: marker, count(u32), [(cell_id:u64, flags:u8)] × count
// flags: bit 0 = has_street, bit 1 = has_addr, bit 2 = has_interp
static constexpr uint32_t CELL_FLAGS_MARKER = 0xFFFFFFF9;

// Secondary ID remap marker: 0xFFFFFFF6
// Additional old→new ID mappings for modified records (recovered by relaxed key matching).
// Both diff and patch tools must apply these to their derived remaps for consistent results.
// Format: marker(4), n_files(4),
//   for each: file_id(4), n_pairs(4), [(old_id:u32, new_id:u32)] × n_pairs
static constexpr uint32_t SECONDARY_REMAP_MARKER = 0xFFFFFFF6;

#pragma once
// Shared definitions for the geocoder patch system.
// Used by geocoder-canonicalize, geocoder-diff, and geocoder-patch.

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_set>
#include <vector>

// --- Binary record structs (must match types.h and server) ---

#pragma pack(push, 1)
struct PatchWayHeader {
    uint32_t node_offset;
    uint8_t node_count;
    uint32_t name_id;
};
static_assert(sizeof(PatchWayHeader) == 9, "WayHeader must be 9 bytes packed");

struct PatchAddrPoint {
    float lat;
    float lng;
    uint32_t housenumber_id;
    uint32_t street_id;
};
static_assert(sizeof(PatchAddrPoint) == 16, "AddrPoint must be 16 bytes");

struct PatchNodeCoord {
    float lat;
    float lng;
};
static_assert(sizeof(PatchNodeCoord) == 8, "NodeCoord must be 8 bytes");
#pragma pack(pop)

// InterpWay and AdminPolygon have compiler-dependent padding.
// Read them field-by-field instead of struct-casting.

struct PatchInterpWay {
    uint32_t node_offset;
    uint8_t node_count;
    uint32_t street_id;
    uint32_t start_number;
    uint32_t end_number;
    uint8_t interpolation;
};

struct PatchAdminPolygon {
    uint32_t vertex_offset;
    uint32_t vertex_count;
    uint32_t name_id;
    uint8_t admin_level;
    float area;
    uint16_t country_code;
};

// --- File I/O helpers ---

inline std::vector<char> read_file(const std::string& path) {
    std::ifstream f(path, std::ios::binary | std::ios::ate);
    if (!f) return {};
    auto size = f.tellg();
    f.seekg(0);
    std::vector<char> data(size);
    f.read(data.data(), size);
    return data;
}

inline bool write_file(const std::string& path, const char* data, size_t size) {
    std::ofstream f(path, std::ios::binary);
    if (!f) return false;
    f.write(data, size);
    return f.good();
}

inline bool write_file(const std::string& path, const std::vector<char>& data) {
    return write_file(path, data.data(), data.size());
}

inline const char* get_string(const std::vector<char>& pool, uint32_t offset) {
    if (offset >= pool.size()) return "";
    return pool.data() + offset;
}

// Read WayHeader array — detect stride from file size vs known struct sizes
inline std::vector<PatchWayHeader> read_ways(const std::string& path) {
    auto data = read_file(path);
    // Determine stride: builder writes sizeof(WayHeader) which may be 9 or 12
    // depending on compiler padding. Try 12 first (common), then 9.
    size_t stride = 12;
    if (data.size() % 12 != 0 || data.size() / 12 == 0) {
        stride = 9;
    }
    size_t count = data.size() / stride;
    std::vector<PatchWayHeader> result(count);
    for (size_t i = 0; i < count; i++) {
        const char* p = data.data() + i * stride;
        memcpy(&result[i].node_offset, p, 4);
        result[i].node_count = static_cast<uint8_t>(p[4]);
        // name_id is at offset 5 (packed) or 8 (padded to 4-byte alignment)
        size_t name_off = (stride == 12) ? 8 : 5;
        memcpy(&result[i].name_id, p + name_off, 4);
    }
    return result;
}

inline size_t detect_way_stride(const std::string& path) {
    auto data = read_file(path);
    return (data.size() % 12 == 0 && data.size() / 12 > 0) ? 12 : 9;
}

template<typename T>
inline std::vector<T> read_structs(const std::string& path) {
    auto data = read_file(path);
    size_t count = data.size() / sizeof(T);
    std::vector<T> result(count);
    if (count > 0) memcpy(result.data(), data.data(), count * sizeof(T));
    return result;
}

// Read InterpWay with field-by-field parsing to handle padding
inline std::vector<PatchInterpWay> read_interps(const std::string& path) {
    auto data = read_file(path);
    // Determine record size from file: try sizeof(InterpWay) as compiled by the builder
    // The builder uses the same struct, so sizeof should match.
    size_t rec_size = sizeof(PatchInterpWay);
    // But the builder's InterpWay has padding. Read with the builder's size.
    // From types.h the struct is: u32 + u8 + u32 + u32 + u32 + u8 = 18 bytes raw
    // but compiler pads to 20 bytes typically. Check by trying both.
    size_t count_20 = data.size() / 20;
    size_t count_18 = data.size() / 18;
    // Use whichever divides evenly
    size_t stride = (data.size() % 20 == 0 && count_20 > 0) ? 20 :
                    (data.size() % 18 == 0 && count_18 > 0) ? 18 : 20;
    size_t count = data.size() / stride;
    std::vector<PatchInterpWay> result(count);
    for (size_t i = 0; i < count; i++) {
        const char* p = data.data() + i * stride;
        memcpy(&result[i].node_offset, p, 4); p += 4;
        result[i].node_count = *reinterpret_cast<const uint8_t*>(p); p += 1;
        // Skip padding to align street_id
        if (stride == 20) p += 3; // 3 bytes padding after node_count
        memcpy(&result[i].street_id, p, 4); p += 4;
        memcpy(&result[i].start_number, p, 4); p += 4;
        memcpy(&result[i].end_number, p, 4); p += 4;
        result[i].interpolation = *reinterpret_cast<const uint8_t*>(p);
    }
    return result;
}

// Read AdminPolygon with field-by-field parsing
inline std::vector<PatchAdminPolygon> read_admin_polygons(const std::string& path) {
    auto data = read_file(path);
    // AdminPolygon: u32 + u32 + u32 + u8 + f32 + u16 = 19 raw bytes
    // Compiler likely pads to 20 or 24 bytes. Try common sizes.
    size_t stride = 0;
    for (size_t s : {24, 20, 19}) {
        if (data.size() % s == 0 && data.size() / s > 0) { stride = s; break; }
    }
    if (stride == 0) stride = 24; // fallback
    size_t count = data.size() / stride;
    std::vector<PatchAdminPolygon> result(count);
    for (size_t i = 0; i < count; i++) {
        const char* p = data.data() + i * stride;
        memcpy(&result[i].vertex_offset, p, 4); p += 4;
        memcpy(&result[i].vertex_count, p, 4); p += 4;
        memcpy(&result[i].name_id, p, 4); p += 4;
        result[i].admin_level = *reinterpret_cast<const uint8_t*>(p); p += 1;
        // padding for area alignment
        if (stride >= 20) {
            size_t skip = (stride == 24) ? 3 : (stride == 20 ? 3 : 0);
            p += skip;
        }
        memcpy(&result[i].area, p, 4); p += 4;
        memcpy(&result[i].country_code, p, 2);
    }
    return result;
}

// --- Patch file format ---

static constexpr char GCPATCH_MAGIC[8] = {'G','C','P','A','T','C','H','\0'};
static constexpr uint32_t GCPATCH_VERSION = 1;

enum class PatchFileId : uint32_t {
    STRINGS = 0,
    STREET_WAYS = 1,
    STREET_NODES = 2,
    ADDR_POINTS = 3,
    INTERP_WAYS = 4,
    INTERP_NODES = 5,
    ADMIN_POLYGONS = 6,
    ADMIN_VERTICES = 7,
    GEO_CELLS = 8,
    STREET_ENTRIES = 9,
    ADDR_ENTRIES = 10,
    INTERP_ENTRIES = 11,
    ADMIN_CELLS = 12,
    ADMIN_ENTRIES = 13,
    COUNT = 14
};

static const char* patch_file_names[] = {
    "strings.bin", "street_ways.bin", "street_nodes.bin", "addr_points.bin",
    "interp_ways.bin", "interp_nodes.bin", "admin_polygons.bin", "admin_vertices.bin",
    "geo_cells.bin", "street_entries.bin", "addr_entries.bin", "interp_entries.bin",
    "admin_cells.bin", "admin_entries.bin"
};

// Encoding types for each section
enum class PatchEncoding : uint32_t {
    RAW_REPLACE = 0,  // Full replacement (zlib compressed)
    ZSTD_DELTA = 1,   // Single zstd delta frame (zstd --patch-from on remapped old)
};

// Patch file header
struct PatchHeader {
    char magic[8];
    uint32_t version;
    uint32_t flags;
    // Per-file sections follow
};

// Section header (one per file)
struct PatchSection {
    uint32_t file_id;       // PatchFileId
    uint32_t encoding;      // PatchEncoding
    uint64_t old_size;      // Expected size of old file
    uint64_t new_size;      // Size of reconstructed file
    uint64_t data_size;     // Size of compressed section data
};

// COPY_INSERT opcodes
enum : uint8_t {
    OP_COPY = 0,    // Copy from old file: [u8 tag][u64 offset][u32 length]
    OP_INSERT = 1,  // Insert new data:    [u8 tag][u32 length][data...]
};

// Offset fixup section marker: 0xFFFFFFFD
// Format: uint32_t marker, uint32_t file_id, uint32_t stride,
//         uint32_t count, [(uint32_t record_index, uint32_t new_offset_value)] * count
// Applied to byte offset 0 of each record (node_offset/vertex_offset field).

static constexpr uint32_t FIXUP_MARKER = 0xFFFFFFFD;

// Cell changes section marker: 0xFFFFFFFB
// Format: marker, num_added(u32), num_removed(u32),
//         [added_cell_id(u64)] * num_added, [removed_cell_id(u64)] * num_removed
// For both geo and admin cell sets.
static constexpr uint32_t CELL_CHANGES_GEO_MARKER = 0xFFFFFFFB;
static constexpr uint32_t CELL_CHANGES_ADMIN_MARKER = 0xFFFFFFFA;

// Entry correction marker: 0xFFFFFFF8
// Cell-level diff of entries: lists cells whose entries differ between derived and new.
// Format: marker(4), file_id(4), count(4),
//   for each: cell_index(4), entry_count(2), [id(4)] * entry_count
// cell_index is the position in the geo_cells/admin_cells array.
static constexpr uint32_t ENTRY_CORRECTION_MARKER = 0xFFFFFFF8;

// Cell flag corrections marker: 0xFFFFFFF9
// Format: marker, count(u32), [(cell_id:u64, flags:u8)] × count
// flags: bit 0 = has_street, bit 1 = has_addr, bit 2 = has_interp
static constexpr uint32_t CELL_FLAGS_MARKER = 0xFFFFFFF9;

// --- Shared entry rebuild logic ---
// Used by both diff and patch tools to produce identical rebuilt entries.
// Takes old geo_cells + old entries + ID remap → produces rebuilt entries + geo_cells.

struct RebuiltGeo {
    std::vector<char> geo_cells_data;
    std::vector<char> street_entries_data;
    std::vector<char> addr_entries_data;
    std::vector<char> interp_entries_data;
};

inline RebuiltGeo rebuild_geo_from_remap(
    const std::vector<char>& old_geo,
    const std::vector<char>& old_se, const std::vector<char>& old_ae, const std::vector<char>& old_ie,
    const std::unordered_map<uint32_t,uint32_t>& way_rm,
    const std::unordered_map<uint32_t,uint32_t>& addr_rm,
    const std::unordered_map<uint32_t,uint32_t>& interp_rm,
    const std::vector<uint64_t>& added_cells = {},
    const std::vector<uint64_t>& removed_cells = {})
{
    size_t n_cells = old_geo.size() / 20;

    auto parse_entry = [](const std::vector<char>& data, uint32_t off) -> std::vector<uint32_t> {
        if (off == 0xFFFFFFFF || off + 2 > data.size()) return {};
        uint16_t count; memcpy(&count, data.data() + off, 2);
        if (off + 2 + count * 4 > data.size()) return {};
        std::vector<uint32_t> ids(count);
        memcpy(ids.data(), data.data() + off + 2, count * 4);
        return ids;
    };

    auto remap_ids = [](std::vector<uint32_t>& ids, const std::unordered_map<uint32_t,uint32_t>& rm) {
        for (auto& id : ids) {
            auto it = rm.find(id);
            if (it != rm.end()) id = it->second;
        }
        std::sort(ids.begin(), ids.end());
    };

    // Parse all cells and remap IDs
    struct CellData {
        uint64_t cell_id;
        std::vector<uint32_t> streets, addrs, interps;
    };
    std::vector<CellData> cells(n_cells);
    for (size_t i = 0; i < n_cells; i++) {
        memcpy(&cells[i].cell_id, old_geo.data() + i * 20, 8);
        uint32_t s_off, a_off, i_off;
        memcpy(&s_off, old_geo.data() + i * 20 + 8, 4);
        memcpy(&a_off, old_geo.data() + i * 20 + 12, 4);
        memcpy(&i_off, old_geo.data() + i * 20 + 16, 4);
        cells[i].streets = parse_entry(old_se, s_off);
        cells[i].addrs = parse_entry(old_ae, a_off);
        cells[i].interps = parse_entry(old_ie, i_off);
        remap_ids(cells[i].streets, way_rm);
        remap_ids(cells[i].addrs, addr_rm);
        remap_ids(cells[i].interps, interp_rm);
    }

    // Apply cell set changes (add new cells, remove old cells)
    if (!removed_cells.empty()) {
        std::unordered_set<uint64_t> removed_set(removed_cells.begin(), removed_cells.end());
        cells.erase(std::remove_if(cells.begin(), cells.end(),
            [&](const CellData& c) { return removed_set.count(c.cell_id); }), cells.end());
    }
    if (!added_cells.empty()) {
        for (uint64_t cid : added_cells) {
            CellData cd; cd.cell_id = cid;
            // New cell entry data is provided via the new_cell_entries parameter (if available)
            cells.push_back(cd);
        }
        // Re-sort to maintain cell_id order
        std::sort(cells.begin(), cells.end(),
            [](const CellData& a, const CellData& b) { return a.cell_id < b.cell_id; });
    }

    // Write rebuilt files
    RebuiltGeo result;
    uint32_t no_data = 0xFFFFFFFF;

    auto write_entries = [&](std::vector<char>& buf, const auto& getter) -> std::unordered_map<uint64_t, uint32_t> {
        std::unordered_map<uint64_t, uint32_t> offsets;
        for (auto& c : cells) {
            const auto& ids = getter(c);
            if (ids.empty()) continue;
            offsets[c.cell_id] = static_cast<uint32_t>(buf.size());
            uint16_t count = static_cast<uint16_t>(ids.size());
            buf.insert(buf.end(), (const char*)&count, (const char*)&count + 2);
            buf.insert(buf.end(), (const char*)ids.data(), (const char*)ids.data() + ids.size() * 4);
        }
        return offsets;
    };

    auto s_off = write_entries(result.street_entries_data, [](const CellData& c) -> const std::vector<uint32_t>& { return c.streets; });
    auto a_off = write_entries(result.addr_entries_data, [](const CellData& c) -> const std::vector<uint32_t>& { return c.addrs; });
    auto i_off = write_entries(result.interp_entries_data, [](const CellData& c) -> const std::vector<uint32_t>& { return c.interps; });

    // Write geo_cells
    for (auto& c : cells) {
        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&c.cell_id, (const char*)&c.cell_id + 8);
        auto get = [&](const auto& m) -> uint32_t {
            auto it = m.find(c.cell_id); return it != m.end() ? it->second : no_data;
        };
        uint32_t sv = get(s_off), av = get(a_off), iv = get(i_off);
        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&sv, (const char*)&sv + 4);
        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&av, (const char*)&av + 4);
        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&iv, (const char*)&iv + 4);
    }

    return result;
}

struct RebuiltAdmin {
    std::vector<char> admin_cells_data;
    std::vector<char> admin_entries_data;
};

inline RebuiltAdmin rebuild_admin_from_remap(
    const std::vector<char>& old_ac, const std::vector<char>& old_ae,
    const std::unordered_map<uint32_t,uint32_t>& admin_rm,
    const std::vector<uint64_t>& added_cells = {},
    const std::vector<uint64_t>& removed_cells = {})
{
    size_t n_cells = old_ac.size() / 12;

    struct CellData { uint64_t cell_id; std::vector<uint32_t> ids; };
    std::vector<CellData> cells(n_cells);
    for (size_t i = 0; i < n_cells; i++) {
        memcpy(&cells[i].cell_id, old_ac.data() + i * 12, 8);
        uint32_t off; memcpy(&off, old_ac.data() + i * 12 + 8, 4);
        if (off != 0xFFFFFFFF && off + 2 <= old_ae.size()) {
            uint16_t count; memcpy(&count, old_ae.data() + off, 2);
            if (off + 2 + count * 4 <= old_ae.size()) {
                cells[i].ids.resize(count);
                memcpy(cells[i].ids.data(), old_ae.data() + off + 2, count * 4);
                for (auto& id : cells[i].ids) {
                    uint32_t flags = id & 0x80000000u;
                    uint32_t masked = id & 0x7FFFFFFFu;
                    auto it = admin_rm.find(masked);
                    if (it != admin_rm.end()) id = it->second | flags;
                }
                std::sort(cells[i].ids.begin(), cells[i].ids.end());
            }
        }
    }

    // Apply cell changes (add/remove)
    if (!removed_cells.empty()) {
        std::unordered_set<uint64_t> removed_set(removed_cells.begin(), removed_cells.end());
        cells.erase(std::remove_if(cells.begin(), cells.end(),
            [&](const CellData& c) { return removed_set.count(c.cell_id); }), cells.end());
    }
    if (!added_cells.empty()) {
        for (uint64_t cid : added_cells) {
            CellData cd; cd.cell_id = cid;
            cells.push_back(cd);
        }
        std::sort(cells.begin(), cells.end(),
            [](const CellData& a, const CellData& b) { return a.cell_id < b.cell_id; });
    }

    RebuiltAdmin result;
    uint32_t no_data = 0xFFFFFFFF;
    std::unordered_map<uint64_t, uint32_t> offsets;
    for (auto& c : cells) {
        if (c.ids.empty()) continue;
        offsets[c.cell_id] = static_cast<uint32_t>(result.admin_entries_data.size());
        uint16_t count = static_cast<uint16_t>(c.ids.size());
        result.admin_entries_data.insert(result.admin_entries_data.end(), (const char*)&count, (const char*)&count + 2);
        result.admin_entries_data.insert(result.admin_entries_data.end(), (const char*)c.ids.data(), (const char*)c.ids.data() + c.ids.size() * 4);
    }
    for (auto& c : cells) {
        result.admin_cells_data.insert(result.admin_cells_data.end(), (const char*)&c.cell_id, (const char*)&c.cell_id + 8);
        auto it = offsets.find(c.cell_id);
        uint32_t off = it != offsets.end() ? it->second : no_data;
        result.admin_cells_data.insert(result.admin_cells_data.end(), (const char*)&off, (const char*)&off + 4);
    }
    return result;
}

// --- Varint encoding for delta-compressed fixup tables ---

inline void write_varint(std::vector<char>& buf, uint32_t value) {
    while (value >= 128) {
        buf.push_back(static_cast<char>((value & 0x7F) | 0x80));
        value >>= 7;
    }
    buf.push_back(static_cast<char>(value));
}

inline uint32_t read_varint(const char* data, size_t& pos) {
    uint32_t result = 0, shift = 0;
    while (true) {
        uint8_t byte = static_cast<uint8_t>(data[pos++]);
        result |= (uint32_t)(byte & 0x7F) << shift;
        if (!(byte & 0x80)) break;
        shift += 7;
    }
    return result;
}

// --- Grid coordinate for fingerprinting ---

inline int to_grid(float v) {
    return (int)(v * 1e5f + (v >= 0 ? 0.5f : -0.5f));
}

// --- Directory creation ---
inline void ensure_dir(const std::string& path) {
    std::string cmd = "mkdir -p '" + path + "'";
    system(cmd.c_str());
}

+// Secondary ID remap marker: 0xFFFFFFF6

+// Additional old→new ID mappings for modified records (recovered by relaxed key matching).

+// Both diff and patch tools must apply these to their derived remaps for consistent results.

+//   for each: file_id(4), n_pairs(4), [(old_id:u32, new_id:u32)] × n_pairs

+static constexpr uint32_t SECONDARY_REMAP_MARKER = 0xFFFFFFF6;


Now update the diff tool to emit secondary remaps as a separate section (before entry corrections), and apply them to the derived remap: