        uint32_t fid, count;
        pf.read(reinterpret_cast<char*>(&fid), 4);
        pf.read(reinterpret_cast<char*>(&count), 4);
        uint64_t comp_size; pf.read(reinterpret_cast<char*>(&comp_size), 8);
        std::vector<char> compressed(comp_size);
        pf.read(compressed.data(), comp_size);
        auto blob = zstd_decompress_blob(compressed, count * 8);
        auto& rm = id_remaps[fid];
        for (uint32_t i = 0; i < count; i++) {
            uint32_t old_id, new_id;
            memcpy(&old_id, blob.data() + i * 8, 4);
            memcpy(&new_id, blob.data() + i * 8 + 4, 4);
            rm[old_id] = new_id;
        }
        std::cerr << "Loaded " << count << " ID remaps for file " << fid << std::endl;

// geocoder-patch: Apply a .gcpatch to a geocoder index directory.
//
// For each file: apply string remap to old, write temp, run zstd -d --patch-from.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>

#include <unistd.h>
#include "patch_format.h"

// --- String remap application ---

static void remap_addr_points(std::vector<char>& data,
                                const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + 16 <= data.size(); i += 16) {
        for (size_t off : {8, 12}) {
            uint32_t v; memcpy(&v, data.data() + i + off, 4);
            auto it = rm.find(v);
            if (it != rm.end()) memcpy(data.data() + i + off, &it->second, 4);
        }
    }
}

static void remap_field(std::vector<char>& data, size_t stride, size_t field_off,
                          const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + stride <= data.size(); i += stride) {
        uint32_t v; memcpy(&v, data.data() + i + field_off, 4);
        auto it = rm.find(v);
        if (it != rm.end()) memcpy(data.data() + i + field_off, &it->second, 4);
    }
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);

    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    std::ifstream pf(patch_path, std::ios::binary);
    if (!pf) { std::cerr << "Cannot open " << patch_path << std::endl; return 1; }

    // Read header
    char magic[8]; pf.read(magic, 8);
    if (memcmp(magic, GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch file" << std::endl; return 1;
    }
    uint32_t version, flags;
    pf.read(reinterpret_cast<char*>(&version), 4);
    pf.read(reinterpret_cast<char*>(&flags), 4);
    if (version != GCPATCH_VERSION) {
        std::cerr << "Unsupported version " << version << std::endl; return 1;
    }

    // Helper: decompress zstd blob via CLI
    auto zstd_decompress_blob = [&](const std::vector<char>& compressed, size_t expected_size) -> std::vector<char> {
        std::string zst_path = tmpdir + "/blob.zst";
        std::string raw_path = tmpdir + "/blob.raw";
        write_file(zst_path, compressed);
        system(("zstd -d '" + zst_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null").c_str());
        auto result = read_file(raw_path);
        remove(zst_path.c_str()); remove(raw_path.c_str());
        return result;
    };

    // Read string remap (compressed)
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t section_id; pf.read(reinterpret_cast<char*>(&section_id), 4);
        if (section_id == 0xFFFFFFFE) {
            uint32_t count; pf.read(reinterpret_cast<char*>(&count), 4);
            uint64_t comp_size; pf.read(reinterpret_cast<char*>(&comp_size), 8);
            std::vector<char> compressed(comp_size);
            pf.read(compressed.data(), comp_size);
            auto blob = zstd_decompress_blob(compressed, count * 8);
            for (uint32_t i = 0; i < count; i++) {
                uint32_t old_off, new_off;
                memcpy(&old_off, blob.data() + i * 8, 4);
                memcpy(&new_off, blob.data() + i * 8 + 4, 4);
                if (old_off != new_off) str_remap[old_off] = new_off;
            }
            std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;
        }
    }

    // Read offset fixup tables
    struct Fixup { uint32_t record_idx; uint32_t new_value; };
    std::unordered_map<uint32_t, std::vector<Fixup>> fixup_tables; // file_id → fixups
    std::unordered_map<uint32_t, uint32_t> fixup_strides; // file_id → stride
    while (pf) {
        uint32_t marker; pf.read(reinterpret_cast<char*>(&marker), 4);
        if (marker != FIXUP_MARKER) {
            pf.seekg(-4, std::ios::cur);
            break;
        }
        uint32_t fid, stride, count;
        pf.read(reinterpret_cast<char*>(&fid), 4);
        pf.read(reinterpret_cast<char*>(&stride), 4);
        pf.read(reinterpret_cast<char*>(&count), 4);
        uint64_t comp_size; pf.read(reinterpret_cast<char*>(&comp_size), 8);
        std::vector<char> compressed(comp_size);
        pf.read(compressed.data(), comp_size);
        auto blob = zstd_decompress_blob(compressed, count * 8);
        auto& vec = fixup_tables[fid];
        fixup_strides[fid] = stride;
        vec.resize(count);
        for (uint32_t i = 0; i < count; i++) {
            memcpy(&vec[i].record_idx, blob.data() + i * 8, 4);
            memcpy(&vec[i].new_value, blob.data() + i * 8 + 4, 4);
        }
        std::cerr << "Loaded " << count << " fixups for file " << fid << std::endl;
    }

    // Detect struct strides from current files
    auto detect_stride = [&](const std::string& name, std::vector<size_t> cs) -> size_t {
        auto d = read_file(cur_dir + "/" + name);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect_stride("street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride("interp_ways.bin", {20, 18});
    size_t admin_stride = detect_stride("admin_polygons.bin", {24, 20, 19});

    // Helper: apply string remap to old data file
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t file_id) {
        if (str_remap.empty()) return;
        if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) {
            for (size_t i = 0; i + 16 <= data.size(); i += 16)
                for (size_t off : {8, 12}) {
                    uint32_t v; memcpy(&v, data.data() + i + off, 4);
                    auto it = str_remap.find(v);
                    if (it != str_remap.end()) memcpy(data.data() + i + off, &it->second, 4);
                }
        } else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) {
            size_t name_off = (way_stride == 12) ? 8 : 5;
            for (size_t i = 0; i + way_stride <= data.size(); i += way_stride) {
                uint32_t v; memcpy(&v, data.data() + i + name_off, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + name_off, &it->second, 4);
            }
        } else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) {
            size_t soff = (interp_stride >= 20) ? 8 : 5;
            for (size_t i = 0; i + interp_stride <= data.size(); i += interp_stride) {
                uint32_t v; memcpy(&v, data.data() + i + soff, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + soff, &it->second, 4);
            }
        } else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) {
            for (size_t i = 0; i + admin_stride <= data.size(); i += admin_stride) {
                uint32_t v; memcpy(&v, data.data() + i + 8, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + 8, &it->second, 4);
            }
        }
    };

    // Helper: apply offset fixups
    auto apply_fixups = [&](std::vector<char>& data, uint32_t file_id) {
        auto it = fixup_tables.find(file_id);
        if (it == fixup_tables.end()) return;
        uint32_t stride = fixup_strides[file_id];
        for (auto& f : it->second) {
            size_t pos = (size_t)f.record_idx * stride;
            if (pos + 4 <= data.size())
                memcpy(data.data() + pos, &f.new_value, 4);
        }
    };

    // Read ID remap tables
    std::unordered_map<uint32_t, std::unordered_map<uint32_t,uint32_t>> id_remaps; // file_id → (old→new)
    while (pf) {
        uint32_t marker; pf.read(reinterpret_cast<char*>(&marker), 4);
        if (marker != 0xFFFFFFFC) {
            pf.seekg(-4, std::ios::cur);
            break;
        }
        uint32_t fid, count;
        pf.read(reinterpret_cast<char*>(&fid), 4);
        pf.read(reinterpret_cast<char*>(&count), 4);
        auto& rm = id_remaps[fid];
        for (uint32_t i = 0; i < count; i++) {
            uint32_t old_id, new_id;
            pf.read(reinterpret_cast<char*>(&old_id), 4);
            pf.read(reinterpret_cast<char*>(&new_id), 4);
            rm[old_id] = new_id;
        }
        std::cerr << "Loaded " << count << " ID remaps for file " << fid << std::endl;
    }

    // Helper: rebuild entry + cell files using ID remap
    auto rebuild_entries = [&](const std::string& cells_file, const std::string& entries_file,
                                size_t cell_stride, size_t offset_pos,
                                const std::unordered_map<uint32_t,uint32_t>& id_rm, bool has_flags)
        -> std::pair<std::vector<char>, std::vector<char>> // (entries_data, cells_data)
    {
        auto old_cells = read_file(cur_dir + "/" + cells_file);
        auto old_entries = read_file(cur_dir + "/" + entries_file);
        size_t n_cells = old_cells.size() / cell_stride;

        // Parse old entries, remap IDs
        struct CellEntry { uint64_t cell_id; std::vector<uint32_t> ids; };
        std::vector<CellEntry> parsed;
        for (size_t i = 0; i < n_cells; i++) {
            uint64_t cid; uint32_t off;
            memcpy(&cid, old_cells.data() + i * cell_stride, 8);
            memcpy(&off, old_cells.data() + i * cell_stride + offset_pos, 4);
            CellEntry ce; ce.cell_id = cid;
            if (off != 0xFFFFFFFF && off + 2 <= old_entries.size()) {
                uint16_t count; memcpy(&count, old_entries.data() + off, 2);
                if (off + 2 + count * 4 <= old_entries.size()) {
                    for (uint16_t j = 0; j < count; j++) {
                        uint32_t id; memcpy(&id, old_entries.data() + off + 2 + j * 4, 4);
                        uint32_t flags = has_flags ? (id & 0x80000000u) : 0;
                        uint32_t masked = id & 0x7FFFFFFFu;
                        auto it = id_rm.find(masked);
                        if (it != id_rm.end())
                            ce.ids.push_back(it->second | flags);
                        else
                            ce.ids.push_back(id); // keep unmapped IDs as-is
                    }
                }
            }
            std::sort(ce.ids.begin(), ce.ids.end());
            parsed.push_back(std::move(ce));
        }

        // Rebuild entries + cells
        std::vector<char> new_entries_buf, new_cells_buf;
        uint32_t no_data = 0xFFFFFFFF;
        for (auto& ce : parsed) {
            new_cells_buf.insert(new_cells_buf.end(), (char*)&ce.cell_id, (char*)&ce.cell_id + 8);
            if (ce.ids.empty()) {
                new_cells_buf.insert(new_cells_buf.end(), (char*)&no_data, (char*)&no_data + 4);
            } else {
                uint32_t off = static_cast<uint32_t>(new_entries_buf.size());
                new_cells_buf.insert(new_cells_buf.end(), (char*)&off, (char*)&off + 4);
                uint16_t count = static_cast<uint16_t>(ce.ids.size());
                new_entries_buf.insert(new_entries_buf.end(), (char*)&count, (char*)&count + 2);
                new_entries_buf.insert(new_entries_buf.end(), (char*)ce.ids.data(),
                                      (char*)ce.ids.data() + ce.ids.size() * 4);
            }
        }
        return {new_entries_buf, new_cells_buf};
    };

    // Pre-rebuild entry/cell files using ID remaps (for use as zstd reference)
    std::unordered_map<std::string, std::string> rebuilt_refs; // filename → temp path
    if (!id_remaps.empty()) {
        // Rebuild geo entries (street, addr, interp)
        auto way_rm = id_remaps.count((uint32_t)PatchFileId::STREET_WAYS) ?
                       id_remaps[(uint32_t)PatchFileId::STREET_WAYS] : std::unordered_map<uint32_t,uint32_t>{};
        auto addr_rm = id_remaps.count((uint32_t)PatchFileId::ADDR_POINTS) ?
                        id_remaps[(uint32_t)PatchFileId::ADDR_POINTS] : std::unordered_map<uint32_t,uint32_t>{};
        auto interp_rm = id_remaps.count((uint32_t)PatchFileId::INTERP_WAYS) ?
                          id_remaps[(uint32_t)PatchFileId::INTERP_WAYS] : std::unordered_map<uint32_t,uint32_t>{};
        auto admin_rm = id_remaps.count((uint32_t)PatchFileId::ADMIN_POLYGONS) ?
                         id_remaps[(uint32_t)PatchFileId::ADMIN_POLYGONS] : std::unordered_map<uint32_t,uint32_t>{};

        // Rebuild geo: street_entries (off 8), addr_entries (off 12), interp_entries (off 16)
        auto old_geo = read_file(cur_dir + "/geo_cells.bin");
        auto old_se = read_file(cur_dir + "/street_entries.bin");
        auto old_ae = read_file(cur_dir + "/addr_entries.bin");
        auto old_ie = read_file(cur_dir + "/interp_entries.bin");

        // Parse + remap each entry type
        auto parse_and_remap = [&](const std::vector<char>& geo, size_t off_pos,
                                     const std::vector<char>& entries,
                                     const std::unordered_map<uint32_t,uint32_t>& rm) {
            size_t n = geo.size() / 20;
            std::vector<std::pair<uint64_t, std::vector<uint32_t>>> result;
            for (size_t i = 0; i < n; i++) {
                uint64_t cid; uint32_t off;
                memcpy(&cid, geo.data() + i * 20, 8);
                memcpy(&off, geo.data() + i * 20 + off_pos, 4);
                std::vector<uint32_t> ids;
                if (off != 0xFFFFFFFF && off + 2 <= entries.size()) {
                    uint16_t count; memcpy(&count, entries.data() + off, 2);
                    if (off + 2 + count * 4 <= entries.size()) {
                        for (uint16_t j = 0; j < count; j++) {
                            uint32_t id; memcpy(&id, entries.data() + off + 2 + j * 4, 4);
                            auto it = rm.find(id);
                            ids.push_back(it != rm.end() ? it->second : id);
                        }
                    }
                }
                std::sort(ids.begin(), ids.end());
                result.push_back({cid, std::move(ids)});
            }
            return result;
        };

        auto sc = parse_and_remap(old_geo, 8, old_se, way_rm);
        auto ac = parse_and_remap(old_geo, 12, old_ae, addr_rm);
        auto ic = parse_and_remap(old_geo, 16, old_ie, interp_rm);

        // Write rebuilt entries + geo_cells
        auto write_rebuilt = [&](const std::string& name,
                                  const std::vector<std::pair<uint64_t, std::vector<uint32_t>>>& entries) -> std::string {
            std::string path = tmpdir + "/" + name;
            std::vector<char> buf;
            std::unordered_map<uint64_t, uint32_t> offsets;
            for (auto& [cid, ids] : entries) {
                if (ids.empty()) continue;
                offsets[cid] = static_cast<uint32_t>(buf.size());
                uint16_t count = static_cast<uint16_t>(ids.size());
                buf.insert(buf.end(), (char*)&count, (char*)&count + 2);
                buf.insert(buf.end(), (char*)ids.data(), (char*)ids.data() + ids.size() * 4);
            }
            write_file(path, buf);
            return path;
        };

        // Entry files: don't use rebuilt refs (diff uses old→new directly)
        // Only geo_cells uses rebuilt ref (proven to match byte-identical)
        write_rebuilt("street_entries.bin", sc); // write for geo_cells offset calculation
        write_rebuilt("addr_entries.bin", ac);
        write_rebuilt("interp_entries.bin", ic);

        // Rebuild geo_cells from the remapped entries
        {
            uint32_t no_data = 0xFFFFFFFF;
            std::vector<char> buf;
            // Build offset maps for each entry type
            std::unordered_map<uint64_t, uint32_t> s_off, a_off, i_off;
            auto entries_data = [](const std::string& path) {
                auto d = read_file(path);
                std::unordered_map<uint64_t, uint32_t> offsets;
                return d;
            };
            // Re-parse the rebuilt entries to get offsets
            auto get_offsets = [](const std::vector<std::pair<uint64_t, std::vector<uint32_t>>>& entries) {
                std::unordered_map<uint64_t, uint32_t> offsets;
                uint32_t pos = 0;
                for (auto& [cid, ids] : entries) {
                    if (ids.empty()) continue;
                    offsets[cid] = pos;
                    pos += 2 + ids.size() * 4;
                }
                return offsets;
            };
            auto so = get_offsets(sc), ao = get_offsets(ac), io = get_offsets(ic);

            size_t n = old_geo.size() / 20;
            for (size_t i = 0; i < n; i++) {
                uint64_t cid; memcpy(&cid, old_geo.data() + i * 20, 8);
                buf.insert(buf.end(), (char*)&cid, (char*)&cid + 8);
                auto get = [&](const auto& m) -> uint32_t {
                    auto it = m.find(cid); return it != m.end() ? it->second : no_data;
                };
                uint32_t sv = get(so), av = get(ao), iv = get(io);
                buf.insert(buf.end(), (char*)&sv, (char*)&sv + 4);
                buf.insert(buf.end(), (char*)&av, (char*)&av + 4);
                buf.insert(buf.end(), (char*)&iv, (char*)&iv + 4);
            }
            std::string path = tmpdir + "/geo_cells.bin";
            write_file(path, buf);
            rebuilt_refs["geo_cells.bin"] = path;
        }

        // Rebuild admin entries
        {
            auto [ae_data, ac_data] = rebuild_entries("admin_cells.bin", "admin_entries.bin",
                                                        12, 8, admin_rm, true);
            std::string ae_path = tmpdir + "/admin_entries.bin";
            std::string ac_path = tmpdir + "/admin_cells.bin";
            write_file(ae_path, ae_data);
            write_file(ac_path, ac_data);
            // Don't add admin entries/cells to rebuilt_refs (diff uses old→new directly)
        }

        std::cerr << "Rebuilt " << rebuilt_refs.size() << " entry/cell files from ID remaps" << std::endl;
    }

    // Process file sections
    while (pf) {
        uint32_t file_id; pf.read(reinterpret_cast<char*>(&file_id), 4);
        if (!pf || file_id == 0xFFFFFFFF) break;
        uint32_t encoding; pf.read(reinterpret_cast<char*>(&encoding), 4);

        if (file_id >= (uint32_t)PatchFileId::COUNT) {
            std::cerr << "  Unknown section " << file_id << std::endl;
            break;
        }
        const char* filename = patch_file_names[file_id];

        auto write_temp = [&](const std::string& path, uint64_t size) {
            std::ofstream zf(path, std::ios::binary);
            std::vector<char> buf(std::min(size, (uint64_t)1024*1024));
            uint64_t remaining = size;
            while (remaining > 0) {
                size_t chunk = std::min(remaining, (uint64_t)buf.size());
                pf.read(buf.data(), chunk);
                zf.write(buf.data(), chunk);
                remaining -= chunk;
            }
        };

        auto zstd_apply = [&](const std::string& ref, const std::string& zst, const std::string& out) -> bool {
            std::string cmd = "zstd --patch-from='" + ref + "' -d '" + zst + "' -o '" + out + "' -f --long=31 --quiet 2>/dev/null";
            return system(cmd.c_str()) == 0;
        };

        if (encoding == (uint32_t)PatchEncoding::ZSTD_DELTA) {
            uint64_t old_size, new_size, delta_size;
            pf.read(reinterpret_cast<char*>(&old_size), 8);
            pf.read(reinterpret_cast<char*>(&new_size), 8);
            pf.read(reinterpret_cast<char*>(&delta_size), 8);

            // Write zstd frame to temp
            std::string zst = tmpdir + "/" + std::string(filename) + ".zst";
            write_temp(zst, delta_size);

            // Check if we have a rebuilt ref (entry/cell files with ID remap)
            std::string ref;
            auto rit = rebuilt_refs.find(std::string(filename));
            if (rit != rebuilt_refs.end()) {
                ref = rit->second;
            } else {
                // Load old file, apply string remap + fixups → write remapped ref
                auto old_data = read_file(cur_dir + "/" + std::string(filename));
                apply_str_remap(old_data, file_id);
                apply_fixups(old_data, file_id);
                ref = tmpdir + "/" + std::string(filename) + ".ref";
                write_file(ref, old_data);
            }

            // Apply zstd delta
            std::string out = out_dir + "/" + std::string(filename);
            if (!zstd_apply(ref, zst, out)) {
                std::cerr << "FAILED: " << filename << std::endl; return 1;
            }
            std::cerr << "  " << filename << ": " << read_file(out).size() << " bytes" << std::endl;
            remove(zst.c_str()); remove(ref.c_str());

        } else {
            std::cerr << "  Unknown encoding " << encoding << " for " << filename << std::endl;
            break;
        }
    }

    // Cleanup
    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());

    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

-            pf.read(reinterpret_cast<char*>(&old_id), 4);

-            pf.read(reinterpret_cast<char*>(&new_id), 4);

+            memcpy(&old_id, blob.data() + i * 8, 4);

+            memcpy(&new_id, blob.data() + i * 8 + 4, 4);


Bash

cmake --build build -j$(nproc) --target geocoder-diff --target geocoder-patch 2>&1 | tail -5

Build with compression


[100%] Linking CXX executable geocoder-diff
[100%] Built target geocoder-diff
[ 50%] Building CXX object CMakeFiles/geocoder-patch.dir/tools/geocoder_patch.cpp.o
[100%] Linking CXX executable geocoder-patch
[100%] Built target geocoder-patch


sshpass -p 'elon-is-insane' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" /root/traccar-geocoder/builder/build/geocoder-diff /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:~/ 2>&1 && sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
echo "=== DIFF (compressed tables) ==="
rm -f /home/michtest/test10.gcpatch
time /home/michtest/geocoder-diff /home/michtest/det-A /home/michtest/det-new -o /home/michtest/test10.gcpatch 2>&1
echo "Patch: $(numfmt --to=iec $(stat -c%s /home/michtest/test10.gcpatch))"
echo ""
echo "=== APPLY ==="
rm -rf /home/michtest/patched10
time /home/michtest/geocoder-patch /home/michtest/det-A /home/michtest/test10.gcpatch -o /home/michtest/patched10 2>&1
echo ""
echo "=== VERIFY ==="
all_match=true
for f in /home/michtest/det-new/*.bin; do
    b=$(basename $f); p=/home/michtest/patched10/$b
    [ -f "$p" ] && cmp -s "$f" "$p" && echo "  $b: MATCH" || { echo "  $b: FAIL"; all_match=false; }
done
$all_match && echo "ALL FILES MATCH" || echo "SOME DIFFER"
' 2>&1

Full test with compressed tables


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
=== DIFF (compressed tables) ===
Building string remap...
  4980078 strings mapped
Building record ID remaps...
    addr matched=85825938/85891830 new=164173
    ways matched=19372891/19452480 new=95957
    interps matched=1016/16785 new=15755
    admin matched=449642/450907 new=1222
  Remapping geo entries...
  Remapping admin entries...
  Remap: 4982052 entries (39856416 -> 11885604 bytes)
  fixup street_ways.bin: 19372891 entries (154983128 -> 41799994)
  fixup interp_ways.bin: 1016 entries (8128 -> 2779)
  fixup admin_polygons.bin: 449642 entries (3597136 -> 1262718)
  id_remap file 1: 19372891 (154983128 -> 38873409)
  id_remap file 3: 85825938 (686607504 -> 146330168)
  id_remap file 4: 1016 (8128 -> 1455)
  id_remap file 6: 449642 (3597136 -> 642236)
  addr_points.bin: 3770551 bytes (0.27%)
  street_ways.bin: 1515482 bytes (0.65%)
  interp_ways.bin: 66381 bytes (19.79%)
  admin_polygons.bin: 39775 bytes (0.37%)
  strings.bin: 71309 bytes (0.09%)
  street_nodes.bin: 6727404 bytes (0.47%)
  interp_nodes.bin: 1228 bytes (0.17%)
  admin_vertices.bin: 535915 bytes (0.18%)
  geo_cells.bin: 191638 bytes (0.01%)
  street_entries.bin: 111330 bytes (0.02%)
  addr_entries.bin: 267581 bytes (0.07%)
  interp_entries.bin: 39 bytes (0.02%)
  admin_cells.bin: 182 bytes (0.01%)
  admin_entries.bin: 1248 bytes (0.02%)

Total patch: 254099062 bytes (242 MiB)

real	7m22.776s
user	10m42.094s
sys	0m13.874s
Patch: 243M

=== APPLY ===
Loaded 4980054 string remap entries
Loaded 19372891 fixups for file 1
Loaded 1016 fixups for file 4
Loaded 449642 fixups for file 6
Loaded 19372891 ID remaps for file 1
Loaded 85825938 ID remaps for file 3
Loaded 1016 ID remaps for file 4
Loaded 449642 ID remaps for file 6
Rebuilt 1 entry/cell files from ID remaps
  addr_points.bin: 1375841776 bytes
  street_ways.bin: 233626176 bytes
  interp_ways.bin: 335424 bytes
  admin_polygons.bin: 10820736 bytes
  strings.bin: 82573397 bytes
  street_nodes.bin: 1445600360 bytes
  interp_nodes.bin: 725184 bytes
  admin_vertices.bin: 291061192 bytes
FAILED: geo_cells.bin

=== VERIFY ===

real	1m59.563s
user	1m52.098s
sys	0m8.209s
  addr_entries.bin: FAIL
  addr_points.bin: MATCH
  admin_cells.bin: FAIL
  admin_entries.bin: FAIL
  admin_polygons.bin: MATCH
  admin_vertices.bin: MATCH
  geo_cells.bin: FAIL
  interp_entries.bin: FAIL
  interp_nodes.bin: MATCH
  interp_ways.bin: MATCH
  street_entries.bin: FAIL
  street_nodes.bin: MATCH
  street_ways.bin: MATCH
  strings.bin: MATCH
SOME DIFFER


Patch is now **243 MiB** (down from 1.9 GiB). The compression is working. But the entry/cell files fail again — the patch tool's rebuild produces different entries.

The issue: the diff tool's rebuilt entries use the overlay approach (which produces nearly-correct but not byte-identical entries). The patch tool rebuilds entries with a different approach (simpler parsing + remapping, no overlay).

Since the diff tool uses rebuilt entries as reference, the patch tool must produce EXACTLY the same rebuilt entries. The fix: make the patch tool's rebuild logic identical to the diff tool's. The diff overlays ALL new entries on top of remapped old. The patch tool only has old data + ID remap — it doesn't have new entries to overlay.

The patch tool needs the new entry data to overlay. But that's what we're trying to produce! Circular.

The solution: **include the rebuilt entry files directly** in the patch (compressed). They're tiny after remapping. The diff tool already writes them to tmpdir — I can include them as raw replacement sections.

Actually simpler: for the zstd deltas of entry/cell files, the diff uses rebuilt entries as the prefix. The patch tool needs the same prefix. Instead of rebuilding, the patch tool could just use the zstd delta against OLD entries (which is what works). But then the deltas are huge.

Or: include a small "entry patch" that transforms old entries → rebuilt entries. This is the string remap + ID remap applied to entries. The patch tool applies this to old entries to get the rebuilt version, then uses that as zstd prefix.

Actually, I already have the ID remap tables. The patch tool already rebuilds entries using them. The issue is just that the rebuild doesn't match the diff's version perfectly.

Let me look at what's different: the diff overlays new entries. The patch doesn't have new entries. The difference is in cells where old had data that new doesn't — the diff removes these (via the `new_sc.count` check), the patch keeps them (no such check).

The patch tool's rebuild needs the same cleanup. It needs to know which cells have data in the new build for each type. This info is in the new entry files, which we don't have.

But we DO have the rebuilt geo_cells from the patch tool. The geo_cells tells us which cells have which types (NO_DATA for missing types). Let me use the rebuilt geo_cells offsets to determine which cells should have entries for each type.

Actually, this won't work because the rebuilt geo_cells uses the rebuilt entries' offsets.

The simplest fix: include a small "cell bitmap" in the patch that says which cells have data for each type. With 84M cells and 3 types, that's 84M × 3 bits = ~31 MiB raw. Compressed: ~5 MiB.

Or even simpler: include the new geo_cells file directly (it's tiny after rebuild — the zstd delta is 192 KB). The patch tool reads the new geo_cells to know which cells have data, and uses that to guide the entry rebuild.

Actually, the most elegant fix: **have the diff tool include rebuilt entry files as the zstd reference instead of using the overlay.** The diff tool writes rebuilt entries to tmpdir. These are the reference for zstd. The patch tool reconstructs the SAME rebuilt entries (by applying ID remap to old entries) and uses them as the zstd prefix.

The mismatch is that the diff's rebuild uses overlay (new entries) while the patch's rebuild uses ID remap (transformed old entries). These produce different results for cells where data was added/removed.

The fix: make the diff tool's rebuild NOT use overlay. Just use pure ID remap like the patch tool does. Then both produce the same output.


Edit

        // Merge: start with remapped old, overlay with new for changed/added cells
        // For cells that exist in new but not old: add from new
        // For cells that exist in old but not new: remove
        for (auto& [cid, ids] : new_sc) sc[cid] = ids;
        for (auto& [cid, ids] : new_ac) ac[cid] = ids;
        for (auto& [cid, ids] : new_ic) ic[cid] = ids;

        // Collect all cells from new build
        std::unordered_map<uint64_t, bool> new_cells;
        size_t nn = new_geo.size() / 20;
        for (size_t i = 0; i < nn; i++) {
            uint64_t cid; memcpy(&cid, new_geo.data() + i * 20, 8);
            new_cells[cid] = true;
        }
        // Remove cells not in new, AND remove stale entries where the new build
        // has no data for that type (cell exists but no addrs/streets/interps)
        for (auto it = sc.begin(); it != sc.end(); )
            (new_cells.count(it->first) && new_sc.count(it->first)) ? ++it : it = sc.erase(it);
        for (auto it = ac.begin(); it != ac.end(); )
            (new_cells.count(it->first) && new_ac.count(it->first)) ? ++it : it = ac.erase(it);
        for (auto it = ic.begin(); it != ic.end(); )
            (new_cells.count(it->first) && new_ic.count(it->first)) ? ++it : it = ic.erase(it);

        // Don't overlay new entries — use pure ID remap only.
        // This ensures the diff tool's rebuild matches the patch tool's rebuild exactly.
        // The zstd delta will encode the remaining differences (new/removed records).
        (void)new_sc; (void)new_ac; (void)new_ic; // not used for overlay


The file /root/traccar-geocoder/builder/tools/geocoder_diff.cpp has been updated successfully.

        // Don't overlay new entries — use pure ID remap only.
        // This ensures the diff tool's rebuild matches the patch tool's rebuild exactly.
        // The zstd delta will encode the remaining differences (new/removed records).
        (void)new_sc; (void)new_ac; (void)new_ic; // not used for overlay

// geocoder-diff: Compare two deterministic geocoder builds, produce a .gcpatch.
//
// Three-level remapping for minimal patches:
// 1. String remap: old string offsets → new string offsets
// 2. Record ID remap: match records by content, map old_id → new_id
// 3. Entry remap: apply record ID remap to entry files + rebuild cell indexes
//
// Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>

#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <functional>
#include <iomanip>
#include <iostream>
#include <string>
#include <unistd.h>
#include <unordered_map>
#include <vector>

#include "patch_format.h"

// --- String remap ---

static std::unordered_map<uint32_t, uint32_t> build_string_remap(
    const std::vector<char>& old_pool, const std::vector<char>& new_pool)
{
    std::unordered_map<std::string, uint32_t> new_idx;
    size_t pos = 0;
    while (pos < new_pool.size()) {
        const char* s = new_pool.data() + pos;
        size_t len = strlen(s);
        new_idx[std::string(s, len)] = static_cast<uint32_t>(pos);
        pos += len + 1;
    }
    std::unordered_map<uint32_t, uint32_t> remap;
    pos = 0;
    while (pos < old_pool.size()) {
        const char* s = old_pool.data() + pos;
        size_t len = strlen(s);
        auto it = new_idx.find(std::string(s, len));
        if (it != new_idx.end())
            remap[static_cast<uint32_t>(pos)] = it->second;
        pos += len + 1;
    }
    return remap;
}

// --- Remap helpers ---

static void remap_addr_points(std::vector<char>& data, const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + 16 <= data.size(); i += 16)
        for (size_t off : {8, 12}) {
            uint32_t v; memcpy(&v, data.data() + i + off, 4);
            auto it = rm.find(v);
            if (it != rm.end()) memcpy(data.data() + i + off, &it->second, 4);
        }
}

static void remap_field(std::vector<char>& data, size_t stride, size_t field_off,
                         const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + stride <= data.size(); i += stride) {
        uint32_t v; memcpy(&v, data.data() + i + field_off, 4);
        auto it = rm.find(v);
        if (it != rm.end()) memcpy(data.data() + i + field_off, &it->second, 4);
    }
}

// --- Content-based record matching ---
// Returns old_id → new_id mapping

static uint64_t fnv_mix(uint64_t h, uint64_t v) { h ^= v; h *= 1099511628211ULL; return h; }

static std::vector<uint32_t> match_addr_points(
    const std::vector<char>& old_data, const std::vector<char>& new_data)
{
    // After string remap, matching records are byte-identical
    size_t old_n = old_data.size() / 16, new_n = new_data.size() / 16;
    // Hash each new record
    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++) {
        uint64_t h = 14695981039346656037ULL;
        const char* p = new_data.data() + i * 16;
        for (int j = 0; j < 16; j++) h = fnv_mix(h, (uint8_t)p[j]);
        new_map.emplace(h, i);
    }
    std::vector<uint32_t> result(old_n, 0xFFFFFFFF);
    size_t matched = 0;
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = 14695981039346656037ULL;
        const char* p = old_data.data() + i * 16;
        for (int j = 0; j < 16; j++) h = fnv_mix(h, (uint8_t)p[j]);
        auto it = new_map.find(h);
        if (it != new_map.end() &&
            memcmp(old_data.data() + i * 16, new_data.data() + it->second * 16, 16) == 0) {
            result[i] = it->second;
            new_map.erase(it);
            matched++;
        }
    }
    std::cerr << "    addr matched=" << matched << "/" << old_n << " new=" << new_map.size() << std::endl;
    return result;
}

static std::vector<uint32_t> match_ways(
    const std::vector<char>& old_ways, const std::vector<char>& old_nodes,
    const std::vector<char>& new_ways, const std::vector<char>& new_nodes,
    size_t stride)
{
    size_t name_off = (stride == 12) ? 8 : 5;
    size_t old_n = old_ways.size() / stride, new_n = new_ways.size() / stride;

    size_t old_node_count = old_nodes.size() / 8, new_node_count = new_nodes.size() / 8;
    auto way_hash = [&](const char* w, const char* nodes, size_t max_nodes) -> uint64_t {
        uint32_t node_offset, name_id; uint8_t node_count;
        memcpy(&node_offset, w, 4); node_count = (uint8_t)w[4];
        memcpy(&name_id, w + name_off, 4);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, node_count);
        for (uint8_t j = 0; j < node_count && (node_offset + j) < max_nodes; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4);
            memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(way_hash(new_ways.data() + i * stride, new_nodes.data(), new_node_count), i);

    std::vector<uint32_t> result(old_n, 0xFFFFFFFF);
    size_t matched = 0;
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = way_hash(old_ways.data() + i * stride, old_nodes.data(), old_node_count);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            result[i] = it->second;
            // Also fix node_offset in old to match new (makes way bytes identical)
            uint32_t new_node_off;
            memcpy(&new_node_off, new_ways.data() + it->second * stride, 4);
            memcpy(const_cast<char*>(old_ways.data()) + i * stride, &new_node_off, 4);
            new_map.erase(it);
            matched++;
        }
    }
    std::cerr << "    ways matched=" << matched << "/" << old_n << " new=" << new_map.size() << std::endl;
    return result;
}

static std::vector<uint32_t> match_admin(
    const std::vector<char>& old_polys, const std::vector<char>& old_verts,
    const std::vector<char>& new_polys, const std::vector<char>& new_verts,
    size_t stride)
{
    size_t old_n = old_polys.size() / stride, new_n = new_polys.size() / stride;

    size_t old_vc = old_verts.size() / 8, new_vc = new_verts.size() / 8;
    auto poly_hash = [&](const char* p, const char* verts, size_t max_v) -> uint64_t {
        uint32_t vert_offset, vert_count, name_id;
        memcpy(&vert_offset, p, 4); memcpy(&vert_count, p + 4, 4);
        memcpy(&name_id, p + 8, 4);
        uint8_t level = (uint8_t)p[12];
        uint16_t cc; memcpy(&cc, p + stride - 2, 2);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, level); h = fnv_mix(h, cc);
        h = fnv_mix(h, vert_count);
        uint32_t n = std::min(vert_count, 10u);
        for (uint32_t j = 0; j < n && (vert_offset + j) < max_v; j++) {
            float lat, lng;
            memcpy(&lat, verts + (vert_offset + j) * 8, 4);
            memcpy(&lng, verts + (vert_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(poly_hash(new_polys.data() + i * stride, new_verts.data(), new_vc), i);

    std::vector<uint32_t> result(old_n, 0xFFFFFFFF);
    size_t matched = 0;
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = poly_hash(old_polys.data() + i * stride, old_verts.data(), old_vc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            result[i] = it->second;
            uint32_t new_vert_off;
            memcpy(&new_vert_off, new_polys.data() + it->second * stride, 4);
            memcpy(const_cast<char*>(old_polys.data()) + i * stride, &new_vert_off, 4);
            new_map.erase(it);
            matched++;
        }
    }
    std::cerr << "    admin matched=" << matched << "/" << old_n << " new=" << new_map.size() << std::endl;
    return result;
}

// Match interp ways by content
static std::vector<uint32_t> match_interps(
    const std::vector<char>& old_data, const std::vector<char>& old_nodes,
    const std::vector<char>& new_data, const std::vector<char>& new_nodes,
    size_t stride)
{
    size_t street_off = (stride >= 20) ? 8 : 5;
    size_t old_n = old_data.size() / stride, new_n = new_data.size() / stride;

    size_t old_nc = old_nodes.size() / 8, new_nc = new_nodes.size() / 8;
    auto ihash = [&](const char* p, const char* nodes, size_t max_n) -> uint64_t {
        uint32_t node_offset, street_id, start, end;
        uint8_t count, itype;
        memcpy(&node_offset, p, 4); count = (uint8_t)p[4];
        memcpy(&street_id, p + street_off, 4);
        memcpy(&start, p + street_off + 4, 4);
        memcpy(&end, p + street_off + 8, 4);
        itype = (street_off + 12 < stride) ? (uint8_t)p[street_off + 12] : 0;
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, street_id); h = fnv_mix(h, start);
        h = fnv_mix(h, end); h = fnv_mix(h, itype); h = fnv_mix(h, count);
        for (uint8_t j = 0; j < count && (node_offset + j) < max_n; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4);
            memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(ihash(new_data.data() + i * stride, new_nodes.data(), new_nc), i);

    std::vector<uint32_t> result(old_n, 0xFFFFFFFF);
    size_t matched = 0;
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = ihash(old_data.data() + i * stride, old_nodes.data(), old_nc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            result[i] = it->second;
            uint32_t new_node_off;
            memcpy(&new_node_off, new_data.data() + it->second * stride, 4);
            memcpy(const_cast<char*>(old_data.data()) + i * stride, &new_node_off, 4);
            new_map.erase(it);
            matched++;
        }
    }
    std::cerr << "    interps matched=" << matched << "/" << old_n << " new=" << new_map.size() << std::endl;
    return result;
}

// --- Entry file remapping ---
// Parse geo_cells + entries, remap IDs, rebuild entry + cell files.

static void remap_and_rebuild_entries(
    const std::string& old_dir, const std::string& new_dir, const std::string& tmpdir,
    const std::vector<uint32_t>& way_remap,
    const std::vector<uint32_t>& addr_remap,
    const std::vector<uint32_t>& interp_remap,
    const std::vector<uint32_t>& admin_remap)
{
    // For geo entries: parse old entries, remap IDs, then copy NEW entries/cells
    // (since the new files have the correct structure for the new build).
    // We write the remapped old entries + new cells to temp for zstd diffing.

    // Simpler approach: just copy new entry/cell files to temp dir.
    // The zstd diff of old vs new entry files won't benefit from ID remapping
    // unless we actually remap the old entries. Let's do the full remap.

    auto parse_entries = [](const std::vector<char>& cells_data, size_t cell_stride, size_t off_pos,
                            const std::vector<char>& entries_data)
        -> std::unordered_map<uint64_t, std::vector<uint32_t>>
    {
        std::unordered_map<uint64_t, std::vector<uint32_t>> result;
        size_t n = cells_data.size() / cell_stride;
        for (size_t i = 0; i < n; i++) {
            uint64_t cell_id; uint32_t off;
            memcpy(&cell_id, cells_data.data() + i * cell_stride, 8);
            memcpy(&off, cells_data.data() + i * cell_stride + off_pos, 4);
            if (off == 0xFFFFFFFF || off + 2 > entries_data.size()) continue;
            uint16_t count;
            memcpy(&count, entries_data.data() + off, 2);
            if (off + 2 + count * 4 > entries_data.size()) continue;
            std::vector<uint32_t> ids(count);
            memcpy(ids.data(), entries_data.data() + off + 2, count * 4);
            result[cell_id] = std::move(ids);
        }
        return result;
    };

    auto remap_ids = [](std::unordered_map<uint64_t, std::vector<uint32_t>>& cells,
                         const std::vector<uint32_t>& id_remap, bool has_flags = false) {
        for (auto& [cid, ids] : cells) {
            for (auto& id : ids) {
                uint32_t flags = has_flags ? (id & 0x80000000u) : 0;
                uint32_t masked = id & 0x7FFFFFFFu;
                if (masked < id_remap.size() && id_remap[masked] != 0xFFFFFFFF)
                    id = id_remap[masked] | flags;
            }
            std::sort(ids.begin(), ids.end());
        }
    };

    auto write_entries_cells = [](const std::string& cells_path, const std::string& entries_path,
                                   const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map,
                                   size_t cell_stride)
    {
        std::vector<uint64_t> sorted_cells;
        for (auto& [id, _] : cell_map) sorted_cells.push_back(id);
        std::sort(sorted_cells.begin(), sorted_cells.end());

        std::vector<char> entries_buf;
        std::unordered_map<uint64_t, uint32_t> offsets;
        for (uint64_t cid : sorted_cells) {
            auto it = cell_map.find(cid);
            if (it == cell_map.end() || it->second.empty()) continue;
            offsets[cid] = static_cast<uint32_t>(entries_buf.size());
            uint16_t count = static_cast<uint16_t>(it->second.size());
            entries_buf.insert(entries_buf.end(), (char*)&count, (char*)&count + 2);
            entries_buf.insert(entries_buf.end(), (char*)it->second.data(),
                              (char*)it->second.data() + it->second.size() * 4);
        }
        write_file(entries_path, entries_buf);

        uint32_t no_data = 0xFFFFFFFF;
        std::vector<char> cells_buf;
        for (uint64_t cid : sorted_cells) {
            cells_buf.insert(cells_buf.end(), (char*)&cid, (char*)&cid + 8);
            auto it = offsets.find(cid);
            uint32_t off = (it != offsets.end()) ? it->second : no_data;
            cells_buf.insert(cells_buf.end(), (char*)&off, (char*)&off + 4);
        }
        write_file(cells_path, cells_buf);
    };

    // Geo cells: parse old, remap, merge with new cell set
    std::cerr << "  Remapping geo entries..." << std::endl;
    {
        auto old_geo = read_file(old_dir + "/geo_cells.bin");
        auto old_se = read_file(old_dir + "/street_entries.bin");
        auto old_ae = read_file(old_dir + "/addr_entries.bin");
        auto old_ie = read_file(old_dir + "/interp_entries.bin");

        auto new_geo = read_file(new_dir + "/geo_cells.bin");
        auto new_se = read_file(new_dir + "/street_entries.bin");
        auto new_ae = read_file(new_dir + "/addr_entries.bin");
        auto new_ie = read_file(new_dir + "/interp_entries.bin");

        // Parse old entries
        auto sc = parse_entries(old_geo, 20, 8, old_se);
        auto ac = parse_entries(old_geo, 20, 12, old_ae);
        auto ic = parse_entries(old_geo, 20, 16, old_ie);

        // Parse new entries (for cells that are new/changed)
        auto new_sc = parse_entries(new_geo, 20, 8, new_se);
        auto new_ac = parse_entries(new_geo, 20, 12, new_ae);
        auto new_ic = parse_entries(new_geo, 20, 16, new_ie);

        // Remap old IDs
        remap_ids(sc, way_remap);
        remap_ids(ac, addr_remap);
        remap_ids(ic, interp_remap);

        // Merge: start with remapped old, overlay with new for changed/added cells
        // For cells that exist in new but not old: add from new
        // For cells that exist in old but not new: remove
        for (auto& [cid, ids] : new_sc) sc[cid] = ids;
        for (auto& [cid, ids] : new_ac) ac[cid] = ids;
        for (auto& [cid, ids] : new_ic) ic[cid] = ids;

        // Collect all cells from new build
        std::unordered_map<uint64_t, bool> new_cells;
        size_t nn = new_geo.size() / 20;
        for (size_t i = 0; i < nn; i++) {
            uint64_t cid; memcpy(&cid, new_geo.data() + i * 20, 8);
            new_cells[cid] = true;
        }
        // Remove cells not in new, AND remove stale entries where the new build
        // has no data for that type (cell exists but no addrs/streets/interps)
        for (auto it = sc.begin(); it != sc.end(); )
            (new_cells.count(it->first) && new_sc.count(it->first)) ? ++it : it = sc.erase(it);
        for (auto it = ac.begin(); it != ac.end(); )
            (new_cells.count(it->first) && new_ac.count(it->first)) ? ++it : it = ac.erase(it);
        for (auto it = ic.begin(); it != ic.end(); )
            (new_cells.count(it->first) && new_ic.count(it->first)) ? ++it : it = ic.erase(it);

        // Write merged entries + geo_cells
        // (This produces files almost identical to new build's entries)
        auto write_geo = [&](const std::string& dir,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& s,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& a,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& ip) {
            // Collect all cells
            std::vector<uint64_t> all_cells;
            for (auto& [id, _] : new_cells) all_cells.push_back(id);
            std::sort(all_cells.begin(), all_cells.end());

            auto write_ent = [&](const std::string& path,
                                  const std::unordered_map<uint64_t, std::vector<uint32_t>>& cm)
                -> std::unordered_map<uint64_t, uint32_t>
            {
                std::vector<char> buf;
                std::unordered_map<uint64_t, uint32_t> offsets;
                for (uint64_t cid : all_cells) {
                    auto it = cm.find(cid);
                    if (it == cm.end() || it->second.empty()) continue;
                    offsets[cid] = static_cast<uint32_t>(buf.size());
                    uint16_t count = static_cast<uint16_t>(it->second.size());
                    buf.insert(buf.end(), (char*)&count, (char*)&count + 2);
                    buf.insert(buf.end(), (char*)it->second.data(),
                              (char*)it->second.data() + it->second.size() * 4);
                }
                write_file(path, buf);
                return offsets;
            };

            auto so = write_ent(dir + "/street_entries.bin", s);
            auto ao = write_ent(dir + "/addr_entries.bin", a);
            auto io = write_ent(dir + "/interp_entries.bin", ip);

            uint32_t no_data = 0xFFFFFFFF;
            std::vector<char> buf;
            for (uint64_t cid : all_cells) {
                buf.insert(buf.end(), (char*)&cid, (char*)&cid + 8);
                auto g = [&](const auto& m) -> uint32_t {
                    auto it = m.find(cid); return (it != m.end()) ? it->second : no_data;
                };
                uint32_t sv = g(so), av = g(ao), iv = g(io);
                buf.insert(buf.end(), (char*)&sv, (char*)&sv + 4);
                buf.insert(buf.end(), (char*)&av, (char*)&av + 4);
                buf.insert(buf.end(), (char*)&iv, (char*)&iv + 4);
            }
            write_file(dir + "/geo_cells.bin", buf);
        };

        write_geo(tmpdir, sc, ac, ic);
    }

    // Admin cells: parse, remap, rebuild
    std::cerr << "  Remapping admin entries..." << std::endl;
    {
        auto old_ac = read_file(old_dir + "/admin_cells.bin");
        auto old_ae = read_file(old_dir + "/admin_entries.bin");
        auto new_ac = read_file(new_dir + "/admin_cells.bin");
        auto new_ae = read_file(new_dir + "/admin_entries.bin");

        auto cells = parse_entries(old_ac, 12, 8, old_ae);
        auto new_cells_map = parse_entries(new_ac, 12, 8, new_ae);
        remap_ids(cells, admin_remap, true);

        // Merge with new
        for (auto& [cid, ids] : new_cells_map) cells[cid] = ids;
        // Remove cells not in new
        std::unordered_map<uint64_t, bool> new_cell_set;
        size_t nn = new_ac.size() / 12;
        for (size_t i = 0; i < nn; i++) {
            uint64_t cid; memcpy(&cid, new_ac.data() + i * 12, 8);
            new_cell_set[cid] = true;
        }
        for (auto it = cells.begin(); it != cells.end(); )
            new_cell_set.count(it->first) ? ++it : it = cells.erase(it);

        write_entries_cells(tmpdir + "/admin_cells.bin", tmpdir + "/admin_entries.bin", cells, 12);
    }
}

// --- Write helpers ---
static void write_val(std::ofstream& f, const void* data, size_t size) {
    f.write(reinterpret_cast<const char*>(data), size);
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>" << std::endl;
        return 1;
    }
    std::string old_dir = argv[1], new_dir = argv[2], patch_path = argv[4];
    std::string tmpdir = "/tmp/geocoder-diff-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    // Build string remap
    std::cerr << "Building string remap..." << std::endl;
    auto old_strings = read_file(old_dir + "/strings.bin");
    auto new_strings = read_file(new_dir + "/strings.bin");
    auto str_remap = build_string_remap(old_strings, new_strings);
    std::cerr << "  " << str_remap.size() << " strings mapped" << std::endl;

    // Detect struct strides
    auto detect = [](const std::string& path, std::vector<size_t> cs) -> size_t {
        auto d = read_file(path);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect(old_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect(old_dir + "/interp_ways.bin", {20, 18});
    size_t admin_stride = detect(old_dir + "/admin_polygons.bin", {24, 20, 19});

    // Build record ID remaps (requires loading data + coordinate files)
    std::cerr << "Building record ID remaps..." << std::endl;
    std::vector<uint32_t> addr_id_remap, way_id_remap, interp_id_remap, admin_id_remap;
    {
        // addr_points: string-remap old, then match by bytes
        auto old_ap = read_file(old_dir + "/addr_points.bin");
        remap_addr_points(old_ap, str_remap);
        auto new_ap = read_file(new_dir + "/addr_points.bin");
        addr_id_remap = match_addr_points(old_ap, new_ap);
        write_file(tmpdir + "/addr_points.bin", old_ap);
    }
    {
        // street_ways: string-remap + content match (fixes node_offset too)
        auto old_w = read_file(old_dir + "/street_ways.bin");
        remap_field(old_w, way_stride, way_stride == 12 ? 8 : 5, str_remap);
        auto old_n = read_file(old_dir + "/street_nodes.bin");
        auto new_w = read_file(new_dir + "/street_ways.bin");
        auto new_n = read_file(new_dir + "/street_nodes.bin");
        way_id_remap = match_ways(old_w, old_n, new_w, new_n, way_stride);
        write_file(tmpdir + "/street_ways.bin", old_w);
        write_file(tmpdir + "/street_nodes.bin", new_n); // use new nodes (offsets now match)
    }
    {
        auto old_i = read_file(old_dir + "/interp_ways.bin");
        remap_field(old_i, interp_stride, interp_stride >= 20 ? 8 : 5, str_remap);
        auto old_n = read_file(old_dir + "/interp_nodes.bin");
        auto new_i = read_file(new_dir + "/interp_ways.bin");
        auto new_n = read_file(new_dir + "/interp_nodes.bin");
        interp_id_remap = match_interps(old_i, old_n, new_i, new_n, interp_stride);
        write_file(tmpdir + "/interp_ways.bin", old_i);
        write_file(tmpdir + "/interp_nodes.bin", new_n);
    }
    {
        auto old_p = read_file(old_dir + "/admin_polygons.bin");
        remap_field(old_p, admin_stride, 8, str_remap);
        auto old_v = read_file(old_dir + "/admin_vertices.bin");
        auto new_p = read_file(new_dir + "/admin_polygons.bin");
        auto new_v = read_file(new_dir + "/admin_vertices.bin");
        admin_id_remap = match_admin(old_p, old_v, new_p, new_v, admin_stride);
        write_file(tmpdir + "/admin_polygons.bin", old_p);
        write_file(tmpdir + "/admin_vertices.bin", new_v);
    }

    // Rebuild entry/cell files with remapped IDs
    remap_and_rebuild_entries(old_dir, new_dir, tmpdir,
                              way_id_remap, addr_id_remap, interp_id_remap, admin_id_remap);

    // Open patch file
    std::ofstream pf(patch_path, std::ios::binary);
    pf.write(GCPATCH_MAGIC, 8);
    uint32_t ver = GCPATCH_VERSION, flags = 0;
    write_val(pf, &ver, 4); write_val(pf, &flags, 4);

    // Helper: compress a blob with zstd via CLI, return compressed data
    auto zstd_compress_blob = [&](const std::vector<char>& data) -> std::vector<char> {
        std::string raw_path = tmpdir + "/blob.raw";
        std::string zst_path = tmpdir + "/blob.zst";
        write_file(raw_path, data);
        system(("zstd -19 '" + raw_path + "' -o '" + zst_path + "' -f --quiet 2>/dev/null").c_str());
        auto result = read_file(zst_path);
        remove(raw_path.c_str()); remove(zst_path.c_str());
        return result;
    };

    // Write string remap (compressed)
    {
        uint32_t marker = 0xFFFFFFFE;
        write_val(pf, &marker, 4);
        std::vector<std::pair<uint32_t,uint32_t>> entries;
        size_t pos = 0;
        while (pos < old_strings.size()) {
            uint32_t old_off = static_cast<uint32_t>(pos);
            auto it = str_remap.find(old_off);
            entries.push_back({old_off, (it != str_remap.end()) ? it->second : old_off});
            pos += strlen(old_strings.data() + pos) + 1;
        }
        // Serialize to blob
        std::vector<char> blob(entries.size() * 8);
        for (size_t i = 0; i < entries.size(); i++) {
            memcpy(blob.data() + i * 8, &entries[i].first, 4);
            memcpy(blob.data() + i * 8 + 4, &entries[i].second, 4);
        }
        auto compressed = zstd_compress_blob(blob);
        uint32_t count = static_cast<uint32_t>(entries.size());
        uint64_t comp_size = compressed.size();
        write_val(pf, &count, 4);
        write_val(pf, &comp_size, 8);
        pf.write(compressed.data(), compressed.size());
        std::cerr << "  Remap: " << count << " entries (" << blob.size() << " -> " << compressed.size() << " bytes)" << std::endl;
    }

    // Write offset fixup tables for ways/interps/admin (node_offset/vertex_offset)
    {
        auto write_fixups = [&](PatchFileId id, const std::string& name,
                                 const std::vector<uint32_t>& id_remap, size_t stride) {
            auto new_data = read_file(new_dir + "/" + name);
            uint32_t marker = FIXUP_MARKER, fid = static_cast<uint32_t>(id);
            uint32_t st = static_cast<uint32_t>(stride);
            std::vector<std::pair<uint32_t, uint32_t>> fixups;
            for (uint32_t i = 0; i < id_remap.size(); i++) {
                if (id_remap[i] != 0xFFFFFFFF) {
                    uint32_t new_val;
                    memcpy(&new_val, new_data.data() + id_remap[i] * stride, 4);
                    fixups.push_back({i, new_val});
                }
            }
            // Serialize + compress
            std::vector<char> blob(fixups.size() * 8);
            for (size_t i = 0; i < fixups.size(); i++) {
                memcpy(blob.data() + i * 8, &fixups[i].first, 4);
                memcpy(blob.data() + i * 8 + 4, &fixups[i].second, 4);
            }
            auto compressed = zstd_compress_blob(blob);
            write_val(pf, &marker, 4); write_val(pf, &fid, 4);
            write_val(pf, &st, 4);
            uint32_t count = static_cast<uint32_t>(fixups.size());
            uint64_t comp_size = compressed.size();
            write_val(pf, &count, 4);
            write_val(pf, &comp_size, 8);
            pf.write(compressed.data(), compressed.size());
            std::cerr << "  fixup " << name << ": " << count << " entries (" << blob.size() << " -> " << compressed.size() << ")" << std::endl;
        };
        write_fixups(PatchFileId::STREET_WAYS, "street_ways.bin", way_id_remap, way_stride);
        write_fixups(PatchFileId::INTERP_WAYS, "interp_ways.bin", interp_id_remap, interp_stride);
        write_fixups(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin", admin_id_remap, admin_stride);
    }

    // Write record ID remap tables (for entry file reconstruction in patch tool)
    // Marker: 0xFFFFFFFC, file_id, count, [(old_id, new_id)] pairs
    {
        auto write_id_remap = [&](PatchFileId id, const std::vector<uint32_t>& remap) {
            uint32_t marker = 0xFFFFFFFC, fid = static_cast<uint32_t>(id);
            std::vector<std::pair<uint32_t,uint32_t>> pairs;
            for (uint32_t i = 0; i < remap.size(); i++)
                if (remap[i] != 0xFFFFFFFF) pairs.push_back({i, remap[i]});
            std::vector<char> blob(pairs.size() * 8);
            for (size_t i = 0; i < pairs.size(); i++) {
                memcpy(blob.data() + i * 8, &pairs[i].first, 4);
                memcpy(blob.data() + i * 8 + 4, &pairs[i].second, 4);
            }
            auto compressed = zstd_compress_blob(blob);
            write_val(pf, &marker, 4); write_val(pf, &fid, 4);
            uint32_t count = static_cast<uint32_t>(pairs.size());
            uint64_t comp_size = compressed.size();
            write_val(pf, &count, 4);
            write_val(pf, &comp_size, 8);
            pf.write(compressed.data(), compressed.size());
            std::cerr << "  id_remap file " << fid << ": " << count << " (" << blob.size() << " -> " << compressed.size() << ")" << std::endl;
        };
        write_id_remap(PatchFileId::STREET_WAYS, way_id_remap);
        write_id_remap(PatchFileId::ADDR_POINTS, addr_id_remap);
        write_id_remap(PatchFileId::INTERP_WAYS, interp_id_remap);
        write_id_remap(PatchFileId::ADMIN_POLYGONS, admin_id_remap);
    }

    // Single-stage diffs: remapped_old → new (zstd --patch-from on temp files)
    auto diff_file = [&](PatchFileId id, const std::string& name, bool use_old_as_ref = false) {
        std::cerr << "  " << name << ": ";
        std::string ref;
        if (use_old_as_ref) {
            ref = old_dir + "/" + name;
        } else {
            ref = tmpdir + "/" + name;
            if (read_file(ref).empty()) ref = old_dir + "/" + name;
        }
        std::string tgt = new_dir + "/" + name;
        std::string zst = tmpdir + "/" + name + ".zst";
        system(("zstd --patch-from='" + ref + "' '" + tgt + "' -o '" + zst + "' -f --quiet 2>/dev/null").c_str());
        auto zst_data = read_file(zst);
        auto old_data = read_file(old_dir + "/" + name);
        auto new_data = read_file(tgt);

        uint32_t fid = static_cast<uint32_t>(id), enc = static_cast<uint32_t>(PatchEncoding::ZSTD_DELTA);
        uint64_t os = old_data.size(), ns = new_data.size(), ds = zst_data.size();
        write_val(pf, &fid, 4); write_val(pf, &enc, 4);
        write_val(pf, &os, 8); write_val(pf, &ns, 8); write_val(pf, &ds, 8);
        pf.write(zst_data.data(), zst_data.size());

        double pct = ns > 0 ? ds * 100.0 / ns : 0;
        std::cerr << ds << " bytes (" << std::fixed << std::setprecision(2) << pct << "%)" << std::endl;
        remove(zst.c_str());
    };

    // Data files with string remap + fixups: use remapped temp as ref
    diff_file(PatchFileId::ADDR_POINTS, "addr_points.bin");
    diff_file(PatchFileId::STREET_WAYS, "street_ways.bin");
    diff_file(PatchFileId::INTERP_WAYS, "interp_ways.bin");
    diff_file(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin");
    // Coord files: use old as ref (no remap needed)
    diff_file(PatchFileId::STRINGS, "strings.bin", true);
    diff_file(PatchFileId::STREET_NODES, "street_nodes.bin", true);
    diff_file(PatchFileId::INTERP_NODES, "interp_nodes.bin", true);
    diff_file(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", true);
    // Entry/cell files: use rebuilt temp as ref (with remapped IDs)
    diff_file(PatchFileId::GEO_CELLS, "geo_cells.bin");
    diff_file(PatchFileId::STREET_ENTRIES, "street_entries.bin");
    diff_file(PatchFileId::ADDR_ENTRIES, "addr_entries.bin");
    diff_file(PatchFileId::INTERP_ENTRIES, "interp_entries.bin");
    diff_file(PatchFileId::ADMIN_CELLS, "admin_cells.bin");
    diff_file(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin");

    uint32_t end = 0xFFFFFFFF;
    write_val(pf, &end, 4);

    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());

    auto total = pf.tellp();
    std::cerr << "\nTotal patch: " << total << " bytes (" << total / 1024 / 1024 << " MiB)" << std::endl;
    return 0;
}

-        // Merge: start with remapped old, overlay with new for changed/added cells

-        // For cells that exist in new but not old: add from new

-        // For cells that exist in old but not new: remove

-        for (auto& [cid, ids] : new_sc) sc[cid] = ids;

-        for (auto& [cid, ids] : new_ac) ac[cid] = ids;

-        for (auto& [cid, ids] : new_ic) ic[cid] = ids;

+        // Don't overlay new entries — use pure ID remap only.

+        // This ensures the diff tool's rebuild matches the patch tool's rebuild exactly.

+        // The zstd delta will encode the remaining differences (new/removed records).

+        (void)new_sc; (void)new_ac; (void)new_ic; // not used for overlay

-        // Collect all cells from new build

-        std::unordered_map<uint64_t, bool> new_cells;

-        size_t nn = new_geo.size() / 20;

-        for (size_t i = 0; i < nn; i++) {

-            uint64_t cid; memcpy(&cid, new_geo.data() + i * 20, 8);

-        // Remove cells not in new, AND remove stale entries where the new build

-        // has no data for that type (cell exists but no addrs/streets/interps)

-        for (auto it = sc.begin(); it != sc.end(); )

-            (new_cells.count(it->first) && new_sc.count(it->first)) ? ++it : it = sc.erase(it);

-        for (auto it = ac.begin(); it != ac.end(); )

-            (new_cells.count(it->first) && new_ac.count(it->first)) ? ++it : it = ac.erase(it);

-        for (auto it = ic.begin(); it != ic.end(); )

-            (new_cells.count(it->first) && new_ic.count(it->first)) ? ++it : it = ic.erase(it);


Also need to update the write section to use the OLD cell set (since we're not merging with new):


   377→        // (This produces files almost identical to new build's entries)
   378→        auto write_geo = [&](const std::string& dir,
   379→                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& s,
   380→                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& a,
   381→                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& ip) {
   382→            // Collect all cells
   383→            std::vector<uint64_t> all_cells;
   384→            for (auto& [id, _] : new_cells) all_cells.push_back(id);
   385→            std::sort(all_cells.begin(), all_cells.end());
   386→

        // (This produces files almost identical to new build's entries)
        auto write_geo = [&](const std::string& dir,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& s,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& a,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& ip) {
            // Collect all cells
            std::vector<uint64_t> all_cells;
            for (auto& [id, _] : new_cells) all_cells.push_back(id);
            std::sort(all_cells.begin(), all_cells.end());


The `new_cells` variable was used for the cell set. But now we removed the overlay and new_cells declaration. Let me use the OLD cell set instead (parsed from old geo_cells):

        // (This produces files almost identical to new build's entries)
        auto write_geo = [&](const std::string& dir,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& s,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& a,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& ip) {
            // Collect all cells
            std::vector<uint64_t> all_cells;
            for (auto& [id, _] : new_cells) all_cells.push_back(id);

        // Write rebuilt entries + geo_cells using old cell set + remapped IDs.
        // This produces the same output as the patch tool's rebuild.
        auto write_geo = [&](const std::string& dir,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& s,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& a,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& ip) {
            // Use OLD cell set (patch tool only has old cells)
            std::vector<uint64_t> all_cells;
            size_t n_old = old_geo.size() / 20;
            for (size_t i = 0; i < n_old; i++) {
                uint64_t cid; memcpy(&cid, old_geo.data() + i * 20, 8);
                all_cells.push_back(cid);
            }