// geocoder-diff: Compare two deterministic geocoder builds, produce a .gcpatch.
//
// Three-level remapping for minimal patches:
// 1. String remap: old string offsets → new string offsets
// 2. Record ID remap: match records by content, map old_id → new_id
// 3. Entry remap: apply record ID remap to entry files + rebuild cell indexes
//
// Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>

#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <functional>
#include <iomanip>
#include <iostream>
#include <string>
#include <unistd.h>
#include <unordered_map>
#include <vector>

#include "patch_format.h"

// --- String remap ---

static std::unordered_map<uint32_t, uint32_t> build_string_remap(
    const std::vector<char>& old_pool, const std::vector<char>& new_pool)
{
    std::unordered_map<std::string, uint32_t> new_idx;
    size_t pos = 0;
    while (pos < new_pool.size()) {
        const char* s = new_pool.data() + pos;
        size_t len = strlen(s);
        new_idx[std::string(s, len)] = static_cast<uint32_t>(pos);
        pos += len + 1;
    }
    std::unordered_map<uint32_t, uint32_t> remap;
    pos = 0;
    while (pos < old_pool.size()) {
        const char* s = old_pool.data() + pos;
        size_t len = strlen(s);
        auto it = new_idx.find(std::string(s, len));
        if (it != new_idx.end())
            remap[static_cast<uint32_t>(pos)] = it->second;
        pos += len + 1;
    }
    return remap;
}

// --- Remap helpers ---

static void remap_addr_points(std::vector<char>& data, const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + 16 <= data.size(); i += 16)
        for (size_t off : {8, 12}) {
            uint32_t v; memcpy(&v, data.data() + i + off, 4);
            auto it = rm.find(v);
            if (it != rm.end()) memcpy(data.data() + i + off, &it->second, 4);
        }
}

static void remap_field(std::vector<char>& data, size_t stride, size_t field_off,
                         const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + stride <= data.size(); i += stride) {
        uint32_t v; memcpy(&v, data.data() + i + field_off, 4);
        auto it = rm.find(v);
        if (it != rm.end()) memcpy(data.data() + i + field_off, &it->second, 4);
    }
}

// --- Content-based record matching ---
// Returns old_id → new_id mapping

static uint64_t fnv_mix(uint64_t h, uint64_t v) { h ^= v; h *= 1099511628211ULL; return h; }

static std::vector<uint32_t> match_addr_points(
    const std::vector<char>& old_data, const std::vector<char>& new_data)
{
    // After string remap, matching records are byte-identical
    size_t old_n = old_data.size() / 16, new_n = new_data.size() / 16;
    // Hash each new record
    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++) {
        uint64_t h = 14695981039346656037ULL;
        const char* p = new_data.data() + i * 16;
        for (int j = 0; j < 16; j++) h = fnv_mix(h, (uint8_t)p[j]);
        new_map.emplace(h, i);
    }
    std::vector<uint32_t> result(old_n, 0xFFFFFFFF);
    size_t matched = 0;
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = 14695981039346656037ULL;
        const char* p = old_data.data() + i * 16;
        for (int j = 0; j < 16; j++) h = fnv_mix(h, (uint8_t)p[j]);
        auto it = new_map.find(h);
        if (it != new_map.end() &&
            memcmp(old_data.data() + i * 16, new_data.data() + it->second * 16, 16) == 0) {
            result[i] = it->second;
            new_map.erase(it);
            matched++;
        }
    }
    std::cerr << "    addr matched=" << matched << "/" << old_n << " new=" << new_map.size() << std::endl;
    return result;
}

static std::vector<uint32_t> match_ways(
    const std::vector<char>& old_ways, const std::vector<char>& old_nodes,
    const std::vector<char>& new_ways, const std::vector<char>& new_nodes,
    size_t stride)
{
    size_t name_off = (stride == 12) ? 8 : 5;
    size_t old_n = old_ways.size() / stride, new_n = new_ways.size() / stride;

    size_t old_node_count = old_nodes.size() / 8, new_node_count = new_nodes.size() / 8;
    auto way_hash = [&](const char* w, const char* nodes, size_t max_nodes) -> uint64_t {
        uint32_t node_offset, name_id; uint8_t node_count;
        memcpy(&node_offset, w, 4); node_count = (uint8_t)w[4];
        memcpy(&name_id, w + name_off, 4);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, node_count);
        for (uint8_t j = 0; j < node_count && (node_offset + j) < max_nodes; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4);
            memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(way_hash(new_ways.data() + i * stride, new_nodes.data(), new_node_count), i);

    std::vector<uint32_t> result(old_n, 0xFFFFFFFF);
    size_t matched = 0;
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = way_hash(old_ways.data() + i * stride, old_nodes.data(), old_node_count);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            result[i] = it->second;
            // Also fix node_offset in old to match new (makes way bytes identical)
            uint32_t new_node_off;
            memcpy(&new_node_off, new_ways.data() + it->second * stride, 4);
            memcpy(const_cast<char*>(old_ways.data()) + i * stride, &new_node_off, 4);
            new_map.erase(it);
            matched++;
        }
    }
    std::cerr << "    ways matched=" << matched << "/" << old_n << " new=" << new_map.size() << std::endl;
    return result;
}

static std::vector<uint32_t> match_admin(
    const std::vector<char>& old_polys, const std::vector<char>& old_verts,
    const std::vector<char>& new_polys, const std::vector<char>& new_verts,
    size_t stride)
{
    size_t old_n = old_polys.size() / stride, new_n = new_polys.size() / stride;

    size_t old_vc = old_verts.size() / 8, new_vc = new_verts.size() / 8;
    auto poly_hash = [&](const char* p, const char* verts, size_t max_v) -> uint64_t {
        uint32_t vert_offset, vert_count, name_id;
        memcpy(&vert_offset, p, 4); memcpy(&vert_count, p + 4, 4);
        memcpy(&name_id, p + 8, 4);
        uint8_t level = (uint8_t)p[12];
        uint16_t cc; memcpy(&cc, p + stride - 2, 2);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, level); h = fnv_mix(h, cc);
        h = fnv_mix(h, vert_count);
        uint32_t n = std::min(vert_count, 10u);
        for (uint32_t j = 0; j < n && (vert_offset + j) < max_v; j++) {
            float lat, lng;
            memcpy(&lat, verts + (vert_offset + j) * 8, 4);
            memcpy(&lng, verts + (vert_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(poly_hash(new_polys.data() + i * stride, new_verts.data(), new_vc), i);

    std::vector<uint32_t> result(old_n, 0xFFFFFFFF);
    size_t matched = 0;
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = poly_hash(old_polys.data() + i * stride, old_verts.data(), old_vc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            result[i] = it->second;
            uint32_t new_vert_off;
            memcpy(&new_vert_off, new_polys.data() + it->second * stride, 4);
            memcpy(const_cast<char*>(old_polys.data()) + i * stride, &new_vert_off, 4);
            new_map.erase(it);
            matched++;
        }
    }
    std::cerr << "    admin matched=" << matched << "/" << old_n << " new=" << new_map.size() << std::endl;
    return result;
}

// Match interp ways by content
static std::vector<uint32_t> match_interps(
    const std::vector<char>& old_data, const std::vector<char>& old_nodes,
    const std::vector<char>& new_data, const std::vector<char>& new_nodes,
    size_t stride)
{
    size_t street_off = (stride >= 20) ? 8 : 5;
    size_t old_n = old_data.size() / stride, new_n = new_data.size() / stride;

    size_t old_nc = old_nodes.size() / 8, new_nc = new_nodes.size() / 8;
    auto ihash = [&](const char* p, const char* nodes, size_t max_n) -> uint64_t {
        uint32_t node_offset, street_id, start, end;
        uint8_t count, itype;
        memcpy(&node_offset, p, 4); count = (uint8_t)p[4];
        memcpy(&street_id, p + street_off, 4);
        memcpy(&start, p + street_off + 4, 4);
        memcpy(&end, p + street_off + 8, 4);
        itype = (street_off + 12 < stride) ? (uint8_t)p[street_off + 12] : 0;
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, street_id); h = fnv_mix(h, start);
        h = fnv_mix(h, end); h = fnv_mix(h, itype); h = fnv_mix(h, count);
        for (uint8_t j = 0; j < count && (node_offset + j) < max_n; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4);
            memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(ihash(new_data.data() + i * stride, new_nodes.data(), new_nc), i);

    std::vector<uint32_t> result(old_n, 0xFFFFFFFF);
    size_t matched = 0;
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = ihash(old_data.data() + i * stride, old_nodes.data(), old_nc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            result[i] = it->second;
            uint32_t new_node_off;
            memcpy(&new_node_off, new_data.data() + it->second * stride, 4);
            memcpy(const_cast<char*>(old_data.data()) + i * stride, &new_node_off, 4);
            new_map.erase(it);
            matched++;
        }
    }
    std::cerr << "    interps matched=" << matched << "/" << old_n << " new=" << new_map.size() << std::endl;
    return result;
}

// --- Entry file remapping ---
// Parse geo_cells + entries, remap IDs, rebuild entry + cell files.

static void remap_and_rebuild_entries(
    const std::string& old_dir, const std::string& new_dir, const std::string& tmpdir,
    const std::vector<uint32_t>& way_remap,
    const std::vector<uint32_t>& addr_remap,
    const std::vector<uint32_t>& interp_remap,
    const std::vector<uint32_t>& admin_remap)
{
    // For geo entries: parse old entries, remap IDs, then copy NEW entries/cells
    // (since the new files have the correct structure for the new build).
    // We write the remapped old entries + new cells to temp for zstd diffing.

    // Simpler approach: just copy new entry/cell files to temp dir.
    // The zstd diff of old vs new entry files won't benefit from ID remapping
    // unless we actually remap the old entries. Let's do the full remap.

    auto parse_entries = [](const std::vector<char>& cells_data, size_t cell_stride, size_t off_pos,
                            const std::vector<char>& entries_data)
        -> std::unordered_map<uint64_t, std::vector<uint32_t>>
    {
        std::unordered_map<uint64_t, std::vector<uint32_t>> result;
        size_t n = cells_data.size() / cell_stride;
        for (size_t i = 0; i < n; i++) {
            uint64_t cell_id; uint32_t off;
            memcpy(&cell_id, cells_data.data() + i * cell_stride, 8);
            memcpy(&off, cells_data.data() + i * cell_stride + off_pos, 4);
            if (off == 0xFFFFFFFF || off + 2 > entries_data.size()) continue;
            uint16_t count;
            memcpy(&count, entries_data.data() + off, 2);
            if (off + 2 + count * 4 > entries_data.size()) continue;
            std::vector<uint32_t> ids(count);
            memcpy(ids.data(), entries_data.data() + off + 2, count * 4);
            result[cell_id] = std::move(ids);
        }
        return result;
    };

    auto remap_ids = [](std::unordered_map<uint64_t, std::vector<uint32_t>>& cells,
                         const std::vector<uint32_t>& id_remap, bool has_flags = false) {
        for (auto& [cid, ids] : cells) {
            for (auto& id : ids) {
                uint32_t flags = has_flags ? (id & 0x80000000u) : 0;
                uint32_t masked = id & 0x7FFFFFFFu;
                if (masked < id_remap.size() && id_remap[masked] != 0xFFFFFFFF)
                    id = id_remap[masked] | flags;
            }
            std::sort(ids.begin(), ids.end());
        }
    };

    auto write_entries_cells = [](const std::string& cells_path, const std::string& entries_path,
                                   const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map,
                                   size_t cell_stride)
    {
        std::vector<uint64_t> sorted_cells;
        for (auto& [id, _] : cell_map) sorted_cells.push_back(id);
        std::sort(sorted_cells.begin(), sorted_cells.end());

        std::vector<char> entries_buf;
        std::unordered_map<uint64_t, uint32_t> offsets;
        for (uint64_t cid : sorted_cells) {
            auto it = cell_map.find(cid);
            if (it == cell_map.end() || it->second.empty()) continue;
            offsets[cid] = static_cast<uint32_t>(entries_buf.size());
            uint16_t count = static_cast<uint16_t>(it->second.size());
            entries_buf.insert(entries_buf.end(), (char*)&count, (char*)&count + 2);
            entries_buf.insert(entries_buf.end(), (char*)it->second.data(),
                              (char*)it->second.data() + it->second.size() * 4);
        }
        write_file(entries_path, entries_buf);

        uint32_t no_data = 0xFFFFFFFF;
        std::vector<char> cells_buf;
        for (uint64_t cid : sorted_cells) {
            cells_buf.insert(cells_buf.end(), (char*)&cid, (char*)&cid + 8);
            auto it = offsets.find(cid);
            uint32_t off = (it != offsets.end()) ? it->second : no_data;
            cells_buf.insert(cells_buf.end(), (char*)&off, (char*)&off + 4);
        }
        write_file(cells_path, cells_buf);
    };

    // Geo cells: parse old, remap, merge with new cell set
    std::cerr << "  Remapping geo entries..." << std::endl;
    {
        auto old_geo = read_file(old_dir + "/geo_cells.bin");
        auto old_se = read_file(old_dir + "/street_entries.bin");
        auto old_ae = read_file(old_dir + "/addr_entries.bin");
        auto old_ie = read_file(old_dir + "/interp_entries.bin");

        auto new_geo = read_file(new_dir + "/geo_cells.bin");
        auto new_se = read_file(new_dir + "/street_entries.bin");
        auto new_ae = read_file(new_dir + "/addr_entries.bin");
        auto new_ie = read_file(new_dir + "/interp_entries.bin");

        // Parse old entries
        auto sc = parse_entries(old_geo, 20, 8, old_se);
        auto ac = parse_entries(old_geo, 20, 12, old_ae);
        auto ic = parse_entries(old_geo, 20, 16, old_ie);

        // Parse new entries (for cells that are new/changed)
        auto new_sc = parse_entries(new_geo, 20, 8, new_se);
        auto new_ac = parse_entries(new_geo, 20, 12, new_ae);
        auto new_ic = parse_entries(new_geo, 20, 16, new_ie);

        // Remap old IDs
        remap_ids(sc, way_remap);
        remap_ids(ac, addr_remap);
        remap_ids(ic, interp_remap);

        // Don't overlay new entries — use pure ID remap only.
        // This ensures the diff tool's rebuild matches the patch tool's rebuild exactly.
        // The zstd delta will encode the remaining differences (new/removed records).
        (void)new_sc; (void)new_ac; (void)new_ic; // not used for overlay

        // Write merged entries + geo_cells
        // Write rebuilt entries + geo_cells using old cell set + remapped IDs.
        // This produces the same output as the patch tool's rebuild.
        auto write_geo = [&](const std::string& dir,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& s,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& a,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& ip) {
            // Use OLD cell set (patch tool only has old cells)
            std::vector<uint64_t> all_cells;
            size_t n_old = old_geo.size() / 20;
            for (size_t i = 0; i < n_old; i++) {
                uint64_t cid; memcpy(&cid, old_geo.data() + i * 20, 8);
                all_cells.push_back(cid);
            }
            std::sort(all_cells.begin(), all_cells.end());

            auto write_ent = [&](const std::string& path,
                                  const std::unordered_map<uint64_t, std::vector<uint32_t>>& cm)
                -> std::unordered_map<uint64_t, uint32_t>
            {
                std::vector<char> buf;
                std::unordered_map<uint64_t, uint32_t> offsets;
                for (uint64_t cid : all_cells) {
                    auto it = cm.find(cid);
                    if (it == cm.end() || it->second.empty()) continue;
                    offsets[cid] = static_cast<uint32_t>(buf.size());
                    uint16_t count = static_cast<uint16_t>(it->second.size());
                    buf.insert(buf.end(), (char*)&count, (char*)&count + 2);
                    buf.insert(buf.end(), (char*)it->second.data(),
                              (char*)it->second.data() + it->second.size() * 4);
                }
                write_file(path, buf);
                return offsets;
            };

            auto so = write_ent(dir + "/street_entries.bin", s);
            auto ao = write_ent(dir + "/addr_entries.bin", a);
            auto io = write_ent(dir + "/interp_entries.bin", ip);

            uint32_t no_data = 0xFFFFFFFF;
            std::vector<char> buf;
            for (uint64_t cid : all_cells) {
                buf.insert(buf.end(), (char*)&cid, (char*)&cid + 8);
                auto g = [&](const auto& m) -> uint32_t {
                    auto it = m.find(cid); return (it != m.end()) ? it->second : no_data;
                };
                uint32_t sv = g(so), av = g(ao), iv = g(io);
                buf.insert(buf.end(), (char*)&sv, (char*)&sv + 4);
                buf.insert(buf.end(), (char*)&av, (char*)&av + 4);
                buf.insert(buf.end(), (char*)&iv, (char*)&iv + 4);
            }
            write_file(dir + "/geo_cells.bin", buf);
        };

        write_geo(tmpdir, sc, ac, ic);
    }

    // Admin cells: parse, remap, rebuild
    std::cerr << "  Remapping admin entries..." << std::endl;
    {
        auto old_ac = read_file(old_dir + "/admin_cells.bin");
        auto old_ae = read_file(old_dir + "/admin_entries.bin");
        auto new_ac = read_file(new_dir + "/admin_cells.bin");
        auto new_ae = read_file(new_dir + "/admin_entries.bin");

        auto cells = parse_entries(old_ac, 12, 8, old_ae);
        (void)new_ac; (void)new_ae; // not used — pure ID remap to match patch tool
        remap_ids(cells, admin_remap, true);

        // No overlay, no removal — pure remap of old data (matches patch tool behavior)
        // The zstd delta encodes the differences from this to new.
        // Remove the old cell-not-in-new check since we don't overlay:
        std::unordered_map<uint64_t, bool> dummy;
        for (auto it = cells.begin(); it != cells.end(); )
            new_cell_set.count(it->first) ? ++it : it = cells.erase(it);

        write_entries_cells(tmpdir + "/admin_cells.bin", tmpdir + "/admin_entries.bin", cells, 12);
    }
}

// --- Write helpers ---
static void write_val(std::ofstream& f, const void* data, size_t size) {
    f.write(reinterpret_cast<const char*>(data), size);
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>" << std::endl;
        return 1;
    }
    std::string old_dir = argv[1], new_dir = argv[2], patch_path = argv[4];
    std::string tmpdir = "/tmp/geocoder-diff-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    // Build string remap
    std::cerr << "Building string remap..." << std::endl;
    auto old_strings = read_file(old_dir + "/strings.bin");
    auto new_strings = read_file(new_dir + "/strings.bin");
    auto str_remap = build_string_remap(old_strings, new_strings);
    std::cerr << "  " << str_remap.size() << " strings mapped" << std::endl;

    // Detect struct strides
    auto detect = [](const std::string& path, std::vector<size_t> cs) -> size_t {
        auto d = read_file(path);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect(old_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect(old_dir + "/interp_ways.bin", {20, 18});
    size_t admin_stride = detect(old_dir + "/admin_polygons.bin", {24, 20, 19});

    // Build record ID remaps (requires loading data + coordinate files)
    std::cerr << "Building record ID remaps..." << std::endl;
    std::vector<uint32_t> addr_id_remap, way_id_remap, interp_id_remap, admin_id_remap;
    {
        // addr_points: string-remap old, then match by bytes
        auto old_ap = read_file(old_dir + "/addr_points.bin");
        remap_addr_points(old_ap, str_remap);
        auto new_ap = read_file(new_dir + "/addr_points.bin");
        addr_id_remap = match_addr_points(old_ap, new_ap);
        write_file(tmpdir + "/addr_points.bin", old_ap);
    }
    {
        // street_ways: string-remap + content match (fixes node_offset too)
        auto old_w = read_file(old_dir + "/street_ways.bin");
        remap_field(old_w, way_stride, way_stride == 12 ? 8 : 5, str_remap);
        auto old_n = read_file(old_dir + "/street_nodes.bin");
        auto new_w = read_file(new_dir + "/street_ways.bin");
        auto new_n = read_file(new_dir + "/street_nodes.bin");
        way_id_remap = match_ways(old_w, old_n, new_w, new_n, way_stride);
        write_file(tmpdir + "/street_ways.bin", old_w);
        write_file(tmpdir + "/street_nodes.bin", new_n); // use new nodes (offsets now match)
    }
    {
        auto old_i = read_file(old_dir + "/interp_ways.bin");
        remap_field(old_i, interp_stride, interp_stride >= 20 ? 8 : 5, str_remap);
        auto old_n = read_file(old_dir + "/interp_nodes.bin");
        auto new_i = read_file(new_dir + "/interp_ways.bin");
        auto new_n = read_file(new_dir + "/interp_nodes.bin");
        interp_id_remap = match_interps(old_i, old_n, new_i, new_n, interp_stride);
        write_file(tmpdir + "/interp_ways.bin", old_i);
        write_file(tmpdir + "/interp_nodes.bin", new_n);
    }
    {
        auto old_p = read_file(old_dir + "/admin_polygons.bin");
        remap_field(old_p, admin_stride, 8, str_remap);
        auto old_v = read_file(old_dir + "/admin_vertices.bin");
        auto new_p = read_file(new_dir + "/admin_polygons.bin");
        auto new_v = read_file(new_dir + "/admin_vertices.bin");
        admin_id_remap = match_admin(old_p, old_v, new_p, new_v, admin_stride);
        write_file(tmpdir + "/admin_polygons.bin", old_p);
        write_file(tmpdir + "/admin_vertices.bin", new_v);
    }

    // Rebuild entry/cell files with remapped IDs
    remap_and_rebuild_entries(old_dir, new_dir, tmpdir,
                              way_id_remap, addr_id_remap, interp_id_remap, admin_id_remap);

    // Open patch file
    std::ofstream pf(patch_path, std::ios::binary);
    pf.write(GCPATCH_MAGIC, 8);
    uint32_t ver = GCPATCH_VERSION, flags = 0;
    write_val(pf, &ver, 4); write_val(pf, &flags, 4);

    // Helper: compress a blob with zstd via CLI, return compressed data
    auto zstd_compress_blob = [&](const std::vector<char>& data) -> std::vector<char> {
        std::string raw_path = tmpdir + "/blob.raw";
        std::string zst_path = tmpdir + "/blob.zst";
        write_file(raw_path, data);
        system(("zstd -19 '" + raw_path + "' -o '" + zst_path + "' -f --quiet 2>/dev/null").c_str());
        auto result = read_file(zst_path);
        remove(raw_path.c_str()); remove(zst_path.c_str());
        return result;
    };

    // Write string remap (compressed)
    {
        uint32_t marker = 0xFFFFFFFE;
        write_val(pf, &marker, 4);
        std::vector<std::pair<uint32_t,uint32_t>> entries;
        size_t pos = 0;
        while (pos < old_strings.size()) {
            uint32_t old_off = static_cast<uint32_t>(pos);
            auto it = str_remap.find(old_off);
            entries.push_back({old_off, (it != str_remap.end()) ? it->second : old_off});
            pos += strlen(old_strings.data() + pos) + 1;
        }
        // Serialize to blob
        std::vector<char> blob(entries.size() * 8);
        for (size_t i = 0; i < entries.size(); i++) {
            memcpy(blob.data() + i * 8, &entries[i].first, 4);
            memcpy(blob.data() + i * 8 + 4, &entries[i].second, 4);
        }
        auto compressed = zstd_compress_blob(blob);
        uint32_t count = static_cast<uint32_t>(entries.size());
        uint64_t comp_size = compressed.size();
        write_val(pf, &count, 4);
        write_val(pf, &comp_size, 8);
        pf.write(compressed.data(), compressed.size());
        std::cerr << "  Remap: " << count << " entries (" << blob.size() << " -> " << compressed.size() << " bytes)" << std::endl;
    }

    // Write offset fixup tables for ways/interps/admin (node_offset/vertex_offset)
    {
        auto write_fixups = [&](PatchFileId id, const std::string& name,
                                 const std::vector<uint32_t>& id_remap, size_t stride) {
            auto new_data = read_file(new_dir + "/" + name);
            uint32_t marker = FIXUP_MARKER, fid = static_cast<uint32_t>(id);
            uint32_t st = static_cast<uint32_t>(stride);
            std::vector<std::pair<uint32_t, uint32_t>> fixups;
            for (uint32_t i = 0; i < id_remap.size(); i++) {
                if (id_remap[i] != 0xFFFFFFFF) {
                    uint32_t new_val;
                    memcpy(&new_val, new_data.data() + id_remap[i] * stride, 4);
                    fixups.push_back({i, new_val});
                }
            }
            // Serialize + compress
            std::vector<char> blob(fixups.size() * 8);
            for (size_t i = 0; i < fixups.size(); i++) {
                memcpy(blob.data() + i * 8, &fixups[i].first, 4);
                memcpy(blob.data() + i * 8 + 4, &fixups[i].second, 4);
            }
            auto compressed = zstd_compress_blob(blob);
            write_val(pf, &marker, 4); write_val(pf, &fid, 4);
            write_val(pf, &st, 4);
            uint32_t count = static_cast<uint32_t>(fixups.size());
            uint64_t comp_size = compressed.size();
            write_val(pf, &count, 4);
            write_val(pf, &comp_size, 8);
            pf.write(compressed.data(), compressed.size());
            std::cerr << "  fixup " << name << ": " << count << " entries (" << blob.size() << " -> " << compressed.size() << ")" << std::endl;
        };
        write_fixups(PatchFileId::STREET_WAYS, "street_ways.bin", way_id_remap, way_stride);
        write_fixups(PatchFileId::INTERP_WAYS, "interp_ways.bin", interp_id_remap, interp_stride);
        write_fixups(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin", admin_id_remap, admin_stride);
    }

    // Write record ID remap tables (for entry file reconstruction in patch tool)
    // Marker: 0xFFFFFFFC, file_id, count, [(old_id, new_id)] pairs
    {
        auto write_id_remap = [&](PatchFileId id, const std::vector<uint32_t>& remap) {
            uint32_t marker = 0xFFFFFFFC, fid = static_cast<uint32_t>(id);
            std::vector<std::pair<uint32_t,uint32_t>> pairs;
            for (uint32_t i = 0; i < remap.size(); i++)
                if (remap[i] != 0xFFFFFFFF) pairs.push_back({i, remap[i]});
            std::vector<char> blob(pairs.size() * 8);
            for (size_t i = 0; i < pairs.size(); i++) {
                memcpy(blob.data() + i * 8, &pairs[i].first, 4);
                memcpy(blob.data() + i * 8 + 4, &pairs[i].second, 4);
            }
            auto compressed = zstd_compress_blob(blob);
            write_val(pf, &marker, 4); write_val(pf, &fid, 4);
            uint32_t count = static_cast<uint32_t>(pairs.size());
            uint64_t comp_size = compressed.size();
            write_val(pf, &count, 4);
            write_val(pf, &comp_size, 8);
            pf.write(compressed.data(), compressed.size());
            std::cerr << "  id_remap file " << fid << ": " << count << " (" << blob.size() << " -> " << compressed.size() << ")" << std::endl;
        };
        write_id_remap(PatchFileId::STREET_WAYS, way_id_remap);
        write_id_remap(PatchFileId::ADDR_POINTS, addr_id_remap);
        write_id_remap(PatchFileId::INTERP_WAYS, interp_id_remap);
        write_id_remap(PatchFileId::ADMIN_POLYGONS, admin_id_remap);
    }

    // Single-stage diffs: remapped_old → new (zstd --patch-from on temp files)
    auto diff_file = [&](PatchFileId id, const std::string& name, bool use_old_as_ref = false) {
        std::cerr << "  " << name << ": ";
        std::string ref;
        if (use_old_as_ref) {
            ref = old_dir + "/" + name;
        } else {
            ref = tmpdir + "/" + name;
            if (read_file(ref).empty()) ref = old_dir + "/" + name;
        }
        std::string tgt = new_dir + "/" + name;
        std::string zst = tmpdir + "/" + name + ".zst";
        system(("zstd --patch-from='" + ref + "' '" + tgt + "' -o '" + zst + "' -f --quiet 2>/dev/null").c_str());
        auto zst_data = read_file(zst);
        auto old_data = read_file(old_dir + "/" + name);
        auto new_data = read_file(tgt);

        uint32_t fid = static_cast<uint32_t>(id), enc = static_cast<uint32_t>(PatchEncoding::ZSTD_DELTA);
        uint64_t os = old_data.size(), ns = new_data.size(), ds = zst_data.size();
        write_val(pf, &fid, 4); write_val(pf, &enc, 4);
        write_val(pf, &os, 8); write_val(pf, &ns, 8); write_val(pf, &ds, 8);
        pf.write(zst_data.data(), zst_data.size());

        double pct = ns > 0 ? ds * 100.0 / ns : 0;
        std::cerr << ds << " bytes (" << std::fixed << std::setprecision(2) << pct << "%)" << std::endl;
        remove(zst.c_str());
    };

    // Data files with string remap + fixups: use remapped temp as ref
    diff_file(PatchFileId::ADDR_POINTS, "addr_points.bin");
    diff_file(PatchFileId::STREET_WAYS, "street_ways.bin");
    diff_file(PatchFileId::INTERP_WAYS, "interp_ways.bin");
    diff_file(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin");
    // Coord files: use old as ref (no remap needed)
    diff_file(PatchFileId::STRINGS, "strings.bin", true);
    diff_file(PatchFileId::STREET_NODES, "street_nodes.bin", true);
    diff_file(PatchFileId::INTERP_NODES, "interp_nodes.bin", true);
    diff_file(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", true);
    // Entry/cell files: use rebuilt temp as ref (with remapped IDs)
    diff_file(PatchFileId::GEO_CELLS, "geo_cells.bin");
    diff_file(PatchFileId::STREET_ENTRIES, "street_entries.bin");
    diff_file(PatchFileId::ADDR_ENTRIES, "addr_entries.bin");
    diff_file(PatchFileId::INTERP_ENTRIES, "interp_entries.bin");
    diff_file(PatchFileId::ADMIN_CELLS, "admin_cells.bin");
    diff_file(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin");

    uint32_t end = 0xFFFFFFFF;
    write_val(pf, &end, 4);

    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());

    auto total = pf.tellp();
    std::cerr << "\nTotal patch: " << total << " bytes (" << total / 1024 / 1024 << " MiB)" << std::endl;
    return 0;
}

-        // No overlay, no removal — pure remap of old data (matches patch tool behavior)

-        // The zstd delta encodes the differences from this to new.

-        // Remove the old cell-not-in-new check since we don't overlay:

-        std::unordered_map<uint64_t, bool> dummy;

-        for (auto it = cells.begin(); it != cells.end(); )

-            new_cell_set.count(it->first) ? ++it : it = cells.erase(it);


Bash

cmake --build build -j$(nproc) --target geocoder-diff --target geocoder-patch 2>&1 | tail -5

Build


[ 50%] Building CXX object CMakeFiles/geocoder-diff.dir/tools/geocoder_diff.cpp.o
[100%] Linking CXX executable geocoder-diff
[100%] Built target geocoder-diff
[100%] Built target geocoder-patch


sshpass -p 'elon-is-insane' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" /root/traccar-geocoder/builder/build/geocoder-diff /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:~/ 2>&1 && sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
rm -f /home/michtest/test11.gcpatch && /home/michtest/geocoder-diff /home/michtest/det-A /home/michtest/det-new -o /home/michtest/test11.gcpatch 2>&1
echo "Patch: $(numfmt --to=iec $(stat -c%s /home/michtest/test11.gcpatch))"
rm -rf /home/michtest/patched11 && /home/michtest/geocoder-patch /home/michtest/det-A /home/michtest/test11.gcpatch -o /home/michtest/patched11 2>&1
all_match=true
for f in /home/michtest/det-new/*.bin; do
    b=$(basename $f); p=/home/michtest/patched11/$b
    [ -f "$p" ] && cmp -s "$f" "$p" && echo "  $b: MATCH" || { echo "  $b: FAIL"; all_match=false; }
done
$all_match && echo "ALL FILES MATCH" || echo "SOME DIFFER"
' 2>&1

Test with matching diff/patch rebuild logic


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Building string remap...
  4980078 strings mapped
Building record ID remaps...
    addr matched=85825938/85891830 new=164173
    ways matched=19372891/19452480 new=95957
    interps matched=1016/16785 new=15755
    admin matched=449642/450907 new=1222
  Remapping geo entries...
  Remapping admin entries...
  Remap: 4982052 entries (39856416 -> 11885604 bytes)
  fixup street_ways.bin: 19372891 entries (154983128 -> 41799994)
  fixup interp_ways.bin: 1016 entries (8128 -> 2779)
  fixup admin_polygons.bin: 449642 entries (3597136 -> 1262718)
  id_remap file 1: 19372891 (154983128 -> 38873409)
  id_remap file 3: 85825938 (686607504 -> 146330168)
  id_remap file 4: 1016 (8128 -> 1455)
  id_remap file 6: 449642 (3597136 -> 642236)
  addr_points.bin: 3770551 bytes (0.27%)
  street_ways.bin: 1515482 bytes (0.65%)
  interp_ways.bin: 66381 bytes (19.79%)
  admin_polygons.bin: 39775 bytes (0.37%)
  strings.bin: 71309 bytes (0.09%)
  street_nodes.bin: 6727404 bytes (0.47%)
  interp_nodes.bin: 1228 bytes (0.17%)
  admin_vertices.bin: 535915 bytes (0.18%)
  geo_cells.bin: 396408051 bytes (23.68%)
  street_entries.bin: 3465195 bytes (0.59%)
  addr_entries.bin: 1871879 bytes (0.47%)
  interp_entries.bin: 60561 bytes (38.00%)
  admin_cells.bin: 469093 bytes (27.72%)
  admin_entries.bin: 305870 bytes (5.00%)

Total patch: 656107693 bytes (625 MiB)
Patch: 626M
Loaded 4980054 string remap entries
Loaded 19372891 fixups for file 1
Loaded 1016 fixups for file 4
Loaded 449642 fixups for file 6
Loaded 19372891 ID remaps for file 1
Loaded 85825938 ID remaps for file 3
Loaded 1016 ID remaps for file 4
Loaded 449642 ID remaps for file 6
Rebuilt 1 entry/cell files from ID remaps
  addr_points.bin: 1375841776 bytes
  street_ways.bin: 233626176 bytes
  interp_ways.bin: 335424 bytes
  admin_polygons.bin: 10820736 bytes
  strings.bin: 82573397 bytes
  street_nodes.bin: 1445600360 bytes
  interp_nodes.bin: 725184 bytes
  admin_vertices.bin: 291061192 bytes
  geo_cells.bin: 1674295440 bytes
FAILED: street_entries.bin
  addr_entries.bin: FAIL
  addr_points.bin: MATCH
  admin_cells.bin: FAIL
  admin_entries.bin: FAIL
  admin_polygons.bin: MATCH
  admin_vertices.bin: MATCH
  geo_cells.bin: MATCH
  interp_entries.bin: FAIL
  interp_nodes.bin: MATCH
  interp_ways.bin: MATCH
  street_entries.bin: FAIL
  street_nodes.bin: MATCH
  street_ways.bin: MATCH
  strings.bin: MATCH
SOME DIFFER


geo_cells MATCHES again but entries fail. The entry deltas are now larger (0.47-0.59% instead of 0.02-0.07%) because we removed the overlay — the remapped entries have more differences from new.

The entry/cell failures are because the diff and patch tool rebuilds still differ. The diff uses `all_cells` from old geo_cells. The patch tool also parses old geo_cells. But the patch tool's `rebuild_entries` function uses a slightly different approach (separate function vs inline lambda).

The builds are SO close. Let me try a different tactic: just have the diff tool verify that its rebuilt temp files match what the patch tool would produce. But that's hard without running the patch tool.

Let me step back and think about this more pragmatically. The current patch is 626 MiB with compressed tables. Without ID remap tables (which are 228 MiB compressed), the data file deltas are 12 MiB, and entry/cell deltas at ~400 MiB (old→new, no remap).

The biggest single component is now **geo_cells at 396 MiB** (using old→new, no remap). The entry deltas with old→new are ~500 MiB.

What if I just drop the entry/cell file rebuild entirely and use old→new for everything? The tables would still be needed for data files (string remap + fixups = ~55 MiB compressed). Total: 55 + 12 (data deltas) + 900 (entry/cell old→new) = 967 MiB. That's worse.

The problem is clear: we need the entry rebuild to work identically in both diff and patch. Let me just literally copy the rebuild function from the diff tool into the patch tool to ensure identical logic.

Actually, let me make both tools call the same `rebuild_from_remap` function from patch_format.h:


git add tools/geocoder_diff.cpp tools/geocoder_patch.cpp tools/patch_format.h && git commit -m "$(cat <<'EOF'
feat: compressed remap tables, matching rebuild logic (WIP)

Compressed string remap, fixup, and ID remap tables with zstd.
Tables: 995 MiB raw → 240 MiB compressed.

Entry rebuild now uses pure ID remap (no overlay) to match patch
tool behavior. Entry/cell files still fail verification due to
subtle differences in rebuild logic between diff and patch tools.

Current patch: 626 MiB (down from 1.9 GiB).
Remaining work: unify rebuild logic between diff and patch.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
EOF
)"

Commit checkpoint


[main 1475388] feat: compressed remap tables, matching rebuild logic (WIP)
 2 files changed, 94 insertions(+), 58 deletions(-)


The entry rebuild mismatch is the last blocker. Let me extract the rebuild logic into a shared function in patch_format.h that both tools use identically. This guarantees the same output.

The shared function takes: old geo_cells data, old entry data, ID remap → produces rebuilt entries + geo_cells.


Edit

// --- Grid coordinate for fingerprinting ---

// --- Shared entry rebuild logic ---
// Used by both diff and patch tools to produce identical rebuilt entries.
// Takes old geo_cells + old entries + ID remap → produces rebuilt entries + geo_cells.

struct RebuiltGeo {
    std::vector<char> geo_cells_data;
    std::vector<char> street_entries_data;
    std::vector<char> addr_entries_data;
    std::vector<char> interp_entries_data;
};

inline RebuiltGeo rebuild_geo_from_remap(
    const std::vector<char>& old_geo,
    const std::vector<char>& old_se, const std::vector<char>& old_ae, const std::vector<char>& old_ie,
    const std::unordered_map<uint32_t,uint32_t>& way_rm,
    const std::unordered_map<uint32_t,uint32_t>& addr_rm,
    const std::unordered_map<uint32_t,uint32_t>& interp_rm)
{
    size_t n_cells = old_geo.size() / 20;

    auto parse_entry = [](const std::vector<char>& data, uint32_t off) -> std::vector<uint32_t> {
        if (off == 0xFFFFFFFF || off + 2 > data.size()) return {};
        uint16_t count; memcpy(&count, data.data() + off, 2);
        if (off + 2 + count * 4 > data.size()) return {};
        std::vector<uint32_t> ids(count);
        memcpy(ids.data(), data.data() + off + 2, count * 4);
        return ids;
    };

    auto remap_ids = [](std::vector<uint32_t>& ids, const std::unordered_map<uint32_t,uint32_t>& rm) {
        for (auto& id : ids) {
            auto it = rm.find(id);
            if (it != rm.end()) id = it->second;
        }
        std::sort(ids.begin(), ids.end());
    };

    // Parse all cells and remap IDs
    struct CellData {
        uint64_t cell_id;
        std::vector<uint32_t> streets, addrs, interps;
    };
    std::vector<CellData> cells(n_cells);
    for (size_t i = 0; i < n_cells; i++) {
        memcpy(&cells[i].cell_id, old_geo.data() + i * 20, 8);
        uint32_t s_off, a_off, i_off;
        memcpy(&s_off, old_geo.data() + i * 20 + 8, 4);
        memcpy(&a_off, old_geo.data() + i * 20 + 12, 4);
        memcpy(&i_off, old_geo.data() + i * 20 + 16, 4);
        cells[i].streets = parse_entry(old_se, s_off);
        cells[i].addrs = parse_entry(old_ae, a_off);
        cells[i].interps = parse_entry(old_ie, i_off);
        remap_ids(cells[i].streets, way_rm);
        remap_ids(cells[i].addrs, addr_rm);
        remap_ids(cells[i].interps, interp_rm);
    }

    // Write rebuilt files
    RebuiltGeo result;
    uint32_t no_data = 0xFFFFFFFF;

    auto write_entries = [&](std::vector<char>& buf, const auto& getter) -> std::unordered_map<uint64_t, uint32_t> {
        std::unordered_map<uint64_t, uint32_t> offsets;
        for (auto& c : cells) {
            const auto& ids = getter(c);
            if (ids.empty()) continue;
            offsets[c.cell_id] = static_cast<uint32_t>(buf.size());
            uint16_t count = static_cast<uint16_t>(ids.size());
            buf.insert(buf.end(), (const char*)&count, (const char*)&count + 2);
            buf.insert(buf.end(), (const char*)ids.data(), (const char*)ids.data() + ids.size() * 4);
        }
        return offsets;
    };

    auto s_off = write_entries(result.street_entries_data, [](const CellData& c) -> const std::vector<uint32_t>& { return c.streets; });
    auto a_off = write_entries(result.addr_entries_data, [](const CellData& c) -> const std::vector<uint32_t>& { return c.addrs; });
    auto i_off = write_entries(result.interp_entries_data, [](const CellData& c) -> const std::vector<uint32_t>& { return c.interps; });

    // Write geo_cells
    for (auto& c : cells) {
        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&c.cell_id, (const char*)&c.cell_id + 8);
        auto get = [&](const auto& m) -> uint32_t {
            auto it = m.find(c.cell_id); return it != m.end() ? it->second : no_data;
        };
        uint32_t sv = get(s_off), av = get(a_off), iv = get(i_off);
        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&sv, (const char*)&sv + 4);
        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&av, (const char*)&av + 4);
        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&iv, (const char*)&iv + 4);
    }

    return result;
}

struct RebuiltAdmin {
    std::vector<char> admin_cells_data;
    std::vector<char> admin_entries_data;
};

inline RebuiltAdmin rebuild_admin_from_remap(
    const std::vector<char>& old_ac, const std::vector<char>& old_ae,
    const std::unordered_map<uint32_t,uint32_t>& admin_rm)
{
    size_t n_cells = old_ac.size() / 12;

    struct CellData { uint64_t cell_id; std::vector<uint32_t> ids; };
    std::vector<CellData> cells(n_cells);
    for (size_t i = 0; i < n_cells; i++) {
        memcpy(&cells[i].cell_id, old_ac.data() + i * 12, 8);
        uint32_t off; memcpy(&off, old_ac.data() + i * 12 + 8, 4);
        if (off != 0xFFFFFFFF && off + 2 <= old_ae.size()) {
            uint16_t count; memcpy(&count, old_ae.data() + off, 2);
            if (off + 2 + count * 4 <= old_ae.size()) {
                cells[i].ids.resize(count);
                memcpy(cells[i].ids.data(), old_ae.data() + off + 2, count * 4);
                for (auto& id : cells[i].ids) {
                    uint32_t flags = id & 0x80000000u;
                    uint32_t masked = id & 0x7FFFFFFFu;
                    auto it = admin_rm.find(masked);
                    if (it != admin_rm.end()) id = it->second | flags;
                }
                std::sort(cells[i].ids.begin(), cells[i].ids.end());
            }
        }
    }

    RebuiltAdmin result;
    uint32_t no_data = 0xFFFFFFFF;
    std::unordered_map<uint64_t, uint32_t> offsets;
    for (auto& c : cells) {
        if (c.ids.empty()) continue;
        offsets[c.cell_id] = static_cast<uint32_t>(result.admin_entries_data.size());
        uint16_t count = static_cast<uint16_t>(c.ids.size());
        result.admin_entries_data.insert(result.admin_entries_data.end(), (const char*)&count, (const char*)&count + 2);
        result.admin_entries_data.insert(result.admin_entries_data.end(), (const char*)c.ids.data(), (const char*)c.ids.data() + c.ids.size() * 4);
    }
    for (auto& c : cells) {
        result.admin_cells_data.insert(result.admin_cells_data.end(), (const char*)&c.cell_id, (const char*)&c.cell_id + 8);
        auto it = offsets.find(c.cell_id);
        uint32_t off = it != offsets.end() ? it->second : no_data;
        result.admin_cells_data.insert(result.admin_cells_data.end(), (const char*)&off, (const char*)&off + 4);
    }
    return result;
}

// --- Grid coordinate for fingerprinting ---


The file /root/traccar-geocoder/builder/tools/patch_format.h has been updated successfully.

// --- Shared entry rebuild logic ---
// Used by both diff and patch tools to produce identical rebuilt entries.
// Takes old geo_cells + old entries + ID remap → produces rebuilt entries + geo_cells.

struct RebuiltGeo {
    std::vector<char> geo_cells_data;
    std::vector<char> street_entries_data;
    std::vector<char> addr_entries_data;
    std::vector<char> interp_entries_data;
};

inline RebuiltGeo rebuild_geo_from_remap(
    const std::vector<char>& old_geo,
    const std::vector<char>& old_se, const std::vector<char>& old_ae, const std::vector<char>& old_ie,
    const std::unordered_map<uint32_t,uint32_t>& way_rm,
    const std::unordered_map<uint32_t,uint32_t>& addr_rm,
    const std::unordered_map<uint32_t,uint32_t>& interp_rm)
{
    size_t n_cells = old_geo.size() / 20;

    auto parse_entry = [](const std::vector<char>& data, uint32_t off) -> std::vector<uint32_t> {
        if (off == 0xFFFFFFFF || off + 2 > data.size()) return {};
        uint16_t count; memcpy(&count, data.data() + off, 2);
        if (off + 2 + count * 4 > data.size()) return {};
        std::vector<uint32_t> ids(count);
        memcpy(ids.data(), data.data() + off + 2, count * 4);
        return ids;
    };

    auto remap_ids = [](std::vector<uint32_t>& ids, const std::unordered_map<uint32_t,uint32_t>& rm) {
        for (auto& id : ids) {
            auto it = rm.find(id);
            if (it != rm.end()) id = it->second;
        }
        std::sort(ids.begin(), ids.end());
    };

    // Parse all cells and remap IDs
    struct CellData {
        uint64_t cell_id;
        std::vector<uint32_t> streets, addrs, interps;
    };
    std::vector<CellData> cells(n_cells);
    for (size_t i = 0; i < n_cells; i++) {
        memcpy(&cells[i].cell_id, old_geo.data() + i * 20, 8);
        uint32_t s_off, a_off, i_off;
        memcpy(&s_off, old_geo.data() + i * 20 + 8, 4);
        memcpy(&a_off, old_geo.data() + i * 20 + 12, 4);
        memcpy(&i_off, old_geo.data() + i * 20 + 16, 4);
        cells[i].streets = parse_entry(old_se, s_off);
        cells[i].addrs = parse_entry(old_ae, a_off);
        cells[i].interps = parse_entry(old_ie, i_off);
        remap_ids(cells[i].streets, way_rm);
        remap_ids(cells[i].addrs, addr_rm);
        remap_ids(cells[i].interps, interp_rm);
    }

    // Write rebuilt files
    RebuiltGeo result;
    uint32_t no_data = 0xFFFFFFFF;

    auto write_entries = [&](std::vector<char>& buf, const auto& getter) -> std::unordered_map<uint64_t, uint32_t> {
        std::unordered_map<uint64_t, uint32_t> offsets;
        for (auto& c : cells) {
            const auto& ids = getter(c);
            if (ids.empty()) continue;
            offsets[c.cell_id] = static_cast<uint32_t>(buf.size());
            uint16_t count = static_cast<uint16_t>(ids.size());
            buf.insert(buf.end(), (const char*)&count, (const char*)&count + 2);
            buf.insert(buf.end(), (const char*)ids.data(), (const char*)ids.data() + ids.size() * 4);
        }
        return offsets;
    };

    auto s_off = write_entries(result.street_entries_data, [](const CellData& c) -> const std::vector<uint32_t>& { return c.streets; });
    auto a_off = write_entries(result.addr_entries_data, [](const CellData& c) -> const std::vector<uint32_t>& { return c.addrs; });
    auto i_off = write_entries(result.interp_entries_data, [](const CellData& c) -> const std::vector<uint32_t>& { return c.interps; });

    // Write geo_cells
    for (auto& c : cells) {
        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&c.cell_id, (const char*)&c.cell_id + 8);
        auto get = [&](const auto& m) -> uint32_t {
            auto it = m.find(c.cell_id); return it != m.end() ? it->second : no_data;
        };
        uint32_t sv = get(s_off), av = get(a_off), iv = get(i_off);
        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&sv, (const char*)&sv + 4);
        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&av, (const char*)&av + 4);
        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&iv, (const char*)&iv + 4);
    }

    return result;
}

struct RebuiltAdmin {
    std::vector<char> admin_cells_data;
    std::vector<char> admin_entries_data;
};

inline RebuiltAdmin rebuild_admin_from_remap(
    const std::vector<char>& old_ac, const std::vector<char>& old_ae,
    const std::unordered_map<uint32_t,uint32_t>& admin_rm)
{
    size_t n_cells = old_ac.size() / 12;

    struct CellData { uint64_t cell_id; std::vector<uint32_t> ids; };
    std::vector<CellData> cells(n_cells);
    for (size_t i = 0; i < n_cells; i++) {
        memcpy(&cells[i].cell_id, old_ac.data() + i * 12, 8);
        uint32_t off; memcpy(&off, old_ac.data() + i * 12 + 8, 4);
        if (off != 0xFFFFFFFF && off + 2 <= old_ae.size()) {
            uint16_t count; memcpy(&count, old_ae.data() + off, 2);
            if (off + 2 + count * 4 <= old_ae.size()) {
                cells[i].ids.resize(count);
                memcpy(cells[i].ids.data(), old_ae.data() + off + 2, count * 4);
                for (auto& id : cells[i].ids) {
                    uint32_t flags = id & 0x80000000u;
                    uint32_t masked = id & 0x7FFFFFFFu;
                    auto it = admin_rm.find(masked);
                    if (it != admin_rm.end()) id = it->second | flags;
                }
                std::sort(cells[i].ids.begin(), cells[i].ids.end());
            }
        }
    }

    RebuiltAdmin result;
    uint32_t no_data = 0xFFFFFFFF;
    std::unordered_map<uint64_t, uint32_t> offsets;
    for (auto& c : cells) {
        if (c.ids.empty()) continue;
        offsets[c.cell_id] = static_cast<uint32_t>(result.admin_entries_data.size());
        uint16_t count = static_cast<uint16_t>(c.ids.size());
        result.admin_entries_data.insert(result.admin_entries_data.end(), (const char*)&count, (const char*)&count + 2);
        result.admin_entries_data.insert(result.admin_entries_data.end(), (const char*)c.ids.data(), (const char*)c.ids.data() + c.ids.size() * 4);
    }
    for (auto& c : cells) {
        result.admin_cells_data.insert(result.admin_cells_data.end(), (const char*)&c.cell_id, (const char*)&c.cell_id + 8);
        auto it = offsets.find(c.cell_id);
        uint32_t off = it != offsets.end() ? it->second : no_data;
        result.admin_cells_data.insert(result.admin_cells_data.end(), (const char*)&off, (const char*)&off + 4);
    }
    return result;
}

// --- Grid coordinate for fingerprinting ---

#pragma once
// Shared definitions for the geocoder patch system.
// Used by geocoder-canonicalize, geocoder-diff, and geocoder-patch.

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>

// --- Binary record structs (must match types.h and server) ---

#pragma pack(push, 1)
struct PatchWayHeader {
    uint32_t node_offset;
    uint8_t node_count;
    uint32_t name_id;
};
static_assert(sizeof(PatchWayHeader) == 9, "WayHeader must be 9 bytes packed");

struct PatchAddrPoint {
    float lat;
    float lng;
    uint32_t housenumber_id;
    uint32_t street_id;
};
static_assert(sizeof(PatchAddrPoint) == 16, "AddrPoint must be 16 bytes");

struct PatchNodeCoord {
    float lat;
    float lng;
};
static_assert(sizeof(PatchNodeCoord) == 8, "NodeCoord must be 8 bytes");
#pragma pack(pop)

// InterpWay and AdminPolygon have compiler-dependent padding.
// Read them field-by-field instead of struct-casting.

struct PatchInterpWay {
    uint32_t node_offset;
    uint8_t node_count;
    uint32_t street_id;
    uint32_t start_number;
    uint32_t end_number;
    uint8_t interpolation;
};

struct PatchAdminPolygon {
    uint32_t vertex_offset;
    uint32_t vertex_count;
    uint32_t name_id;
    uint8_t admin_level;
    float area;
    uint16_t country_code;
};

// --- File I/O helpers ---

inline std::vector<char> read_file(const std::string& path) {
    std::ifstream f(path, std::ios::binary | std::ios::ate);
    if (!f) return {};
    auto size = f.tellg();
    f.seekg(0);
    std::vector<char> data(size);
    f.read(data.data(), size);
    return data;
}

inline bool write_file(const std::string& path, const char* data, size_t size) {
    std::ofstream f(path, std::ios::binary);
    if (!f) return false;
    f.write(data, size);
    return f.good();
}

inline bool write_file(const std::string& path, const std::vector<char>& data) {
    return write_file(path, data.data(), data.size());
}

inline const char* get_string(const std::vector<char>& pool, uint32_t offset) {
    if (offset >= pool.size()) return "";
    return pool.data() + offset;
}

// Read WayHeader array — detect stride from file size vs known struct sizes
inline std::vector<PatchWayHeader> read_ways(const std::string& path) {
    auto data = read_file(path);
    // Determine stride: builder writes sizeof(WayHeader) which may be 9 or 12
    // depending on compiler padding. Try 12 first (common), then 9.
    size_t stride = 12;
    if (data.size() % 12 != 0 || data.size() / 12 == 0) {
        stride = 9;
    }
    size_t count = data.size() / stride;
    std::vector<PatchWayHeader> result(count);
    for (size_t i = 0; i < count; i++) {
        const char* p = data.data() + i * stride;
        memcpy(&result[i].node_offset, p, 4);
        result[i].node_count = static_cast<uint8_t>(p[4]);
        // name_id is at offset 5 (packed) or 8 (padded to 4-byte alignment)
        size_t name_off = (stride == 12) ? 8 : 5;
        memcpy(&result[i].name_id, p + name_off, 4);
    }
    return result;
}

inline size_t detect_way_stride(const std::string& path) {
    auto data = read_file(path);
    return (data.size() % 12 == 0 && data.size() / 12 > 0) ? 12 : 9;
}

template<typename T>
inline std::vector<T> read_structs(const std::string& path) {
    auto data = read_file(path);
    size_t count = data.size() / sizeof(T);
    std::vector<T> result(count);
    if (count > 0) memcpy(result.data(), data.data(), count * sizeof(T));
    return result;
}

// Read InterpWay with field-by-field parsing to handle padding
inline std::vector<PatchInterpWay> read_interps(const std::string& path) {
    auto data = read_file(path);
    // Determine record size from file: try sizeof(InterpWay) as compiled by the builder
    // The builder uses the same struct, so sizeof should match.
    size_t rec_size = sizeof(PatchInterpWay);
    // But the builder's InterpWay has padding. Read with the builder's size.
    // From types.h the struct is: u32 + u8 + u32 + u32 + u32 + u8 = 18 bytes raw
    // but compiler pads to 20 bytes typically. Check by trying both.
    size_t count_20 = data.size() / 20;
    size_t count_18 = data.size() / 18;
    // Use whichever divides evenly
    size_t stride = (data.size() % 20 == 0 && count_20 > 0) ? 20 :
                    (data.size() % 18 == 0 && count_18 > 0) ? 18 : 20;
    size_t count = data.size() / stride;
    std::vector<PatchInterpWay> result(count);
    for (size_t i = 0; i < count; i++) {
        const char* p = data.data() + i * stride;
        memcpy(&result[i].node_offset, p, 4); p += 4;
        result[i].node_count = *reinterpret_cast<const uint8_t*>(p); p += 1;
        // Skip padding to align street_id
        if (stride == 20) p += 3; // 3 bytes padding after node_count
        memcpy(&result[i].street_id, p, 4); p += 4;
        memcpy(&result[i].start_number, p, 4); p += 4;
        memcpy(&result[i].end_number, p, 4); p += 4;
        result[i].interpolation = *reinterpret_cast<const uint8_t*>(p);
    }
    return result;
}

// Read AdminPolygon with field-by-field parsing
inline std::vector<PatchAdminPolygon> read_admin_polygons(const std::string& path) {
    auto data = read_file(path);
    // AdminPolygon: u32 + u32 + u32 + u8 + f32 + u16 = 19 raw bytes
    // Compiler likely pads to 20 or 24 bytes. Try common sizes.
    size_t stride = 0;
    for (size_t s : {24, 20, 19}) {
        if (data.size() % s == 0 && data.size() / s > 0) { stride = s; break; }
    }
    if (stride == 0) stride = 24; // fallback
    size_t count = data.size() / stride;
    std::vector<PatchAdminPolygon> result(count);
    for (size_t i = 0; i < count; i++) {
        const char* p = data.data() + i * stride;
        memcpy(&result[i].vertex_offset, p, 4); p += 4;
        memcpy(&result[i].vertex_count, p, 4); p += 4;
        memcpy(&result[i].name_id, p, 4); p += 4;
        result[i].admin_level = *reinterpret_cast<const uint8_t*>(p); p += 1;
        // padding for area alignment
        if (stride >= 20) {
            size_t skip = (stride == 24) ? 3 : (stride == 20 ? 3 : 0);
            p += skip;
        }
        memcpy(&result[i].area, p, 4); p += 4;
        memcpy(&result[i].country_code, p, 2);
    }
    return result;
}

// --- Patch file format ---

static constexpr char GCPATCH_MAGIC[8] = {'G','C','P','A','T','C','H','\0'};
static constexpr uint32_t GCPATCH_VERSION = 1;

enum class PatchFileId : uint32_t {
    STRINGS = 0,
    STREET_WAYS = 1,
    STREET_NODES = 2,
    ADDR_POINTS = 3,
    INTERP_WAYS = 4,
    INTERP_NODES = 5,
    ADMIN_POLYGONS = 6,
    ADMIN_VERTICES = 7,
    GEO_CELLS = 8,
    STREET_ENTRIES = 9,
    ADDR_ENTRIES = 10,
    INTERP_ENTRIES = 11,
    ADMIN_CELLS = 12,
    ADMIN_ENTRIES = 13,
    COUNT = 14
};

static const char* patch_file_names[] = {
    "strings.bin", "street_ways.bin", "street_nodes.bin", "addr_points.bin",
    "interp_ways.bin", "interp_nodes.bin", "admin_polygons.bin", "admin_vertices.bin",
    "geo_cells.bin", "street_entries.bin", "addr_entries.bin", "interp_entries.bin",
    "admin_cells.bin", "admin_entries.bin"
};

// Encoding types for each section
enum class PatchEncoding : uint32_t {
    RAW_REPLACE = 0,  // Full replacement (zlib compressed)
    ZSTD_DELTA = 1,   // Single zstd delta frame (zstd --patch-from on remapped old)
};

// Patch file header
struct PatchHeader {
    char magic[8];
    uint32_t version;
    uint32_t flags;
    // Per-file sections follow
};

// Section header (one per file)
struct PatchSection {
    uint32_t file_id;       // PatchFileId
    uint32_t encoding;      // PatchEncoding
    uint64_t old_size;      // Expected size of old file
    uint64_t new_size;      // Size of reconstructed file
    uint64_t data_size;     // Size of compressed section data
};

// COPY_INSERT opcodes
enum : uint8_t {
    OP_COPY = 0,    // Copy from old file: [u8 tag][u64 offset][u32 length]
    OP_INSERT = 1,  // Insert new data:    [u8 tag][u32 length][data...]
};

// Offset fixup section marker: 0xFFFFFFFD
// Format: uint32_t marker, uint32_t file_id, uint32_t stride,
//         uint32_t count, [(uint32_t record_index, uint32_t new_offset_value)] * count
// Applied to byte offset 0 of each record (node_offset/vertex_offset field).

static constexpr uint32_t FIXUP_MARKER = 0xFFFFFFFD;

// --- Grid coordinate for fingerprinting ---

inline int to_grid(float v) {
    return (int)(v * 1e5f + (v >= 0 ? 0.5f : -0.5f));
}

// --- Directory creation ---
inline void ensure_dir(const std::string& path) {
    std::string cmd = "mkdir -p '" + path + "'";
    system(cmd.c_str());
}

+// Used by both diff and patch tools to produce identical rebuilt entries.

+// Takes old geo_cells + old entries + ID remap → produces rebuilt entries + geo_cells.

+    std::vector<char> street_entries_data;

+    std::vector<char> addr_entries_data;

+    std::vector<char> interp_entries_data;

+    const std::vector<char>& old_se, const std::vector<char>& old_ae, const std::vector<char>& old_ie,

+    const std::unordered_map<uint32_t,uint32_t>& way_rm,

+    const std::unordered_map<uint32_t,uint32_t>& addr_rm,

+    const std::unordered_map<uint32_t,uint32_t>& interp_rm)

+    size_t n_cells = old_geo.size() / 20;

+    auto parse_entry = [](const std::vector<char>& data, uint32_t off) -> std::vector<uint32_t> {

+        if (off == 0xFFFFFFFF || off + 2 > data.size()) return {};

+        uint16_t count; memcpy(&count, data.data() + off, 2);

+        if (off + 2 + count * 4 > data.size()) return {};

+        std::vector<uint32_t> ids(count);

+        memcpy(ids.data(), data.data() + off + 2, count * 4);

+    auto remap_ids = [](std::vector<uint32_t>& ids, const std::unordered_map<uint32_t,uint32_t>& rm) {

+            if (it != rm.end()) id = it->second;

+        std::sort(ids.begin(), ids.end());

+        std::vector<uint32_t> streets, addrs, interps;

+    std::vector<CellData> cells(n_cells);

+    for (size_t i = 0; i < n_cells; i++) {

+        memcpy(&cells[i].cell_id, old_geo.data() + i * 20, 8);

+        memcpy(&s_off, old_geo.data() + i * 20 + 8, 4);

+        memcpy(&a_off, old_geo.data() + i * 20 + 12, 4);

+        memcpy(&i_off, old_geo.data() + i * 20 + 16, 4);

+        cells[i].streets = parse_entry(old_se, s_off);

+        cells[i].addrs = parse_entry(old_ae, a_off);

+        cells[i].interps = parse_entry(old_ie, i_off);

+        remap_ids(cells[i].streets, way_rm);

+        remap_ids(cells[i].addrs, addr_rm);

+        remap_ids(cells[i].interps, interp_rm);

+    auto write_entries = [&](std::vector<char>& buf, const auto& getter) -> std::unordered_map<uint64_t, uint32_t> {

+        std::unordered_map<uint64_t, uint32_t> offsets;

+            const auto& ids = getter(c);

+            offsets[c.cell_id] = static_cast<uint32_t>(buf.size());

+            uint16_t count = static_cast<uint16_t>(ids.size());

+            buf.insert(buf.end(), (const char*)&count, (const char*)&count + 2);

+            buf.insert(buf.end(), (const char*)ids.data(), (const char*)ids.data() + ids.size() * 4);

+    auto s_off = write_entries(result.street_entries_data, [](const CellData& c) -> const std::vector<uint32_t>& { return c.streets; });

+    auto a_off = write_entries(result.addr_entries_data, [](const CellData& c) -> const std::vector<uint32_t>& { return c.addrs; });

+    auto i_off = write_entries(result.interp_entries_data, [](const CellData& c) -> const std::vector<uint32_t>& { return c.interps; });

+        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&c.cell_id, (const char*)&c.cell_id + 8);

+        auto get = [&](const auto& m) -> uint32_t {

+            auto it = m.find(c.cell_id); return it != m.end() ? it->second : no_data;

+        uint32_t sv = get(s_off), av = get(a_off), iv = get(i_off);

+        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&sv, (const char*)&sv + 4);

+        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&av, (const char*)&av + 4);

+        result.geo_cells_data.insert(result.geo_cells_data.end(), (const char*)&iv, (const char*)&iv + 4);

+    std::vector<char> admin_cells_data;

+    std::vector<char> admin_entries_data;

+    const std::vector<char>& old_ac, const std::vector<char>& old_ae,

+    const std::unordered_map<uint32_t,uint32_t>& admin_rm)

+    size_t n_cells = old_ac.size() / 12;

+    struct CellData { uint64_t cell_id; std::vector<uint32_t> ids; };

+    std::vector<CellData> cells(n_cells);

+    for (size_t i = 0; i < n_cells; i++) {

+        memcpy(&cells[i].cell_id, old_ac.data() + i * 12, 8);

+        uint32_t off; memcpy(&off, old_ac.data() + i * 12 + 8, 4);

+        if (off != 0xFFFFFFFF && off + 2 <= old_ae.size()) {

+            uint16_t count; memcpy(&count, old_ae.data() + off, 2);

+            if (off + 2 + count * 4 <= old_ae.size()) {

+                cells[i].ids.resize(count);

+                memcpy(cells[i].ids.data(), old_ae.data() + off + 2, count * 4);

+                for (auto& id : cells[i].ids) {

+                    uint32_t masked = id & 0x7FFFFFFFu;

+                    auto it = admin_rm.find(masked);

+                    if (it != admin_rm.end()) id = it->second | flags;

+                std::sort(cells[i].ids.begin(), cells[i].ids.end());

+        offsets[c.cell_id] = static_cast<uint32_t>(result.admin_entries_data.size());

+        uint16_t count = static_cast<uint16_t>(c.ids.size());

+        result.admin_entries_data.insert(result.admin_entries_data.end(), (const char*)&count, (const char*)&count + 2);

+        result.admin_entries_data.insert(result.admin_entries_data.end(), (const char*)c.ids.data(), (const char*)c.ids.data() + c.ids.size() * 4);

+        result.admin_cells_data.insert(result.admin_cells_data.end(), (const char*)&c.cell_id, (const char*)&c.cell_id + 8);

+        auto it = offsets.find(c.cell_id);

+        uint32_t off = it != offsets.end() ? it->second : no_data;

+        result.admin_cells_data.insert(result.admin_cells_data.end(), (const char*)&off, (const char*)&off + 4);


Now update the diff tool to use the shared function:


    // Rebuild entry/cell files with remapped IDs
    remap_and_rebuild_entries(old_dir, new_dir, tmpdir,
                              way_id_remap, addr_id_remap, interp_id_remap, admin_id_remap);

    // Rebuild entry/cell files using shared function (identical to patch tool)
    {
        auto old_geo = read_file(old_dir + "/geo_cells.bin");
        auto old_se = read_file(old_dir + "/street_entries.bin");
        auto old_ae = read_file(old_dir + "/addr_entries.bin");
        auto old_ie = read_file(old_dir + "/interp_entries.bin");
        // Convert vector remaps to unordered_maps
        std::unordered_map<uint32_t,uint32_t> w_rm, a_rm, i_rm, ad_rm;
        for (uint32_t j = 0; j < way_id_remap.size(); j++)
            if (way_id_remap[j] != 0xFFFFFFFF) w_rm[j] = way_id_remap[j];
        for (uint32_t j = 0; j < addr_id_remap.size(); j++)
            if (addr_id_remap[j] != 0xFFFFFFFF) a_rm[j] = addr_id_remap[j];
        for (uint32_t j = 0; j < interp_id_remap.size(); j++)
            if (interp_id_remap[j] != 0xFFFFFFFF) i_rm[j] = interp_id_remap[j];
        for (uint32_t j = 0; j < admin_id_remap.size(); j++)
            if (admin_id_remap[j] != 0xFFFFFFFF) ad_rm[j] = admin_id_remap[j];

        auto geo = rebuild_geo_from_remap(old_geo, old_se, old_ae, old_ie, w_rm, a_rm, i_rm);
        write_file(tmpdir + "/geo_cells.bin", geo.geo_cells_data);
        write_file(tmpdir + "/street_entries.bin", geo.street_entries_data);
        write_file(tmpdir + "/addr_entries.bin", geo.addr_entries_data);
        write_file(tmpdir + "/interp_entries.bin", geo.interp_entries_data);

        auto old_ac = read_file(old_dir + "/admin_cells.bin");
        auto old_admin_e = read_file(old_dir + "/admin_entries.bin");
        auto admin = rebuild_admin_from_remap(old_ac, old_admin_e, ad_rm);
        write_file(tmpdir + "/admin_cells.bin", admin.admin_cells_data);
        write_file(tmpdir + "/admin_entries.bin", admin.admin_entries_data);
        std::cerr << "  Rebuilt entry/cell files" << std::endl;
    }