// geocoder-diff: Compare two deterministic geocoder builds, produce a .gcpatch.
//
// Three-level remapping for minimal patches:
// 1. String remap: old string offsets → new string offsets
// 2. Record ID remap: match records by content, map old_id → new_id
// 3. Entry remap: apply record ID remap to entry files + rebuild cell indexes
//
// Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>

#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <functional>
#include <iomanip>
#include <iostream>
#include <string>
#include <unistd.h>
#include <unordered_map>
#include <vector>

#include "patch_format.h"

// --- String remap ---

static std::unordered_map<uint32_t, uint32_t> build_string_remap(
    const std::vector<char>& old_pool, const std::vector<char>& new_pool)
{
    std::unordered_map<std::string, uint32_t> new_idx;
    size_t pos = 0;
    while (pos < new_pool.size()) {
        const char* s = new_pool.data() + pos;
        size_t len = strlen(s);
        new_idx[std::string(s, len)] = static_cast<uint32_t>(pos);
        pos += len + 1;
    }
    std::unordered_map<uint32_t, uint32_t> remap;
    pos = 0;
    while (pos < old_pool.size()) {
        const char* s = old_pool.data() + pos;
        size_t len = strlen(s);
        auto it = new_idx.find(std::string(s, len));
        if (it != new_idx.end())
            remap[static_cast<uint32_t>(pos)] = it->second;
        pos += len + 1;
    }
    return remap;
}

// --- Remap helpers ---

static void remap_addr_points(std::vector<char>& data, const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + 16 <= data.size(); i += 16)
        for (size_t off : {8, 12}) {
            uint32_t v; memcpy(&v, data.data() + i + off, 4);
            auto it = rm.find(v);
            if (it != rm.end()) memcpy(data.data() + i + off, &it->second, 4);
        }
}

static void remap_field(std::vector<char>& data, size_t stride, size_t field_off,
                         const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + stride <= data.size(); i += stride) {
        uint32_t v; memcpy(&v, data.data() + i + field_off, 4);
        auto it = rm.find(v);
        if (it != rm.end()) memcpy(data.data() + i + field_off, &it->second, 4);
    }
}

// --- Content-based record matching ---
// Returns old_id → new_id mapping

static uint64_t fnv_mix(uint64_t h, uint64_t v) { h ^= v; h *= 1099511628211ULL; return h; }

static std::vector<uint32_t> match_addr_points(
    const std::vector<char>& old_data, const std::vector<char>& new_data)
{
    // After string remap, matching records are byte-identical
    size_t old_n = old_data.size() / 16, new_n = new_data.size() / 16;
    // Hash each new record
    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++) {
        uint64_t h = 14695981039346656037ULL;
        const char* p = new_data.data() + i * 16;
        for (int j = 0; j < 16; j++) h = fnv_mix(h, (uint8_t)p[j]);
        new_map.emplace(h, i);
    }
    std::vector<uint32_t> result(old_n, 0xFFFFFFFF);
    size_t matched = 0;
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = 14695981039346656037ULL;
        const char* p = old_data.data() + i * 16;
        for (int j = 0; j < 16; j++) h = fnv_mix(h, (uint8_t)p[j]);
        auto it = new_map.find(h);
        if (it != new_map.end() &&
            memcmp(old_data.data() + i * 16, new_data.data() + it->second * 16, 16) == 0) {
            result[i] = it->second;
            new_map.erase(it);
            matched++;
        }
    }
    std::cerr << "    addr matched=" << matched << "/" << old_n << " new=" << new_map.size() << std::endl;
    return result;
}

static std::vector<uint32_t> match_ways(
    const std::vector<char>& old_ways, const std::vector<char>& old_nodes,
    const std::vector<char>& new_ways, const std::vector<char>& new_nodes,
    size_t stride)
{
    size_t name_off = (stride == 12) ? 8 : 5;
    size_t old_n = old_ways.size() / stride, new_n = new_ways.size() / stride;

    size_t old_node_count = old_nodes.size() / 8, new_node_count = new_nodes.size() / 8;
    auto way_hash = [&](const char* w, const char* nodes, size_t max_nodes) -> uint64_t {
        uint32_t node_offset, name_id; uint8_t node_count;
        memcpy(&node_offset, w, 4); node_count = (uint8_t)w[4];
        memcpy(&name_id, w + name_off, 4);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, node_count);
        for (uint8_t j = 0; j < node_count && (node_offset + j) < max_nodes; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4);
            memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(way_hash(new_ways.data() + i * stride, new_nodes.data(), new_node_count), i);

    std::vector<uint32_t> result(old_n, 0xFFFFFFFF);
    size_t matched = 0;
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = way_hash(old_ways.data() + i * stride, old_nodes.data(), old_node_count);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            result[i] = it->second;
            // Also fix node_offset in old to match new (makes way bytes identical)
            uint32_t new_node_off;
            memcpy(&new_node_off, new_ways.data() + it->second * stride, 4);
            memcpy(const_cast<char*>(old_ways.data()) + i * stride, &new_node_off, 4);
            new_map.erase(it);
            matched++;
        }
    }
    std::cerr << "    ways matched=" << matched << "/" << old_n << " new=" << new_map.size() << std::endl;
    return result;
}

static std::vector<uint32_t> match_admin(
    const std::vector<char>& old_polys, const std::vector<char>& old_verts,
    const std::vector<char>& new_polys, const std::vector<char>& new_verts,
    size_t stride)
{
    size_t old_n = old_polys.size() / stride, new_n = new_polys.size() / stride;

    size_t old_vc = old_verts.size() / 8, new_vc = new_verts.size() / 8;
    auto poly_hash = [&](const char* p, const char* verts, size_t max_v) -> uint64_t {
        uint32_t vert_offset, vert_count, name_id;
        memcpy(&vert_offset, p, 4); memcpy(&vert_count, p + 4, 4);
        memcpy(&name_id, p + 8, 4);
        uint8_t level = (uint8_t)p[12];
        uint16_t cc; memcpy(&cc, p + stride - 2, 2);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, level); h = fnv_mix(h, cc);
        h = fnv_mix(h, vert_count);
        uint32_t n = std::min(vert_count, 10u);
        for (uint32_t j = 0; j < n && (vert_offset + j) < max_v; j++) {
            float lat, lng;
            memcpy(&lat, verts + (vert_offset + j) * 8, 4);
            memcpy(&lng, verts + (vert_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(poly_hash(new_polys.data() + i * stride, new_verts.data(), new_vc), i);

    std::vector<uint32_t> result(old_n, 0xFFFFFFFF);
    size_t matched = 0;
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = poly_hash(old_polys.data() + i * stride, old_verts.data(), old_vc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            result[i] = it->second;
            uint32_t new_vert_off;
            memcpy(&new_vert_off, new_polys.data() + it->second * stride, 4);
            memcpy(const_cast<char*>(old_polys.data()) + i * stride, &new_vert_off, 4);
            new_map.erase(it);
            matched++;
        }
    }
    std::cerr << "    admin matched=" << matched << "/" << old_n << " new=" << new_map.size() << std::endl;
    return result;
}

// Match interp ways by content
static std::vector<uint32_t> match_interps(
    const std::vector<char>& old_data, const std::vector<char>& old_nodes,
    const std::vector<char>& new_data, const std::vector<char>& new_nodes,
    size_t stride)
{
    size_t street_off = (stride >= 20) ? 8 : 5;
    size_t old_n = old_data.size() / stride, new_n = new_data.size() / stride;

    size_t old_nc = old_nodes.size() / 8, new_nc = new_nodes.size() / 8;
    auto ihash = [&](const char* p, const char* nodes, size_t max_n) -> uint64_t {
        uint32_t node_offset, street_id, start, end;
        uint8_t count, itype;
        memcpy(&node_offset, p, 4); count = (uint8_t)p[4];
        memcpy(&street_id, p + street_off, 4);
        memcpy(&start, p + street_off + 4, 4);
        memcpy(&end, p + street_off + 8, 4);
        itype = (street_off + 12 < stride) ? (uint8_t)p[street_off + 12] : 0;
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, street_id); h = fnv_mix(h, start);
        h = fnv_mix(h, end); h = fnv_mix(h, itype); h = fnv_mix(h, count);
        for (uint8_t j = 0; j < count && (node_offset + j) < max_n; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4);
            memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(ihash(new_data.data() + i * stride, new_nodes.data(), new_nc), i);

    std::vector<uint32_t> result(old_n, 0xFFFFFFFF);
    size_t matched = 0;
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = ihash(old_data.data() + i * stride, old_nodes.data(), old_nc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            result[i] = it->second;
            uint32_t new_node_off;
            memcpy(&new_node_off, new_data.data() + it->second * stride, 4);
            memcpy(const_cast<char*>(old_data.data()) + i * stride, &new_node_off, 4);
            new_map.erase(it);
            matched++;
        }
    }
    std::cerr << "    interps matched=" << matched << "/" << old_n << " new=" << new_map.size() << std::endl;
    return result;
}

// --- Entry file remapping ---
// Parse geo_cells + entries, remap IDs, rebuild entry + cell files.

static void remap_and_rebuild_entries(
    const std::string& old_dir, const std::string& new_dir, const std::string& tmpdir,
    const std::vector<uint32_t>& way_remap,
    const std::vector<uint32_t>& addr_remap,
    const std::vector<uint32_t>& interp_remap,
    const std::vector<uint32_t>& admin_remap)
{
    // For geo entries: parse old entries, remap IDs, then copy NEW entries/cells
    // (since the new files have the correct structure for the new build).
    // We write the remapped old entries + new cells to temp for zstd diffing.

    // Simpler approach: just copy new entry/cell files to temp dir.
    // The zstd diff of old vs new entry files won't benefit from ID remapping
    // unless we actually remap the old entries. Let's do the full remap.

    auto parse_entries = [](const std::vector<char>& cells_data, size_t cell_stride, size_t off_pos,
                            const std::vector<char>& entries_data)
        -> std::unordered_map<uint64_t, std::vector<uint32_t>>
    {
        std::unordered_map<uint64_t, std::vector<uint32_t>> result;
        size_t n = cells_data.size() / cell_stride;
        for (size_t i = 0; i < n; i++) {
            uint64_t cell_id; uint32_t off;
            memcpy(&cell_id, cells_data.data() + i * cell_stride, 8);
            memcpy(&off, cells_data.data() + i * cell_stride + off_pos, 4);
            if (off == 0xFFFFFFFF || off + 2 > entries_data.size()) continue;
            uint16_t count;
            memcpy(&count, entries_data.data() + off, 2);
            if (off + 2 + count * 4 > entries_data.size()) continue;
            std::vector<uint32_t> ids(count);
            memcpy(ids.data(), entries_data.data() + off + 2, count * 4);
            result[cell_id] = std::move(ids);
        }
        return result;
    };

    auto remap_ids = [](std::unordered_map<uint64_t, std::vector<uint32_t>>& cells,
                         const std::vector<uint32_t>& id_remap, bool has_flags = false) {
        for (auto& [cid, ids] : cells) {
            for (auto& id : ids) {
                uint32_t flags = has_flags ? (id & 0x80000000u) : 0;
                uint32_t masked = id & 0x7FFFFFFFu;
                if (masked < id_remap.size() && id_remap[masked] != 0xFFFFFFFF)
                    id = id_remap[masked] | flags;
            }
            std::sort(ids.begin(), ids.end());
        }
    };

    auto write_entries_cells = [](const std::string& cells_path, const std::string& entries_path,
                                   const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map,
                                   size_t cell_stride)
    {
        std::vector<uint64_t> sorted_cells;
        for (auto& [id, _] : cell_map) sorted_cells.push_back(id);
        std::sort(sorted_cells.begin(), sorted_cells.end());

        std::vector<char> entries_buf;
        std::unordered_map<uint64_t, uint32_t> offsets;
        for (uint64_t cid : sorted_cells) {
            auto it = cell_map.find(cid);
            if (it == cell_map.end() || it->second.empty()) continue;
            offsets[cid] = static_cast<uint32_t>(entries_buf.size());
            uint16_t count = static_cast<uint16_t>(it->second.size());
            entries_buf.insert(entries_buf.end(), (char*)&count, (char*)&count + 2);
            entries_buf.insert(entries_buf.end(), (char*)it->second.data(),
                              (char*)it->second.data() + it->second.size() * 4);
        }
        write_file(entries_path, entries_buf);

        uint32_t no_data = 0xFFFFFFFF;
        std::vector<char> cells_buf;
        for (uint64_t cid : sorted_cells) {
            cells_buf.insert(cells_buf.end(), (char*)&cid, (char*)&cid + 8);
            auto it = offsets.find(cid);
            uint32_t off = (it != offsets.end()) ? it->second : no_data;
            cells_buf.insert(cells_buf.end(), (char*)&off, (char*)&off + 4);
        }
        write_file(cells_path, cells_buf);
    };

    // Geo cells: parse old, remap, merge with new cell set
    std::cerr << "  Remapping geo entries..." << std::endl;
    {
        auto old_geo = read_file(old_dir + "/geo_cells.bin");
        auto old_se = read_file(old_dir + "/street_entries.bin");
        auto old_ae = read_file(old_dir + "/addr_entries.bin");
        auto old_ie = read_file(old_dir + "/interp_entries.bin");

        auto new_geo = read_file(new_dir + "/geo_cells.bin");
        auto new_se = read_file(new_dir + "/street_entries.bin");
        auto new_ae = read_file(new_dir + "/addr_entries.bin");
        auto new_ie = read_file(new_dir + "/interp_entries.bin");

        // Parse old entries
        auto sc = parse_entries(old_geo, 20, 8, old_se);
        auto ac = parse_entries(old_geo, 20, 12, old_ae);
        auto ic = parse_entries(old_geo, 20, 16, old_ie);

        // Parse new entries (for cells that are new/changed)
        auto new_sc = parse_entries(new_geo, 20, 8, new_se);
        auto new_ac = parse_entries(new_geo, 20, 12, new_ae);
        auto new_ic = parse_entries(new_geo, 20, 16, new_ie);

        // Remap old IDs
        remap_ids(sc, way_remap);
        remap_ids(ac, addr_remap);
        remap_ids(ic, interp_remap);

        // Merge: start with remapped old, overlay with new for changed/added cells
        // For cells that exist in new but not old: add from new
        // For cells that exist in old but not new: remove
        for (auto& [cid, ids] : new_sc) sc[cid] = ids;
        for (auto& [cid, ids] : new_ac) ac[cid] = ids;
        for (auto& [cid, ids] : new_ic) ic[cid] = ids;

        // Collect all cells from new build
        std::unordered_map<uint64_t, bool> new_cells;
        size_t nn = new_geo.size() / 20;
        for (size_t i = 0; i < nn; i++) {
            uint64_t cid; memcpy(&cid, new_geo.data() + i * 20, 8);
            new_cells[cid] = true;
        }
        // Remove cells not in new
        for (auto it = sc.begin(); it != sc.end(); )
            new_cells.count(it->first) ? ++it : it = sc.erase(it);
        for (auto it = ac.begin(); it != ac.end(); )
            new_cells.count(it->first) ? ++it : it = ac.erase(it);
        for (auto it = ic.begin(); it != ic.end(); )
            new_cells.count(it->first) ? ++it : it = ic.erase(it);

        // Write merged entries + geo_cells
        // (This produces files almost identical to new build's entries)
        auto write_geo = [&](const std::string& dir,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& s,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& a,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& ip) {
            // Collect all cells
            std::vector<uint64_t> all_cells;
            for (auto& [id, _] : new_cells) all_cells.push_back(id);
            std::sort(all_cells.begin(), all_cells.end());

            auto write_ent = [&](const std::string& path,
                                  const std::unordered_map<uint64_t, std::vector<uint32_t>>& cm)
                -> std::unordered_map<uint64_t, uint32_t>
            {
                std::vector<char> buf;
                std::unordered_map<uint64_t, uint32_t> offsets;
                for (uint64_t cid : all_cells) {
                    auto it = cm.find(cid);
                    if (it == cm.end() || it->second.empty()) continue;
                    offsets[cid] = static_cast<uint32_t>(buf.size());
                    uint16_t count = static_cast<uint16_t>(it->second.size());
                    buf.insert(buf.end(), (char*)&count, (char*)&count + 2);
                    buf.insert(buf.end(), (char*)it->second.data(),
                              (char*)it->second.data() + it->second.size() * 4);
                }
                write_file(path, buf);
                return offsets;
            };

            auto so = write_ent(dir + "/street_entries.bin", s);
            auto ao = write_ent(dir + "/addr_entries.bin", a);
            auto io = write_ent(dir + "/interp_entries.bin", ip);

            uint32_t no_data = 0xFFFFFFFF;
            std::vector<char> buf;
            for (uint64_t cid : all_cells) {
                buf.insert(buf.end(), (char*)&cid, (char*)&cid + 8);
                auto g = [&](const auto& m) -> uint32_t {
                    auto it = m.find(cid); return (it != m.end()) ? it->second : no_data;
                };
                uint32_t sv = g(so), av = g(ao), iv = g(io);
                buf.insert(buf.end(), (char*)&sv, (char*)&sv + 4);
                buf.insert(buf.end(), (char*)&av, (char*)&av + 4);
                buf.insert(buf.end(), (char*)&iv, (char*)&iv + 4);
            }
            write_file(dir + "/geo_cells.bin", buf);
        };

        write_geo(tmpdir, sc, ac, ic);
    }

    // Admin cells: parse, remap, rebuild
    std::cerr << "  Remapping admin entries..." << std::endl;
    {
        auto old_ac = read_file(old_dir + "/admin_cells.bin");
        auto old_ae = read_file(old_dir + "/admin_entries.bin");
        auto new_ac = read_file(new_dir + "/admin_cells.bin");
        auto new_ae = read_file(new_dir + "/admin_entries.bin");

        auto cells = parse_entries(old_ac, 12, 8, old_ae);
        auto new_cells_map = parse_entries(new_ac, 12, 8, new_ae);
        remap_ids(cells, admin_remap, true);

        // Merge with new
        for (auto& [cid, ids] : new_cells_map) cells[cid] = ids;
        // Remove cells not in new
        std::unordered_map<uint64_t, bool> new_cell_set;
        size_t nn = new_ac.size() / 12;
        for (size_t i = 0; i < nn; i++) {
            uint64_t cid; memcpy(&cid, new_ac.data() + i * 12, 8);
            new_cell_set[cid] = true;
        }
        for (auto it = cells.begin(); it != cells.end(); )
            new_cell_set.count(it->first) ? ++it : it = cells.erase(it);

        write_entries_cells(tmpdir + "/admin_cells.bin", tmpdir + "/admin_entries.bin", cells, 12);
    }
}

// --- Write helpers ---
static void write_val(std::ofstream& f, const void* data, size_t size) {
    f.write(reinterpret_cast<const char*>(data), size);
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>" << std::endl;
        return 1;
    }
    std::string old_dir = argv[1], new_dir = argv[2], patch_path = argv[4];
    std::string tmpdir = "/tmp/geocoder-diff-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    // Build string remap
    std::cerr << "Building string remap..." << std::endl;
    auto old_strings = read_file(old_dir + "/strings.bin");
    auto new_strings = read_file(new_dir + "/strings.bin");
    auto str_remap = build_string_remap(old_strings, new_strings);
    std::cerr << "  " << str_remap.size() << " strings mapped" << std::endl;

    // Detect struct strides
    auto detect = [](const std::string& path, std::vector<size_t> cs) -> size_t {
        auto d = read_file(path);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect(old_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect(old_dir + "/interp_ways.bin", {20, 18});
    size_t admin_stride = detect(old_dir + "/admin_polygons.bin", {24, 20, 19});

    // Build record ID remaps (requires loading data + coordinate files)
    std::cerr << "Building record ID remaps..." << std::endl;
    std::vector<uint32_t> addr_id_remap, way_id_remap, interp_id_remap, admin_id_remap;
    {
        // addr_points: string-remap old, then match by bytes
        auto old_ap = read_file(old_dir + "/addr_points.bin");
        remap_addr_points(old_ap, str_remap);
        auto new_ap = read_file(new_dir + "/addr_points.bin");
        addr_id_remap = match_addr_points(old_ap, new_ap);
        write_file(tmpdir + "/addr_points.bin", old_ap);
    }
    {
        // street_ways: string-remap + content match (fixes node_offset too)
        auto old_w = read_file(old_dir + "/street_ways.bin");
        remap_field(old_w, way_stride, way_stride == 12 ? 8 : 5, str_remap);
        auto old_n = read_file(old_dir + "/street_nodes.bin");
        auto new_w = read_file(new_dir + "/street_ways.bin");
        auto new_n = read_file(new_dir + "/street_nodes.bin");
        way_id_remap = match_ways(old_w, old_n, new_w, new_n, way_stride);
        write_file(tmpdir + "/street_ways.bin", old_w);
        write_file(tmpdir + "/street_nodes.bin", new_n); // use new nodes (offsets now match)
    }
    {
        auto old_i = read_file(old_dir + "/interp_ways.bin");
        remap_field(old_i, interp_stride, interp_stride >= 20 ? 8 : 5, str_remap);
        auto old_n = read_file(old_dir + "/interp_nodes.bin");
        auto new_i = read_file(new_dir + "/interp_ways.bin");
        auto new_n = read_file(new_dir + "/interp_nodes.bin");
        interp_id_remap = match_interps(old_i, old_n, new_i, new_n, interp_stride);
        write_file(tmpdir + "/interp_ways.bin", old_i);
        write_file(tmpdir + "/interp_nodes.bin", new_n);
    }
    {
        auto old_p = read_file(old_dir + "/admin_polygons.bin");
        remap_field(old_p, admin_stride, 8, str_remap);
        auto old_v = read_file(old_dir + "/admin_vertices.bin");
        auto new_p = read_file(new_dir + "/admin_polygons.bin");
        auto new_v = read_file(new_dir + "/admin_vertices.bin");
        admin_id_remap = match_admin(old_p, old_v, new_p, new_v, admin_stride);
        write_file(tmpdir + "/admin_polygons.bin", old_p);
        write_file(tmpdir + "/admin_vertices.bin", new_v);
    }

    // Rebuild entry/cell files with remapped IDs
    remap_and_rebuild_entries(old_dir, new_dir, tmpdir,
                              way_id_remap, addr_id_remap, interp_id_remap, admin_id_remap);

    // Open patch file
    std::ofstream pf(patch_path, std::ios::binary);
    pf.write(GCPATCH_MAGIC, 8);
    uint32_t ver = GCPATCH_VERSION, flags = 0;
    write_val(pf, &ver, 4); write_val(pf, &flags, 4);

    // Write string remap
    {
        uint32_t marker = 0xFFFFFFFE;
        write_val(pf, &marker, 4);
        std::vector<std::pair<uint32_t,uint32_t>> entries;
        size_t pos = 0;
        while (pos < old_strings.size()) {
            uint32_t old_off = static_cast<uint32_t>(pos);
            auto it = str_remap.find(old_off);
            entries.push_back({old_off, (it != str_remap.end()) ? it->second : old_off});
            pos += strlen(old_strings.data() + pos) + 1;
        }
        uint32_t count = static_cast<uint32_t>(entries.size());
        write_val(pf, &count, 4);
        for (auto& [o, n] : entries) { write_val(pf, &o, 4); write_val(pf, &n, 4); }
        std::cerr << "  Remap: " << count << " entries" << std::endl;
    }

    // Write offset fixup tables for ways/interps/admin (node_offset/vertex_offset)
    {
        auto write_fixups = [&](PatchFileId id, const std::string& name,
                                 const std::vector<uint32_t>& id_remap, size_t stride) {
            auto new_data = read_file(new_dir + "/" + name);
            uint32_t marker = FIXUP_MARKER, fid = static_cast<uint32_t>(id);
            uint32_t st = static_cast<uint32_t>(stride);
            std::vector<std::pair<uint32_t, uint32_t>> fixups;
            for (uint32_t i = 0; i < id_remap.size(); i++) {
                if (id_remap[i] != 0xFFFFFFFF) {
                    uint32_t new_val;
                    memcpy(&new_val, new_data.data() + id_remap[i] * stride, 4);
                    fixups.push_back({i, new_val});
                }
            }
            write_val(pf, &marker, 4); write_val(pf, &fid, 4);
            write_val(pf, &st, 4);
            uint32_t count = static_cast<uint32_t>(fixups.size());
            write_val(pf, &count, 4);
            for (auto& [idx, val] : fixups) { write_val(pf, &idx, 4); write_val(pf, &val, 4); }
            std::cerr << "  fixup " << name << ": " << count << " entries" << std::endl;
        };
        write_fixups(PatchFileId::STREET_WAYS, "street_ways.bin", way_id_remap, way_stride);
        write_fixups(PatchFileId::INTERP_WAYS, "interp_ways.bin", interp_id_remap, interp_stride);
        write_fixups(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin", admin_id_remap, admin_stride);
    }

    // Single-stage diffs: remapped_old → new (zstd --patch-from on temp files)
    auto diff_file = [&](PatchFileId id, const std::string& name) {
        std::cerr << "  " << name << ": ";
        std::string ref = tmpdir + "/" + name;
        if (read_file(ref).empty()) ref = old_dir + "/" + name;
        std::string tgt = new_dir + "/" + name;
        std::string zst = tmpdir + "/" + name + ".zst";
        system(("zstd --patch-from='" + ref + "' '" + tgt + "' -o '" + zst + "' -f --quiet 2>/dev/null").c_str());
        auto zst_data = read_file(zst);
        auto old_data = read_file(old_dir + "/" + name);
        auto new_data = read_file(tgt);

        uint32_t fid = static_cast<uint32_t>(id), enc = static_cast<uint32_t>(PatchEncoding::ZSTD_DELTA);
        uint64_t os = old_data.size(), ns = new_data.size(), ds = zst_data.size();
        write_val(pf, &fid, 4); write_val(pf, &enc, 4);
        write_val(pf, &os, 8); write_val(pf, &ns, 8); write_val(pf, &ds, 8);
        pf.write(zst_data.data(), zst_data.size());

        double pct = ns > 0 ? ds * 100.0 / ns : 0;
        std::cerr << ds << " bytes (" << std::fixed << std::setprecision(2) << pct << "%)" << std::endl;
        remove(zst.c_str());
    };

    diff_file(PatchFileId::STRINGS, "strings.bin");
    diff_file(PatchFileId::ADDR_POINTS, "addr_points.bin");
    diff_file(PatchFileId::STREET_WAYS, "street_ways.bin");
    diff_file(PatchFileId::INTERP_WAYS, "interp_ways.bin");
    diff_file(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin");
    diff_file(PatchFileId::STREET_NODES, "street_nodes.bin");
    diff_file(PatchFileId::INTERP_NODES, "interp_nodes.bin");
    diff_file(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin");
    diff_file(PatchFileId::GEO_CELLS, "geo_cells.bin");
    diff_file(PatchFileId::STREET_ENTRIES, "street_entries.bin");
    diff_file(PatchFileId::ADDR_ENTRIES, "addr_entries.bin");
    diff_file(PatchFileId::INTERP_ENTRIES, "interp_entries.bin");
    diff_file(PatchFileId::ADMIN_CELLS, "admin_cells.bin");
    diff_file(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin");

    // Remove the old two-stage diff→remapped (stage1), remapped→new (stage2)
    auto diff_two_stage = [&](PatchFileId id, const std::string& name) {
        std::cerr << "  " << name << ": ";
        std::string old_path = old_dir + "/" + name;
        std::string temp_path = tmpdir + "/" + name;
        std::string new_path = new_dir + "/" + name;
        std::string zst1 = tmpdir + "/" + name + ".s1.zst";
        std::string zst2 = tmpdir + "/" + name + ".s2.zst";

        system(("zstd --patch-from='" + old_path + "' '" + temp_path + "' -o '" + zst1 + "' -f --quiet 2>/dev/null").c_str());
        system(("zstd --patch-from='" + temp_path + "' '" + new_path + "' -o '" + zst2 + "' -f --quiet 2>/dev/null").c_str());

        auto s1 = read_file(zst1), s2 = read_file(zst2);
        auto old_data = read_file(old_path), remapped = read_file(temp_path), new_data = read_file(new_path);

        uint32_t fid = static_cast<uint32_t>(id), enc = static_cast<uint32_t>(PatchEncoding::TWO_STAGE);
        uint64_t os = old_data.size(), rs = remapped.size(), ns = new_data.size();
        uint64_t s1s = s1.size(), s2s = s2.size();
        write_val(pf, &fid, 4); write_val(pf, &enc, 4);
        write_val(pf, &os, 8); write_val(pf, &rs, 8); write_val(pf, &ns, 8);
        write_val(pf, &s1s, 8); write_val(pf, &s2s, 8);
        pf.write(s1.data(), s1.size());
        pf.write(s2.data(), s2.size());

        double pct = ns > 0 ? (s1s + s2s) * 100.0 / ns : 0;
        std::cerr << "s1=" << s1s << " s2=" << s2s << " (" << std::fixed << std::setprecision(2) << pct << "%)" << std::endl;
        remove(zst1.c_str()); remove(zst2.c_str());
    };

    // Single-stage diff for files without remapping
    auto diff_single = [&](PatchFileId id, const std::string& name) {
        std::cerr << "  " << name << ": ";
        std::string old_path = old_dir + "/" + name;
        std::string new_path = new_dir + "/" + name;
        std::string zst = tmpdir + "/" + name + ".zst";
        system(("zstd --patch-from='" + old_path + "' '" + new_path + "' -o '" + zst + "' -f --quiet 2>/dev/null").c_str());
        auto zst_data = read_file(zst);
        auto old_data = read_file(old_path), new_data = read_file(new_path);

        uint32_t fid = static_cast<uint32_t>(id), enc = static_cast<uint32_t>(PatchEncoding::COPY_INSERT);
        uint64_t os = old_data.size(), ns = new_data.size(), ds = zst_data.size();
        write_val(pf, &fid, 4); write_val(pf, &enc, 4);
        write_val(pf, &os, 8); write_val(pf, &ns, 8); write_val(pf, &ds, 8);
        pf.write(zst_data.data(), zst_data.size());

        double pct = ns > 0 ? ds * 100.0 / ns : 0;
        std::cerr << ds << " bytes (" << std::fixed << std::setprecision(2) << pct << "%)" << std::endl;
        remove(zst.c_str());
    };

    // All remapped files use two-stage
    diff_two_stage(PatchFileId::ADDR_POINTS, "addr_points.bin");
    diff_two_stage(PatchFileId::STREET_WAYS, "street_ways.bin");
    diff_two_stage(PatchFileId::INTERP_WAYS, "interp_ways.bin");
    diff_two_stage(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin");
    diff_two_stage(PatchFileId::STREET_NODES, "street_nodes.bin");
    diff_two_stage(PatchFileId::INTERP_NODES, "interp_nodes.bin");
    diff_two_stage(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin");
    diff_two_stage(PatchFileId::GEO_CELLS, "geo_cells.bin");
    diff_two_stage(PatchFileId::STREET_ENTRIES, "street_entries.bin");
    diff_two_stage(PatchFileId::ADDR_ENTRIES, "addr_entries.bin");
    diff_two_stage(PatchFileId::INTERP_ENTRIES, "interp_entries.bin");
    diff_two_stage(PatchFileId::ADMIN_CELLS, "admin_cells.bin");
    diff_two_stage(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin");
    // strings.bin: single stage (no remap)
    diff_single(PatchFileId::STRINGS, "strings.bin");

    uint32_t end = 0xFFFFFFFF;
    write_val(pf, &end, 4);

    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());

    auto total = pf.tellp();
    std::cerr << "\nTotal patch: " << total << " bytes (" << total / 1024 / 1024 << " MiB)" << std::endl;
    return 0;
}

-    // Remove the old two-stage diff→remapped (stage1), remapped→new (stage2)

-    auto diff_two_stage = [&](PatchFileId id, const std::string& name) {

-        std::string old_path = old_dir + "/" + name;

-        std::string temp_path = tmpdir + "/" + name;

-        std::string new_path = new_dir + "/" + name;

-        std::string zst1 = tmpdir + "/" + name + ".s1.zst";

-        std::string zst2 = tmpdir + "/" + name + ".s2.zst";

-        system(("zstd --patch-from='" + old_path + "' '" + temp_path + "' -o '" + zst1 + "' -f --quiet 2>/dev/null").c_str());

-        system(("zstd --patch-from='" + temp_path + "' '" + new_path + "' -o '" + zst2 + "' -f --quiet 2>/dev/null").c_str());

-        auto s1 = read_file(zst1), s2 = read_file(zst2);

-        auto old_data = read_file(old_path), remapped = read_file(temp_path), new_data = read_file(new_path);

-        uint32_t fid = static_cast<uint32_t>(id), enc = static_cast<uint32_t>(PatchEncoding::TWO_STAGE);

-        uint64_t os = old_data.size(), rs = remapped.size(), ns = new_data.size();

-        uint64_t s1s = s1.size(), s2s = s2.size();

-        write_val(pf, &fid, 4); write_val(pf, &enc, 4);

-        write_val(pf, &os, 8); write_val(pf, &rs, 8); write_val(pf, &ns, 8);

-        write_val(pf, &s1s, 8); write_val(pf, &s2s, 8);

-        pf.write(s1.data(), s1.size());

-        pf.write(s2.data(), s2.size());

-        remove(zst1.c_str()); remove(zst2.c_str());

-    // Single-stage diff for files without remapping

-    auto diff_single = [&](PatchFileId id, const std::string& name) {

-        std::string zst = tmpdir + "/" + name + ".zst";

-        system(("zstd --patch-from='" + old_path + "' '" + new_path + "' -o '" + zst + "' -f --quiet 2>/dev/null").c_str());

-        auto zst_data = read_file(zst);

-        auto old_data = read_file(old_path), new_data = read_file(new_path);

-        uint32_t fid = static_cast<uint32_t>(id), enc = static_cast<uint32_t>(PatchEncoding::COPY_INSERT);

-        uint64_t os = old_data.size(), ns = new_data.size(), ds = zst_data.size();

-        write_val(pf, &os, 8); write_val(pf, &ns, 8); write_val(pf, &ds, 8);

-        pf.write(zst_data.data(), zst_data.size());

-        double pct = ns > 0 ? ds * 100.0 / ns : 0;

-        std::cerr << ds << " bytes (" << std::fixed << std::setprecision(2) << pct << "%)" << std::endl;

-    // All remapped files use two-stage

-    diff_two_stage(PatchFileId::ADDR_POINTS, "addr_points.bin");

-    diff_two_stage(PatchFileId::STREET_WAYS, "street_ways.bin");

-    diff_two_stage(PatchFileId::INTERP_WAYS, "interp_ways.bin");

-    diff_two_stage(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin");

-    diff_two_stage(PatchFileId::STREET_NODES, "street_nodes.bin");

-    diff_two_stage(PatchFileId::INTERP_NODES, "interp_nodes.bin");

-    diff_two_stage(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin");

-    diff_two_stage(PatchFileId::GEO_CELLS, "geo_cells.bin");

-    diff_two_stage(PatchFileId::STREET_ENTRIES, "street_entries.bin");

-    diff_two_stage(PatchFileId::ADDR_ENTRIES, "addr_entries.bin");

-    diff_two_stage(PatchFileId::INTERP_ENTRIES, "interp_entries.bin");

-    diff_two_stage(PatchFileId::ADMIN_CELLS, "admin_cells.bin");

-    diff_two_stage(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin");

-    // strings.bin: single stage (no remap)

-    diff_single(PatchFileId::STRINGS, "strings.bin");


default-src 'none'; frame-ancestors 'none'

_cfuvid=SDt9a5.J5PkMNCLBfPDueacRJ07iQUJoUfdyyj3RrWI-1774613038.6512787-1.0.1.1-p4RdyUVpTChhs8hLmxpSviI0JHs_b9igHcsHh6aUw14; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com

Overloaded. https://docs.claude.com/en/api/errors


Now update the patch tool to: 1) read string remap, 2) read fixup tables, 3) apply both to old files before zstd decompression:


Edit

    // Skip string remap section (two-stage encoding handles remapping internally)
    {
        uint32_t section_id; pf.read(reinterpret_cast<char*>(&section_id), 4);
        if (section_id == 0xFFFFFFFE) {
            uint32_t count; pf.read(reinterpret_cast<char*>(&count), 4);
            pf.seekg(count * 8, std::ios::cur);
            std::cerr << "Skipped string remap (" << count << " entries, handled by two-stage)" << std::endl;
        } else {
            std::cerr << "Warning: expected remap section, got " << section_id << std::endl;
            // Seek back so section processing can handle it
            pf.seekg(-4, std::ios::cur);
        }
    }

    // Read string remap
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t section_id; pf.read(reinterpret_cast<char*>(&section_id), 4);
        if (section_id == 0xFFFFFFFE) {
            uint32_t count; pf.read(reinterpret_cast<char*>(&count), 4);
            for (uint32_t i = 0; i < count; i++) {
                uint32_t old_off, new_off;
                pf.read(reinterpret_cast<char*>(&old_off), 4);
                pf.read(reinterpret_cast<char*>(&new_off), 4);
                if (old_off != new_off) str_remap[old_off] = new_off;
            }
            std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;
        }
    }

    // Read offset fixup tables
    struct Fixup { uint32_t record_idx; uint32_t new_value; };
    std::unordered_map<uint32_t, std::vector<Fixup>> fixup_tables; // file_id → fixups
    std::unordered_map<uint32_t, uint32_t> fixup_strides; // file_id → stride
    while (pf) {
        uint32_t marker; pf.read(reinterpret_cast<char*>(&marker), 4);
        if (marker != FIXUP_MARKER) {
            pf.seekg(-4, std::ios::cur);
            break;
        }
        uint32_t fid, stride, count;
        pf.read(reinterpret_cast<char*>(&fid), 4);
        pf.read(reinterpret_cast<char*>(&stride), 4);
        pf.read(reinterpret_cast<char*>(&count), 4);
        auto& vec = fixup_tables[fid];
        fixup_strides[fid] = stride;
        vec.resize(count);
        for (uint32_t i = 0; i < count; i++) {
            pf.read(reinterpret_cast<char*>(&vec[i].record_idx), 4);
            pf.read(reinterpret_cast<char*>(&vec[i].new_value), 4);
        }
        std::cerr << "Loaded " << count << " fixups for file " << fid << std::endl;
    }

    // Detect struct strides from current files
    auto detect_stride = [&](const std::string& name, std::vector<size_t> cs) -> size_t {
        auto d = read_file(cur_dir + "/" + name);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect_stride("street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride("interp_ways.bin", {20, 18});
    size_t admin_stride = detect_stride("admin_polygons.bin", {24, 20, 19});

    // Helper: apply string remap to old data file
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t file_id) {
        if (str_remap.empty()) return;
        if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) {
            for (size_t i = 0; i + 16 <= data.size(); i += 16)
                for (size_t off : {8, 12}) {
                    uint32_t v; memcpy(&v, data.data() + i + off, 4);
                    auto it = str_remap.find(v);
                    if (it != str_remap.end()) memcpy(data.data() + i + off, &it->second, 4);
                }
        } else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) {
            size_t name_off = (way_stride == 12) ? 8 : 5;
            for (size_t i = 0; i + way_stride <= data.size(); i += way_stride) {
                uint32_t v; memcpy(&v, data.data() + i + name_off, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + name_off, &it->second, 4);
            }
        } else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) {
            size_t soff = (interp_stride >= 20) ? 8 : 5;
            for (size_t i = 0; i + interp_stride <= data.size(); i += interp_stride) {
                uint32_t v; memcpy(&v, data.data() + i + soff, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + soff, &it->second, 4);
            }
        } else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) {
            for (size_t i = 0; i + admin_stride <= data.size(); i += admin_stride) {
                uint32_t v; memcpy(&v, data.data() + i + 8, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + 8, &it->second, 4);
            }
        }
    };

    // Helper: apply offset fixups
    auto apply_fixups = [&](std::vector<char>& data, uint32_t file_id) {
        auto it = fixup_tables.find(file_id);
        if (it == fixup_tables.end()) return;
        uint32_t stride = fixup_strides[file_id];
        for (auto& f : it->second) {
            size_t pos = (size_t)f.record_idx * stride;
            if (pos + 4 <= data.size())
                memcpy(data.data() + pos, &f.new_value, 4);
        }
    };


The file /root/traccar-geocoder/builder/tools/geocoder_patch.cpp has been updated successfully.

    // Read string remap
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t section_id; pf.read(reinterpret_cast<char*>(&section_id), 4);
        if (section_id == 0xFFFFFFFE) {
            uint32_t count; pf.read(reinterpret_cast<char*>(&count), 4);
            for (uint32_t i = 0; i < count; i++) {
                uint32_t old_off, new_off;
                pf.read(reinterpret_cast<char*>(&old_off), 4);
                pf.read(reinterpret_cast<char*>(&new_off), 4);
                if (old_off != new_off) str_remap[old_off] = new_off;
            }
            std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;
        }
    }

    // Read offset fixup tables
    struct Fixup { uint32_t record_idx; uint32_t new_value; };
    std::unordered_map<uint32_t, std::vector<Fixup>> fixup_tables; // file_id → fixups
    std::unordered_map<uint32_t, uint32_t> fixup_strides; // file_id → stride
    while (pf) {
        uint32_t marker; pf.read(reinterpret_cast<char*>(&marker), 4);
        if (marker != FIXUP_MARKER) {
            pf.seekg(-4, std::ios::cur);
            break;
        }
        uint32_t fid, stride, count;
        pf.read(reinterpret_cast<char*>(&fid), 4);
        pf.read(reinterpret_cast<char*>(&stride), 4);
        pf.read(reinterpret_cast<char*>(&count), 4);
        auto& vec = fixup_tables[fid];
        fixup_strides[fid] = stride;
        vec.resize(count);
        for (uint32_t i = 0; i < count; i++) {
            pf.read(reinterpret_cast<char*>(&vec[i].record_idx), 4);
            pf.read(reinterpret_cast<char*>(&vec[i].new_value), 4);
        }
        std::cerr << "Loaded " << count << " fixups for file " << fid << std::endl;
    }

    // Detect struct strides from current files
    auto detect_stride = [&](const std::string& name, std::vector<size_t> cs) -> size_t {
        auto d = read_file(cur_dir + "/" + name);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect_stride("street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride("interp_ways.bin", {20, 18});
    size_t admin_stride = detect_stride("admin_polygons.bin", {24, 20, 19});

    // Helper: apply string remap to old data file
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t file_id) {
        if (str_remap.empty()) return;
        if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) {
            for (size_t i = 0; i + 16 <= data.size(); i += 16)
                for (size_t off : {8, 12}) {
                    uint32_t v; memcpy(&v, data.data() + i + off, 4);
                    auto it = str_remap.find(v);
                    if (it != str_remap.end()) memcpy(data.data() + i + off, &it->second, 4);
                }
        } else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) {
            size_t name_off = (way_stride == 12) ? 8 : 5;
            for (size_t i = 0; i + way_stride <= data.size(); i += way_stride) {
                uint32_t v; memcpy(&v, data.data() + i + name_off, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + name_off, &it->second, 4);
            }
        } else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) {
            size_t soff = (interp_stride >= 20) ? 8 : 5;
            for (size_t i = 0; i + interp_stride <= data.size(); i += interp_stride) {
                uint32_t v; memcpy(&v, data.data() + i + soff, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + soff, &it->second, 4);
            }
        } else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) {
            for (size_t i = 0; i + admin_stride <= data.size(); i += admin_stride) {
                uint32_t v; memcpy(&v, data.data() + i + 8, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + 8, &it->second, 4);
            }
        }
    };

    // Helper: apply offset fixups
    auto apply_fixups = [&](std::vector<char>& data, uint32_t file_id) {
        auto it = fixup_tables.find(file_id);
        if (it == fixup_tables.end()) return;
        uint32_t stride = fixup_strides[file_id];
        for (auto& f : it->second) {
            size_t pos = (size_t)f.record_idx * stride;
            if (pos + 4 <= data.size())
                memcpy(data.data() + pos, &f.new_value, 4);
        }
    };

// geocoder-patch: Apply a .gcpatch to a geocoder index directory.
//
// For each file: apply string remap to old, write temp, run zstd -d --patch-from.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>

#include <unistd.h>
#include "patch_format.h"

// --- String remap application ---

static void remap_addr_points(std::vector<char>& data,
                                const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + 16 <= data.size(); i += 16) {
        for (size_t off : {8, 12}) {
            uint32_t v; memcpy(&v, data.data() + i + off, 4);
            auto it = rm.find(v);
            if (it != rm.end()) memcpy(data.data() + i + off, &it->second, 4);
        }
    }
}

static void remap_field(std::vector<char>& data, size_t stride, size_t field_off,
                          const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + stride <= data.size(); i += stride) {
        uint32_t v; memcpy(&v, data.data() + i + field_off, 4);
        auto it = rm.find(v);
        if (it != rm.end()) memcpy(data.data() + i + field_off, &it->second, 4);
    }
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);

    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    std::ifstream pf(patch_path, std::ios::binary);
    if (!pf) { std::cerr << "Cannot open " << patch_path << std::endl; return 1; }

    // Read header
    char magic[8]; pf.read(magic, 8);
    if (memcmp(magic, GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch file" << std::endl; return 1;
    }
    uint32_t version, flags;
    pf.read(reinterpret_cast<char*>(&version), 4);
    pf.read(reinterpret_cast<char*>(&flags), 4);
    if (version != GCPATCH_VERSION) {
        std::cerr << "Unsupported version " << version << std::endl; return 1;
    }

    // Skip string remap section (two-stage encoding handles remapping internally)
    {
        uint32_t section_id; pf.read(reinterpret_cast<char*>(&section_id), 4);
        if (section_id == 0xFFFFFFFE) {
            uint32_t count; pf.read(reinterpret_cast<char*>(&count), 4);
            pf.seekg(count * 8, std::ios::cur);
            std::cerr << "Skipped string remap (" << count << " entries, handled by two-stage)" << std::endl;
        } else {
            std::cerr << "Warning: expected remap section, got " << section_id << std::endl;
            // Seek back so section processing can handle it
            pf.seekg(-4, std::ios::cur);
        }
    }

    // Process file sections
    while (pf) {
        uint32_t file_id; pf.read(reinterpret_cast<char*>(&file_id), 4);
        if (!pf || file_id == 0xFFFFFFFF) break;
        uint32_t encoding; pf.read(reinterpret_cast<char*>(&encoding), 4);

        if (file_id >= (uint32_t)PatchFileId::COUNT) {
            std::cerr << "  Unknown section " << file_id << std::endl;
            break;
        }
        const char* filename = patch_file_names[file_id];

        auto write_temp = [&](const std::string& path, uint64_t size) {
            std::ofstream zf(path, std::ios::binary);
            std::vector<char> buf(std::min(size, (uint64_t)1024*1024));
            uint64_t remaining = size;
            while (remaining > 0) {
                size_t chunk = std::min(remaining, (uint64_t)buf.size());
                pf.read(buf.data(), chunk);
                zf.write(buf.data(), chunk);
                remaining -= chunk;
            }
        };

        auto zstd_apply = [&](const std::string& ref, const std::string& zst, const std::string& out) -> bool {
            std::string cmd = "zstd --patch-from='" + ref + "' -d '" + zst + "' -o '" + out + "' -f --long=31 --quiet 2>/dev/null";
            return system(cmd.c_str()) == 0;
        };

        if (encoding == (uint32_t)PatchEncoding::COPY_INSERT) {
            // Single-stage: old → new
            uint64_t old_size, new_size, delta_size;
            pf.read(reinterpret_cast<char*>(&old_size), 8);
            pf.read(reinterpret_cast<char*>(&new_size), 8);
            pf.read(reinterpret_cast<char*>(&delta_size), 8);

            std::string zst = tmpdir + "/" + std::string(filename) + ".zst";
            write_temp(zst, delta_size);

            std::string ref = cur_dir + "/" + std::string(filename);
            std::string out = out_dir + "/" + std::string(filename);
            if (!zstd_apply(ref, zst, out)) {
                std::cerr << "FAILED: " << filename << std::endl; return 1;
            }
            std::cerr << "  " << filename << ": " << read_file(out).size() << " bytes" << std::endl;
            remove(zst.c_str());

        } else if (encoding == (uint32_t)PatchEncoding::TWO_STAGE) {
            // Two-stage: old → remapped → new
            uint64_t old_size, remap_size, new_size, s1_size, s2_size;
            pf.read(reinterpret_cast<char*>(&old_size), 8);
            pf.read(reinterpret_cast<char*>(&remap_size), 8);
            pf.read(reinterpret_cast<char*>(&new_size), 8);
            pf.read(reinterpret_cast<char*>(&s1_size), 8);
            pf.read(reinterpret_cast<char*>(&s2_size), 8);

            std::string zst1 = tmpdir + "/" + std::string(filename) + ".s1.zst";
            std::string zst2 = tmpdir + "/" + std::string(filename) + ".s2.zst";
            std::string mid = tmpdir + "/" + std::string(filename) + ".mid";
            std::string out = out_dir + "/" + std::string(filename);

            write_temp(zst1, s1_size);
            write_temp(zst2, s2_size);

            // Stage 1: old → remapped
            std::string ref = cur_dir + "/" + std::string(filename);
            if (!zstd_apply(ref, zst1, mid)) {
                std::cerr << "FAILED stage1: " << filename << std::endl; return 1;
            }
            // Stage 2: remapped → new
            if (!zstd_apply(mid, zst2, out)) {
                std::cerr << "FAILED stage2: " << filename << std::endl; return 1;
            }
            std::cerr << "  " << filename << ": " << read_file(out).size() << " bytes" << std::endl;
            remove(zst1.c_str()); remove(zst2.c_str()); remove(mid.c_str());

        } else {
            std::cerr << "  Unknown encoding " << encoding << " for " << filename << std::endl;
            break;
        }
    }

    // Cleanup
    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());

    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

-    // Skip string remap section (two-stage encoding handles remapping internally)

+    std::unordered_map<uint32_t, uint32_t> str_remap;

-            pf.seekg(count * 8, std::ios::cur);

-            std::cerr << "Skipped string remap (" << count << " entries, handled by two-stage)" << std::endl;

-            std::cerr << "Warning: expected remap section, got " << section_id << std::endl;

-            // Seek back so section processing can handle it

+            for (uint32_t i = 0; i < count; i++) {

+                uint32_t old_off, new_off;

+                pf.read(reinterpret_cast<char*>(&old_off), 4);

+                pf.read(reinterpret_cast<char*>(&new_off), 4);

+                if (old_off != new_off) str_remap[old_off] = new_off;

+            std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;

+    struct Fixup { uint32_t record_idx; uint32_t new_value; };

+    std::unordered_map<uint32_t, std::vector<Fixup>> fixup_tables; // file_id → fixups

+    std::unordered_map<uint32_t, uint32_t> fixup_strides; // file_id → stride

+        uint32_t marker; pf.read(reinterpret_cast<char*>(&marker), 4);

+        pf.read(reinterpret_cast<char*>(&fid), 4);

+        pf.read(reinterpret_cast<char*>(&stride), 4);

+        pf.read(reinterpret_cast<char*>(&count), 4);

+            pf.read(reinterpret_cast<char*>(&vec[i].record_idx), 4);

+            pf.read(reinterpret_cast<char*>(&vec[i].new_value), 4);

+        std::cerr << "Loaded " << count << " fixups for file " << fid << std::endl;

+    // Detect struct strides from current files

+    auto detect_stride = [&](const std::string& name, std::vector<size_t> cs) -> size_t {

+        auto d = read_file(cur_dir + "/" + name);

+        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;

+    size_t way_stride = detect_stride("street_ways.bin", {12, 9});

+    size_t interp_stride = detect_stride("interp_ways.bin", {20, 18});

+    size_t admin_stride = detect_stride("admin_polygons.bin", {24, 20, 19});

+    // Helper: apply string remap to old data file

+    auto apply_str_remap = [&](std::vector<char>& data, uint32_t file_id) {

+        if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) {

+                    uint32_t v; memcpy(&v, data.data() + i + off, 4);

+                    auto it = str_remap.find(v);

+                    if (it != str_remap.end()) memcpy(data.data() + i + off, &it->second, 4);

+        } else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) {

+            size_t name_off = (way_stride == 12) ? 8 : 5;

+            for (size_t i = 0; i + way_stride <= data.size(); i += way_stride) {

+                uint32_t v; memcpy(&v, data.data() + i + name_off, 4);

+                auto it = str_remap.find(v);

+                if (it != str_remap.end()) memcpy(data.data() + i + name_off, &it->second, 4);

+        } else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) {

+            size_t soff = (interp_stride >= 20) ? 8 : 5;

+            for (size_t i = 0; i + interp_stride <= data.size(); i += interp_stride) {

+                uint32_t v; memcpy(&v, data.data() + i + soff, 4);

+                if (it != str_remap.end()) memcpy(data.data() + i + soff, &it->second, 4);

+        } else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) {

+            for (size_t i = 0; i + admin_stride <= data.size(); i += admin_stride) {

+                uint32_t v; memcpy(&v, data.data() + i + 8, 4);

+                if (it != str_remap.end()) memcpy(data.data() + i + 8, &it->second, 4);

+    auto apply_fixups = [&](std::vector<char>& data, uint32_t file_id) {

+        auto it = fixup_tables.find(file_id);

+        if (it == fixup_tables.end()) return;

+        uint32_t stride = fixup_strides[file_id];

+            size_t pos = (size_t)f.record_idx * stride;

+            if (pos + 4 <= data.size())

+                memcpy(data.data() + pos, &f.new_value, 4);


Now update the section processing to apply remap + fixups before zstd delta:


        if (encoding == (uint32_t)PatchEncoding::COPY_INSERT) {
            // Single-stage: old → new
            uint64_t old_size, new_size, delta_size;
            pf.read(reinterpret_cast<char*>(&old_size), 8);
            pf.read(reinterpret_cast<char*>(&new_size), 8);
            pf.read(reinterpret_cast<char*>(&delta_size), 8);

            std::string zst = tmpdir + "/" + std::string(filename) + ".zst";
            write_temp(zst, delta_size);

            std::string ref = cur_dir + "/" + std::string(filename);
            std::string out = out_dir + "/" + std::string(filename);
            if (!zstd_apply(ref, zst, out)) {
                std::cerr << "FAILED: " << filename << std::endl; return 1;
            }
            std::cerr << "  " << filename << ": " << read_file(out).size() << " bytes" << std::endl;
            remove(zst.c_str());

        } else if (encoding == (uint32_t)PatchEncoding::TWO_STAGE) {
            // Two-stage: old → remapped → new
            uint64_t old_size, remap_size, new_size, s1_size, s2_size;
            pf.read(reinterpret_cast<char*>(&old_size), 8);
            pf.read(reinterpret_cast<char*>(&remap_size), 8);
            pf.read(reinterpret_cast<char*>(&new_size), 8);
            pf.read(reinterpret_cast<char*>(&s1_size), 8);
            pf.read(reinterpret_cast<char*>(&s2_size), 8);

            std::string zst1 = tmpdir + "/" + std::string(filename) + ".s1.zst";
            std::string zst2 = tmpdir + "/" + std::string(filename) + ".s2.zst";
            std::string mid = tmpdir + "/" + std::string(filename) + ".mid";
            std::string out = out_dir + "/" + std::string(filename);

            write_temp(zst1, s1_size);
            write_temp(zst2, s2_size);

            // Stage 1: old → remapped
            std::string ref = cur_dir + "/" + std::string(filename);
            if (!zstd_apply(ref, zst1, mid)) {
                std::cerr << "FAILED stage1: " << filename << std::endl; return 1;
            }
            // Stage 2: remapped → new
            if (!zstd_apply(mid, zst2, out)) {
                std::cerr << "FAILED stage2: " << filename << std::endl; return 1;
            }
            std::cerr << "  " << filename << ": " << read_file(out).size() << " bytes" << std::endl;
            remove(zst1.c_str()); remove(zst2.c_str()); remove(mid.c_str());

        } else {

        if (encoding == (uint32_t)PatchEncoding::ZSTD_DELTA) {
            uint64_t old_size, new_size, delta_size;
            pf.read(reinterpret_cast<char*>(&old_size), 8);
            pf.read(reinterpret_cast<char*>(&new_size), 8);
            pf.read(reinterpret_cast<char*>(&delta_size), 8);

            // Write zstd frame to temp
            std::string zst = tmpdir + "/" + std::string(filename) + ".zst";
            write_temp(zst, delta_size);

            // Load old file, apply string remap + fixups → write remapped ref
            auto old_data = read_file(cur_dir + "/" + std::string(filename));
            apply_str_remap(old_data, file_id);
            apply_fixups(old_data, file_id);
            std::string ref = tmpdir + "/" + std::string(filename) + ".ref";
            write_file(ref, old_data);

            // Apply zstd delta
            std::string out = out_dir + "/" + std::string(filename);
            if (!zstd_apply(ref, zst, out)) {
                std::cerr << "FAILED: " << filename << std::endl; return 1;
            }
            std::cerr << "  " << filename << ": " << read_file(out).size() << " bytes" << std::endl;
            remove(zst.c_str()); remove(ref.c_str());

        } else {


        if (encoding == (uint32_t)PatchEncoding::ZSTD_DELTA) {
            uint64_t old_size, new_size, delta_size;
            pf.read(reinterpret_cast<char*>(&old_size), 8);
            pf.read(reinterpret_cast<char*>(&new_size), 8);
            pf.read(reinterpret_cast<char*>(&delta_size), 8);

            // Write zstd frame to temp
            std::string zst = tmpdir + "/" + std::string(filename) + ".zst";
            write_temp(zst, delta_size);

            // Load old file, apply string remap + fixups → write remapped ref
            auto old_data = read_file(cur_dir + "/" + std::string(filename));
            apply_str_remap(old_data, file_id);
            apply_fixups(old_data, file_id);
            std::string ref = tmpdir + "/" + std::string(filename) + ".ref";
            write_file(ref, old_data);

            // Apply zstd delta
            std::string out = out_dir + "/" + std::string(filename);
            if (!zstd_apply(ref, zst, out)) {
                std::cerr << "FAILED: " << filename << std::endl; return 1;
            }
            std::cerr << "  " << filename << ": " << read_file(out).size() << " bytes" << std::endl;
            remove(zst.c_str()); remove(ref.c_str());

        } else {

// geocoder-patch: Apply a .gcpatch to a geocoder index directory.
//
// For each file: apply string remap to old, write temp, run zstd -d --patch-from.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>

#include <unistd.h>
#include "patch_format.h"

// --- String remap application ---

static void remap_addr_points(std::vector<char>& data,
                                const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + 16 <= data.size(); i += 16) {
        for (size_t off : {8, 12}) {
            uint32_t v; memcpy(&v, data.data() + i + off, 4);
            auto it = rm.find(v);
            if (it != rm.end()) memcpy(data.data() + i + off, &it->second, 4);
        }
    }
}

static void remap_field(std::vector<char>& data, size_t stride, size_t field_off,
                          const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + stride <= data.size(); i += stride) {
        uint32_t v; memcpy(&v, data.data() + i + field_off, 4);
        auto it = rm.find(v);
        if (it != rm.end()) memcpy(data.data() + i + field_off, &it->second, 4);
    }
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);

    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    std::ifstream pf(patch_path, std::ios::binary);
    if (!pf) { std::cerr << "Cannot open " << patch_path << std::endl; return 1; }

    // Read header
    char magic[8]; pf.read(magic, 8);
    if (memcmp(magic, GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch file" << std::endl; return 1;
    }
    uint32_t version, flags;
    pf.read(reinterpret_cast<char*>(&version), 4);
    pf.read(reinterpret_cast<char*>(&flags), 4);
    if (version != GCPATCH_VERSION) {
        std::cerr << "Unsupported version " << version << std::endl; return 1;
    }

    // Read string remap
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t section_id; pf.read(reinterpret_cast<char*>(&section_id), 4);
        if (section_id == 0xFFFFFFFE) {
            uint32_t count; pf.read(reinterpret_cast<char*>(&count), 4);
            for (uint32_t i = 0; i < count; i++) {
                uint32_t old_off, new_off;
                pf.read(reinterpret_cast<char*>(&old_off), 4);
                pf.read(reinterpret_cast<char*>(&new_off), 4);
                if (old_off != new_off) str_remap[old_off] = new_off;
            }
            std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;
        }
    }

    // Read offset fixup tables
    struct Fixup { uint32_t record_idx; uint32_t new_value; };
    std::unordered_map<uint32_t, std::vector<Fixup>> fixup_tables; // file_id → fixups
    std::unordered_map<uint32_t, uint32_t> fixup_strides; // file_id → stride
    while (pf) {
        uint32_t marker; pf.read(reinterpret_cast<char*>(&marker), 4);
        if (marker != FIXUP_MARKER) {
            pf.seekg(-4, std::ios::cur);
            break;
        }
        uint32_t fid, stride, count;
        pf.read(reinterpret_cast<char*>(&fid), 4);
        pf.read(reinterpret_cast<char*>(&stride), 4);
        pf.read(reinterpret_cast<char*>(&count), 4);
        auto& vec = fixup_tables[fid];
        fixup_strides[fid] = stride;
        vec.resize(count);
        for (uint32_t i = 0; i < count; i++) {
            pf.read(reinterpret_cast<char*>(&vec[i].record_idx), 4);
            pf.read(reinterpret_cast<char*>(&vec[i].new_value), 4);
        }
        std::cerr << "Loaded " << count << " fixups for file " << fid << std::endl;
    }

    // Detect struct strides from current files
    auto detect_stride = [&](const std::string& name, std::vector<size_t> cs) -> size_t {
        auto d = read_file(cur_dir + "/" + name);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect_stride("street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride("interp_ways.bin", {20, 18});
    size_t admin_stride = detect_stride("admin_polygons.bin", {24, 20, 19});

    // Helper: apply string remap to old data file
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t file_id) {
        if (str_remap.empty()) return;
        if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) {
            for (size_t i = 0; i + 16 <= data.size(); i += 16)
                for (size_t off : {8, 12}) {
                    uint32_t v; memcpy(&v, data.data() + i + off, 4);
                    auto it = str_remap.find(v);
                    if (it != str_remap.end()) memcpy(data.data() + i + off, &it->second, 4);
                }
        } else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) {
            size_t name_off = (way_stride == 12) ? 8 : 5;
            for (size_t i = 0; i + way_stride <= data.size(); i += way_stride) {
                uint32_t v; memcpy(&v, data.data() + i + name_off, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + name_off, &it->second, 4);
            }
        } else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) {
            size_t soff = (interp_stride >= 20) ? 8 : 5;
            for (size_t i = 0; i + interp_stride <= data.size(); i += interp_stride) {
                uint32_t v; memcpy(&v, data.data() + i + soff, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + soff, &it->second, 4);
            }
        } else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) {
            for (size_t i = 0; i + admin_stride <= data.size(); i += admin_stride) {
                uint32_t v; memcpy(&v, data.data() + i + 8, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + 8, &it->second, 4);
            }
        }
    };

    // Helper: apply offset fixups
    auto apply_fixups = [&](std::vector<char>& data, uint32_t file_id) {
        auto it = fixup_tables.find(file_id);
        if (it == fixup_tables.end()) return;
        uint32_t stride = fixup_strides[file_id];
        for (auto& f : it->second) {
            size_t pos = (size_t)f.record_idx * stride;
            if (pos + 4 <= data.size())
                memcpy(data.data() + pos, &f.new_value, 4);
        }
    };

    // Process file sections
    while (pf) {
        uint32_t file_id; pf.read(reinterpret_cast<char*>(&file_id), 4);
        if (!pf || file_id == 0xFFFFFFFF) break;
        uint32_t encoding; pf.read(reinterpret_cast<char*>(&encoding), 4);

        if (file_id >= (uint32_t)PatchFileId::COUNT) {
            std::cerr << "  Unknown section " << file_id << std::endl;
            break;
        }
        const char* filename = patch_file_names[file_id];

        auto write_temp = [&](const std::string& path, uint64_t size) {
            std::ofstream zf(path, std::ios::binary);
            std::vector<char> buf(std::min(size, (uint64_t)1024*1024));
            uint64_t remaining = size;
            while (remaining > 0) {
                size_t chunk = std::min(remaining, (uint64_t)buf.size());
                pf.read(buf.data(), chunk);
                zf.write(buf.data(), chunk);
                remaining -= chunk;
            }
        };

        auto zstd_apply = [&](const std::string& ref, const std::string& zst, const std::string& out) -> bool {
            std::string cmd = "zstd --patch-from='" + ref + "' -d '" + zst + "' -o '" + out + "' -f --long=31 --quiet 2>/dev/null";
            return system(cmd.c_str()) == 0;
        };

        if (encoding == (uint32_t)PatchEncoding::COPY_INSERT) {
            // Single-stage: old → new
            uint64_t old_size, new_size, delta_size;
            pf.read(reinterpret_cast<char*>(&old_size), 8);
            pf.read(reinterpret_cast<char*>(&new_size), 8);
            pf.read(reinterpret_cast<char*>(&delta_size), 8);

            std::string zst = tmpdir + "/" + std::string(filename) + ".zst";
            write_temp(zst, delta_size);

            std::string ref = cur_dir + "/" + std::string(filename);
            std::string out = out_dir + "/" + std::string(filename);
            if (!zstd_apply(ref, zst, out)) {
                std::cerr << "FAILED: " << filename << std::endl; return 1;
            }
            std::cerr << "  " << filename << ": " << read_file(out).size() << " bytes" << std::endl;
            remove(zst.c_str());

        } else if (encoding == (uint32_t)PatchEncoding::TWO_STAGE) {
            // Two-stage: old → remapped → new
            uint64_t old_size, remap_size, new_size, s1_size, s2_size;
            pf.read(reinterpret_cast<char*>(&old_size), 8);
            pf.read(reinterpret_cast<char*>(&remap_size), 8);
            pf.read(reinterpret_cast<char*>(&new_size), 8);
            pf.read(reinterpret_cast<char*>(&s1_size), 8);
            pf.read(reinterpret_cast<char*>(&s2_size), 8);

            std::string zst1 = tmpdir + "/" + std::string(filename) + ".s1.zst";
            std::string zst2 = tmpdir + "/" + std::string(filename) + ".s2.zst";
            std::string mid = tmpdir + "/" + std::string(filename) + ".mid";
            std::string out = out_dir + "/" + std::string(filename);

            write_temp(zst1, s1_size);
            write_temp(zst2, s2_size);

            // Stage 1: old → remapped
            std::string ref = cur_dir + "/" + std::string(filename);
            if (!zstd_apply(ref, zst1, mid)) {
                std::cerr << "FAILED stage1: " << filename << std::endl; return 1;
            }
            // Stage 2: remapped → new
            if (!zstd_apply(mid, zst2, out)) {
                std::cerr << "FAILED stage2: " << filename << std::endl; return 1;
            }
            std::cerr << "  " << filename << ": " << read_file(out).size() << " bytes" << std::endl;
            remove(zst1.c_str()); remove(zst2.c_str()); remove(mid.c_str());

        } else {
            std::cerr << "  Unknown encoding " << encoding << " for " << filename << std::endl;
            break;
        }
    }

    // Cleanup
    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());

    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

-        if (encoding == (uint32_t)PatchEncoding::COPY_INSERT) {

+        if (encoding == (uint32_t)PatchEncoding::ZSTD_DELTA) {

+            // Write zstd frame to temp

-            std::string ref = cur_dir + "/" + std::string(filename);

+            // Load old file, apply string remap + fixups → write remapped ref

+            auto old_data = read_file(cur_dir + "/" + std::string(filename));

+            apply_str_remap(old_data, file_id);

+            apply_fixups(old_data, file_id);

+            std::string ref = tmpdir + "/" + std::string(filename) + ".ref";

+            remove(zst.c_str()); remove(ref.c_str());

-        } else if (encoding == (uint32_t)PatchEncoding::TWO_STAGE) {

-            // Two-stage: old → remapped → new

-            uint64_t old_size, remap_size, new_size, s1_size, s2_size;

-            pf.read(reinterpret_cast<char*>(&old_size), 8);

-            pf.read(reinterpret_cast<char*>(&remap_size), 8);

-            pf.read(reinterpret_cast<char*>(&new_size), 8);

-            pf.read(reinterpret_cast<char*>(&s1_size), 8);

-            pf.read(reinterpret_cast<char*>(&s2_size), 8);

-            std::string zst1 = tmpdir + "/" + std::string(filename) + ".s1.zst";

-            std::string zst2 = tmpdir + "/" + std::string(filename) + ".s2.zst";

-            std::string mid = tmpdir + "/" + std::string(filename) + ".mid";

-            std::string out = out_dir + "/" + std::string(filename);

-            if (!zstd_apply(ref, zst1, mid)) {

-                std::cerr << "FAILED stage1: " << filename << std::endl; return 1;

-            if (!zstd_apply(mid, zst2, out)) {

-                std::cerr << "FAILED stage2: " << filename << std::endl; return 1;

-            std::cerr << "  " << filename << ": " << read_file(out).size() << " bytes" << std::endl;

-            remove(zst1.c_str()); remove(zst2.c_str()); remove(mid.c_str());


Bash

cmake --build build -j$(nproc) --target geocoder-diff --target geocoder-patch 2>&1 | tail -5

Build both tools


[100%] Linking CXX executable geocoder-diff
[100%] Built target geocoder-diff
[ 50%] Building CXX object CMakeFiles/geocoder-patch.dir/tools/geocoder_patch.cpp.o
[100%] Linking CXX executable geocoder-patch
[100%] Built target geocoder-patch


_cfuvid=Us_0Oe.VxvZgK5VGFoOmjJYLwp4M2HdFvo8rlGJoEgA-1774613114.423246-1.0.1.1-LjZLHvj1k5vlu_alkcmjXkXyqDUbk9NxwH82tvwsBKA; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com


_cfuvid=TyyJoGAw7h.gEZ7Gn78c9zTHJSNDZLevuWEGb2NRJeo-1774613115.8307347-1.0.1.1-rVVmbH33qfjI0tIkWbBIGJtfeV45jWCb7G7NCFc6Nbc; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com


Deploy and test:


sshpass -p 'elon-is-insane' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" /root/traccar-geocoder/builder/build/geocoder-diff /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:~/ 2>&1 && sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
echo "=== DIFF ==="
rm -f /home/michtest/test5.gcpatch
time /home/michtest/geocoder-diff /home/michtest/det-A /home/michtest/det-new -o /home/michtest/test5.gcpatch 2>&1
echo "Patch: $(numfmt --to=iec $(stat -c%s /home/michtest/test5.gcpatch 2>/dev/null))"
echo ""
echo "=== APPLY ==="
rm -rf /home/michtest/patched5
time /home/michtest/geocoder-patch /home/michtest/det-A /home/michtest/test5.gcpatch -o /home/michtest/patched5 2>&1
echo ""
echo "=== VERIFY ==="
all_match=true
for f in /home/michtest/det-new/*.bin; do
    b=$(basename $f); p=/home/michtest/patched5/$b
    [ -f "$p" ] && cmp -s "$f" "$p" && echo "  $b: MATCH" || { echo "  $b: FAIL"; all_match=false; }
done
$all_match && echo "ALL FILES MATCH" || echo "SOME DIFFER"
' 2>&1

Full test with remap + fixups + single-stage delta


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
=== DIFF ===
Building string remap...
  4980078 strings mapped
Building record ID remaps...
    addr matched=85825938/85891830 new=164173
    ways matched=19372891/19452480 new=95957
    interps matched=1016/16785 new=15755
    admin matched=449642/450907 new=1222
  Remapping geo entries...
  Remapping admin entries...
  Remap: 4982052 entries
  fixup street_ways.bin: 19372891 entries
  fixup interp_ways.bin: 1016 entries
  fixup admin_polygons.bin: 449642 entries
  strings.bin: 71309 bytes (0.09%)
  addr_points.bin: 3770551 bytes (0.27%)
  street_ways.bin: 1515482 bytes (0.65%)
  interp_ways.bin: 66381 bytes (19.79%)
  admin_polygons.bin: 39775 bytes (0.37%)
  street_nodes.bin: 906243 bytes (0.06%)
  interp_nodes.bin: 338 bytes (0.05%)
  admin_vertices.bin: 249709 bytes (0.09%)
  geo_cells.bin: 396021062 bytes (23.65%)
  street_entries.bin: 120945 bytes (0.02%)
  addr_entries.bin: 273895 bytes (0.07%)
  interp_entries.bin: 82 bytes (0.05%)
  admin_cells.bin: 182 bytes (0.01%)
  admin_entries.bin: 1248 bytes (0.02%)

Total patch: 601482534 bytes (573 MiB)

real	6m19.798s
user	6m3.515s
sys	0m22.022s
Patch: 574M

=== APPLY ===
Loaded 4980054 string remap entries
Loaded 19372891 fixups for file 1
Loaded 1016 fixups for file 4
Loaded 449642 fixups for file 6
  strings.bin: 82573397 bytes
  addr_points.bin: 1375841776 bytes
  street_ways.bin: 233626176 bytes
  interp_ways.bin: 335424 bytes
  admin_polygons.bin: 10820736 bytes
FAILED: street_nodes.bin

=== VERIFY ===
  addr_entries.bin: FAIL

real	0m5.744s
user	0m3.502s
sys	0m2.483s
  addr_points.bin: MATCH
  admin_cells.bin: FAIL
  admin_entries.bin: FAIL
  admin_polygons.bin: MATCH
  admin_vertices.bin: FAIL
  geo_cells.bin: FAIL
  interp_entries.bin: FAIL
  interp_nodes.bin: FAIL
  interp_ways.bin: MATCH
  street_entries.bin: FAIL
  street_nodes.bin: FAIL
  street_ways.bin: MATCH
  strings.bin: MATCH
SOME DIFFER


_cfuvid=Q6u5iUce4MnPWxXSKqkyCztF8Hz03RjYoVJud1RSUeI-1774613522.8218448-1.0.1.1-HXVrBdLrDceHl28E6mGoHfyBGLuTjIbLrLDxiitkwMc; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com


_cfuvid=DihDdD9STNLcT8xC8Twppow18uzrzSnnRjEj7qxpgTs-1774613524.45473-1.0.1.1-B1VsThk_psLCy9sBxwTTvvQeAifCz6T.qW5lVeNMDtg; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com


_cfuvid=NXSv3DMG18fx8UJpbT2DxP4iBd01WPbX4Eas0R2aBr4-1774613526.6807034-1.0.1.1-se_9kegi0iqT6ZVyMrBbCSWU1SCdDxmvigRPXgRViT4; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com


Progress! addr_points, street_ways, interp_ways, admin_polygons, and strings all **MATCH**. The fixup approach works for data files.

`street_nodes.bin` failed because the diff tool wrote new nodes to temp (for matched ways with fixed offsets), but the patch tool applies string remap + fixups to the old street_nodes — which doesn't make sense for node files (they have no string refs or offset fields). The issue: the diff tool uses `tmpdir/street_nodes.bin` (new nodes) as the zstd reference, but the patch tool creates a reference from old street_nodes with no transformations.

For coordinate files: the diff created the reference from the NEW data (because matched ways point to new node positions). But the patch tool only has the OLD data. The fix: for coordinate files, the diff should use OLD data as reference (not remapped temp), since the patch tool uses OLD data.

But wait — the ways in the remapped old have NEW node_offsets (we fixed them). So the ways point to the NEW node array, not the old. The diff tool writes the NEW nodes as the temp file to be consistent. But the patch tool doesn't have access to new nodes.

The solution: for coordinate files, the diff should NOT use the remapped temp. Use old→new directly. The zstd --patch-from should compare old nodes vs new nodes without any remapping:


   606→        if (read_file(ref).empty()) ref = old_dir + "/" + name;
   607→        std::string tgt = new_dir + "/" + name;
   608→        std::string zst = tmpdir + "/" + name + ".zst";
   609→        system(("zstd --patch-from='" + ref + "' '" + tgt + "' -o '" + zst + "' -f --quiet 2>/dev/null").c_str());
   610→        auto zst_data = read_file(zst);
   611→        auto old_data = read_file(old_dir + "/" + name);
   612→        auto new_data = read_file(tgt);
   613→
   614→        uint32_t fid = static_cast<uint32_t>(id), enc = static_cast<uint32_t>(PatchEncoding::ZSTD_DELTA);
   615→        uint64_t os = old_data.size(), ns = new_data.size(), ds = zst_data.size();

        if (read_file(ref).empty()) ref = old_dir + "/" + name;
        std::string tgt = new_dir + "/" + name;
        std::string zst = tmpdir + "/" + name + ".zst";
        system(("zstd --patch-from='" + ref + "' '" + tgt + "' -o '" + zst + "' -f --quiet 2>/dev/null").c_str());
        auto zst_data = read_file(zst);
        auto old_data = read_file(old_dir + "/" + name);
        auto new_data = read_file(tgt);

        uint32_t fid = static_cast<uint32_t>(id), enc = static_cast<uint32_t>(PatchEncoding::ZSTD_DELTA);
        uint64_t os = old_data.size(), ns = new_data.size(), ds = zst_data.size();


_cfuvid=MemCoZxGnDnrW4L9kH.pCB405ne7UtozzYQSJM6t.hE-1774613561.5386639-1.0.1.1-j6X6xflylyrSl_1Ow9Cs244vac6C5SMs8RTaifAN9Bc; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com


_cfuvid=4Q1AxtPBzU2Lc.OymWHoqWrzIHfonPR9dQapFx4l2wc-1774613563.8293767-1.0.1.1-wG_A796SJiz7qn3h0x.gM8sBjLdevfqdMHeniwathJg; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com


The diff_file function checks `if (read_file(ref).empty()) ref = old_dir/name`. For coord files, the temp dir has the NEW data written by the match functions. But we need old→new for the patch tool. Let me add a flag to force using old dir:


Edit

    auto diff_file = [&](PatchFileId id, const std::string& name) {
        std::cerr << "  " << name << ": ";
        std::string ref = tmpdir + "/" + name;
        if (read_file(ref).empty()) ref = old_dir + "/" + name;

    auto diff_file = [&](PatchFileId id, const std::string& name, bool use_old_as_ref = false) {
        std::cerr << "  " << name << ": ";
        std::string ref;
        if (use_old_as_ref) {
            ref = old_dir + "/" + name;
        } else {
            ref = tmpdir + "/" + name;
            if (read_file(ref).empty()) ref = old_dir + "/" + name;
        }