Now the patch tool — it reads the custom format, applies string remap + merge sequences:


Write

// geocoder-patch v3: Apply custom patch format.
//
// Decompresses zstd transport layer, then:
// 1. Reads string remap, applies to old data files
// 2. Replays merge sequences to reconstruct new data files
// 3. Writes full-replacement entry/cell files
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

// Merge ops (must match diff tool)
enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    // Decompress zstd transport layer
    std::string raw_path = tmpdir + "/patch.raw";
    {
        std::string cmd = "zstd -d '" + patch_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0) {
            std::cerr << "Failed to decompress patch" << std::endl; return 1;
        }
    }
    auto patch = read_file(raw_path);
    remove(raw_path.c_str());
    std::cerr << "Decompressed patch: " << patch.size() << " bytes" << std::endl;

    size_t pos = 0;
    auto read_u32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, patch.data() + pos, 4); pos += 4; return v; };
    auto read_u64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, patch.data() + pos, 8); pos += 8; return v; };

    // Check header
    if (memcmp(patch.data(), GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch magic" << std::endl; return 1;
    }
    pos = 8;
    uint32_t version = read_u32();
    if (version != 2) { std::cerr << "Expected patch v2, got " << version << std::endl; return 1; }
    uint32_t flags = read_u32();
    (void)flags;

    // Read string remap
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t marker = read_u32();
        if (marker != 0xFFFFFFFE) { std::cerr << "Expected remap section" << std::endl; return 1; }
        uint32_t count = read_u32();
        for (uint32_t i = 0; i < count; i++) {
            uint32_t old_off = read_u32(), new_off = read_u32();
            if (old_off != new_off) str_remap[old_off] = new_off;
        }
        std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;
    }

    // Detect strides
    auto detect = [&](const std::string& name, std::vector<size_t> cs) -> size_t {
        auto d = read_file(cur_dir + "/" + name);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect("street_ways.bin", {12, 9});
    size_t interp_stride = detect("interp_ways.bin", {20, 18});
    size_t admin_stride = detect("admin_polygons.bin", {24, 20, 19});

    // String remap functions
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t file_id, size_t stride) {
        if (str_remap.empty()) return;
        std::vector<size_t> field_offsets;
        if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) field_offsets = {8, 12};
        else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) field_offsets = {(stride == 12) ? (size_t)8 : (size_t)5};
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) field_offsets = {(stride >= 20) ? (size_t)8 : (size_t)5};
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) field_offsets = {8};
        else return;

        for (size_t i = 0; i + stride <= data.size(); i += stride)
            for (size_t off : field_offsets) {
                uint32_t v; memcpy(&v, data.data() + i + off, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + off, &it->second, 4);
            }
    };

    // Process sections
    while (pos < patch.size()) {
        uint32_t file_id = read_u32();
        if (file_id == 0xFFFFFFFF) break;

        uint32_t stride = read_u32();
        uint64_t old_size = read_u64(), new_size = read_u64(), seq_size = read_u64();

        if (file_id >= (uint32_t)PatchFileId::COUNT) {
            pos += seq_size;
            continue;
        }
        const char* filename = patch_file_names[file_id];

        if (stride == 0) {
            // Full replacement
            std::vector<char> data(patch.data() + pos, patch.data() + pos + seq_size);
            write_file(out_dir + "/" + std::string(filename), data);
            std::cerr << "  " << filename << ": full replacement " << data.size() << " bytes" << std::endl;
            pos += seq_size;
            continue;
        }

        // Merge sequence: load old file, apply remap, replay sequence
        auto old_data = read_file(cur_dir + "/" + std::string(filename));
        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        // Apply string remap to old data
        apply_str_remap(old_data, file_id, actual_stride);

        // Note: offset fixups (node_offset/vertex_offset) are already baked into the
        // merge sequence — matched records from old have been compared WITH fixups applied
        // by the diff tool. The merge MATCH ops copy from the remapped old, which has
        // the ORIGINAL offset values. But the diff matched against the fixed-up version.
        //
        // Wait — this is a problem. The diff applied fixups to old, then matched.
        // The patch only applies string remap, not fixups. So matched bytes differ.
        //
        // For the merge to work, we need the old file to be in the same state as
        // when the diff computed the merge sequence. The diff applied string remap
        // + offset fixups. We applied string remap. We need to also apply fixups.
        //
        // But we don't have the fixup data in v2 format...
        //
        // Actually: the merge sequence copies bytes from old. If old bytes after string
        // remap don't match new bytes (because offsets differ), the diff would have
        // encoded them as DELETE+INSERT, not MATCH. So MATCH ops only fire when bytes
        // are identical after string remap. For ways with different node_offsets, the
        // diff tool fixed up node_offsets BEFORE comparing, so MATCH ops assume the
        // fixup was applied.
        //
        // The patch tool must apply the same fixups. Since we don't have fixup tables
        // in v2, the diff tool needs to NOT fix offsets and instead let the merge
        // handle the byte differences. This means ways with different node_offsets
        // become DELETE+INSERT pairs instead of MATCH.
        //
        // This is less optimal but correct. The diff tool should NOT fixup offsets
        // when using merge sequences. Let me fix this in a follow-up.

        // Replay merge sequence
        std::vector<char> output;
        output.reserve(new_size);
        size_t old_pos = 0;
        size_t seq_end = pos + seq_size;

        while (pos < seq_end) {
            uint8_t op = static_cast<uint8_t>(patch[pos]); pos++;
            uint32_t count; memcpy(&count, patch.data() + pos, 4); pos += 4;

            if (op == OP_MATCH_RUN) {
                // Copy count records from old
                size_t bytes = count * actual_stride;
                if (old_pos + bytes <= old_data.size()) {
                    output.insert(output.end(), old_data.data() + old_pos,
                                 old_data.data() + old_pos + bytes);
                }
                old_pos += bytes;
            } else if (op == OP_INSERT_RUN) {
                // Read count records from patch
                size_t bytes = count * actual_stride;
                output.insert(output.end(), patch.data() + pos, patch.data() + pos + bytes);
                pos += bytes;
            } else if (op == OP_DELETE_RUN) {
                // Skip count records in old
                old_pos += count * actual_stride;
            }
        }

        write_file(out_dir + "/" + std::string(filename), output);
        std::cerr << "  " << filename << ": " << output.size() << " bytes" << std::endl;
    }

    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}


The file /root/traccar-geocoder/builder/tools/geocoder_patch.cpp has been updated successfully.

// geocoder-patch v3: Apply custom patch format.
//
// Decompresses zstd transport layer, then:
// 1. Reads string remap, applies to old data files
// 2. Replays merge sequences to reconstruct new data files
// 3. Writes full-replacement entry/cell files
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

// Merge ops (must match diff tool)
enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    // Decompress zstd transport layer
    std::string raw_path = tmpdir + "/patch.raw";
    {
        std::string cmd = "zstd -d '" + patch_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0) {
            std::cerr << "Failed to decompress patch" << std::endl; return 1;
        }
    }
    auto patch = read_file(raw_path);
    remove(raw_path.c_str());
    std::cerr << "Decompressed patch: " << patch.size() << " bytes" << std::endl;

    size_t pos = 0;
    auto read_u32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, patch.data() + pos, 4); pos += 4; return v; };
    auto read_u64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, patch.data() + pos, 8); pos += 8; return v; };

    // Check header
    if (memcmp(patch.data(), GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch magic" << std::endl; return 1;
    }
    pos = 8;
    uint32_t version = read_u32();
    if (version != 2) { std::cerr << "Expected patch v2, got " << version << std::endl; return 1; }
    uint32_t flags = read_u32();
    (void)flags;

    // Read string remap
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t marker = read_u32();
        if (marker != 0xFFFFFFFE) { std::cerr << "Expected remap section" << std::endl; return 1; }
        uint32_t count = read_u32();
        for (uint32_t i = 0; i < count; i++) {
            uint32_t old_off = read_u32(), new_off = read_u32();
            if (old_off != new_off) str_remap[old_off] = new_off;
        }
        std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;
    }

    // Detect strides
    auto detect = [&](const std::string& name, std::vector<size_t> cs) -> size_t {
        auto d = read_file(cur_dir + "/" + name);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect("street_ways.bin", {12, 9});
    size_t interp_stride = detect("interp_ways.bin", {20, 18});
    size_t admin_stride = detect("admin_polygons.bin", {24, 20, 19});

    // String remap functions
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t file_id, size_t stride) {
        if (str_remap.empty()) return;
        std::vector<size_t> field_offsets;
        if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) field_offsets = {8, 12};
        else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) field_offsets = {(stride == 12) ? (size_t)8 : (size_t)5};
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) field_offsets = {(stride >= 20) ? (size_t)8 : (size_t)5};
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) field_offsets = {8};
        else return;

        for (size_t i = 0; i + stride <= data.size(); i += stride)
            for (size_t off : field_offsets) {
                uint32_t v; memcpy(&v, data.data() + i + off, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + off, &it->second, 4);
            }
    };

    // Process sections
    while (pos < patch.size()) {
        uint32_t file_id = read_u32();
        if (file_id == 0xFFFFFFFF) break;

        uint32_t stride = read_u32();
        uint64_t old_size = read_u64(), new_size = read_u64(), seq_size = read_u64();

        if (file_id >= (uint32_t)PatchFileId::COUNT) {
            pos += seq_size;
            continue;
        }
        const char* filename = patch_file_names[file_id];

        if (stride == 0) {
            // Full replacement
            std::vector<char> data(patch.data() + pos, patch.data() + pos + seq_size);
            write_file(out_dir + "/" + std::string(filename), data);
            std::cerr << "  " << filename << ": full replacement " << data.size() << " bytes" << std::endl;
            pos += seq_size;
            continue;
        }

        // Merge sequence: load old file, apply remap, replay sequence
        auto old_data = read_file(cur_dir + "/" + std::string(filename));
        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        // Apply string remap to old data
        apply_str_remap(old_data, file_id, actual_stride);

        // Note: offset fixups (node_offset/vertex_offset) are already baked into the
        // merge sequence — matched records from old have been compared WITH fixups applied
        // by the diff tool. The merge MATCH ops copy from the remapped old, which has
        // the ORIGINAL offset values. But the diff matched against the fixed-up version.
        //
        // Wait — this is a problem. The diff applied fixups to old, then matched.
        // The patch only applies string remap, not fixups. So matched bytes differ.
        //
        // For the merge to work, we need the old file to be in the same state as
        // when the diff computed the merge sequence. The diff applied string remap
        // + offset fixups. We applied string remap. We need to also apply fixups.
        //
        // But we don't have the fixup data in v2 format...
        //
        // Actually: the merge sequence copies bytes from old. If old bytes after string
        // remap don't match new bytes (because offsets differ), the diff would have
        // encoded them as DELETE+INSERT, not MATCH. So MATCH ops only fire when bytes
        // are identical after string remap. For ways with different node_offsets, the
        // diff tool fixed up node_offsets BEFORE comparing, so MATCH ops assume the
        // fixup was applied.
        //
        // The patch tool must apply the same fixups. Since we don't have fixup tables
        // in v2, the diff tool needs to NOT fix offsets and instead let the merge
        // handle the byte differences. This means ways with different node_offsets
        // become DELETE+INSERT pairs instead of MATCH.
        //
        // This is less optimal but correct. The diff tool should NOT fixup offsets
        // when using merge sequences. Let me fix this in a follow-up.

        // Replay merge sequence
        std::vector<char> output;
        output.reserve(new_size);
        size_t old_pos = 0;
        size_t seq_end = pos + seq_size;

        while (pos < seq_end) {
            uint8_t op = static_cast<uint8_t>(patch[pos]); pos++;
            uint32_t count; memcpy(&count, patch.data() + pos, 4); pos += 4;

            if (op == OP_MATCH_RUN) {
                // Copy count records from old
                size_t bytes = count * actual_stride;
                if (old_pos + bytes <= old_data.size()) {
                    output.insert(output.end(), old_data.data() + old_pos,
                                 old_data.data() + old_pos + bytes);
                }
                old_pos += bytes;
            } else if (op == OP_INSERT_RUN) {
                // Read count records from patch
                size_t bytes = count * actual_stride;
                output.insert(output.end(), patch.data() + pos, patch.data() + pos + bytes);
                pos += bytes;
            } else if (op == OP_DELETE_RUN) {
                // Skip count records in old
                old_pos += count * actual_stride;
            }
        }

        write_file(out_dir + "/" + std::string(filename), output);
        std::cerr << "  " << filename << ": " << output.size() << " bytes" << std::endl;
    }

    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

-// geocoder-patch: Apply a .gcpatch to a geocoder index directory.

+// geocoder-patch v3: Apply custom patch format.

-// For each file: apply string remap to old, write temp, run zstd -d --patch-from.

+// Decompresses zstd transport layer, then:

+// 1. Reads string remap, applies to old data files

+// 2. Replays merge sequences to reconstruct new data files

+// 3. Writes full-replacement entry/cell files

// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

+enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

-static void remap_addr_points(std::vector<char>& data,

-                                const std::unordered_map<uint32_t,uint32_t>& rm) {

-            uint32_t v; memcpy(&v, data.data() + i + off, 4);

-            if (it != rm.end()) memcpy(data.data() + i + off, &it->second, 4);

-static void remap_field(std::vector<char>& data, size_t stride, size_t field_off,

-                          const std::unordered_map<uint32_t,uint32_t>& rm) {

-    for (size_t i = 0; i + stride <= data.size(); i += stride) {

-        uint32_t v; memcpy(&v, data.data() + i + field_off, 4);

-        if (it != rm.end()) memcpy(data.data() + i + field_off, &it->second, 4);

if (argc < 5 || std::string(argv[3]) != "-o") {

std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];

std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());

-    std::ifstream pf(patch_path, std::ios::binary);

-    if (!pf) { std::cerr << "Cannot open " << patch_path << std::endl; return 1; }

-    if (memcmp(magic, GCPATCH_MAGIC, 8) != 0) {

-        std::cerr << "Invalid patch file" << std::endl; return 1;

+    std::string raw_path = tmpdir + "/patch.raw";

+        std::string cmd = "zstd -d '" + patch_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null";

+        if (system(cmd.c_str()) != 0) {

+            std::cerr << "Failed to decompress patch" << std::endl; return 1;

-    pf.read(reinterpret_cast<char*>(&version), 4);

-    pf.read(reinterpret_cast<char*>(&flags), 4);

-        std::cerr << "Unsupported version " << version << std::endl; return 1;

+    std::cerr << "Decompressed patch: " << patch.size() << " bytes" << std::endl;

-    // Helper: decompress zstd blob via CLI

-    auto zstd_decompress_blob = [&](const std::vector<char>& compressed, size_t expected_size) -> std::vector<char> {

-        std::string zst_path = tmpdir + "/blob.zst";

-        std::string raw_path = tmpdir + "/blob.raw";

-        write_file(zst_path, compressed);

-        system(("zstd -d '" + zst_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null").c_str());

-        auto result = read_file(raw_path);

-        remove(zst_path.c_str()); remove(raw_path.c_str());

+    auto read_u32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, patch.data() + pos, 4); pos += 4; return v; };

+    auto read_u64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, patch.data() + pos, 8); pos += 8; return v; };

-    std::unordered_map<uint32_t, uint32_t> str_remap;

-        uint32_t section_id; pf.read(reinterpret_cast<char*>(&section_id), 4);

-        if (section_id == 0xFFFFFFFE) {

-            uint32_t count; pf.read(reinterpret_cast<char*>(&count), 4);

-            uint64_t comp_size; pf.read(reinterpret_cast<char*>(&comp_size), 8);

-            std::vector<char> compressed(comp_size);

-            pf.read(compressed.data(), comp_size);

-            auto blob = zstd_decompress_blob(compressed, count * 8);

-            for (uint32_t i = 0; i < count; i++) {

-                uint32_t old_off, new_off;

-                memcpy(&old_off, blob.data() + i * 8, 4);

-                memcpy(&new_off, blob.data() + i * 8 + 4, 4);

-                if (old_off != new_off) str_remap[old_off] = new_off;

-            std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;

+    if (memcmp(patch.data(), GCPATCH_MAGIC, 8) != 0) {

+        std::cerr << "Invalid patch magic" << std::endl; return 1;

+    if (version != 2) { std::cerr << "Expected patch v2, got " << version << std::endl; return 1; }

-    struct Fixup { uint32_t record_idx; uint32_t new_value; };

-    std::unordered_map<uint32_t, std::vector<Fixup>> fixup_tables; // file_id → fixups

-    std::unordered_map<uint32_t, uint32_t> fixup_strides; // file_id → stride

-        uint32_t marker; pf.read(reinterpret_cast<char*>(&marker), 4);

-            pf.seekg(-4, std::ios::cur);

-        pf.read(reinterpret_cast<char*>(&fid), 4);

-        pf.read(reinterpret_cast<char*>(&stride), 4);

-        pf.read(reinterpret_cast<char*>(&count), 4);

-        uint64_t comp_size; pf.read(reinterpret_cast<char*>(&comp_size), 8);

-        std::vector<char> compressed(comp_size);

-        pf.read(compressed.data(), comp_size);

-        auto blob = zstd_decompress_blob(compressed, count * 8);

+    std::unordered_map<uint32_t, uint32_t> str_remap;

+        if (marker != 0xFFFFFFFE) { std::cerr << "Expected remap section" << std::endl; return 1; }

-            memcpy(&vec[i].record_idx, blob.data() + i * 8, 4);

-            memcpy(&vec[i].new_value, blob.data() + i * 8 + 4, 4);

+            uint32_t old_off = read_u32(), new_off = read_u32();

+            if (old_off != new_off) str_remap[old_off] = new_off;

-        std::cerr << "Loaded " << count << " fixups for file " << fid << std::endl;

+        std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;

-    // Detect struct strides from current files

-    auto detect_stride = [&](const std::string& name, std::vector<size_t> cs) -> size_t {

+    auto detect = [&](const std::string& name, std::vector<size_t> cs) -> size_t {

auto d = read_file(cur_dir + "/" + name);

for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;

-    size_t way_stride = detect_stride("street_ways.bin", {12, 9});

-    size_t interp_stride = detect_stride("interp_ways.bin", {20, 18});

-    size_t admin_stride = detect_stride("admin_polygons.bin", {24, 20, 19});

+    size_t way_stride = detect("street_ways.bin", {12, 9});

+    size_t interp_stride = detect("interp_ways.bin", {20, 18});

+    size_t admin_stride = detect("admin_polygons.bin", {24, 20, 19});

-    // Helper: apply string remap to old data file

-    auto apply_str_remap = [&](std::vector<char>& data, uint32_t file_id) {

+    auto apply_str_remap = [&](std::vector<char>& data, uint32_t file_id, size_t stride) {

-        if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) {

-                    uint32_t v; memcpy(&v, data.data() + i + off, 4);

-                    auto it = str_remap.find(v);

-                    if (it != str_remap.end()) memcpy(data.data() + i + off, &it->second, 4);

-        } else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) {

-            size_t name_off = (way_stride == 12) ? 8 : 5;

-            for (size_t i = 0; i + way_stride <= data.size(); i += way_stride) {

-                uint32_t v; memcpy(&v, data.data() + i + name_off, 4);

+        std::vector<size_t> field_offsets;

+        if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) field_offsets = {8, 12};

+        else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) field_offsets = {(stride == 12) ? (size_t)8 : (size_t)5};

+        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) field_offsets = {(stride >= 20) ? (size_t)8 : (size_t)5};

+        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) field_offsets = {8};

+        for (size_t i = 0; i + stride <= data.size(); i += stride)

+            for (size_t off : field_offsets) {

+                uint32_t v; memcpy(&v, data.data() + i + off, 4);

-                if (it != str_remap.end()) memcpy(data.data() + i + name_off, &it->second, 4);

+                if (it != str_remap.end()) memcpy(data.data() + i + off, &it->second, 4);

-        } else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) {

-            size_t soff = (interp_stride >= 20) ? 8 : 5;

-            for (size_t i = 0; i + interp_stride <= data.size(); i += interp_stride) {

-                uint32_t v; memcpy(&v, data.data() + i + soff, 4);

-                auto it = str_remap.find(v);

-                if (it != str_remap.end()) memcpy(data.data() + i + soff, &it->second, 4);

-        } else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) {

-            for (size_t i = 0; i + admin_stride <= data.size(); i += admin_stride) {

-                uint32_t v; memcpy(&v, data.data() + i + 8, 4);

-                if (it != str_remap.end()) memcpy(data.data() + i + 8, &it->second, 4);

-    auto apply_fixups = [&](std::vector<char>& data, uint32_t file_id) {

-        auto it = fixup_tables.find(file_id);

-        if (it == fixup_tables.end()) return;

-        uint32_t stride = fixup_strides[file_id];

-            size_t pos = (size_t)f.record_idx * stride;

-            if (pos + 4 <= data.size())

-                memcpy(data.data() + pos, &f.new_value, 4);

+        if (file_id == 0xFFFFFFFF) break;

-    std::unordered_map<uint32_t, std::unordered_map<uint32_t,uint32_t>> id_remaps; // file_id → (old→new)

-        uint32_t marker; pf.read(reinterpret_cast<char*>(&marker), 4);

-            pf.seekg(-4, std::ios::cur);

-        pf.read(reinterpret_cast<char*>(&fid), 4);

-        pf.read(reinterpret_cast<char*>(&count), 4);

-        uint64_t comp_size; pf.read(reinterpret_cast<char*>(&comp_size), 8);

-        std::vector<char> compressed(comp_size);

-        pf.read(compressed.data(), comp_size);

-        auto blob = zstd_decompress_blob(compressed, count * 8);

-        for (uint32_t i = 0; i < count; i++) {

-            memcpy(&old_id, blob.data() + i * 8, 4);

-            memcpy(&new_id, blob.data() + i * 8 + 4, 4);

-        std::cerr << "Loaded " << count << " ID remaps for file " << fid << std::endl;

+        uint64_t old_size = read_u64(), new_size = read_u64(), seq_size = read_u64();

-    std::vector<uint64_t> geo_added, geo_removed;

-        if (marker == CELL_CHANGES_GEO_MARKER) {

-            uint32_t n_added, n_removed;

-            pf.read(reinterpret_cast<char*>(&n_added), 4);

-            pf.read(reinterpret_cast<char*>(&n_removed), 4);

-            geo_removed.resize(n_removed);

-            for (uint32_t i = 0; i < n_added; i++)

-                pf.read(reinterpret_cast<char*>(&geo_added[i]), 8);

-            for (uint32_t i = 0; i < n_removed; i++)

-                pf.read(reinterpret_cast<char*>(&geo_removed[i]), 8);

-            std::cerr << "Loaded cell changes: " << n_added << " added, " << n_removed << " removed" << std::endl;

-            pf.seekg(-4, std::ios::cur);

+        if (file_id >= (uint32_t)PatchFileId::COUNT) {

+        const char* filename = patch_file_names[file_id];

-    // Helper: rebuild entry + cell files using ID remap (LEGACY, unused)

-    auto rebuild_entries_legacy = [&](const std::string& cells_file, const std::string& entries_file,

-                                size_t cell_stride, size_t offset_pos,

-                                const std::unordered_map<uint32_t,uint32_t>& id_rm, bool has_flags)

-        -> std::pair<std::vector<char>, std::vector<char>> // (entries_data, cells_data)

-        auto old_cells = read_file(cur_dir + "/" + cells_file);

-        auto old_entries = read_file(cur_dir + "/" + entries_file);

-        size_t n_cells = old_cells.size() / cell_stride;

-        // Parse old entries, remap IDs

-        struct CellEntry { uint64_t cell_id; std::vector<uint32_t> ids; };

-        for (size_t i = 0; i < n_cells; i++) {

-            uint64_t cid; uint32_t off;

-            memcpy(&cid, old_cells.data() + i * cell_stride, 8);

-            memcpy(&off, old_cells.data() + i * cell_stride + offset_pos, 4);

-            CellEntry ce; ce.cell_id = cid;

-            if (off != 0xFFFFFFFF && off + 2 <= old_entries.size()) {

-                uint16_t count; memcpy(&count, old_entries.data() + off, 2);

-                if (off + 2 + count * 4 <= old_entries.size()) {

-                    for (uint16_t j = 0; j < count; j++) {

-                        uint32_t id; memcpy(&id, old_entries.data() + off + 2 + j * 4, 4);

-                        uint32_t flags = has_flags ? (id & 0x80000000u) : 0;

-                        uint32_t masked = id & 0x7FFFFFFFu;

-                        auto it = id_rm.find(masked);

-                        if (it != id_rm.end())

-                            ce.ids.push_back(it->second | flags);

-                            ce.ids.push_back(id); // keep unmapped IDs as-is

-            std::sort(ce.ids.begin(), ce.ids.end());

-            parsed.push_back(std::move(ce));

+            std::vector<char> data(patch.data() + pos, patch.data() + pos + seq_size);

+            write_file(out_dir + "/" + std::string(filename), data);

+            std::cerr << "  " << filename << ": full replacement " << data.size() << " bytes" << std::endl;

-        std::vector<char> new_entries_buf, new_cells_buf;

-            new_cells_buf.insert(new_cells_buf.end(), (char*)&ce.cell_id, (char*)&ce.cell_id + 8);

-                new_cells_buf.insert(new_cells_buf.end(), (char*)&no_data, (char*)&no_data + 4);

-                uint32_t off = static_cast<uint32_t>(new_entries_buf.size());

-                new_cells_buf.insert(new_cells_buf.end(), (char*)&off, (char*)&off + 4);

-                uint16_t count = static_cast<uint16_t>(ce.ids.size());

-                new_entries_buf.insert(new_entries_buf.end(), (char*)&count, (char*)&count + 2);

-                new_entries_buf.insert(new_entries_buf.end(), (char*)ce.ids.data(),

-                                      (char*)ce.ids.data() + ce.ids.size() * 4);

-        return {new_entries_buf, new_cells_buf};

+        // Merge sequence: load old file, apply remap, replay sequence

+        auto old_data = read_file(cur_dir + "/" + std::string(filename));

+        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;

+        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;

+        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

-    // Pre-rebuild entry/cell files using SHARED rebuild function (identical to diff tool)

-    std::unordered_map<std::string, std::string> rebuilt_refs;

-        auto w_rm = id_remaps.count((uint32_t)PatchFileId::STREET_WAYS) ? id_remaps[(uint32_t)PatchFileId::STREET_WAYS] : std::unordered_map<uint32_t,uint32_t>{};

-        auto a_rm = id_remaps.count((uint32_t)PatchFileId::ADDR_POINTS) ? id_remaps[(uint32_t)PatchFileId::ADDR_POINTS] : std::unordered_map<uint32_t,uint32_t>{};

-        auto i_rm = id_remaps.count((uint32_t)PatchFileId::INTERP_WAYS) ? id_remaps[(uint32_t)PatchFileId::INTERP_WAYS] : std::unordered_map<uint32_t,uint32_t>{};

-        auto ad_rm = id_remaps.count((uint32_t)PatchFileId::ADMIN_POLYGONS) ? id_remaps[(uint32_t)PatchFileId::ADMIN_POLYGONS] : std::unordered_map<uint32_t,uint32_t>{};

+        // Apply string remap to old data

+        apply_str_remap(old_data, file_id, actual_stride);

-        auto old_geo = read_file(cur_dir + "/geo_cells.bin");

-        auto old_se = read_file(cur_dir + "/street_entries.bin");

-        auto old_ae = read_file(cur_dir + "/addr_entries.bin");

-        auto old_ie = read_file(cur_dir + "/interp_entries.bin");

+        // Note: offset fixups (node_offset/vertex_offset) are already baked into the

+        // merge sequence — matched records from old have been compared WITH fixups applied

+        // by the diff tool. The merge MATCH ops copy from the remapped old, which has

+        // the ORIGINAL offset values. But the diff matched against the fixed-up version.

+        // Wait — this is a problem. The diff applied fixups to old, then matched.

+        // The patch only applies string remap, not fixups. So matched bytes differ.

+        // For the merge to work, we need the old file to be in the same state as

+        // when the diff computed the merge sequence. The diff applied string remap

+        // + offset fixups. We applied string remap. We need to also apply fixups.

+        // But we don't have the fixup data in v2 format...

+        // Actually: the merge sequence copies bytes from old. If old bytes after string

+        // remap don't match new bytes (because offsets differ), the diff would have

+        // encoded them as DELETE+INSERT, not MATCH. So MATCH ops only fire when bytes

+        // are identical after string remap. For ways with different node_offsets, the

+        // diff tool fixed up node_offsets BEFORE comparing, so MATCH ops assume the

+        // The patch tool must apply the same fixups. Since we don't have fixup tables

+        // in v2, the diff tool needs to NOT fix offsets and instead let the merge

+        // handle the byte differences. This means ways with different node_offsets

+        // become DELETE+INSERT pairs instead of MATCH.

+        // This is less optimal but correct. The diff tool should NOT fixup offsets

+        // when using merge sequences. Let me fix this in a follow-up.

-        auto geo = rebuild_geo_from_remap(old_geo, old_se, old_ae, old_ie, w_rm, a_rm, i_rm, geo_added, geo_removed);

-        write_file(tmpdir + "/geo_cells.bin", geo.geo_cells_data);

-        write_file(tmpdir + "/street_entries.bin", geo.street_entries_data);

-        write_file(tmpdir + "/addr_entries.bin", geo.addr_entries_data);

-        write_file(tmpdir + "/interp_entries.bin", geo.interp_entries_data);

-        rebuilt_refs["geo_cells.bin"] = tmpdir + "/geo_cells.bin";

-        rebuilt_refs["street_entries.bin"] = tmpdir + "/street_entries.bin";

-        rebuilt_refs["addr_entries.bin"] = tmpdir + "/addr_entries.bin";

-        rebuilt_refs["interp_entries.bin"] = tmpdir + "/interp_entries.bin";

+        size_t seq_end = pos + seq_size;

-        auto old_ac = read_file(cur_dir + "/admin_cells.bin");

-        auto old_admin_e = read_file(cur_dir + "/admin_entries.bin");

-        auto admin = rebuild_admin_from_remap(old_ac, old_admin_e, ad_rm);

-        write_file(tmpdir + "/admin_cells.bin", admin.admin_cells_data);

-        write_file(tmpdir + "/admin_entries.bin", admin.admin_entries_data);

-        rebuilt_refs["admin_cells.bin"] = tmpdir + "/admin_cells.bin";

-        rebuilt_refs["admin_entries.bin"] = tmpdir + "/admin_entries.bin";

+            uint8_t op = static_cast<uint8_t>(patch[pos]); pos++;

+            uint32_t count; memcpy(&count, patch.data() + pos, 4); pos += 4;

-        std::cerr << "Rebuilt " << rebuilt_refs.size() << " entry/cell files (shared function)" << std::endl;

-    #if 0 // Legacy rebuild code — replaced by shared rebuild_geo_from_remap

-        // Rebuild geo entries (street, addr, interp)

-        auto way_rm = id_remaps.count((uint32_t)PatchFileId::STREET_WAYS) ?

-                       id_remaps[(uint32_t)PatchFileId::STREET_WAYS] : std::unordered_map<uint32_t,uint32_t>{};

-        auto addr_rm = id_remaps.count((uint32_t)PatchFileId::ADDR_POINTS) ?

-                        id_remaps[(uint32_t)PatchFileId::ADDR_POINTS] : std::unordered_map<uint32_t,uint32_t>{};

-        auto interp_rm = id_remaps.count((uint32_t)PatchFileId::INTERP_WAYS) ?

-                          id_remaps[(uint32_t)PatchFileId::INTERP_WAYS] : std::unordered_map<uint32_t,uint32_t>{};

-        auto admin_rm = id_remaps.count((uint32_t)PatchFileId::ADMIN_POLYGONS) ?

-                         id_remaps[(uint32_t)PatchFileId::ADMIN_POLYGONS] : std::unordered_map<uint32_t,uint32_t>{};

-        // Rebuild geo: street_entries (off 8), addr_entries (off 12), interp_entries (off 16)

-        auto old_geo = read_file(cur_dir + "/geo_cells.bin");

-        auto old_se = read_file(cur_dir + "/street_entries.bin");

-        auto old_ae = read_file(cur_dir + "/addr_entries.bin");

-        auto old_ie = read_file(cur_dir + "/interp_entries.bin");

-        // Parse + remap each entry type

-        auto parse_and_remap = [&](const std::vector<char>& geo, size_t off_pos,

-                                     const std::vector<char>& entries,

-                                     const std::unordered_map<uint32_t,uint32_t>& rm) {

-            size_t n = geo.size() / 20;

-            std::vector<std::pair<uint64_t, std::vector<uint32_t>>> result;

-            for (size_t i = 0; i < n; i++) {

-                uint64_t cid; uint32_t off;

-                memcpy(&cid, geo.data() + i * 20, 8);

-                memcpy(&off, geo.data() + i * 20 + off_pos, 4);

-                std::vector<uint32_t> ids;

-                if (off != 0xFFFFFFFF && off + 2 <= entries.size()) {

-                    uint16_t count; memcpy(&count, entries.data() + off, 2);

-                    if (off + 2 + count * 4 <= entries.size()) {

-                        for (uint16_t j = 0; j < count; j++) {

-                            uint32_t id; memcpy(&id, entries.data() + off + 2 + j * 4, 4);

-                            auto it = rm.find(id);

-                            ids.push_back(it != rm.end() ? it->second : id);

+                // Copy count records from old

+                size_t bytes = count * actual_stride;

+                if (old_pos + bytes <= old_data.size()) {

+                    output.insert(output.end(), old_data.data() + old_pos,

+                                 old_data.data() + old_pos + bytes);

-                std::sort(ids.begin(), ids.end());

-                result.push_back({cid, std::move(ids)});

+            } else if (op == OP_INSERT_RUN) {

+                // Read count records from patch

+                output.insert(output.end(), patch.data() + pos, patch.data() + pos + bytes);

+            } else if (op == OP_DELETE_RUN) {

+                // Skip count records in old

+                old_pos += count * actual_stride;

-        auto sc = parse_and_remap(old_geo, 8, old_se, way_rm);

-        auto ac = parse_and_remap(old_geo, 12, old_ae, addr_rm);

-        auto ic = parse_and_remap(old_geo, 16, old_ie, interp_rm);

-        // Write rebuilt entries + geo_cells

-        auto write_rebuilt = [&](const std::string& name,

-                                  const std::vector<std::pair<uint64_t, std::vector<uint32_t>>>& entries) -> std::string {

-            std::string path = tmpdir + "/" + name;

-            std::unordered_map<uint64_t, uint32_t> offsets;

-            for (auto& [cid, ids] : entries) {

-                if (ids.empty()) continue;

-                offsets[cid] = static_cast<uint32_t>(buf.size());

-                uint16_t count = static_cast<uint16_t>(ids.size());

-                buf.insert(buf.end(), (char*)&count, (char*)&count + 2);

-                buf.insert(buf.end(), (char*)ids.data(), (char*)ids.data() + ids.size() * 4);

-        // Entry files: don't use rebuilt refs (diff uses old→new directly)

-        // Only geo_cells uses rebuilt ref (proven to match byte-identical)

-        write_rebuilt("street_entries.bin", sc); // write for geo_cells offset calculation

-        write_rebuilt("addr_entries.bin", ac);

-        write_rebuilt("interp_entries.bin", ic);

-        // Rebuild geo_cells from the remapped entries

-            uint32_t no_data = 0xFFFFFFFF;

-            // Build offset maps for each entry type

-            std::unordered_map<uint64_t, uint32_t> s_off, a_off, i_off;

-            auto entries_data = [](const std::string& path) {

-                auto d = read_file(path);

-                std::unordered_map<uint64_t, uint32_t> offsets;

-            // Re-parse the rebuilt entries to get offsets

-            auto get_offsets = [](const std::vector<std::pair<uint64_t, std::vector<uint32_t>>>& entries) {

-                for (auto& [cid, ids] : entries) {

-                    if (ids.empty()) continue;

-                    offsets[cid] = pos;

-            auto so = get_offsets(sc), ao = get_offsets(ac), io = get_offsets(ic);

-            size_t n = old_geo.size() / 20;

-            for (size_t i = 0; i < n; i++) {

-                uint64_t cid; memcpy(&cid, old_geo.data() + i * 20, 8);

-                buf.insert(buf.end(), (char*)&cid, (char*)&cid + 8);

-                auto get = [&](const auto& m) -> uint32_t {

-                    auto it = m.find(cid); return it != m.end() ? it->second : no_data;

-                uint32_t sv = get(so), av = get(ao), iv = get(io);

-                buf.insert(buf.end(), (char*)&sv, (char*)&sv + 4);

-                buf.insert(buf.end(), (char*)&av, (char*)&av + 4);

-                buf.insert(buf.end(), (char*)&iv, (char*)&iv + 4);

-            std::string path = tmpdir + "/geo_cells.bin";

-            rebuilt_refs["geo_cells.bin"] = path;

-            auto [ae_data, ac_data] = rebuild_entries("admin_cells.bin", "admin_entries.bin",

-                                                        12, 8, admin_rm, true);

-            std::string ae_path = tmpdir + "/admin_entries.bin";

-            std::string ac_path = tmpdir + "/admin_cells.bin";

-            write_file(ae_path, ae_data);

-            write_file(ac_path, ac_data);

-            // Don't add admin entries/cells to rebuilt_refs (diff uses old→new directly)

-        uint32_t file_id; pf.read(reinterpret_cast<char*>(&file_id), 4);

-        if (!pf || file_id == 0xFFFFFFFF) break;

-        uint32_t encoding; pf.read(reinterpret_cast<char*>(&encoding), 4);

-        if (file_id >= (uint32_t)PatchFileId::COUNT) {

-            std::cerr << "  Unknown section " << file_id << std::endl;

-        const char* filename = patch_file_names[file_id];

-        auto write_temp = [&](const std::string& path, uint64_t size) {

-            std::ofstream zf(path, std::ios::binary);

-            std::vector<char> buf(std::min(size, (uint64_t)1024*1024));

-                size_t chunk = std::min(remaining, (uint64_t)buf.size());

-                pf.read(buf.data(), chunk);

-                zf.write(buf.data(), chunk);

-        auto zstd_apply = [&](const std::string& ref, const std::string& zst, const std::string& out) -> bool {

-            std::string cmd = "zstd --patch-from='" + ref + "' -d '" + zst + "' -o '" + out + "' -f --long=31 --quiet 2>/dev/null";

-            return system(cmd.c_str()) == 0;

-        if (encoding == (uint32_t)PatchEncoding::ZSTD_DELTA) {

-            uint64_t old_size, new_size, delta_size;

-            pf.read(reinterpret_cast<char*>(&old_size), 8);

-            pf.read(reinterpret_cast<char*>(&new_size), 8);

-            pf.read(reinterpret_cast<char*>(&delta_size), 8);

-            // Write zstd frame to temp

-            std::string zst = tmpdir + "/" + std::string(filename) + ".zst";

-            write_temp(zst, delta_size);

-            // Check if we have a rebuilt ref (entry/cell files with ID remap)

-            auto rit = rebuilt_refs.find(std::string(filename));

-            if (rit != rebuilt_refs.end()) {

-                // Load old file, apply string remap + fixups → write remapped ref

-                auto old_data = read_file(cur_dir + "/" + std::string(filename));

-                apply_str_remap(old_data, file_id);

-                apply_fixups(old_data, file_id);

-                ref = tmpdir + "/" + std::string(filename) + ".ref";

-                write_file(ref, old_data);

-            std::string out = out_dir + "/" + std::string(filename);

-            if (!zstd_apply(ref, zst, out)) {

-                std::cerr << "FAILED: " << filename << std::endl; return 1;

-            std::cerr << "  " << filename << ": " << read_file(out).size() << " bytes" << std::endl;

-            remove(zst.c_str()); remove(ref.c_str());

-            std::cerr << "  Unknown encoding " << encoding << " for " << filename << std::endl;

+        write_file(out_dir + "/" + std::string(filename), output);

+        std::cerr << "  " << filename << ": " << output.size() << " bytes" << std::endl;

std::string rm_cmd = "rm -rf '" + tmpdir + "'";

std::cerr << "Patch applied. Output in " << out_dir << std::endl;

// geocoder-patch: Apply a .gcpatch to a geocoder index directory.
//
// For each file: apply string remap to old, write temp, run zstd -d --patch-from.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>

#include <unistd.h>
#include "patch_format.h"

// --- String remap application ---

static void remap_addr_points(std::vector<char>& data,
                                const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + 16 <= data.size(); i += 16) {
        for (size_t off : {8, 12}) {
            uint32_t v; memcpy(&v, data.data() + i + off, 4);
            auto it = rm.find(v);
            if (it != rm.end()) memcpy(data.data() + i + off, &it->second, 4);
        }
    }
}

static void remap_field(std::vector<char>& data, size_t stride, size_t field_off,
                          const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + stride <= data.size(); i += stride) {
        uint32_t v; memcpy(&v, data.data() + i + field_off, 4);
        auto it = rm.find(v);
        if (it != rm.end()) memcpy(data.data() + i + field_off, &it->second, 4);
    }
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);

    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    std::ifstream pf(patch_path, std::ios::binary);
    if (!pf) { std::cerr << "Cannot open " << patch_path << std::endl; return 1; }

    // Read header
    char magic[8]; pf.read(magic, 8);
    if (memcmp(magic, GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch file" << std::endl; return 1;
    }
    uint32_t version, flags;
    pf.read(reinterpret_cast<char*>(&version), 4);
    pf.read(reinterpret_cast<char*>(&flags), 4);
    if (version != GCPATCH_VERSION) {
        std::cerr << "Unsupported version " << version << std::endl; return 1;
    }

    // Helper: decompress zstd blob via CLI
    auto zstd_decompress_blob = [&](const std::vector<char>& compressed, size_t expected_size) -> std::vector<char> {
        std::string zst_path = tmpdir + "/blob.zst";
        std::string raw_path = tmpdir + "/blob.raw";
        write_file(zst_path, compressed);
        system(("zstd -d '" + zst_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null").c_str());
        auto result = read_file(raw_path);
        remove(zst_path.c_str()); remove(raw_path.c_str());
        return result;
    };

    // Read string remap (compressed)
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t section_id; pf.read(reinterpret_cast<char*>(&section_id), 4);
        if (section_id == 0xFFFFFFFE) {
            uint32_t count; pf.read(reinterpret_cast<char*>(&count), 4);
            uint64_t comp_size; pf.read(reinterpret_cast<char*>(&comp_size), 8);
            std::vector<char> compressed(comp_size);
            pf.read(compressed.data(), comp_size);
            auto blob = zstd_decompress_blob(compressed, count * 8);
            for (uint32_t i = 0; i < count; i++) {
                uint32_t old_off, new_off;
                memcpy(&old_off, blob.data() + i * 8, 4);
                memcpy(&new_off, blob.data() + i * 8 + 4, 4);
                if (old_off != new_off) str_remap[old_off] = new_off;
            }
            std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;
        }
    }

    // Read offset fixup tables
    struct Fixup { uint32_t record_idx; uint32_t new_value; };
    std::unordered_map<uint32_t, std::vector<Fixup>> fixup_tables; // file_id → fixups
    std::unordered_map<uint32_t, uint32_t> fixup_strides; // file_id → stride
    while (pf) {
        uint32_t marker; pf.read(reinterpret_cast<char*>(&marker), 4);
        if (marker != FIXUP_MARKER) {
            pf.seekg(-4, std::ios::cur);
            break;
        }
        uint32_t fid, stride, count;
        pf.read(reinterpret_cast<char*>(&fid), 4);
        pf.read(reinterpret_cast<char*>(&stride), 4);
        pf.read(reinterpret_cast<char*>(&count), 4);
        uint64_t comp_size; pf.read(reinterpret_cast<char*>(&comp_size), 8);
        std::vector<char> compressed(comp_size);
        pf.read(compressed.data(), comp_size);
        auto blob = zstd_decompress_blob(compressed, count * 8);
        auto& vec = fixup_tables[fid];
        fixup_strides[fid] = stride;
        vec.resize(count);
        for (uint32_t i = 0; i < count; i++) {
            memcpy(&vec[i].record_idx, blob.data() + i * 8, 4);
            memcpy(&vec[i].new_value, blob.data() + i * 8 + 4, 4);
        }
        std::cerr << "Loaded " << count << " fixups for file " << fid << std::endl;
    }

    // Detect struct strides from current files
    auto detect_stride = [&](const std::string& name, std::vector<size_t> cs) -> size_t {
        auto d = read_file(cur_dir + "/" + name);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect_stride("street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride("interp_ways.bin", {20, 18});
    size_t admin_stride = detect_stride("admin_polygons.bin", {24, 20, 19});

    // Helper: apply string remap to old data file
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t file_id) {
        if (str_remap.empty()) return;
        if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) {
            for (size_t i = 0; i + 16 <= data.size(); i += 16)
                for (size_t off : {8, 12}) {
                    uint32_t v; memcpy(&v, data.data() + i + off, 4);
                    auto it = str_remap.find(v);
                    if (it != str_remap.end()) memcpy(data.data() + i + off, &it->second, 4);
                }
        } else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) {
            size_t name_off = (way_stride == 12) ? 8 : 5;
            for (size_t i = 0; i + way_stride <= data.size(); i += way_stride) {
                uint32_t v; memcpy(&v, data.data() + i + name_off, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + name_off, &it->second, 4);
            }
        } else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) {
            size_t soff = (interp_stride >= 20) ? 8 : 5;
            for (size_t i = 0; i + interp_stride <= data.size(); i += interp_stride) {
                uint32_t v; memcpy(&v, data.data() + i + soff, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + soff, &it->second, 4);
            }
        } else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) {
            for (size_t i = 0; i + admin_stride <= data.size(); i += admin_stride) {
                uint32_t v; memcpy(&v, data.data() + i + 8, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + 8, &it->second, 4);
            }
        }
    };

    // Helper: apply offset fixups
    auto apply_fixups = [&](std::vector<char>& data, uint32_t file_id) {
        auto it = fixup_tables.find(file_id);
        if (it == fixup_tables.end()) return;
        uint32_t stride = fixup_strides[file_id];
        for (auto& f : it->second) {
            size_t pos = (size_t)f.record_idx * stride;
            if (pos + 4 <= data.size())
                memcpy(data.data() + pos, &f.new_value, 4);
        }
    };

    // Read ID remap tables
    std::unordered_map<uint32_t, std::unordered_map<uint32_t,uint32_t>> id_remaps; // file_id → (old→new)
    while (pf) {
        uint32_t marker; pf.read(reinterpret_cast<char*>(&marker), 4);
        if (marker != 0xFFFFFFFC) {
            pf.seekg(-4, std::ios::cur);
            break;
        }
        uint32_t fid, count;
        pf.read(reinterpret_cast<char*>(&fid), 4);
        pf.read(reinterpret_cast<char*>(&count), 4);
        uint64_t comp_size; pf.read(reinterpret_cast<char*>(&comp_size), 8);
        std::vector<char> compressed(comp_size);
        pf.read(compressed.data(), comp_size);
        auto blob = zstd_decompress_blob(compressed, count * 8);
        auto& rm = id_remaps[fid];
        for (uint32_t i = 0; i < count; i++) {
            uint32_t old_id, new_id;
            memcpy(&old_id, blob.data() + i * 8, 4);
            memcpy(&new_id, blob.data() + i * 8 + 4, 4);
            rm[old_id] = new_id;
        }
        std::cerr << "Loaded " << count << " ID remaps for file " << fid << std::endl;
    }

    // Read cell changes
    std::vector<uint64_t> geo_added, geo_removed;
    {
        uint32_t marker; pf.read(reinterpret_cast<char*>(&marker), 4);
        if (marker == CELL_CHANGES_GEO_MARKER) {
            uint32_t n_added, n_removed;
            pf.read(reinterpret_cast<char*>(&n_added), 4);
            pf.read(reinterpret_cast<char*>(&n_removed), 4);
            geo_added.resize(n_added);
            geo_removed.resize(n_removed);
            for (uint32_t i = 0; i < n_added; i++)
                pf.read(reinterpret_cast<char*>(&geo_added[i]), 8);
            for (uint32_t i = 0; i < n_removed; i++)
                pf.read(reinterpret_cast<char*>(&geo_removed[i]), 8);
            std::cerr << "Loaded cell changes: " << n_added << " added, " << n_removed << " removed" << std::endl;
        } else {
            pf.seekg(-4, std::ios::cur);
        }
    }

    // Helper: rebuild entry + cell files using ID remap (LEGACY, unused)
    auto rebuild_entries_legacy = [&](const std::string& cells_file, const std::string& entries_file,
                                size_t cell_stride, size_t offset_pos,
                                const std::unordered_map<uint32_t,uint32_t>& id_rm, bool has_flags)
        -> std::pair<std::vector<char>, std::vector<char>> // (entries_data, cells_data)
    {
        auto old_cells = read_file(cur_dir + "/" + cells_file);
        auto old_entries = read_file(cur_dir + "/" + entries_file);
        size_t n_cells = old_cells.size() / cell_stride;

        // Parse old entries, remap IDs
        struct CellEntry { uint64_t cell_id; std::vector<uint32_t> ids; };
        std::vector<CellEntry> parsed;
        for (size_t i = 0; i < n_cells; i++) {
            uint64_t cid; uint32_t off;
            memcpy(&cid, old_cells.data() + i * cell_stride, 8);
            memcpy(&off, old_cells.data() + i * cell_stride + offset_pos, 4);
            CellEntry ce; ce.cell_id = cid;
            if (off != 0xFFFFFFFF && off + 2 <= old_entries.size()) {
                uint16_t count; memcpy(&count, old_entries.data() + off, 2);
                if (off + 2 + count * 4 <= old_entries.size()) {
                    for (uint16_t j = 0; j < count; j++) {
                        uint32_t id; memcpy(&id, old_entries.data() + off + 2 + j * 4, 4);
                        uint32_t flags = has_flags ? (id & 0x80000000u) : 0;
                        uint32_t masked = id & 0x7FFFFFFFu;
                        auto it = id_rm.find(masked);
                        if (it != id_rm.end())
                            ce.ids.push_back(it->second | flags);
                        else
                            ce.ids.push_back(id); // keep unmapped IDs as-is
                    }
                }
            }
            std::sort(ce.ids.begin(), ce.ids.end());
            parsed.push_back(std::move(ce));
        }

        // Rebuild entries + cells
        std::vector<char> new_entries_buf, new_cells_buf;
        uint32_t no_data = 0xFFFFFFFF;
        for (auto& ce : parsed) {
            new_cells_buf.insert(new_cells_buf.end(), (char*)&ce.cell_id, (char*)&ce.cell_id + 8);
            if (ce.ids.empty()) {
                new_cells_buf.insert(new_cells_buf.end(), (char*)&no_data, (char*)&no_data + 4);
            } else {
                uint32_t off = static_cast<uint32_t>(new_entries_buf.size());
                new_cells_buf.insert(new_cells_buf.end(), (char*)&off, (char*)&off + 4);
                uint16_t count = static_cast<uint16_t>(ce.ids.size());
                new_entries_buf.insert(new_entries_buf.end(), (char*)&count, (char*)&count + 2);
                new_entries_buf.insert(new_entries_buf.end(), (char*)ce.ids.data(),
                                      (char*)ce.ids.data() + ce.ids.size() * 4);
            }
        }
        return {new_entries_buf, new_cells_buf};
    };

    // Pre-rebuild entry/cell files using SHARED rebuild function (identical to diff tool)
    std::unordered_map<std::string, std::string> rebuilt_refs;
    if (!id_remaps.empty()) {
        auto w_rm = id_remaps.count((uint32_t)PatchFileId::STREET_WAYS) ? id_remaps[(uint32_t)PatchFileId::STREET_WAYS] : std::unordered_map<uint32_t,uint32_t>{};
        auto a_rm = id_remaps.count((uint32_t)PatchFileId::ADDR_POINTS) ? id_remaps[(uint32_t)PatchFileId::ADDR_POINTS] : std::unordered_map<uint32_t,uint32_t>{};
        auto i_rm = id_remaps.count((uint32_t)PatchFileId::INTERP_WAYS) ? id_remaps[(uint32_t)PatchFileId::INTERP_WAYS] : std::unordered_map<uint32_t,uint32_t>{};
        auto ad_rm = id_remaps.count((uint32_t)PatchFileId::ADMIN_POLYGONS) ? id_remaps[(uint32_t)PatchFileId::ADMIN_POLYGONS] : std::unordered_map<uint32_t,uint32_t>{};

        auto old_geo = read_file(cur_dir + "/geo_cells.bin");
        auto old_se = read_file(cur_dir + "/street_entries.bin");
        auto old_ae = read_file(cur_dir + "/addr_entries.bin");
        auto old_ie = read_file(cur_dir + "/interp_entries.bin");

        auto geo = rebuild_geo_from_remap(old_geo, old_se, old_ae, old_ie, w_rm, a_rm, i_rm, geo_added, geo_removed);
        write_file(tmpdir + "/geo_cells.bin", geo.geo_cells_data);
        write_file(tmpdir + "/street_entries.bin", geo.street_entries_data);
        write_file(tmpdir + "/addr_entries.bin", geo.addr_entries_data);
        write_file(tmpdir + "/interp_entries.bin", geo.interp_entries_data);
        rebuilt_refs["geo_cells.bin"] = tmpdir + "/geo_cells.bin";
        rebuilt_refs["street_entries.bin"] = tmpdir + "/street_entries.bin";
        rebuilt_refs["addr_entries.bin"] = tmpdir + "/addr_entries.bin";
        rebuilt_refs["interp_entries.bin"] = tmpdir + "/interp_entries.bin";

        auto old_ac = read_file(cur_dir + "/admin_cells.bin");
        auto old_admin_e = read_file(cur_dir + "/admin_entries.bin");
        auto admin = rebuild_admin_from_remap(old_ac, old_admin_e, ad_rm);
        write_file(tmpdir + "/admin_cells.bin", admin.admin_cells_data);
        write_file(tmpdir + "/admin_entries.bin", admin.admin_entries_data);
        rebuilt_refs["admin_cells.bin"] = tmpdir + "/admin_cells.bin";
        rebuilt_refs["admin_entries.bin"] = tmpdir + "/admin_entries.bin";

        std::cerr << "Rebuilt " << rebuilt_refs.size() << " entry/cell files (shared function)" << std::endl;
    }
    #if 0 // Legacy rebuild code — replaced by shared rebuild_geo_from_remap
        // Rebuild geo entries (street, addr, interp)
        auto way_rm = id_remaps.count((uint32_t)PatchFileId::STREET_WAYS) ?
                       id_remaps[(uint32_t)PatchFileId::STREET_WAYS] : std::unordered_map<uint32_t,uint32_t>{};
        auto addr_rm = id_remaps.count((uint32_t)PatchFileId::ADDR_POINTS) ?
                        id_remaps[(uint32_t)PatchFileId::ADDR_POINTS] : std::unordered_map<uint32_t,uint32_t>{};
        auto interp_rm = id_remaps.count((uint32_t)PatchFileId::INTERP_WAYS) ?
                          id_remaps[(uint32_t)PatchFileId::INTERP_WAYS] : std::unordered_map<uint32_t,uint32_t>{};
        auto admin_rm = id_remaps.count((uint32_t)PatchFileId::ADMIN_POLYGONS) ?
                         id_remaps[(uint32_t)PatchFileId::ADMIN_POLYGONS] : std::unordered_map<uint32_t,uint32_t>{};

        // Rebuild geo: street_entries (off 8), addr_entries (off 12), interp_entries (off 16)
        auto old_geo = read_file(cur_dir + "/geo_cells.bin");
        auto old_se = read_file(cur_dir + "/street_entries.bin");
        auto old_ae = read_file(cur_dir + "/addr_entries.bin");
        auto old_ie = read_file(cur_dir + "/interp_entries.bin");

        // Parse + remap each entry type
        auto parse_and_remap = [&](const std::vector<char>& geo, size_t off_pos,
                                     const std::vector<char>& entries,
                                     const std::unordered_map<uint32_t,uint32_t>& rm) {
            size_t n = geo.size() / 20;
            std::vector<std::pair<uint64_t, std::vector<uint32_t>>> result;
            for (size_t i = 0; i < n; i++) {
                uint64_t cid; uint32_t off;
                memcpy(&cid, geo.data() + i * 20, 8);
                memcpy(&off, geo.data() + i * 20 + off_pos, 4);
                std::vector<uint32_t> ids;
                if (off != 0xFFFFFFFF && off + 2 <= entries.size()) {
                    uint16_t count; memcpy(&count, entries.data() + off, 2);
                    if (off + 2 + count * 4 <= entries.size()) {
                        for (uint16_t j = 0; j < count; j++) {
                            uint32_t id; memcpy(&id, entries.data() + off + 2 + j * 4, 4);
                            auto it = rm.find(id);
                            ids.push_back(it != rm.end() ? it->second : id);
                        }
                    }
                }
                std::sort(ids.begin(), ids.end());
                result.push_back({cid, std::move(ids)});
            }
            return result;
        };

        auto sc = parse_and_remap(old_geo, 8, old_se, way_rm);
        auto ac = parse_and_remap(old_geo, 12, old_ae, addr_rm);
        auto ic = parse_and_remap(old_geo, 16, old_ie, interp_rm);

        // Write rebuilt entries + geo_cells
        auto write_rebuilt = [&](const std::string& name,
                                  const std::vector<std::pair<uint64_t, std::vector<uint32_t>>>& entries) -> std::string {
            std::string path = tmpdir + "/" + name;
            std::vector<char> buf;
            std::unordered_map<uint64_t, uint32_t> offsets;
            for (auto& [cid, ids] : entries) {
                if (ids.empty()) continue;
                offsets[cid] = static_cast<uint32_t>(buf.size());
                uint16_t count = static_cast<uint16_t>(ids.size());
                buf.insert(buf.end(), (char*)&count, (char*)&count + 2);
                buf.insert(buf.end(), (char*)ids.data(), (char*)ids.data() + ids.size() * 4);
            }
            write_file(path, buf);
            return path;
        };

        // Entry files: don't use rebuilt refs (diff uses old→new directly)
        // Only geo_cells uses rebuilt ref (proven to match byte-identical)
        write_rebuilt("street_entries.bin", sc); // write for geo_cells offset calculation
        write_rebuilt("addr_entries.bin", ac);
        write_rebuilt("interp_entries.bin", ic);

        // Rebuild geo_cells from the remapped entries
        {
            uint32_t no_data = 0xFFFFFFFF;
            std::vector<char> buf;
            // Build offset maps for each entry type
            std::unordered_map<uint64_t, uint32_t> s_off, a_off, i_off;
            auto entries_data = [](const std::string& path) {
                auto d = read_file(path);
                std::unordered_map<uint64_t, uint32_t> offsets;
                return d;
            };
            // Re-parse the rebuilt entries to get offsets
            auto get_offsets = [](const std::vector<std::pair<uint64_t, std::vector<uint32_t>>>& entries) {
                std::unordered_map<uint64_t, uint32_t> offsets;
                uint32_t pos = 0;
                for (auto& [cid, ids] : entries) {
                    if (ids.empty()) continue;
                    offsets[cid] = pos;
                    pos += 2 + ids.size() * 4;
                }
                return offsets;
            };
            auto so = get_offsets(sc), ao = get_offsets(ac), io = get_offsets(ic);

            size_t n = old_geo.size() / 20;
            for (size_t i = 0; i < n; i++) {
                uint64_t cid; memcpy(&cid, old_geo.data() + i * 20, 8);
                buf.insert(buf.end(), (char*)&cid, (char*)&cid + 8);
                auto get = [&](const auto& m) -> uint32_t {
                    auto it = m.find(cid); return it != m.end() ? it->second : no_data;
                };
                uint32_t sv = get(so), av = get(ao), iv = get(io);
                buf.insert(buf.end(), (char*)&sv, (char*)&sv + 4);
                buf.insert(buf.end(), (char*)&av, (char*)&av + 4);
                buf.insert(buf.end(), (char*)&iv, (char*)&iv + 4);
            }
            std::string path = tmpdir + "/geo_cells.bin";
            write_file(path, buf);
            rebuilt_refs["geo_cells.bin"] = path;
        }

        // Rebuild admin entries
        {
            auto [ae_data, ac_data] = rebuild_entries("admin_cells.bin", "admin_entries.bin",
                                                        12, 8, admin_rm, true);
            std::string ae_path = tmpdir + "/admin_entries.bin";
            std::string ac_path = tmpdir + "/admin_cells.bin";
            write_file(ae_path, ae_data);
            write_file(ac_path, ac_data);
            // Don't add admin entries/cells to rebuilt_refs (diff uses old→new directly)
        }

    #endif // Legacy rebuild code

    // Process file sections
    while (pf) {
        uint32_t file_id; pf.read(reinterpret_cast<char*>(&file_id), 4);
        if (!pf || file_id == 0xFFFFFFFF) break;
        uint32_t encoding; pf.read(reinterpret_cast<char*>(&encoding), 4);

        if (file_id >= (uint32_t)PatchFileId::COUNT) {
            std::cerr << "  Unknown section " << file_id << std::endl;
            break;
        }
        const char* filename = patch_file_names[file_id];

        auto write_temp = [&](const std::string& path, uint64_t size) {
            std::ofstream zf(path, std::ios::binary);
            std::vector<char> buf(std::min(size, (uint64_t)1024*1024));
            uint64_t remaining = size;
            while (remaining > 0) {
                size_t chunk = std::min(remaining, (uint64_t)buf.size());
                pf.read(buf.data(), chunk);
                zf.write(buf.data(), chunk);
                remaining -= chunk;
            }
        };

        auto zstd_apply = [&](const std::string& ref, const std::string& zst, const std::string& out) -> bool {
            std::string cmd = "zstd --patch-from='" + ref + "' -d '" + zst + "' -o '" + out + "' -f --long=31 --quiet 2>/dev/null";
            return system(cmd.c_str()) == 0;
        };

        if (encoding == (uint32_t)PatchEncoding::ZSTD_DELTA) {
            uint64_t old_size, new_size, delta_size;
            pf.read(reinterpret_cast<char*>(&old_size), 8);
            pf.read(reinterpret_cast<char*>(&new_size), 8);
            pf.read(reinterpret_cast<char*>(&delta_size), 8);

            // Write zstd frame to temp
            std::string zst = tmpdir + "/" + std::string(filename) + ".zst";
            write_temp(zst, delta_size);

            // Check if we have a rebuilt ref (entry/cell files with ID remap)
            std::string ref;
            auto rit = rebuilt_refs.find(std::string(filename));
            if (rit != rebuilt_refs.end()) {
                ref = rit->second;
            } else {
                // Load old file, apply string remap + fixups → write remapped ref
                auto old_data = read_file(cur_dir + "/" + std::string(filename));
                apply_str_remap(old_data, file_id);
                apply_fixups(old_data, file_id);
                ref = tmpdir + "/" + std::string(filename) + ".ref";
                write_file(ref, old_data);
            }

            // Apply zstd delta
            std::string out = out_dir + "/" + std::string(filename);
            if (!zstd_apply(ref, zst, out)) {
                std::cerr << "FAILED: " << filename << std::endl; return 1;
            }
            std::cerr << "  " << filename << ": " << read_file(out).size() << " bytes" << std::endl;
            remove(zst.c_str()); remove(ref.c_str());

        } else {
            std::cerr << "  Unknown encoding " << encoding << " for " << filename << std::endl;
            break;
        }
    }

    // Cleanup
    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());

    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}


I noticed the comment in the patch tool about the offset fixup problem. The diff tool applies fixups before matching, but the patch tool doesn't have them. The simplest fix: **don't apply offset fixups in the diff tool**. Let the merge sequence handle offset differences as DELETE+INSERT pairs. This is slightly less optimal (more INSERT data for changed ways) but correct and eliminates the fixup tables entirely.

Let me update the diff tool to NOT fixup offsets:


Edit

    // street_ways.bin (with node_offset fixup)
    {
        auto old_w = read_file(old_dir + "/street_ways.bin");
        remap_field(old_w, way_stride, way_stride == 12 ? 8 : 5, str_remap);
        auto new_w = read_file(new_dir + "/street_ways.bin");
        auto old_n = read_file(old_dir + "/street_nodes.bin");
        auto new_n = read_file(new_dir + "/street_nodes.bin");
        fixup_way_offsets(old_w, old_n, new_w, new_n, way_stride);
        write_merge(PatchFileId::STREET_WAYS, "street_ways.bin", old_w, new_w, way_stride);
    }

    // street_ways.bin (string remap only, no offset fixup — merge handles byte diffs)
    {
        auto old_w = read_file(old_dir + "/street_ways.bin");
        remap_field(old_w, way_stride, way_stride == 12 ? 8 : 5, str_remap);
        auto new_w = read_file(new_dir + "/street_ways.bin");
        write_merge(PatchFileId::STREET_WAYS, "street_ways.bin", old_w, new_w, way_stride);
    }


The file /root/traccar-geocoder/builder/tools/geocoder_diff.cpp has been updated successfully.

    // street_ways.bin (string remap only, no offset fixup — merge handles byte diffs)
    {
        auto old_w = read_file(old_dir + "/street_ways.bin");
        remap_field(old_w, way_stride, way_stride == 12 ? 8 : 5, str_remap);
        auto new_w = read_file(new_dir + "/street_ways.bin");
        write_merge(PatchFileId::STREET_WAYS, "street_ways.bin", old_w, new_w, way_stride);
    }

// geocoder-diff v3: Fully custom patch format using merge-sequence encoding.
//
// For each data file: walk old (string-remapped) and new in parallel,
// match records by content, emit MATCH/INSERT/DELETE operations.
// For entry/cell files: include changed cell entry data directly.
// geo_cells rebuilt by patch tool from entries (not delta-patched).
//
// Patch format: custom binary, zstd-compressed as a whole for transport.
//
// Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>

#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <functional>
#include <iomanip>
#include <iostream>
#include <string>
#include <unistd.h>
#include <unordered_map>
#include <unordered_set>
#include <vector>

#include "patch_format.h"

// --- Merge sequence encoding ---
// Walk two sorted arrays (old remapped, new), matching by record bytes.
// Emit: MATCH(n) = copy n records from old, INSERT(n,data) = add n new records,
//       DELETE(n) = skip n old records.

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

struct MergeSequence {
    std::vector<char> data; // serialized ops

    void add_match(uint32_t count) {
        uint8_t op = OP_MATCH_RUN;
        data.insert(data.end(), (char*)&op, (char*)&op + 1);
        data.insert(data.end(), (char*)&count, (char*)&count + 4);
    }
    void add_delete(uint32_t count) {
        uint8_t op = OP_DELETE_RUN;
        data.insert(data.end(), (char*)&op, (char*)&op + 1);
        data.insert(data.end(), (char*)&count, (char*)&count + 4);
    }
    void add_insert(const char* records, uint32_t count, size_t stride) {
        uint8_t op = OP_INSERT_RUN;
        data.insert(data.end(), (char*)&op, (char*)&op + 1);
        data.insert(data.end(), (char*)&count, (char*)&count + 4);
        data.insert(data.end(), records, records + count * stride);
    }
};

// Build merge sequence for a data file.
// old_data has string remap already applied.
// Records are compared by byte equality (stride bytes).
static MergeSequence build_merge_seq(
    const std::vector<char>& old_data, const std::vector<char>& new_data,
    size_t stride)
{
    size_t old_n = old_data.size() / stride;
    size_t new_n = new_data.size() / stride;
    size_t oi = 0, ni = 0;
    MergeSequence seq;
    uint32_t match_run = 0, del_run = 0;

    auto flush_match = [&]() { if (match_run > 0) { seq.add_match(match_run); match_run = 0; } };
    auto flush_del = [&]() { if (del_run > 0) { seq.add_delete(del_run); del_run = 0; } };

    while (oi < old_n && ni < new_n) {
        const char* op = old_data.data() + oi * stride;
        const char* np = new_data.data() + ni * stride;

        if (memcmp(op, np, stride) == 0) {
            flush_del();
            match_run++;
            oi++; ni++;
        } else {
            // Mismatch: look ahead to determine if it's a deletion in old or insertion in new.
            // Check: does the OLD record appear later in new? If so, there are inserts before it.
            // Check: does the NEW record appear later in old? If so, there are deletes before it.
            // Simple heuristic: look ahead up to 100 records in each direction.
            bool found_old_in_new = false, found_new_in_old = false;
            size_t lookahead = std::min((size_t)200, std::min(old_n - oi, new_n - ni));
            for (size_t k = 1; k <= lookahead; k++) {
                if (ni + k < new_n && memcmp(op, new_data.data() + (ni + k) * stride, stride) == 0) {
                    found_old_in_new = true;
                    // Insert k records from new before continuing
                    flush_match(); flush_del();
                    seq.add_insert(new_data.data() + ni * stride, k, stride);
                    ni += k;
                    break;
                }
                if (oi + k < old_n && memcmp(np, old_data.data() + (oi + k) * stride, stride) == 0) {
                    found_new_in_old = true;
                    // Delete k records from old
                    flush_match();
                    del_run += k;
                    oi += k;
                    break;
                }
            }
            if (!found_old_in_new && !found_new_in_old) {
                // Both records are different and neither appears nearby.
                // Treat as delete old + insert new.
                flush_match();
                del_run++;
                oi++;
                flush_del();
                seq.add_insert(np, 1, stride);
                ni++;
            }
        }
    }

    flush_match(); flush_del();
    // Remaining old records: deletions
    if (oi < old_n) seq.add_delete(old_n - oi);
    // Remaining new records: insertions
    if (ni < new_n) seq.add_insert(new_data.data() + ni * stride, new_n - ni, stride);

    return seq;
}

// --- String remap ---

static std::unordered_map<uint32_t, uint32_t> build_string_remap(
    const std::vector<char>& old_pool, const std::vector<char>& new_pool)
{
    std::unordered_map<std::string, uint32_t> new_idx;
    size_t pos = 0;
    while (pos < new_pool.size()) {
        const char* s = new_pool.data() + pos;
        size_t len = strlen(s);
        new_idx[std::string(s, len)] = static_cast<uint32_t>(pos);
        pos += len + 1;
    }
    std::unordered_map<uint32_t, uint32_t> remap;
    pos = 0;
    while (pos < old_pool.size()) {
        const char* s = old_pool.data() + pos;
        size_t len = strlen(s);
        auto it = new_idx.find(std::string(s, len));
        if (it != new_idx.end())
            remap[static_cast<uint32_t>(pos)] = it->second;
        pos += len + 1;
    }
    return remap;
}

// --- Remap + fixup helpers ---

static void remap_addr_points(std::vector<char>& data, const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + 16 <= data.size(); i += 16)
        for (size_t off : {8, 12}) {
            uint32_t v; memcpy(&v, data.data() + i + off, 4);
            auto it = rm.find(v); if (it != rm.end()) memcpy(data.data() + i + off, &it->second, 4);
        }
}
static void remap_field(std::vector<char>& data, size_t stride, size_t field_off,
                         const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + stride <= data.size(); i += stride) {
        uint32_t v; memcpy(&v, data.data() + i + field_off, 4);
        auto it = rm.find(v); if (it != rm.end()) memcpy(data.data() + i + field_off, &it->second, 4);
    }
}

// Content matching for ways (by name + nodes, ignoring node_offset)
static uint64_t fnv_mix(uint64_t h, uint64_t v) { h ^= v; h *= 1099511628211ULL; return h; }

static void fixup_way_offsets(std::vector<char>& old_ways, const std::vector<char>& old_nodes,
                                const std::vector<char>& new_ways, const std::vector<char>& new_nodes,
                                size_t stride) {
    size_t name_off = (stride == 12) ? 8 : 5;
    size_t old_n = old_ways.size() / stride, new_n = new_ways.size() / stride;
    size_t old_nc = old_nodes.size() / 8, new_nc = new_nodes.size() / 8;

    auto way_hash = [&](const char* w, const char* nodes, size_t max_n) -> uint64_t {
        uint32_t node_offset, name_id; uint8_t node_count;
        memcpy(&node_offset, w, 4); node_count = (uint8_t)w[4]; memcpy(&name_id, w + name_off, 4);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, node_count);
        for (uint8_t j = 0; j < node_count && (node_offset + j) < max_n; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4);
            memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(way_hash(new_ways.data() + i * stride, new_nodes.data(), new_nc), i);

    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = way_hash(old_ways.data() + i * stride, old_nodes.data(), old_nc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            // Fix node_offset to match new way
            uint32_t new_node_off;
            memcpy(&new_node_off, new_ways.data() + it->second * stride, 4);
            memcpy(old_ways.data() + i * stride, &new_node_off, 4);
            new_map.erase(it);
        }
    }
}

// Same for admin polygons (vertex_offset)
static void fixup_admin_offsets(std::vector<char>& old_polys, const std::vector<char>& old_verts,
                                  const std::vector<char>& new_polys, const std::vector<char>& new_verts,
                                  size_t stride) {
    size_t old_n = old_polys.size() / stride, new_n = new_polys.size() / stride;
    size_t old_vc = old_verts.size() / 8, new_vc = new_verts.size() / 8;

    auto poly_hash = [&](const char* p, const char* verts, size_t max_v) -> uint64_t {
        uint32_t vert_offset, vert_count, name_id;
        memcpy(&vert_offset, p, 4); memcpy(&vert_count, p + 4, 4); memcpy(&name_id, p + 8, 4);
        uint8_t level = (uint8_t)p[12]; uint16_t cc; memcpy(&cc, p + stride - 2, 2);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, level); h = fnv_mix(h, cc); h = fnv_mix(h, vert_count);
        for (uint32_t j = 0; j < std::min(vert_count, 10u) && (vert_offset + j) < max_v; j++) {
            float lat, lng;
            memcpy(&lat, verts + (vert_offset + j) * 8, 4); memcpy(&lng, verts + (vert_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(poly_hash(new_polys.data() + i * stride, new_verts.data(), new_vc), i);

    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = poly_hash(old_polys.data() + i * stride, old_verts.data(), old_vc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            uint32_t new_vert_off;
            memcpy(&new_vert_off, new_polys.data() + it->second * stride, 4);
            memcpy(old_polys.data() + i * stride, &new_vert_off, 4);
            new_map.erase(it);
        }
    }
}

// Same for interp ways
static void fixup_interp_offsets(std::vector<char>& old_data, const std::vector<char>& old_nodes,
                                   const std::vector<char>& new_data, const std::vector<char>& new_nodes,
                                   size_t stride) {
    size_t street_off = (stride >= 20) ? 8 : 5;
    size_t old_n = old_data.size() / stride, new_n = new_data.size() / stride;
    size_t old_nc = old_nodes.size() / 8, new_nc = new_nodes.size() / 8;

    auto ihash = [&](const char* p, const char* nodes, size_t max_n) -> uint64_t {
        uint32_t node_offset, street_id, start, end; uint8_t count, itype;
        memcpy(&node_offset, p, 4); count = (uint8_t)p[4];
        memcpy(&street_id, p + street_off, 4); memcpy(&start, p + street_off + 4, 4);
        memcpy(&end, p + street_off + 8, 4);
        itype = (street_off + 12 < stride) ? (uint8_t)p[street_off + 12] : 0;
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, street_id); h = fnv_mix(h, start); h = fnv_mix(h, end);
        h = fnv_mix(h, itype); h = fnv_mix(h, count);
        for (uint8_t j = 0; j < count && (node_offset + j) < max_n; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4); memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(ihash(new_data.data() + i * stride, new_nodes.data(), new_nc), i);
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = ihash(old_data.data() + i * stride, old_nodes.data(), old_nc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            uint32_t new_off; memcpy(&new_off, new_data.data() + it->second * stride, 4);
            memcpy(old_data.data() + i * stride, &new_off, 4);
            new_map.erase(it);
        }
    }
}

// --- Write helpers ---
static void wval(std::vector<char>& buf, const void* data, size_t size) {
    buf.insert(buf.end(), (const char*)data, (const char*)data + size);
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>" << std::endl;
        return 1;
    }
    std::string old_dir = argv[1], new_dir = argv[2], patch_path = argv[4];
    std::string tmpdir = "/tmp/geocoder-diff-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    // Build string remap
    std::cerr << "Building string remap..." << std::endl;
    auto old_strings = read_file(old_dir + "/strings.bin");
    auto new_strings = read_file(new_dir + "/strings.bin");
    auto str_remap = build_string_remap(old_strings, new_strings);

    // Detect strides
    auto detect = [](const std::string& path, std::vector<size_t> cs) -> size_t {
        auto d = read_file(path);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect(old_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect(old_dir + "/interp_ways.bin", {20, 18});
    size_t admin_stride = detect(old_dir + "/admin_polygons.bin", {24, 20, 19});

    // Build patch data (uncompressed, will be zstd-compressed at the end)
    std::vector<char> patch;
    patch.insert(patch.end(), GCPATCH_MAGIC, GCPATCH_MAGIC + 8);
    uint32_t ver = 2, flags = 0; // version 2 = custom format
    wval(patch, &ver, 4); wval(patch, &flags, 4);

    // --- Section: String remap ---
    {
        std::vector<std::pair<uint32_t,uint32_t>> entries;
        size_t pos = 0;
        while (pos < old_strings.size()) {
            uint32_t old_off = static_cast<uint32_t>(pos);
            auto it = str_remap.find(old_off);
            entries.push_back({old_off, (it != str_remap.end()) ? it->second : old_off});
            pos += strlen(old_strings.data() + pos) + 1;
        }
        uint32_t marker = 0xFFFFFFFE, count = static_cast<uint32_t>(entries.size());
        wval(patch, &marker, 4); wval(patch, &count, 4);
        for (auto& [o, n] : entries) { wval(patch, &o, 4); wval(patch, &n, 4); }
        std::cerr << "  String remap: " << count << " entries (" << count * 8 << " bytes)" << std::endl;
    }

    // --- Section: Per-file merge sequences ---
    auto write_merge = [&](PatchFileId id, const std::string& name,
                             const std::vector<char>& old_data, const std::vector<char>& new_data,
                             size_t stride) {
        auto seq = build_merge_seq(old_data, new_data, stride);
        uint32_t fid = static_cast<uint32_t>(id);
        uint64_t os = old_data.size(), ns = new_data.size(), ss = seq.data.size();
        wval(patch, &fid, 4);
        uint32_t st = static_cast<uint32_t>(stride);
        wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8); wval(patch, &ss, 8);
        patch.insert(patch.end(), seq.data.begin(), seq.data.end());
        std::cerr << "  " << name << ": seq=" << ss << " bytes (" << std::fixed
                  << std::setprecision(2) << (ns > 0 ? ss * 100.0 / ns : 0) << "% of "
                  << ns / 1024 / 1024 << " MiB)" << std::endl;
    };

    // strings.bin: full new (small file)
    {
        uint32_t fid = static_cast<uint32_t>(PatchFileId::STRINGS);
        uint32_t st = 1; // byte stride
        uint64_t os = old_strings.size(), ns = new_strings.size();
        auto seq = build_merge_seq(old_strings, new_strings, 1);
        uint64_t ss = seq.data.size();
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8); wval(patch, &ss, 8);
        patch.insert(patch.end(), seq.data.begin(), seq.data.end());
        std::cerr << "  strings.bin: seq=" << ss << " bytes" << std::endl;
    }

    // addr_points.bin
    {
        auto old_data = read_file(old_dir + "/addr_points.bin");
        remap_addr_points(old_data, str_remap);
        auto new_data = read_file(new_dir + "/addr_points.bin");
        write_merge(PatchFileId::ADDR_POINTS, "addr_points.bin", old_data, new_data, 16);
    }

    // street_ways.bin (with node_offset fixup)
    {
        auto old_w = read_file(old_dir + "/street_ways.bin");
        remap_field(old_w, way_stride, way_stride == 12 ? 8 : 5, str_remap);
        auto new_w = read_file(new_dir + "/street_ways.bin");
        auto old_n = read_file(old_dir + "/street_nodes.bin");
        auto new_n = read_file(new_dir + "/street_nodes.bin");
        fixup_way_offsets(old_w, old_n, new_w, new_n, way_stride);
        write_merge(PatchFileId::STREET_WAYS, "street_ways.bin", old_w, new_w, way_stride);
    }

    // street_nodes.bin (no remap)
    {
        auto old_data = read_file(old_dir + "/street_nodes.bin");
        auto new_data = read_file(new_dir + "/street_nodes.bin");
        write_merge(PatchFileId::STREET_NODES, "street_nodes.bin", old_data, new_data, 8);
    }

    // interp_ways.bin
    {
        auto old_data = read_file(old_dir + "/interp_ways.bin");
        remap_field(old_data, interp_stride, interp_stride >= 20 ? 8 : 5, str_remap);
        auto new_data = read_file(new_dir + "/interp_ways.bin");
        auto old_n = read_file(old_dir + "/interp_nodes.bin");
        auto new_n = read_file(new_dir + "/interp_nodes.bin");
        fixup_interp_offsets(old_data, old_n, new_data, new_n, interp_stride);
        write_merge(PatchFileId::INTERP_WAYS, "interp_ways.bin", old_data, new_data, interp_stride);
    }

    // interp_nodes.bin
    {
        auto old_data = read_file(old_dir + "/interp_nodes.bin");
        auto new_data = read_file(new_dir + "/interp_nodes.bin");
        write_merge(PatchFileId::INTERP_NODES, "interp_nodes.bin", old_data, new_data, 8);
    }

    // admin_polygons.bin
    {
        auto old_data = read_file(old_dir + "/admin_polygons.bin");
        remap_field(old_data, admin_stride, 8, str_remap);
        auto new_data = read_file(new_dir + "/admin_polygons.bin");
        auto old_v = read_file(old_dir + "/admin_vertices.bin");
        auto new_v = read_file(new_dir + "/admin_vertices.bin");
        fixup_admin_offsets(old_data, old_v, new_data, new_v, admin_stride);
        write_merge(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin", old_data, new_data, admin_stride);
    }

    // admin_vertices.bin
    {
        auto old_data = read_file(old_dir + "/admin_vertices.bin");
        auto new_data = read_file(new_dir + "/admin_vertices.bin");
        write_merge(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", old_data, new_data, 8);
    }

    // --- Section: Entry/cell files as full new data ---
    // These are small relative to data files and rebuilt by patch tool isn't reliable.
    // Include them as full replacement. zstd transport compression handles the rest.
    for (auto& [id, name] : std::vector<std::pair<PatchFileId, std::string>>{
        {PatchFileId::GEO_CELLS, "geo_cells.bin"},
        {PatchFileId::STREET_ENTRIES, "street_entries.bin"},
        {PatchFileId::ADDR_ENTRIES, "addr_entries.bin"},
        {PatchFileId::INTERP_ENTRIES, "interp_entries.bin"},
        {PatchFileId::ADMIN_CELLS, "admin_cells.bin"},
        {PatchFileId::ADMIN_ENTRIES, "admin_entries.bin"}}) {
        auto data = read_file(new_dir + "/" + name);
        uint32_t fid = static_cast<uint32_t>(id);
        uint32_t st = 0; // stride 0 = full replacement
        uint64_t os = 0, ns = data.size(), ss = data.size();
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8); wval(patch, &ss, 8);
        patch.insert(patch.end(), data.begin(), data.end());
        std::cerr << "  " << name << ": full " << data.size() << " bytes" << std::endl;
    }

    // End marker
    uint32_t end_marker = 0xFFFFFFFF;
    wval(patch, &end_marker, 4);

    std::cerr << "\nUncompressed patch: " << patch.size() << " bytes ("
              << patch.size() / 1024 / 1024 << " MiB)" << std::endl;

    // Compress whole patch with zstd for transport
    {
        std::string raw_path = tmpdir + "/patch.raw";
        write_file(raw_path, patch);
        std::string cmd = "zstd -19 '" + raw_path + "' -o '" + patch_path + "' -f --quiet 2>/dev/null";
        system(cmd.c_str());
        auto compressed = read_file(patch_path);
        std::cerr << "Compressed patch: " << compressed.size() << " bytes ("
                  << compressed.size() / 1024 / 1024 << " MiB)" << std::endl;
        remove(raw_path.c_str());
    }

    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());
    return 0;
}

-    // street_ways.bin (with node_offset fixup)

+    // street_ways.bin (string remap only, no offset fixup — merge handles byte diffs)

-        auto old_n = read_file(old_dir + "/street_nodes.bin");

-        auto new_n = read_file(new_dir + "/street_nodes.bin");

-        fixup_way_offsets(old_w, old_n, new_w, new_n, way_stride);


Same for interps and admin:


    // interp_ways.bin
    {
        auto old_data = read_file(old_dir + "/interp_ways.bin");
        remap_field(old_data, interp_stride, interp_stride >= 20 ? 8 : 5, str_remap);
        auto new_data = read_file(new_dir + "/interp_ways.bin");
        auto old_n = read_file(old_dir + "/interp_nodes.bin");
        auto new_n = read_file(new_dir + "/interp_nodes.bin");
        fixup_interp_offsets(old_data, old_n, new_data, new_n, interp_stride);
        write_merge(PatchFileId::INTERP_WAYS, "interp_ways.bin", old_data, new_data, interp_stride);
    }

    // interp_ways.bin (string remap only)
    {
        auto old_data = read_file(old_dir + "/interp_ways.bin");
        remap_field(old_data, interp_stride, interp_stride >= 20 ? 8 : 5, str_remap);
        auto new_data = read_file(new_dir + "/interp_ways.bin");
        write_merge(PatchFileId::INTERP_WAYS, "interp_ways.bin", old_data, new_data, interp_stride);
    }