        uint32_t stride = read_u32();
        uint64_t old_size = read_u64(), new_size = read_u64();

        if (file_id >= (uint32_t)PatchFileId::COUNT) {
            std::cerr << "  Unknown file " << file_id << std::endl;
            break;
        }
        const char* filename = patch_file_names[file_id];

        if (stride == 0) {
            // Full replacement: n_fixups=0, then seq_size = data size
            uint32_t n_fix = read_u32(); (void)n_fix; // always 0 for full replacement
            uint64_t data_size = read_u64();
            std::vector<char> data(patch.data() + pos, patch.data() + pos + data_size);
            write_file(out_dir + "/" + std::string(filename), data);
            std::cerr << "  " << filename << ": full replacement " << data.size() << " bytes" << std::endl;
            pos += data_size;
            continue;
        }

// geocoder-patch v3: Apply custom patch format.
//
// Decompresses zstd transport layer, then:
// 1. Reads string remap, applies to old data files
// 2. Replays merge sequences to reconstruct new data files
// 3. Writes full-replacement entry/cell files
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

// Merge ops (must match diff tool)
enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    // Decompress zstd transport layer
    std::string raw_path = tmpdir + "/patch.raw";
    {
        std::string cmd = "zstd -d '" + patch_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0) {
            std::cerr << "Failed to decompress patch" << std::endl; return 1;
        }
    }
    auto patch = read_file(raw_path);
    remove(raw_path.c_str());
    std::cerr << "Decompressed patch: " << patch.size() << " bytes" << std::endl;

    size_t pos = 0;
    auto read_u32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, patch.data() + pos, 4); pos += 4; return v; };
    auto read_u64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, patch.data() + pos, 8); pos += 8; return v; };

    // Check header
    if (memcmp(patch.data(), GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch magic" << std::endl; return 1;
    }
    pos = 8;
    uint32_t version = read_u32();
    if (version != 2) { std::cerr << "Expected patch v2, got " << version << std::endl; return 1; }
    uint32_t flags = read_u32();
    (void)flags;

    // Read string remap
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t marker = read_u32();
        if (marker != 0xFFFFFFFE) { std::cerr << "Expected remap section" << std::endl; return 1; }
        uint32_t count = read_u32();
        for (uint32_t i = 0; i < count; i++) {
            uint32_t old_off = read_u32(), new_off = read_u32();
            if (old_off != new_off) str_remap[old_off] = new_off;
        }
        std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;
    }

    // Detect strides
    auto detect = [&](const std::string& name, std::vector<size_t> cs) -> size_t {
        auto d = read_file(cur_dir + "/" + name);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect("street_ways.bin", {12, 9});
    size_t interp_stride = detect("interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect("admin_polygons.bin", {24, 20, 19});

    // String remap functions
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t file_id, size_t stride) {
        if (str_remap.empty()) return;
        std::vector<size_t> field_offsets;
        if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) field_offsets = {8, 12};
        else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) field_offsets = {(stride == 12) ? (size_t)8 : (size_t)5};
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) field_offsets = {(stride >= 20) ? (size_t)8 : (size_t)5};
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) field_offsets = {8};
        else return;

        for (size_t i = 0; i + stride <= data.size(); i += stride)
            for (size_t off : field_offsets) {
                uint32_t v; memcpy(&v, data.data() + i + off, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + off, &it->second, 4);
            }
    };

    // Process sections
    while (pos < patch.size()) {
        uint32_t file_id = read_u32();
        if (file_id == 0xFFFFFFFF) break;

        uint32_t stride = read_u32();
        uint64_t old_size = read_u64(), new_size = read_u64(), seq_size = read_u64();

        if (file_id >= (uint32_t)PatchFileId::COUNT) {
            pos += seq_size;
            continue;
        }
        const char* filename = patch_file_names[file_id];

        if (stride == 0) {
            // Full replacement
            std::vector<char> data(patch.data() + pos, patch.data() + pos + seq_size);
            write_file(out_dir + "/" + std::string(filename), data);
            std::cerr << "  " << filename << ": full replacement " << data.size() << " bytes" << std::endl;
            pos += seq_size;
            continue;
        }

        // Merge sequence: load old file, apply remap + fixups, replay sequence
        auto old_data = read_file(cur_dir + "/" + std::string(filename));
        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        // Apply string remap to old data
        apply_str_remap(old_data, file_id, actual_stride);

        // Read and apply fixups (node_offset/vertex_offset fixes)
        uint32_t n_fixups; memcpy(&n_fixups, patch.data() + pos, 4); pos += 4;
        for (uint32_t i = 0; i < n_fixups; i++) {
            uint32_t idx, val;
            memcpy(&idx, patch.data() + pos, 4); pos += 4;
            memcpy(&val, patch.data() + pos, 4); pos += 4;
            size_t byte_pos = (size_t)idx * actual_stride;
            if (byte_pos + 4 <= old_data.size())
                memcpy(old_data.data() + byte_pos, &val, 4);
        }

        // Note: offset fixups (node_offset/vertex_offset) are already baked into the
        // merge sequence — matched records from old have been compared WITH fixups applied
        // by the diff tool. The merge MATCH ops copy from the remapped old, which has
        // the ORIGINAL offset values. But the diff matched against the fixed-up version.
        //
        // Wait — this is a problem. The diff applied fixups to old, then matched.
        // The patch only applies string remap, not fixups. So matched bytes differ.
        //
        // For the merge to work, we need the old file to be in the same state as
        // when the diff computed the merge sequence. The diff applied string remap
        // + offset fixups. We applied string remap. We need to also apply fixups.
        //
        // But we don't have the fixup data in v2 format...
        //
        // Actually: the merge sequence copies bytes from old. If old bytes after string
        // remap don't match new bytes (because offsets differ), the diff would have
        // encoded them as DELETE+INSERT, not MATCH. So MATCH ops only fire when bytes
        // are identical after string remap. For ways with different node_offsets, the
        // diff tool fixed up node_offsets BEFORE comparing, so MATCH ops assume the
        // fixup was applied.
        //
        // The patch tool must apply the same fixups. Since we don't have fixup tables
        // in v2, the diff tool needs to NOT fix offsets and instead let the merge
        // handle the byte differences. This means ways with different node_offsets
        // become DELETE+INSERT pairs instead of MATCH.
        //
        // This is less optimal but correct. The diff tool should NOT fixup offsets
        // when using merge sequences. Let me fix this in a follow-up.

        // Replay merge sequence
        std::vector<char> output;
        output.reserve(new_size);
        size_t old_pos = 0;
        size_t seq_end = pos + seq_size;

        while (pos < seq_end) {
            uint8_t op = static_cast<uint8_t>(patch[pos]); pos++;
            uint32_t count; memcpy(&count, patch.data() + pos, 4); pos += 4;

            if (op == OP_MATCH_RUN) {
                // Copy count records from old
                size_t bytes = count * actual_stride;
                if (old_pos + bytes <= old_data.size()) {
                    output.insert(output.end(), old_data.data() + old_pos,
                                 old_data.data() + old_pos + bytes);
                }
                old_pos += bytes;
            } else if (op == OP_INSERT_RUN) {
                // Read count records from patch
                size_t bytes = count * actual_stride;
                output.insert(output.end(), patch.data() + pos, patch.data() + pos + bytes);
                pos += bytes;
            } else if (op == OP_DELETE_RUN) {
                // Skip count records in old
                old_pos += count * actual_stride;
            }
        }

        write_file(out_dir + "/" + std::string(filename), output);
        std::cerr << "  " << filename << ": " << output.size() << " bytes" << std::endl;
    }

    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

-        uint64_t old_size = read_u64(), new_size = read_u64(), seq_size = read_u64();

+        uint64_t old_size = read_u64(), new_size = read_u64();

+            std::cerr << "  Unknown file " << file_id << std::endl;

-            std::vector<char> data(patch.data() + pos, patch.data() + pos + seq_size);

+            // Full replacement: n_fixups=0, then seq_size = data size

+            uint32_t n_fix = read_u32(); (void)n_fix; // always 0 for full replacement

+            uint64_t data_size = read_u64();

+            std::vector<char> data(patch.data() + pos, patch.data() + pos + data_size);


Now read seq_size after fixups in the merge path:


Edit

        // Read and apply fixups (node_offset/vertex_offset fixes)
        uint32_t n_fixups; memcpy(&n_fixups, patch.data() + pos, 4); pos += 4;
        for (uint32_t i = 0; i < n_fixups; i++) {
            uint32_t idx, val;
            memcpy(&idx, patch.data() + pos, 4); pos += 4;
            memcpy(&val, patch.data() + pos, 4); pos += 4;
            size_t byte_pos = (size_t)idx * actual_stride;
            if (byte_pos + 4 <= old_data.size())
                memcpy(old_data.data() + byte_pos, &val, 4);
        }

        // Replay merge sequence
        std::vector<char> output;
        output.reserve(new_size);
        size_t old_pos = 0;
        size_t seq_end = pos + seq_size;

        // Read and apply fixups (node_offset/vertex_offset fixes)
        uint32_t n_fixups = read_u32();
        for (uint32_t i = 0; i < n_fixups; i++) {
            uint32_t idx = read_u32(), val = read_u32();
            size_t byte_pos = (size_t)idx * actual_stride;
            if (byte_pos + 4 <= old_data.size())
                memcpy(old_data.data() + byte_pos, &val, 4);
        }
        if (n_fixups > 0) std::cerr << "    Applied " << n_fixups << " fixups" << std::endl;

        // Read merge sequence size and replay
        uint64_t seq_size = read_u64();
        std::vector<char> output;
        output.reserve(new_size);
        size_t old_pos = 0;
        size_t seq_end = pos + seq_size;


<tool_use_error>String to replace not found in file.
String:         // Read and apply fixups (node_offset/vertex_offset fixes)
        uint32_t n_fixups; memcpy(&n_fixups, patch.data() + pos, 4); pos += 4;
        for (uint32_t i = 0; i < n_fixups; i++) {
            uint32_t idx, val;
            memcpy(&idx, patch.data() + pos, 4); pos += 4;
            memcpy(&val, patch.data() + pos, 4); pos += 4;
            size_t byte_pos = (size_t)idx * actual_stride;
            if (byte_pos + 4 <= old_data.size())
                memcpy(old_data.data() + byte_pos, &val, 4);
        }

        // Replay merge sequence
        std::vector<char> output;
        output.reserve(new_size);
        size_t old_pos = 0;
        size_t seq_end = pos + seq_size;</tool_use_error>

Error: String to replace not found in file.
String:         // Read and apply fixups (node_offset/vertex_offset fixes)
        uint32_t n_fixups; memcpy(&n_fixups, patch.data() + pos, 4); pos += 4;
        for (uint32_t i = 0; i < n_fixups; i++) {
            uint32_t idx, val;
            memcpy(&idx, patch.data() + pos, 4); pos += 4;
            memcpy(&val, patch.data() + pos, 4); pos += 4;
            size_t byte_pos = (size_t)idx * actual_stride;
            if (byte_pos + 4 <= old_data.size())
                memcpy(old_data.data() + byte_pos, &val, 4);
        }

        // Replay merge sequence
        std::vector<char> output;
        output.reserve(new_size);
        size_t old_pos = 0;
        size_t seq_end = pos + seq_size;


   138→        uint32_t n_fixups; memcpy(&n_fixups, patch.data() + pos, 4); pos += 4;
   139→        for (uint32_t i = 0; i < n_fixups; i++) {
   140→            uint32_t idx, val;
   141→            memcpy(&idx, patch.data() + pos, 4); pos += 4;
   142→            memcpy(&val, patch.data() + pos, 4); pos += 4;
   143→            size_t byte_pos = (size_t)idx * actual_stride;
   144→            if (byte_pos + 4 <= old_data.size())
   145→                memcpy(old_data.data() + byte_pos, &val, 4);
   146→        }
   147→
   148→        // Note: offset fixups (node_offset/vertex_offset) are already baked into the
   149→        // merge sequence — matched records from old have been compared WITH fixups applied
   150→        // by the diff tool. The merge MATCH ops copy from the remapped old, which has
   151→        // the ORIGINAL offset values. But the diff matched against the fixed-up version.
   152→        //
   153→        // Wait — this is a problem. The diff applied fixups to old, then matched.
   154→        // The patch only applies string remap, not fixups. So matched bytes differ.
   155→        //
   156→        // For the merge to work, we need the old file to be in the same state as
   157→        // when the diff computed the merge sequence. The diff applied string remap
   158→        // + offset fixups. We applied string remap. We need to also apply fixups.
   159→        //
   160→        // But we don't have the fixup data in v2 format...
   161→        //
   162→        // Actually: the merge sequence copies bytes from old. If old bytes after string

        uint32_t n_fixups; memcpy(&n_fixups, patch.data() + pos, 4); pos += 4;
        for (uint32_t i = 0; i < n_fixups; i++) {
            uint32_t idx, val;
            memcpy(&idx, patch.data() + pos, 4); pos += 4;
            memcpy(&val, patch.data() + pos, 4); pos += 4;
            size_t byte_pos = (size_t)idx * actual_stride;
            if (byte_pos + 4 <= old_data.size())
                memcpy(old_data.data() + byte_pos, &val, 4);
        }

        // Note: offset fixups (node_offset/vertex_offset) are already baked into the
        // merge sequence — matched records from old have been compared WITH fixups applied
        // by the diff tool. The merge MATCH ops copy from the remapped old, which has
        // the ORIGINAL offset values. But the diff matched against the fixed-up version.
        //
        // Wait — this is a problem. The diff applied fixups to old, then matched.
        // The patch only applies string remap, not fixups. So matched bytes differ.
        //
        // For the merge to work, we need the old file to be in the same state as
        // when the diff computed the merge sequence. The diff applied string remap
        // + offset fixups. We applied string remap. We need to also apply fixups.
        //
        // But we don't have the fixup data in v2 format...
        //
        // Actually: the merge sequence copies bytes from old. If old bytes after string


I see — the old comment block is still there. Let me replace the whole section:


        uint32_t n_fixups; memcpy(&n_fixups, patch.data() + pos, 4); pos += 4;
        for (uint32_t i = 0; i < n_fixups; i++) {
            uint32_t idx, val;
            memcpy(&idx, patch.data() + pos, 4); pos += 4;
            memcpy(&val, patch.data() + pos, 4); pos += 4;
            size_t byte_pos = (size_t)idx * actual_stride;
            if (byte_pos + 4 <= old_data.size())
                memcpy(old_data.data() + byte_pos, &val, 4);
        }

        // Note: offset fixups (node_offset/vertex_offset) are already baked into the
        // merge sequence — matched records from old have been compared WITH fixups applied
        // by the diff tool. The merge MATCH ops copy from the remapped old, which has
        // the ORIGINAL offset values. But the diff matched against the fixed-up version.
        //
        // Wait — this is a problem. The diff applied fixups to old, then matched.
        // The patch only applies string remap, not fixups. So matched bytes differ.
        //
        // For the merge to work, we need the old file to be in the same state as
        // when the diff computed the merge sequence. The diff applied string remap
        // + offset fixups. We applied string remap. We need to also apply fixups.
        //
        // But we don't have the fixup data in v2 format...
        //
        // Actually: the merge sequence copies bytes from old. If old bytes after string
        // remap don't match new bytes (because offsets differ), the diff would have
        // encoded them as DELETE+INSERT, not MATCH. So MATCH ops only fire when bytes
        // are identical after string remap. For ways with different node_offsets, the
        // diff tool fixed up node_offsets BEFORE comparing, so MATCH ops assume the
        // fixup was applied.
        //
        // The patch tool must apply the same fixups. Since we don't have fixup tables
        // in v2, the diff tool needs to NOT fix offsets and instead let the merge
        // handle the byte differences. This means ways with different node_offsets
        // become DELETE+INSERT pairs instead of MATCH.
        //
        // This is less optimal but correct. The diff tool should NOT fixup offsets
        // when using merge sequences. Let me fix this in a follow-up.

        // Replay merge sequence
        std::vector<char> output;
        output.reserve(new_size);
        size_t old_pos = 0;
        size_t seq_end = pos + seq_size;

        // Read and apply fixups (offset field patches for matched records)
        uint32_t n_fixups = read_u32();
        for (uint32_t i = 0; i < n_fixups; i++) {
            uint32_t idx = read_u32(), val = read_u32();
            size_t byte_pos = (size_t)idx * actual_stride;
            if (byte_pos + 4 <= old_data.size())
                memcpy(old_data.data() + byte_pos, &val, 4);
        }
        if (n_fixups > 0) std::cerr << "    Applied " << n_fixups << " fixups" << std::endl;

        // Read merge sequence size and replay
        uint64_t seq_size = read_u64();
        std::vector<char> output;
        output.reserve(new_size);
        size_t old_pos = 0;
        size_t seq_end = pos + seq_size;


The file /root/traccar-geocoder/builder/tools/geocoder_patch.cpp has been updated successfully.

        // Read and apply fixups (offset field patches for matched records)
        uint32_t n_fixups = read_u32();
        for (uint32_t i = 0; i < n_fixups; i++) {
            uint32_t idx = read_u32(), val = read_u32();
            size_t byte_pos = (size_t)idx * actual_stride;
            if (byte_pos + 4 <= old_data.size())
                memcpy(old_data.data() + byte_pos, &val, 4);
        }
        if (n_fixups > 0) std::cerr << "    Applied " << n_fixups << " fixups" << std::endl;

        // Read merge sequence size and replay
        uint64_t seq_size = read_u64();
        std::vector<char> output;
        output.reserve(new_size);
        size_t old_pos = 0;
        size_t seq_end = pos + seq_size;

// geocoder-patch v3: Apply custom patch format.
//
// Decompresses zstd transport layer, then:
// 1. Reads string remap, applies to old data files
// 2. Replays merge sequences to reconstruct new data files
// 3. Writes full-replacement entry/cell files
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

// Merge ops (must match diff tool)
enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    // Decompress zstd transport layer
    std::string raw_path = tmpdir + "/patch.raw";
    {
        std::string cmd = "zstd -d '" + patch_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0) {
            std::cerr << "Failed to decompress patch" << std::endl; return 1;
        }
    }
    auto patch = read_file(raw_path);
    remove(raw_path.c_str());
    std::cerr << "Decompressed patch: " << patch.size() << " bytes" << std::endl;

    size_t pos = 0;
    auto read_u32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, patch.data() + pos, 4); pos += 4; return v; };
    auto read_u64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, patch.data() + pos, 8); pos += 8; return v; };

    // Check header
    if (memcmp(patch.data(), GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch magic" << std::endl; return 1;
    }
    pos = 8;
    uint32_t version = read_u32();
    if (version != 2) { std::cerr << "Expected patch v2, got " << version << std::endl; return 1; }
    uint32_t flags = read_u32();
    (void)flags;

    // Read string remap
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t marker = read_u32();
        if (marker != 0xFFFFFFFE) { std::cerr << "Expected remap section" << std::endl; return 1; }
        uint32_t count = read_u32();
        for (uint32_t i = 0; i < count; i++) {
            uint32_t old_off = read_u32(), new_off = read_u32();
            if (old_off != new_off) str_remap[old_off] = new_off;
        }
        std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;
    }

    // Detect strides
    auto detect = [&](const std::string& name, std::vector<size_t> cs) -> size_t {
        auto d = read_file(cur_dir + "/" + name);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect("street_ways.bin", {12, 9});
    size_t interp_stride = detect("interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect("admin_polygons.bin", {24, 20, 19});

    // String remap functions
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t file_id, size_t stride) {
        if (str_remap.empty()) return;
        std::vector<size_t> field_offsets;
        if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) field_offsets = {8, 12};
        else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) field_offsets = {(stride == 12) ? (size_t)8 : (size_t)5};
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) field_offsets = {(stride >= 20) ? (size_t)8 : (size_t)5};
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) field_offsets = {8};
        else return;

        for (size_t i = 0; i + stride <= data.size(); i += stride)
            for (size_t off : field_offsets) {
                uint32_t v; memcpy(&v, data.data() + i + off, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + off, &it->second, 4);
            }
    };

    // Process sections
    while (pos < patch.size()) {
        uint32_t file_id = read_u32();
        if (file_id == 0xFFFFFFFF) break;

        uint32_t stride = read_u32();
        uint64_t old_size = read_u64(), new_size = read_u64();

        if (file_id >= (uint32_t)PatchFileId::COUNT) {
            std::cerr << "  Unknown file " << file_id << std::endl;
            break;
        }
        const char* filename = patch_file_names[file_id];

        if (stride == 0) {
            // Full replacement: n_fixups=0, then seq_size = data size
            uint32_t n_fix = read_u32(); (void)n_fix; // always 0 for full replacement
            uint64_t data_size = read_u64();
            std::vector<char> data(patch.data() + pos, patch.data() + pos + data_size);
            write_file(out_dir + "/" + std::string(filename), data);
            std::cerr << "  " << filename << ": full replacement " << data.size() << " bytes" << std::endl;
            pos += data_size;
            continue;
        }

        // Merge sequence: load old file, apply remap + fixups, replay sequence
        auto old_data = read_file(cur_dir + "/" + std::string(filename));
        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        // Apply string remap to old data
        apply_str_remap(old_data, file_id, actual_stride);

        // Read and apply fixups (node_offset/vertex_offset fixes)
        uint32_t n_fixups; memcpy(&n_fixups, patch.data() + pos, 4); pos += 4;
        for (uint32_t i = 0; i < n_fixups; i++) {
            uint32_t idx, val;
            memcpy(&idx, patch.data() + pos, 4); pos += 4;
            memcpy(&val, patch.data() + pos, 4); pos += 4;
            size_t byte_pos = (size_t)idx * actual_stride;
            if (byte_pos + 4 <= old_data.size())
                memcpy(old_data.data() + byte_pos, &val, 4);
        }

        // Note: offset fixups (node_offset/vertex_offset) are already baked into the
        // merge sequence — matched records from old have been compared WITH fixups applied
        // by the diff tool. The merge MATCH ops copy from the remapped old, which has
        // the ORIGINAL offset values. But the diff matched against the fixed-up version.
        //
        // Wait — this is a problem. The diff applied fixups to old, then matched.
        // The patch only applies string remap, not fixups. So matched bytes differ.
        //
        // For the merge to work, we need the old file to be in the same state as
        // when the diff computed the merge sequence. The diff applied string remap
        // + offset fixups. We applied string remap. We need to also apply fixups.
        //
        // But we don't have the fixup data in v2 format...
        //
        // Actually: the merge sequence copies bytes from old. If old bytes after string
        // remap don't match new bytes (because offsets differ), the diff would have
        // encoded them as DELETE+INSERT, not MATCH. So MATCH ops only fire when bytes
        // are identical after string remap. For ways with different node_offsets, the
        // diff tool fixed up node_offsets BEFORE comparing, so MATCH ops assume the
        // fixup was applied.
        //
        // The patch tool must apply the same fixups. Since we don't have fixup tables
        // in v2, the diff tool needs to NOT fix offsets and instead let the merge
        // handle the byte differences. This means ways with different node_offsets
        // become DELETE+INSERT pairs instead of MATCH.
        //
        // This is less optimal but correct. The diff tool should NOT fixup offsets
        // when using merge sequences. Let me fix this in a follow-up.

        // Replay merge sequence
        std::vector<char> output;
        output.reserve(new_size);
        size_t old_pos = 0;
        size_t seq_end = pos + seq_size;

        while (pos < seq_end) {
            uint8_t op = static_cast<uint8_t>(patch[pos]); pos++;
            uint32_t count; memcpy(&count, patch.data() + pos, 4); pos += 4;

            if (op == OP_MATCH_RUN) {
                // Copy count records from old
                size_t bytes = count * actual_stride;
                if (old_pos + bytes <= old_data.size()) {
                    output.insert(output.end(), old_data.data() + old_pos,
                                 old_data.data() + old_pos + bytes);
                }
                old_pos += bytes;
            } else if (op == OP_INSERT_RUN) {
                // Read count records from patch
                size_t bytes = count * actual_stride;
                output.insert(output.end(), patch.data() + pos, patch.data() + pos + bytes);
                pos += bytes;
            } else if (op == OP_DELETE_RUN) {
                // Skip count records in old
                old_pos += count * actual_stride;
            }
        }

        write_file(out_dir + "/" + std::string(filename), output);
        std::cerr << "  " << filename << ": " << output.size() << " bytes" << std::endl;
    }

    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

-        uint32_t n_fixups; memcpy(&n_fixups, patch.data() + pos, 4); pos += 4;

+        // Read and apply fixups (offset field patches for matched records)

+        uint32_t n_fixups = read_u32();

-            memcpy(&idx, patch.data() + pos, 4); pos += 4;

-            memcpy(&val, patch.data() + pos, 4); pos += 4;

+            uint32_t idx = read_u32(), val = read_u32();

+        if (n_fixups > 0) std::cerr << "    Applied " << n_fixups << " fixups" << std::endl;

-        // Note: offset fixups (node_offset/vertex_offset) are already baked into the

-        // merge sequence — matched records from old have been compared WITH fixups applied

-        // by the diff tool. The merge MATCH ops copy from the remapped old, which has

-        // the ORIGINAL offset values. But the diff matched against the fixed-up version.

-        // Wait — this is a problem. The diff applied fixups to old, then matched.

-        // The patch only applies string remap, not fixups. So matched bytes differ.

-        // For the merge to work, we need the old file to be in the same state as

-        // when the diff computed the merge sequence. The diff applied string remap

-        // + offset fixups. We applied string remap. We need to also apply fixups.

-        // But we don't have the fixup data in v2 format...

-        // Actually: the merge sequence copies bytes from old. If old bytes after string

-        // remap don't match new bytes (because offsets differ), the diff would have

-        // encoded them as DELETE+INSERT, not MATCH. So MATCH ops only fire when bytes

-        // are identical after string remap. For ways with different node_offsets, the

-        // diff tool fixed up node_offsets BEFORE comparing, so MATCH ops assume the

-        // The patch tool must apply the same fixups. Since we don't have fixup tables

-        // in v2, the diff tool needs to NOT fix offsets and instead let the merge

-        // handle the byte differences. This means ways with different node_offsets

-        // become DELETE+INSERT pairs instead of MATCH.

-        // This is less optimal but correct. The diff tool should NOT fixup offsets

-        // when using merge sequences. Let me fix this in a follow-up.

+        // Read merge sequence size and replay

+        uint64_t seq_size = read_u64();


Now build and test:


Bash

cmake --build build -j$(nproc) --target geocoder-diff --target geocoder-patch 2>&1 | grep -E "(error:|Built)" | head -5

Build v3


[100%] Built target geocoder-diff
[100%] Built target geocoder-patch


sshpass -p 'elon-is-insane' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" /root/traccar-geocoder/builder/build/geocoder-diff /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:~/ 2>&1 && sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
echo "=== DIFF v3 ==="
rm -f /home/michtest/test-v3b.gcpatch
time /home/michtest/geocoder-diff /home/michtest/det-A /home/michtest/det-new -o /home/michtest/test-v3b.gcpatch 2>&1
echo ""
echo "=== APPLY v3 ==="
rm -rf /home/michtest/patched-v3b
time /home/michtest/geocoder-patch /home/michtest/det-A /home/michtest/test-v3b.gcpatch -o /home/michtest/patched-v3b 2>&1
echo ""
echo "=== VERIFY ==="
all_match=true
for f in /home/michtest/det-new/*.bin; do
    b=$(basename $f); p=/home/michtest/patched-v3b/$b
    [ -f "$p" ] && cmp -s "$f" "$p" && echo "  $b: MATCH" || { echo "  $b: FAIL"; all_match=false; }
done
$all_match && echo "ALL FILES MATCH" || echo "SOME DIFFER"
' 2>&1

Full v3 test with fixups


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
=== DIFF v3 ===
Building string remap...
  String remap: 4982052 entries (39856416 bytes)
  strings.bin: seq=106752118 bytes
  addr_points.bin: seq=1066728884 fixups=0 (77.53%)
  street_ways.bin: seq=2485811 fixups=19372497 (1.06%)
  street_nodes.bin: seq=10898537 fixups=0 (0.75%)
  interp_ways.bin: seq=590 fixups=13395 (0.18%)
  interp_nodes.bin: seq=2085 fixups=0 (0.29%)
  admin_polygons.bin: seq=82674 fixups=449077 (0.76%)
  admin_vertices.bin: seq=82545588 fixups=0 (28.36%)
  geo_cells.bin: full 1674295440 bytes
  street_entries.bin: full 586571766 bytes
  addr_entries.bin: full 395005236 bytes
  interp_entries.bin: full 159386 bytes
  admin_cells.bin: full 1692000 bytes
  admin_entries.bin: full 6122260 bytes

Uncompressed patch: 4131879047 bytes (3940 MiB)
Compressed patch: 1019916344 bytes (972 MiB)

=== APPLY v3 ===

real	6m54.703s
user	24m53.366s
sys	0m5.370s
Decompressed patch: 4131879047 bytes
Loaded 4980054 string remap entries
    Applied 106752118 fixups

=== VERIFY ===
bash: line 8: 129851 Segmentation fault      /home/michtest/geocoder-patch /home/michtest/det-A /home/michtest/test-v3b.gcpatch -o /home/michtest/patched-v3b 2>&1

real	0m6.564s
user	0m5.679s
sys	0m1.954s
  addr_entries.bin: FAIL
  addr_points.bin: FAIL
  admin_cells.bin: FAIL
  admin_entries.bin: FAIL
  admin_polygons.bin: FAIL
  admin_vertices.bin: FAIL
  geo_cells.bin: FAIL
  interp_entries.bin: FAIL
  interp_nodes.bin: FAIL
  interp_ways.bin: FAIL
  street_entries.bin: FAIL
  street_nodes.bin: FAIL
  street_ways.bin: FAIL
  strings.bin: FAIL
SOME DIFFER


Segfault during apply. "Applied 106752118 fixups" — that's the string merge sequence being misinterpreted as fixups! The strings.bin section has stride=1 (byte-level merge), but the patch tool tries to read fixups before it. The format expects n_fixups ONLY for merge sections, but strings.bin is also a merge section.

The issue: strings.bin doesn't need fixups. Let me make fixups only apply for files with stride > 1, or always write n_fixups=0 for files without fixups:

The diff tool's `write_merge` function checks `file_fixups` — for strings.bin there are no fixups. `n_fixups = 0`. So it writes `n_fixups=0`. The patch tool reads `n_fixups=0`, skips the loop, reads `seq_size`. This should be fine.

Let me look at the diff tool's entry file writing — the full replacement sections need to also write n_fixups=0 and seq_size:


   330→        while (pos < old_strings.size()) {
   331→            uint32_t old_off = static_cast<uint32_t>(pos);
   332→            auto it = str_remap.find(old_off);
   333→            entries.push_back({old_off, (it != str_remap.end()) ? it->second : old_off});
   334→            pos += strlen(old_strings.data() + pos) + 1;
   335→        }
   336→        uint32_t marker = 0xFFFFFFFE, count = static_cast<uint32_t>(entries.size());
   337→        wval(patch, &marker, 4); wval(patch, &count, 4);
   338→        for (auto& [o, n] : entries) { wval(patch, &o, 4); wval(patch, &n, 4); }
   339→        std::cerr << "  String remap: " << count << " entries (" << count * 8 << " bytes)" << std::endl;
   340→    }
   341→
   342→    // --- Section: Per-file merge sequences ---
   343→    // Track fixups per file (applied before merge, included in patch for patch tool)
   344→    std::unordered_map<uint32_t, std::vector<std::pair<uint32_t,uint32_t>>> file_fixups;
   345→
   346→    auto write_merge = [&](PatchFileId id, const std::string& name,
   347→                             const std::vector<char>& old_data, const std::vector<char>& new_data,
   348→                             size_t stride) {
   349→        auto seq = build_merge_seq(old_data, new_data, stride);

        while (pos < old_strings.size()) {
            uint32_t old_off = static_cast<uint32_t>(pos);
            auto it = str_remap.find(old_off);
            entries.push_back({old_off, (it != str_remap.end()) ? it->second : old_off});
            pos += strlen(old_strings.data() + pos) + 1;
        }
        uint32_t marker = 0xFFFFFFFE, count = static_cast<uint32_t>(entries.size());
        wval(patch, &marker, 4); wval(patch, &count, 4);
        for (auto& [o, n] : entries) { wval(patch, &o, 4); wval(patch, &n, 4); }
        std::cerr << "  String remap: " << count << " entries (" << count * 8 << " bytes)" << std::endl;
    }

    // --- Section: Per-file merge sequences ---
    // Track fixups per file (applied before merge, included in patch for patch tool)
    std::unordered_map<uint32_t, std::vector<std::pair<uint32_t,uint32_t>>> file_fixups;

    auto write_merge = [&](PatchFileId id, const std::string& name,
                             const std::vector<char>& old_data, const std::vector<char>& new_data,
                             size_t stride) {
        auto seq = build_merge_seq(old_data, new_data, stride);


491:        std::cerr << "  " << name << ": full " << data.size() << " bytes" << std::endl;


   479→        {PatchFileId::STREET_ENTRIES, "street_entries.bin"},
   480→        {PatchFileId::ADDR_ENTRIES, "addr_entries.bin"},
   481→        {PatchFileId::INTERP_ENTRIES, "interp_entries.bin"},
   482→        {PatchFileId::ADMIN_CELLS, "admin_cells.bin"},
   483→        {PatchFileId::ADMIN_ENTRIES, "admin_entries.bin"}}) {
   484→        auto data = read_file(new_dir + "/" + name);
   485→        uint32_t fid = static_cast<uint32_t>(id);
   486→        uint32_t st = 0; // stride 0 = full replacement
   487→        uint64_t os = 0, ns = data.size(), ss = data.size();
   488→        wval(patch, &fid, 4); wval(patch, &st, 4);
   489→        wval(patch, &os, 8); wval(patch, &ns, 8); wval(patch, &ss, 8);
   490→        patch.insert(patch.end(), data.begin(), data.end());
   491→        std::cerr << "  " << name << ": full " << data.size() << " bytes" << std::endl;
   492→    }
   493→

        {PatchFileId::STREET_ENTRIES, "street_entries.bin"},
        {PatchFileId::ADDR_ENTRIES, "addr_entries.bin"},
        {PatchFileId::INTERP_ENTRIES, "interp_entries.bin"},
        {PatchFileId::ADMIN_CELLS, "admin_cells.bin"},
        {PatchFileId::ADMIN_ENTRIES, "admin_entries.bin"}}) {
        auto data = read_file(new_dir + "/" + name);
        uint32_t fid = static_cast<uint32_t>(id);
        uint32_t st = 0; // stride 0 = full replacement
        uint64_t os = 0, ns = data.size(), ss = data.size();
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8); wval(patch, &ss, 8);
        patch.insert(patch.end(), data.begin(), data.end());
        std::cerr << "  " << name << ": full " << data.size() << " bytes" << std::endl;
    }


The full replacement sections write: fid, stride=0, old_size=0, new_size, data_size, data. But the patch tool expects: fid, stride, old_size, new_size, then for stride=0: n_fixups(u32), seq_size(u64), data.

The diff writes old_size(u64), new_size(u64), data_size(u64) = 24 bytes of header. The patch reads old_size(u64), new_size(u64) = 16 bytes, then n_fixups(u32) = 4 bytes — but the diff wrote data_size(u64) at that position. Mismatch!

Let me fix the full replacement sections to write the same format: n_fixups=0, then data_size, then data:


Edit

        auto data = read_file(new_dir + "/" + name);
        uint32_t fid = static_cast<uint32_t>(id);
        uint32_t st = 0; // stride 0 = full replacement
        uint64_t os = 0, ns = data.size(), ss = data.size();
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8); wval(patch, &ss, 8);
        patch.insert(patch.end(), data.begin(), data.end());
        std::cerr << "  " << name << ": full " << data.size() << " bytes" << std::endl;

        auto data = read_file(new_dir + "/" + name);
        uint32_t fid = static_cast<uint32_t>(id);
        uint32_t st = 0; // stride 0 = full replacement
        uint64_t os = 0, ns = data.size();
        uint32_t nfix = 0;
        uint64_t ss = data.size();
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8);
        wval(patch, &nfix, 4); // no fixups
        wval(patch, &ss, 8);   // data size
        patch.insert(patch.end(), data.begin(), data.end());
        std::cerr << "  " << name << ": full " << data.size() << " bytes" << std::endl;


The file /root/traccar-geocoder/builder/tools/geocoder_diff.cpp has been updated successfully.

        auto data = read_file(new_dir + "/" + name);
        uint32_t fid = static_cast<uint32_t>(id);
        uint32_t st = 0; // stride 0 = full replacement
        uint64_t os = 0, ns = data.size();
        uint32_t nfix = 0;
        uint64_t ss = data.size();
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8);
        wval(patch, &nfix, 4); // no fixups
        wval(patch, &ss, 8);   // data size
        patch.insert(patch.end(), data.begin(), data.end());
        std::cerr << "  " << name << ": full " << data.size() << " bytes" << std::endl;

// geocoder-diff v3: Fully custom patch format using merge-sequence encoding.
//
// For each data file: walk old (string-remapped) and new in parallel,
// match records by content, emit MATCH/INSERT/DELETE operations.
// For entry/cell files: include changed cell entry data directly.
// geo_cells rebuilt by patch tool from entries (not delta-patched).
//
// Patch format: custom binary, zstd-compressed as a whole for transport.
//
// Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>

#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <functional>
#include <iomanip>
#include <iostream>
#include <string>
#include <unistd.h>
#include <unordered_map>
#include <unordered_set>
#include <vector>

#include "patch_format.h"

// --- Merge sequence encoding ---
// Walk two sorted arrays (old remapped, new), matching by record bytes.
// Emit: MATCH(n) = copy n records from old, INSERT(n,data) = add n new records,
//       DELETE(n) = skip n old records.

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

struct MergeSequence {
    std::vector<char> data; // serialized ops

    void add_match(uint32_t count) {
        uint8_t op = OP_MATCH_RUN;
        data.insert(data.end(), (char*)&op, (char*)&op + 1);
        data.insert(data.end(), (char*)&count, (char*)&count + 4);
    }
    void add_delete(uint32_t count) {
        uint8_t op = OP_DELETE_RUN;
        data.insert(data.end(), (char*)&op, (char*)&op + 1);
        data.insert(data.end(), (char*)&count, (char*)&count + 4);
    }
    void add_insert(const char* records, uint32_t count, size_t stride) {
        uint8_t op = OP_INSERT_RUN;
        data.insert(data.end(), (char*)&op, (char*)&op + 1);
        data.insert(data.end(), (char*)&count, (char*)&count + 4);
        data.insert(data.end(), records, records + count * stride);
    }
};

// Build merge sequence for a data file.
// old_data has string remap already applied.
// Records are compared by byte equality (stride bytes).
static MergeSequence build_merge_seq(
    const std::vector<char>& old_data, const std::vector<char>& new_data,
    size_t stride)
{
    size_t old_n = old_data.size() / stride;
    size_t new_n = new_data.size() / stride;
    size_t oi = 0, ni = 0;
    MergeSequence seq;
    uint32_t match_run = 0, del_run = 0;

    auto flush_match = [&]() { if (match_run > 0) { seq.add_match(match_run); match_run = 0; } };
    auto flush_del = [&]() { if (del_run > 0) { seq.add_delete(del_run); del_run = 0; } };

    while (oi < old_n && ni < new_n) {
        const char* op = old_data.data() + oi * stride;
        const char* np = new_data.data() + ni * stride;

        if (memcmp(op, np, stride) == 0) {
            flush_del();
            match_run++;
            oi++; ni++;
        } else {
            // Mismatch: look ahead to determine if it's a deletion in old or insertion in new.
            // Check: does the OLD record appear later in new? If so, there are inserts before it.
            // Check: does the NEW record appear later in old? If so, there are deletes before it.
            // Simple heuristic: look ahead up to 100 records in each direction.
            bool found_old_in_new = false, found_new_in_old = false;
            size_t lookahead = std::min((size_t)200, std::min(old_n - oi, new_n - ni));
            for (size_t k = 1; k <= lookahead; k++) {
                if (ni + k < new_n && memcmp(op, new_data.data() + (ni + k) * stride, stride) == 0) {
                    found_old_in_new = true;
                    // Insert k records from new before continuing
                    flush_match(); flush_del();
                    seq.add_insert(new_data.data() + ni * stride, k, stride);
                    ni += k;
                    break;
                }
                if (oi + k < old_n && memcmp(np, old_data.data() + (oi + k) * stride, stride) == 0) {
                    found_new_in_old = true;
                    // Delete k records from old
                    flush_match();
                    del_run += k;
                    oi += k;
                    break;
                }
            }
            if (!found_old_in_new && !found_new_in_old) {
                // Both records are different and neither appears nearby.
                // Treat as delete old + insert new.
                flush_match();
                del_run++;
                oi++;
                flush_del();
                seq.add_insert(np, 1, stride);
                ni++;
            }
        }
    }

    flush_match(); flush_del();
    // Remaining old records: deletions
    if (oi < old_n) seq.add_delete(old_n - oi);
    // Remaining new records: insertions
    if (ni < new_n) seq.add_insert(new_data.data() + ni * stride, new_n - ni, stride);

    return seq;
}

// --- String remap ---

static std::unordered_map<uint32_t, uint32_t> build_string_remap(
    const std::vector<char>& old_pool, const std::vector<char>& new_pool)
{
    std::unordered_map<std::string, uint32_t> new_idx;
    size_t pos = 0;
    while (pos < new_pool.size()) {
        const char* s = new_pool.data() + pos;
        size_t len = strlen(s);
        new_idx[std::string(s, len)] = static_cast<uint32_t>(pos);
        pos += len + 1;
    }
    std::unordered_map<uint32_t, uint32_t> remap;
    pos = 0;
    while (pos < old_pool.size()) {
        const char* s = old_pool.data() + pos;
        size_t len = strlen(s);
        auto it = new_idx.find(std::string(s, len));
        if (it != new_idx.end())
            remap[static_cast<uint32_t>(pos)] = it->second;
        pos += len + 1;
    }
    return remap;
}

// --- Remap + fixup helpers ---

static void remap_addr_points(std::vector<char>& data, const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + 16 <= data.size(); i += 16)
        for (size_t off : {8, 12}) {
            uint32_t v; memcpy(&v, data.data() + i + off, 4);
            auto it = rm.find(v); if (it != rm.end()) memcpy(data.data() + i + off, &it->second, 4);
        }
}
static void remap_field(std::vector<char>& data, size_t stride, size_t field_off,
                         const std::unordered_map<uint32_t,uint32_t>& rm) {
    for (size_t i = 0; i + stride <= data.size(); i += stride) {
        uint32_t v; memcpy(&v, data.data() + i + field_off, 4);
        auto it = rm.find(v); if (it != rm.end()) memcpy(data.data() + i + field_off, &it->second, 4);
    }
}

// Content matching for ways (by name + nodes, ignoring node_offset)
static uint64_t fnv_mix(uint64_t h, uint64_t v) { h ^= v; h *= 1099511628211ULL; return h; }

static void fixup_way_offsets(std::vector<char>& old_ways, const std::vector<char>& old_nodes,
                                const std::vector<char>& new_ways, const std::vector<char>& new_nodes,
                                size_t stride) {
    size_t name_off = (stride == 12) ? 8 : 5;
    size_t old_n = old_ways.size() / stride, new_n = new_ways.size() / stride;
    size_t old_nc = old_nodes.size() / 8, new_nc = new_nodes.size() / 8;

    auto way_hash = [&](const char* w, const char* nodes, size_t max_n) -> uint64_t {
        uint32_t node_offset, name_id; uint8_t node_count;
        memcpy(&node_offset, w, 4); node_count = (uint8_t)w[4]; memcpy(&name_id, w + name_off, 4);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, node_count);
        for (uint8_t j = 0; j < node_count && (node_offset + j) < max_n; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4);
            memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(way_hash(new_ways.data() + i * stride, new_nodes.data(), new_nc), i);

    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = way_hash(old_ways.data() + i * stride, old_nodes.data(), old_nc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            // Fix node_offset to match new way
            uint32_t new_node_off;
            memcpy(&new_node_off, new_ways.data() + it->second * stride, 4);
            memcpy(old_ways.data() + i * stride, &new_node_off, 4);
            new_map.erase(it);
        }
    }
}

// Same for admin polygons (vertex_offset)
static void fixup_admin_offsets(std::vector<char>& old_polys, const std::vector<char>& old_verts,
                                  const std::vector<char>& new_polys, const std::vector<char>& new_verts,
                                  size_t stride) {
    size_t old_n = old_polys.size() / stride, new_n = new_polys.size() / stride;
    size_t old_vc = old_verts.size() / 8, new_vc = new_verts.size() / 8;

    auto poly_hash = [&](const char* p, const char* verts, size_t max_v) -> uint64_t {
        uint32_t vert_offset, vert_count, name_id;
        memcpy(&vert_offset, p, 4); memcpy(&vert_count, p + 4, 4); memcpy(&name_id, p + 8, 4);
        uint8_t level = (uint8_t)p[12]; uint16_t cc; memcpy(&cc, p + stride - 2, 2);
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, name_id); h = fnv_mix(h, level); h = fnv_mix(h, cc); h = fnv_mix(h, vert_count);
        for (uint32_t j = 0; j < std::min(vert_count, 10u) && (vert_offset + j) < max_v; j++) {
            float lat, lng;
            memcpy(&lat, verts + (vert_offset + j) * 8, 4); memcpy(&lng, verts + (vert_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(poly_hash(new_polys.data() + i * stride, new_verts.data(), new_vc), i);

    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = poly_hash(old_polys.data() + i * stride, old_verts.data(), old_vc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            uint32_t new_vert_off;
            memcpy(&new_vert_off, new_polys.data() + it->second * stride, 4);
            memcpy(old_polys.data() + i * stride, &new_vert_off, 4);
            new_map.erase(it);
        }
    }
}

// Same for interp ways
static void fixup_interp_offsets(std::vector<char>& old_data, const std::vector<char>& old_nodes,
                                   const std::vector<char>& new_data, const std::vector<char>& new_nodes,
                                   size_t stride) {
    size_t street_off = (stride >= 20) ? 8 : 5;
    size_t old_n = old_data.size() / stride, new_n = new_data.size() / stride;
    size_t old_nc = old_nodes.size() / 8, new_nc = new_nodes.size() / 8;

    auto ihash = [&](const char* p, const char* nodes, size_t max_n) -> uint64_t {
        uint32_t node_offset, street_id, start, end; uint8_t count, itype;
        memcpy(&node_offset, p, 4); count = (uint8_t)p[4];
        memcpy(&street_id, p + street_off, 4); memcpy(&start, p + street_off + 4, 4);
        memcpy(&end, p + street_off + 8, 4);
        itype = (street_off + 12 < stride) ? (uint8_t)p[street_off + 12] : 0;
        uint64_t h = 14695981039346656037ULL;
        h = fnv_mix(h, street_id); h = fnv_mix(h, start); h = fnv_mix(h, end);
        h = fnv_mix(h, itype); h = fnv_mix(h, count);
        for (uint8_t j = 0; j < count && (node_offset + j) < max_n; j++) {
            float lat, lng;
            memcpy(&lat, nodes + (node_offset + j) * 8, 4); memcpy(&lng, nodes + (node_offset + j) * 8 + 4, 4);
            h = fnv_mix(h, to_grid(lat)); h = fnv_mix(h, to_grid(lng));
        }
        return h;
    };

    std::unordered_multimap<uint64_t, uint32_t> new_map;
    new_map.reserve(new_n);
    for (uint32_t i = 0; i < new_n; i++)
        new_map.emplace(ihash(new_data.data() + i * stride, new_nodes.data(), new_nc), i);
    for (uint32_t i = 0; i < old_n; i++) {
        uint64_t h = ihash(old_data.data() + i * stride, old_nodes.data(), old_nc);
        auto it = new_map.find(h);
        if (it != new_map.end()) {
            uint32_t new_off; memcpy(&new_off, new_data.data() + it->second * stride, 4);
            memcpy(old_data.data() + i * stride, &new_off, 4);
            new_map.erase(it);
        }
    }
}

// --- Write helpers ---
static void wval(std::vector<char>& buf, const void* data, size_t size) {
    buf.insert(buf.end(), (const char*)data, (const char*)data + size);
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>" << std::endl;
        return 1;
    }
    std::string old_dir = argv[1], new_dir = argv[2], patch_path = argv[4];
    std::string tmpdir = "/tmp/geocoder-diff-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    // Build string remap
    std::cerr << "Building string remap..." << std::endl;
    auto old_strings = read_file(old_dir + "/strings.bin");
    auto new_strings = read_file(new_dir + "/strings.bin");
    auto str_remap = build_string_remap(old_strings, new_strings);

    // Detect strides
    auto detect = [](const std::string& path, std::vector<size_t> cs) -> size_t {
        auto d = read_file(path);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect(old_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect(old_dir + "/interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect(old_dir + "/admin_polygons.bin", {24, 20, 19});

    // Build patch data (uncompressed, will be zstd-compressed at the end)
    std::vector<char> patch;
    patch.insert(patch.end(), GCPATCH_MAGIC, GCPATCH_MAGIC + 8);
    uint32_t ver = 2, flags = 0; // version 2 = custom format
    wval(patch, &ver, 4); wval(patch, &flags, 4);

    // --- Section: String remap ---
    {
        std::vector<std::pair<uint32_t,uint32_t>> entries;
        size_t pos = 0;
        while (pos < old_strings.size()) {
            uint32_t old_off = static_cast<uint32_t>(pos);
            auto it = str_remap.find(old_off);
            entries.push_back({old_off, (it != str_remap.end()) ? it->second : old_off});
            pos += strlen(old_strings.data() + pos) + 1;
        }
        uint32_t marker = 0xFFFFFFFE, count = static_cast<uint32_t>(entries.size());
        wval(patch, &marker, 4); wval(patch, &count, 4);
        for (auto& [o, n] : entries) { wval(patch, &o, 4); wval(patch, &n, 4); }
        std::cerr << "  String remap: " << count << " entries (" << count * 8 << " bytes)" << std::endl;
    }

    // --- Section: Per-file merge sequences ---
    // Track fixups per file (applied before merge, included in patch for patch tool)
    std::unordered_map<uint32_t, std::vector<std::pair<uint32_t,uint32_t>>> file_fixups;

    auto write_merge = [&](PatchFileId id, const std::string& name,
                             const std::vector<char>& old_data, const std::vector<char>& new_data,
                             size_t stride) {
        auto seq = build_merge_seq(old_data, new_data, stride);
        uint32_t fid = static_cast<uint32_t>(id);
        uint64_t os = old_data.size(), ns = new_data.size(), ss = seq.data.size();

        // Check for fixups
        auto fit = file_fixups.find(fid);
        uint32_t n_fixups = fit != file_fixups.end() ? static_cast<uint32_t>(fit->second.size()) : 0;

        wval(patch, &fid, 4);
        uint32_t st = static_cast<uint32_t>(stride);
        wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8);
        // Fixup count + data before merge sequence
        wval(patch, &n_fixups, 4);
        if (n_fixups > 0) {
            for (auto& [idx, val] : fit->second) { wval(patch, &idx, 4); wval(patch, &val, 4); }
        }
        wval(patch, &ss, 8);
        patch.insert(patch.end(), seq.data.begin(), seq.data.end());
        std::cerr << "  " << name << ": seq=" << ss << " fixups=" << n_fixups
                  << " (" << std::fixed << std::setprecision(2)
                  << (ns > 0 ? ss * 100.0 / ns : 0) << "%)" << std::endl;
    };

    // strings.bin: full new (small file)
    {
        uint32_t fid = static_cast<uint32_t>(PatchFileId::STRINGS);
        uint32_t st = 1; // byte stride
        uint64_t os = old_strings.size(), ns = new_strings.size();
        auto seq = build_merge_seq(old_strings, new_strings, 1);
        uint64_t ss = seq.data.size();
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8); wval(patch, &ss, 8);
        patch.insert(patch.end(), seq.data.begin(), seq.data.end());
        std::cerr << "  strings.bin: seq=" << ss << " bytes" << std::endl;
    }

    // addr_points.bin
    {
        auto old_data = read_file(old_dir + "/addr_points.bin");
        remap_addr_points(old_data, str_remap);
        auto new_data = read_file(new_dir + "/addr_points.bin");
        write_merge(PatchFileId::ADDR_POINTS, "addr_points.bin", old_data, new_data, 16);
    }

    // street_ways.bin (string remap + offset fixup for node_offset)
    {
        auto old_w = read_file(old_dir + "/street_ways.bin");
        remap_field(old_w, way_stride, way_stride == 12 ? 8 : 5, str_remap);
        auto new_w = read_file(new_dir + "/street_ways.bin");
        auto old_n = read_file(old_dir + "/street_nodes.bin");
        auto new_n = read_file(new_dir + "/street_nodes.bin");
        // Record fixups: (record_index, new_node_offset)
        auto& fixups = file_fixups[static_cast<uint32_t>(PatchFileId::STREET_WAYS)];
        // Save old offsets, apply fixups, then record changes
        size_t wn = old_w.size() / way_stride;
        std::vector<uint32_t> old_offsets(wn);
        for (size_t i = 0; i < wn; i++) memcpy(&old_offsets[i], old_w.data() + i * way_stride, 4);
        fixup_way_offsets(old_w, old_n, new_w, new_n, way_stride);
        for (size_t i = 0; i < wn; i++) {
            uint32_t new_off; memcpy(&new_off, old_w.data() + i * way_stride, 4);
            if (new_off != old_offsets[i]) fixups.push_back({static_cast<uint32_t>(i), new_off});
        }
        write_merge(PatchFileId::STREET_WAYS, "street_ways.bin", old_w, new_w, way_stride);
    }

    // street_nodes.bin (no remap)
    {
        auto old_data = read_file(old_dir + "/street_nodes.bin");
        auto new_data = read_file(new_dir + "/street_nodes.bin");
        write_merge(PatchFileId::STREET_NODES, "street_nodes.bin", old_data, new_data, 8);
    }

    // interp_ways.bin
    {
        auto old_data = read_file(old_dir + "/interp_ways.bin");
        remap_field(old_data, interp_stride, interp_stride >= 20 ? 8 : 5, str_remap);
        auto old_n = read_file(old_dir + "/interp_nodes.bin");
        auto new_data = read_file(new_dir + "/interp_ways.bin");
        auto new_n = read_file(new_dir + "/interp_nodes.bin");
        auto& fixups = file_fixups[static_cast<uint32_t>(PatchFileId::INTERP_WAYS)];
        size_t in_count = old_data.size() / interp_stride;
        std::vector<uint32_t> old_offsets(in_count);
        for (size_t i = 0; i < in_count; i++) memcpy(&old_offsets[i], old_data.data() + i * interp_stride, 4);
        fixup_interp_offsets(old_data, old_n, new_data, new_n, interp_stride);
        for (size_t i = 0; i < in_count; i++) {
            uint32_t new_off; memcpy(&new_off, old_data.data() + i * interp_stride, 4);
            if (new_off != old_offsets[i]) fixups.push_back({static_cast<uint32_t>(i), new_off});
        }
        write_merge(PatchFileId::INTERP_WAYS, "interp_ways.bin", old_data, new_data, interp_stride);
    }

    // interp_nodes.bin
    {
        auto old_data = read_file(old_dir + "/interp_nodes.bin");
        auto new_data = read_file(new_dir + "/interp_nodes.bin");
        write_merge(PatchFileId::INTERP_NODES, "interp_nodes.bin", old_data, new_data, 8);
    }

    // admin_polygons.bin
    {
        auto old_data = read_file(old_dir + "/admin_polygons.bin");
        remap_field(old_data, admin_stride, 8, str_remap);
        auto old_v = read_file(old_dir + "/admin_vertices.bin");
        auto new_data = read_file(new_dir + "/admin_polygons.bin");
        auto new_v = read_file(new_dir + "/admin_vertices.bin");
        auto& fixups = file_fixups[static_cast<uint32_t>(PatchFileId::ADMIN_POLYGONS)];
        size_t an = old_data.size() / admin_stride;
        std::vector<uint32_t> old_offsets(an);
        for (size_t i = 0; i < an; i++) memcpy(&old_offsets[i], old_data.data() + i * admin_stride, 4);
        fixup_admin_offsets(old_data, old_v, new_data, new_v, admin_stride);
        for (size_t i = 0; i < an; i++) {
            uint32_t new_off; memcpy(&new_off, old_data.data() + i * admin_stride, 4);
            if (new_off != old_offsets[i]) fixups.push_back({static_cast<uint32_t>(i), new_off});
        }
        write_merge(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin", old_data, new_data, admin_stride);
    }

    // admin_vertices.bin
    {
        auto old_data = read_file(old_dir + "/admin_vertices.bin");
        auto new_data = read_file(new_dir + "/admin_vertices.bin");
        write_merge(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", old_data, new_data, 8);
    }

    // --- Section: Entry/cell files as full new data ---
    // These are small relative to data files and rebuilt by patch tool isn't reliable.
    // Include them as full replacement. zstd transport compression handles the rest.
    for (auto& [id, name] : std::vector<std::pair<PatchFileId, std::string>>{
        {PatchFileId::GEO_CELLS, "geo_cells.bin"},
        {PatchFileId::STREET_ENTRIES, "street_entries.bin"},
        {PatchFileId::ADDR_ENTRIES, "addr_entries.bin"},
        {PatchFileId::INTERP_ENTRIES, "interp_entries.bin"},
        {PatchFileId::ADMIN_CELLS, "admin_cells.bin"},
        {PatchFileId::ADMIN_ENTRIES, "admin_entries.bin"}}) {
        auto data = read_file(new_dir + "/" + name);
        uint32_t fid = static_cast<uint32_t>(id);
        uint32_t st = 0; // stride 0 = full replacement
        uint64_t os = 0, ns = data.size(), ss = data.size();
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8); wval(patch, &ss, 8);
        patch.insert(patch.end(), data.begin(), data.end());
        std::cerr << "  " << name << ": full " << data.size() << " bytes" << std::endl;
    }

    // End marker
    uint32_t end_marker = 0xFFFFFFFF;
    wval(patch, &end_marker, 4);

    std::cerr << "\nUncompressed patch: " << patch.size() << " bytes ("
              << patch.size() / 1024 / 1024 << " MiB)" << std::endl;

    // Compress whole patch with zstd for transport
    {
        std::string raw_path = tmpdir + "/patch.raw";
        write_file(raw_path, patch);
        std::string cmd = "zstd -19 '" + raw_path + "' -o '" + patch_path + "' -f --quiet 2>/dev/null";
        system(cmd.c_str());
        auto compressed = read_file(patch_path);
        std::cerr << "Compressed patch: " << compressed.size() << " bytes ("
                  << compressed.size() / 1024 / 1024 << " MiB)" << std::endl;
        remove(raw_path.c_str());
    }

    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());
    return 0;
}

-        uint64_t os = 0, ns = data.size(), ss = data.size();

+        uint64_t os = 0, ns = data.size();

-        wval(patch, &os, 8); wval(patch, &ns, 8); wval(patch, &ss, 8);

+        wval(patch, &os, 8); wval(patch, &ns, 8);

+        wval(patch, &nfix, 4); // no fixups

+        wval(patch, &ss, 8);   // data size


Bash

cmake --build build -j$(nproc) --target geocoder-diff --target geocoder-patch 2>&1 | grep -E "(error:|Built)" | head -5

Build


sshpass -p 'elon-is-insane' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" /root/traccar-geocoder/builder/build/geocoder-diff /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:~/ 2>&1 && sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
rm -f /home/michtest/test-v3c.gcpatch
time /home/michtest/geocoder-diff /home/michtest/det-A /home/michtest/det-new -o /home/michtest/test-v3c.gcpatch 2>&1
echo ""
rm -rf /home/michtest/patched-v3c
time /home/michtest/geocoder-patch /home/michtest/det-A /home/michtest/test-v3c.gcpatch -o /home/michtest/patched-v3c 2>&1
echo ""
all_match=true
for f in /home/michtest/det-new/*.bin; do
    b=$(basename $f); p=/home/michtest/patched-v3c/$b
    [ -f "$p" ] && cmp -s "$f" "$p" && echo "  $b: MATCH" || { echo "  $b: FAIL"; all_match=false; }
done
$all_match && echo "ALL FILES MATCH" || echo "SOME DIFFER"
' 2>&1

Full v3 test with format fix


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Building string remap...
  String remap: 4982052 entries (39856416 bytes)
  strings.bin: seq=106752118 bytes
  addr_points.bin: seq=1066728884 fixups=0 (77.53%)
  street_ways.bin: seq=2485811 fixups=19372497 (1.06%)
  street_nodes.bin: seq=10898537 fixups=0 (0.75%)
  interp_ways.bin: seq=590 fixups=13395 (0.18%)
  interp_nodes.bin: seq=2085 fixups=0 (0.29%)
  admin_polygons.bin: seq=82674 fixups=449077 (0.76%)
  admin_vertices.bin: seq=82545588 fixups=0 (28.36%)
  geo_cells.bin: full 1674295440 bytes
  street_entries.bin: full 586571766 bytes
  addr_entries.bin: full 395005236 bytes
  interp_entries.bin: full 159386 bytes
  admin_cells.bin: full 1692000 bytes
  admin_entries.bin: full 6122260 bytes

Uncompressed patch: 4131879071 bytes (3940 MiB)
Compressed patch: 1022019598 bytes (974 MiB)

real	6m50.600s
user	24m44.986s
sys	0m5.417s
Decompressed patch: 4131879071 bytes
Loaded 4980054 string remap entries
    Applied 106752118 fixups

  addr_entries.bin: FAIL
bash: line 6: 129939 Segmentation fault      /home/michtest/geocoder-patch /home/michtest/det-A /home/michtest/test-v3c.gcpatch -o /home/michtest/patched-v3c 2>&1

real	0m7.153s
user	0m5.973s
sys	0m2.276s
  addr_points.bin: FAIL
  admin_cells.bin: FAIL
  admin_entries.bin: FAIL
  admin_polygons.bin: FAIL
  admin_vertices.bin: FAIL
  geo_cells.bin: FAIL
  interp_entries.bin: FAIL
  interp_nodes.bin: FAIL
  interp_ways.bin: FAIL
  street_entries.bin: FAIL
  street_nodes.bin: FAIL
  street_ways.bin: FAIL
  strings.bin: FAIL
SOME DIFFER


"Applied 106752118 fixups" — that's 107M fixups which is way too many. It's reading the strings merge sequence data as fixups. The string remap section (marker 0xFFFFFFFE) is read, then the first file section starts. The patch tool reads fid, stride, old_size, new_size — but the diff writes the remap INSIDE the general patch blob, while file sections follow.

The issue: the patch tool reads the remap section correctly, then encounters the first file section (strings.bin, fid=0, stride=1). It reads fid=0, stride=1 correctly. Then old_size, new_size. Then n_fixups — but the string merge has 106M bytes of sequence data that the patch mistakenly reads as fixups.

Wait, n_fixups for strings.bin should be 0 (no fixups in `file_fixups`). The diff writes n_fixups=0 for strings. Let me check: the diff's `write_merge` function checks `file_fixups[fid]`. For strings (fid=0), there are no fixups. So n_fixups=0 is written. Then seq_size=106752118, then the seq data.

The patch reads: fid(u32), stride(u32), old_size(u64), new_size(u64), n_fixups(u32)=0, seq_size(u64), then 106M bytes of seq data. This should be correct.

But the patch output says "Applied 106752118 fixups" — that means it read n_fixups=106752118 from the wrong position. The offset is off.

The issue is in the string remap format. The diff writes: marker(4) + count(4) + [count × 8 bytes]. The patch reads: marker(4) + count(4) + skips(count × 8 bytes). After this, pos should be at the first file section.

But wait — in the diff, the remap is NOT compressed (v2 writes raw). Let me verify the positions match:

String remap: 16 (header) + 4 (marker) + 4 (count=4982052) + 4982052×8 = 16 + 8 + 39856416 = 39856440 bytes.

First file section at pos=39856440: fid(4) + stride(4) + old_size(8) + new_size(8) + n_fixups(4) + seq_size(8) + seq_data.

This should be correct. Let me check if the patch tool's `read_u32/read_u64` are reading at the right positions. They use `pos` which is incremented manually. Let me add debug output:

Actually, I think the issue might be simpler — the string remap entry format includes old_off and new_off. The patch tool filters `old_off != new_off`. But it reads ALL entries to advance `pos`. Let me verify:


    69→            if (old_off != new_off) str_remap[old_off] = new_off;
    70→        }
    71→        std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;
    72→    }
    73→
    74→    // Detect strides
    75→    auto detect = [&](const std::string& name, std::vector<size_t> cs) -> size_t {
    76→        auto d = read_file(cur_dir + "/" + name);
    77→        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
    78→        return cs[0];
    79→    };
    80→    size_t way_stride = detect("street_ways.bin", {12, 9});
    81→    size_t interp_stride = detect("interp_ways.bin", {24, 20, 18});
    82→    size_t admin_stride = detect("admin_polygons.bin", {24, 20, 19});
    83→

            if (old_off != new_off) str_remap[old_off] = new_off;
        }
        std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;
    }

    // Detect strides
    auto detect = [&](const std::string& name, std::vector<size_t> cs) -> size_t {
        auto d = read_file(cur_dir + "/" + name);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect("street_ways.bin", {12, 9});
    size_t interp_stride = detect("interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect("admin_polygons.bin", {24, 20, 19});


The remap loop reads `count` entries using `read_u32()` which advances `pos`. After the loop, `pos` should be at position 39856440. Then the first file section is read.

Let me add a debug line to check pos after remap:


Edit

        std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;
    }

        std::cerr << "Loaded " << str_remap.size() << " string remap entries (pos=" << pos << ")" << std::endl;
    }


        std::cerr << "Loaded " << str_remap.size() << " string remap entries (pos=" << pos << ")" << std::endl;
    }

// geocoder-patch v3: Apply custom patch format.
//
// Decompresses zstd transport layer, then:
// 1. Reads string remap, applies to old data files
// 2. Replays merge sequences to reconstruct new data files
// 3. Writes full-replacement entry/cell files
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

// Merge ops (must match diff tool)
enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    // Decompress zstd transport layer
    std::string raw_path = tmpdir + "/patch.raw";
    {
        std::string cmd = "zstd -d '" + patch_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0) {
            std::cerr << "Failed to decompress patch" << std::endl; return 1;
        }
    }
    auto patch = read_file(raw_path);
    remove(raw_path.c_str());
    std::cerr << "Decompressed patch: " << patch.size() << " bytes" << std::endl;

    size_t pos = 0;
    auto read_u32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, patch.data() + pos, 4); pos += 4; return v; };
    auto read_u64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, patch.data() + pos, 8); pos += 8; return v; };

    // Check header
    if (memcmp(patch.data(), GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch magic" << std::endl; return 1;
    }
    pos = 8;
    uint32_t version = read_u32();
    if (version != 2) { std::cerr << "Expected patch v2, got " << version << std::endl; return 1; }
    uint32_t flags = read_u32();
    (void)flags;

    // Read string remap
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t marker = read_u32();
        if (marker != 0xFFFFFFFE) { std::cerr << "Expected remap section" << std::endl; return 1; }
        uint32_t count = read_u32();
        for (uint32_t i = 0; i < count; i++) {
            uint32_t old_off = read_u32(), new_off = read_u32();
            if (old_off != new_off) str_remap[old_off] = new_off;
        }
        std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;
    }

    // Detect strides
    auto detect = [&](const std::string& name, std::vector<size_t> cs) -> size_t {
        auto d = read_file(cur_dir + "/" + name);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect("street_ways.bin", {12, 9});
    size_t interp_stride = detect("interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect("admin_polygons.bin", {24, 20, 19});

    // String remap functions
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t file_id, size_t stride) {
        if (str_remap.empty()) return;
        std::vector<size_t> field_offsets;
        if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) field_offsets = {8, 12};
        else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) field_offsets = {(stride == 12) ? (size_t)8 : (size_t)5};
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) field_offsets = {(stride >= 20) ? (size_t)8 : (size_t)5};
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) field_offsets = {8};
        else return;

        for (size_t i = 0; i + stride <= data.size(); i += stride)
            for (size_t off : field_offsets) {
                uint32_t v; memcpy(&v, data.data() + i + off, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + off, &it->second, 4);
            }
    };

    // Process sections
    while (pos < patch.size()) {
        uint32_t file_id = read_u32();
        if (file_id == 0xFFFFFFFF) break;

        uint32_t stride = read_u32();
        uint64_t old_size = read_u64(), new_size = read_u64();

        if (file_id >= (uint32_t)PatchFileId::COUNT) {
            std::cerr << "  Unknown file " << file_id << std::endl;
            break;
        }
        const char* filename = patch_file_names[file_id];

        if (stride == 0) {
            // Full replacement: n_fixups=0, then seq_size = data size
            uint32_t n_fix = read_u32(); (void)n_fix; // always 0 for full replacement
            uint64_t data_size = read_u64();
            std::vector<char> data(patch.data() + pos, patch.data() + pos + data_size);
            write_file(out_dir + "/" + std::string(filename), data);
            std::cerr << "  " << filename << ": full replacement " << data.size() << " bytes" << std::endl;
            pos += data_size;
            continue;
        }

        // Merge sequence: load old file, apply remap + fixups, replay sequence
        auto old_data = read_file(cur_dir + "/" + std::string(filename));
        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        // Apply string remap to old data
        apply_str_remap(old_data, file_id, actual_stride);

        // Read and apply fixups (node_offset/vertex_offset fixes)
        // Read and apply fixups (offset field patches for matched records)
        uint32_t n_fixups = read_u32();
        for (uint32_t i = 0; i < n_fixups; i++) {
            uint32_t idx = read_u32(), val = read_u32();
            size_t byte_pos = (size_t)idx * actual_stride;
            if (byte_pos + 4 <= old_data.size())
                memcpy(old_data.data() + byte_pos, &val, 4);
        }
        if (n_fixups > 0) std::cerr << "    Applied " << n_fixups << " fixups" << std::endl;

        // Read merge sequence size and replay
        uint64_t seq_size = read_u64();
        std::vector<char> output;
        output.reserve(new_size);
        size_t old_pos = 0;
        size_t seq_end = pos + seq_size;

        while (pos < seq_end) {
            uint8_t op = static_cast<uint8_t>(patch[pos]); pos++;
            uint32_t count; memcpy(&count, patch.data() + pos, 4); pos += 4;

            if (op == OP_MATCH_RUN) {
                // Copy count records from old
                size_t bytes = count * actual_stride;
                if (old_pos + bytes <= old_data.size()) {
                    output.insert(output.end(), old_data.data() + old_pos,
                                 old_data.data() + old_pos + bytes);
                }
                old_pos += bytes;
            } else if (op == OP_INSERT_RUN) {
                // Read count records from patch
                size_t bytes = count * actual_stride;
                output.insert(output.end(), patch.data() + pos, patch.data() + pos + bytes);
                pos += bytes;
            } else if (op == OP_DELETE_RUN) {
                // Skip count records in old
                old_pos += count * actual_stride;
            }
        }

        write_file(out_dir + "/" + std::string(filename), output);
        std::cerr << "  " << filename << ": " << output.size() << " bytes" << std::endl;
    }

    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

-        std::cerr << "Loaded " << str_remap.size() << " string remap entries" << std::endl;

+        std::cerr << "Loaded " << str_remap.size() << " string remap entries (pos=" << pos << ")" << std::endl;


Bash

cmake --build build -j$(nproc) --target geocoder-patch 2>&1 | tail -2 && sshpass -p 'elon-is-insane' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:~/geocoder-patch 2>&1 && sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
rm -rf /home/michtest/patched-v3c
/home/michtest/geocoder-patch /home/michtest/det-A /home/michtest/test-v3c.gcpatch -o /home/michtest/patched-v3c 2>&1 | head -15
' 2>&1

Debug apply


[100%] Linking CXX executable geocoder-patch
[100%] Built target geocoder-patch
Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Decompressed patch: 4131879071 bytes
Loaded 4980054 string remap entries (pos=39856440)
    Applied 106752118 fixups


pos=39856440 is correct. Then it reads fid(4), stride(4), old_size(8), new_size(8) at pos 39856440. Let me check what bytes are at that position:

The diff writes strings.bin section: fid=0(4), stride=1(4), old_size(8), new_size(8), n_fixups=0(4), seq_size(8), seq_data.

After remap at pos=39856440:
- pos+0: fid = 0 (PatchFileId::STRINGS)
- pos+4: stride = 1
- pos+8: old_size = 82546573 (old strings size)
- pos+16: new_size = 82573397 (new strings size)
- pos+24: n_fixups = 0
- pos+28: seq_size = 106752118
- pos+36: seq_data starts

The patch reads: fid(4)=0, stride(4)=1, old_size(8), new_size(8). Then since stride != 0, it does the merge path. It reads actual_stride = 1 (strings). Then apply_str_remap — for fid=0 (STRINGS), the function returns (no string fields). Then reads n_fixups(4) at pos+24. n_fixups should be 0.

BUT: let me check — does `read_u32` at position 39856464 (39856440 + 24) read the correct value?

39856440 + 4 (fid) + 4 (stride) + 8 (old) + 8 (new) = 39856464. At this position, the patch has n_fixups = 0.

But the output says "Applied 106752118 fixups" which is the seq_size value! That means pos is off by 4 or 8 bytes.

Wait — `read_u32` and `read_u64` use the `pos` variable. But between reading the remap and the file section, the code does `detect_stride` which calls `read_file`. These don't touch `pos`. So pos stays at 39856440.

Then `file_id = read_u32()` → pos becomes 39856444.
`stride = read_u32()` → pos becomes 39856448.
`old_size = read_u64()` → pos becomes 39856456.
`new_size = read_u64()` → pos becomes 39856464.

Then for stride != 0: it reads actual_stride, applies str_remap, then reads n_fixups:
`n_fixups = read_u32()` at pos 39856464.

But at pos 39856464, the data is: n_fixups=0 (4 bytes). So n_fixups should be 0.

Unless the bytes at that position are NOT 0. Let me check by printing the raw bytes:


sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 'python3 -c "
import struct
# Decompress patch
import subprocess
subprocess.run([\"zstd\", \"-d\", \"/home/michtest/test-v3c.gcpatch\", \"-o\", \"/tmp/raw.bin\", \"-f\", \"--quiet\"])
data = open(\"/tmp/raw.bin\", \"rb\").read()
pos = 39856440
fid = struct.unpack_from(\"<I\", data, pos)[0]; pos += 4
stride = struct.unpack_from(\"<I\", data, pos)[0]; pos += 4
old_size = struct.unpack_from(\"<Q\", data, pos)[0]; pos += 8
new_size = struct.unpack_from(\"<Q\", data, pos)[0]; pos += 8
n_fixups = struct.unpack_from(\"<I\", data, pos)[0]; pos += 4
seq_size = struct.unpack_from(\"<Q\", data, pos)[0]; pos += 8
print(f\"fid={fid} stride={stride} old={old_size} new={new_size} n_fixups={n_fixups} seq_size={seq_size}\")
print(f\"pos after header: {pos}\")
import os; os.remove(\"/tmp/raw.bin\")
"' 2>&1

Verify patch file structure


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
fid=0 stride=1 old=82546573 new=82573397 n_fixups=106752118 seq_size=206708186021888
pos after header: 39856476


`n_fixups=106752118` — that's the seq_size value! The issue: the diff tool writes `n_fixups(4)` then `seq_size(8)`, but the value read as n_fixups is actually the first 4 bytes of seq_size (which is 106752118 = 0x065CC5D6, the sequence size).

Wait — 106752118 IS what the diff should write as seq_size. But it's being read at the n_fixups position. Let me check the diff's write order:

In `write_merge`: writes fid(4), stride(4), old_size(8), new_size(8), **n_fixups(4)**, seq_size(8), seq_data.

But the Python reads: fid(4), stride(4), old_size(8), new_size(8), n_fixups(4), seq_size(8). And gets n_fixups=106752118.

The value should be 0. Let me check: does the diff write `n_fixups` at the right position for strings.bin?

For strings.bin, `file_fixups` has no entry for fid=0. So `n_fixups = 0`. The write should be: ...new_size(8), **0(4)**, seq_size(8).

But the Python read shows n_fixups=106752118 at that position. This means the bytes at that position are NOT 0.

The issue might be that the diff's `write_merge` for strings writes the data in a DIFFERENT order than I think. Let me read the actual write_merge code again:

The diff writes `wval(patch, &os, 8); wval(patch, &ns, 8);` then `wval(patch, &n_fixups, 4);` then fixup data then `wval(patch, &ss, 8);` then seq data.

For strings: os=82546573, ns=82573397, n_fixups=0, ss=106752118.

At pos 39856440: fid(4)=0, stride(4)=1, os(8), ns(8) → pos+24. Then n_fixups(4)=0 → pos+28. Then ss(8)=106752118 → pos+36.

Python at pos+24 (39856464) reads 4 bytes as n_fixups and gets 106752118. But those bytes should be 0 (n_fixups=0).

Unless the write is producing wrong bytes. Let me check: does `wval(patch, &n_fixups, 4)` write 0? `n_fixups` is initialized from `file_fixups.find(fid)` — for fid=0, no entry, so `n_fixups=0`. Yes, it should write 0.

Wait — I see the issue! Look at the write_merge code more carefully. The diff writes to a `std::vector<char> patch` buffer. Let me check if the position offsets are correct by checking what's in the buffer at position 39856464.

Actually, the problem might be that `wval` writes to the vector by appending, and some earlier write wrote extra data. Let me count the exact bytes:

Header: 8 (magic) + 4 (ver) + 4 (flags) = 16
Remap: 4 (marker) + 4 (count) + 4982052 × 8 = 8 + 39856416 = 39856424
Total before first file: 16 + 39856424 = 39856440 ✓

First file (strings.bin):
- fid(4) + stride(4) + os(8) + ns(8) = 24 bytes
- Then n_fixups(4) = 4 bytes
- Then (0 fixups, 0 bytes)
- Then ss(8) + seq_data(106752118)

At byte 39856440 + 24 = 39856464, we expect n_fixups = 0 (4 bytes of zeros).

But Python reads 106752118 there. This means something else was written at that position.

OH WAIT. I see it now. The diff's write_merge has strings.bin written DIFFERENTLY. The strings.bin section is NOT written via `write_merge` — it has its own dedicated block:

```cpp
    // strings.bin: full new (small file)
    {
        uint32_t fid = static_cast<uint32_t>(PatchFileId::STRINGS);
        uint32_t st = 1; // byte stride
        uint64_t os = old_strings.size(), ns = new_strings.size();
        auto seq = build_merge_seq(old_strings, new_strings, 1);
        uint64_t ss = seq.data.size();
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8); wval(patch, &ss, 8);
        patch.insert(patch.end(), seq.data.begin(), seq.data.end());
```

It writes: fid, stride, old_size, new_size, **seq_size**, seq_data. It does NOT write n_fixups! It's using the old format without fixups!

The `write_merge` lambda writes n_fixups, but the strings.bin block doesn't use the lambda — it has inline code. Let me fix it to use the lambda:


Edit

    // strings.bin: full new (small file)
    {
        uint32_t fid = static_cast<uint32_t>(PatchFileId::STRINGS);
        uint32_t st = 1; // byte stride
        uint64_t os = old_strings.size(), ns = new_strings.size();
        auto seq = build_merge_seq(old_strings, new_strings, 1);
        uint64_t ss = seq.data.size();
        wval(patch, &fid, 4); wval(patch, &st, 4);
        wval(patch, &os, 8); wval(patch, &ns, 8); wval(patch, &ss, 8);
        patch.insert(patch.end(), seq.data.begin(), seq.data.end());
        std::cerr << "  strings.bin: seq=" << ss << " bytes" << std::endl;
    }

    // strings.bin: byte-level merge (using write_merge lambda)
    write_merge(PatchFileId::STRINGS, "strings.bin", old_strings, new_strings, 1);