// geocoder-patch v3: Apply custom patch format.
//
// Decompresses zstd transport layer, then:
// 1. Reads string remap, applies to old data files
// 2. Replays merge sequences to reconstruct new data files
// 3. Writes full-replacement entry/cell files
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

// Merge ops (must match diff tool)
enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);

    // Decompress zstd transport layer
    std::string raw_path = tmpdir + "/patch.raw";
    {
        std::string cmd = "zstd -d '" + patch_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0) {
            std::cerr << "Failed to decompress patch" << std::endl; return 1;
        }
    }
    auto patch = read_file(raw_path);
    remove(raw_path.c_str());
    std::cerr << "Decompressed patch: " << patch.size() << " bytes" << std::endl;

    size_t pos = 0;
    auto read_u32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, patch.data() + pos, 4); pos += 4; return v; };
    auto read_u64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, patch.data() + pos, 8); pos += 8; return v; };

    // Check header
    if (memcmp(patch.data(), GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch magic" << std::endl; return 1;
    }
    pos = 8;
    uint32_t version = read_u32();
    if (version != 2) { std::cerr << "Expected patch v2, got " << version << std::endl; return 1; }
    uint32_t flags = read_u32();
    (void)flags;

    // Read string remap
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t marker = read_u32();
        if (marker != 0xFFFFFFFE) { std::cerr << "Expected remap section" << std::endl; return 1; }
        uint32_t count = read_u32();
        for (uint32_t i = 0; i < count; i++) {
            uint32_t old_off = read_u32(), new_off = read_u32();
            if (old_off != new_off) str_remap[old_off] = new_off;
        }
        std::cerr << "Loaded " << str_remap.size() << " string remap entries (pos=" << pos << ")" << std::endl;
    }

    // Detect strides
    auto detect = [&](const std::string& name, std::vector<size_t> cs) -> size_t {
        auto d = read_file(cur_dir + "/" + name);
        for (size_t s : cs) if (d.size() % s == 0 && d.size() > 0) return s;
        return cs[0];
    };
    size_t way_stride = detect("street_ways.bin", {12, 9});
    size_t interp_stride = detect("interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect("admin_polygons.bin", {24, 20, 19});

    // String remap functions
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t file_id, size_t stride) {
        if (str_remap.empty()) return;
        std::vector<size_t> field_offsets;
        if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) field_offsets = {8, 12};
        else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) field_offsets = {(stride == 12) ? (size_t)8 : (size_t)5};
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) field_offsets = {(stride >= 20) ? (size_t)8 : (size_t)5};
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) field_offsets = {8};
        else return;

        for (size_t i = 0; i + stride <= data.size(); i += stride)
            for (size_t off : field_offsets) {
                uint32_t v; memcpy(&v, data.data() + i + off, 4);
                auto it = str_remap.find(v);
                if (it != str_remap.end()) memcpy(data.data() + i + off, &it->second, 4);
            }
    };

    // ID remaps derived from merge sequences (populated during merge replay)
    std::unordered_map<uint32_t, std::vector<uint32_t>> derived_id_remaps;

    // Buffered correction sections (applied after entry reconstruction)
    struct PendingCorrection { uint32_t file_id; std::string filename; std::vector<char> delta; };
    std::vector<PendingCorrection> pending_corrections;

    // Read cell changes (may appear before end marker)
    std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;

    // Process sections
    while (pos < patch.size()) {
        uint32_t file_id = read_u32();
        if (file_id == 0xFFFFFFFF) break;

        // Cell change sections
        if (file_id == CELL_CHANGES_GEO_MARKER) {
            uint32_t na = read_u32(), nr = read_u32();
            geo_added.resize(na); geo_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; geo_added[i]=c; }
            for (uint32_t i = 0; i < nr; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; geo_removed[i]=c; }
            std::cerr << "  Geo cell changes: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_ADMIN_MARKER) {
            uint32_t na = read_u32(), nr = read_u32();
            admin_added.resize(na); admin_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; admin_added[i]=c; }
            for (uint32_t i = 0; i < nr; i++) { uint64_t c; memcpy(&c, patch.data()+pos, 8); pos+=8; admin_removed[i]=c; }
            std::cerr << "  Admin cell changes: +" << na << " -" << nr << std::endl;
            continue;
        }

        uint32_t stride = read_u32();
        uint64_t old_size = read_u64(), new_size = read_u64();

        if (file_id >= (uint32_t)PatchFileId::COUNT) {
            std::cerr << "  Unknown file " << file_id << std::endl;
            break;
        }
        const char* filename = patch_file_names[file_id];

        if (stride == 0) {
            // Full replacement
            uint32_t n_fix = read_u32(); (void)n_fix;
            uint64_t data_size = read_u64();
            std::vector<char> data(patch.data() + pos, patch.data() + pos + data_size);
            write_file(out_dir + "/" + std::string(filename), data);
            std::cerr << "  " << filename << ": full replacement " << data.size() << " bytes" << std::endl;
            pos += data_size;
            continue;
        }
        if (stride == 0xFE) {
            // Correction section: buffer for processing AFTER entry reconstruction
            uint32_t n_fix = read_u32(); (void)n_fix;
            uint64_t delta_size = read_u64();
            pending_corrections.push_back({file_id, std::string(filename),
                std::vector<char>(patch.data() + pos, patch.data() + pos + delta_size)});
            pos += delta_size;
            std::cerr << "  " << filename << ": correction buffered (" << delta_size << " bytes)" << std::endl;
            continue;
        }

        // Merge sequence: load old file, apply remap + fixups, replay sequence
        auto old_data = read_file(cur_dir + "/" + std::string(filename));
        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        // Apply string remap to old data
        apply_str_remap(old_data, file_id, actual_stride);

        // Read and apply fixups (node_offset/vertex_offset fixes)
        // Read and apply fixups (offset field patches for matched records)
        uint32_t n_fixups = read_u32();
        for (uint32_t i = 0; i < n_fixups; i++) {
            uint32_t idx = read_u32(), val = read_u32();
            size_t byte_pos = (size_t)idx * actual_stride;
            if (byte_pos + 4 <= old_data.size())
                memcpy(old_data.data() + byte_pos, &val, 4);
        }
        if (n_fixups > 0) std::cerr << "    Applied " << n_fixups << " fixups" << std::endl;

        // Read merge sequence size and replay, tracking old→new ID mapping
        uint64_t seq_size = read_u64();
        std::vector<char> output;
        output.reserve(new_size);
        size_t old_rec = 0, new_rec = 0; // record indices
        size_t old_pos_bytes = 0;
        size_t seq_end = pos + seq_size;

        // ID remap for this file (old_record_index → new_record_index)
        std::vector<uint32_t> id_map;
        bool track_ids = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
                          file_id == (uint32_t)PatchFileId::STREET_WAYS ||
                          file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
                          file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
        if (track_ids) id_map.assign(old_data.size() / actual_stride, 0xFFFFFFFF);

        while (pos < seq_end) {
            uint8_t op = static_cast<uint8_t>(patch[pos]); pos++;
            uint32_t count; memcpy(&count, patch.data() + pos, 4); pos += 4;

            if (op == OP_MATCH_RUN) {
                size_t bytes = count * actual_stride;
                if (old_pos_bytes + bytes <= old_data.size())
                    output.insert(output.end(), old_data.data() + old_pos_bytes,
                                 old_data.data() + old_pos_bytes + bytes);
                if (track_ids)
                    for (uint32_t k = 0; k < count; k++)
                        if (old_rec + k < id_map.size())
                            id_map[old_rec + k] = static_cast<uint32_t>(new_rec + k);
                old_rec += count; new_rec += count;
                old_pos_bytes += bytes;
            } else if (op == OP_INSERT_RUN) {
                size_t bytes = count * actual_stride;
                output.insert(output.end(), patch.data() + pos, patch.data() + pos + bytes);
                pos += bytes;
                new_rec += count;
            } else if (op == OP_DELETE_RUN) {
                old_rec += count;
                old_pos_bytes += count * actual_stride;
            }
        }

        write_file(out_dir + "/" + std::string(filename), output);

        // Store ID remap for entry reconstruction
        if (track_ids) {
            derived_id_remaps[file_id] = std::move(id_map);
            size_t matched = 0;
            for (auto v : derived_id_remaps[file_id]) if (v != 0xFFFFFFFF) matched++;
            std::cerr << "  " << filename << ": " << output.size() << " bytes (mapped "
                      << matched << "/" << derived_id_remaps[file_id].size() << " IDs)" << std::endl;
        } else {
            std::cerr << "  " << filename << ": " << output.size() << " bytes" << std::endl;
        }
    }

    // Reconstruct entry/cell files from derived ID remaps (if not provided as full replacement)
    {
        bool have_geo = false, have_admin = false;
        // Check if geo_cells was already written (full replacement)
        auto check = [&](const std::string& name) -> bool {
            auto d = read_file(out_dir + "/" + name);
            return !d.empty();
        };
        have_geo = check("geo_cells.bin");
        have_admin = check("admin_cells.bin");

        if (!derived_id_remaps.empty()) { // always reconstruct entries (needed for corrections)
            std::cerr << "Reconstructing entry/cell files from derived ID remaps..." << std::endl;
            auto convert = [](const std::vector<uint32_t>& vec) {
                std::unordered_map<uint32_t,uint32_t> m;
                for (uint32_t i = 0; i < vec.size(); i++)
                    if (vec[i] != 0xFFFFFFFF) m[i] = vec[i];
                return m;
            };
            std::unordered_map<uint32_t,uint32_t> w_rm, a_rm, i_rm;
            if (derived_id_remaps.count((uint32_t)PatchFileId::STREET_WAYS))
                w_rm = convert(derived_id_remaps[(uint32_t)PatchFileId::STREET_WAYS]);
            if (derived_id_remaps.count((uint32_t)PatchFileId::ADDR_POINTS))
                a_rm = convert(derived_id_remaps[(uint32_t)PatchFileId::ADDR_POINTS]);
            if (derived_id_remaps.count((uint32_t)PatchFileId::INTERP_WAYS))
                i_rm = convert(derived_id_remaps[(uint32_t)PatchFileId::INTERP_WAYS]);

            auto old_geo = read_file(cur_dir + "/geo_cells.bin");
            auto old_se = read_file(cur_dir + "/street_entries.bin");
            auto old_ae = read_file(cur_dir + "/addr_entries.bin");
            auto old_ie = read_file(cur_dir + "/interp_entries.bin");

            auto geo = rebuild_geo_from_remap(old_geo, old_se, old_ae, old_ie, w_rm, a_rm, i_rm,
                                               geo_added, geo_removed);
            // Always write geo_cells (needed as correction prefix)
            write_file(out_dir + "/geo_cells.bin", geo.geo_cells_data);
            write_file(out_dir + "/street_entries.bin", geo.street_entries_data);
            write_file(out_dir + "/addr_entries.bin", geo.addr_entries_data);
            write_file(out_dir + "/interp_entries.bin", geo.interp_entries_data);
            std::cerr << "  Rebuilt 3 entry files" << (have_geo ? "" : " + geo_cells") << std::endl;
        }

        if (!derived_id_remaps.empty()) { // always reconstruct admin entries for corrections
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            if (derived_id_remaps.count((uint32_t)PatchFileId::ADMIN_POLYGONS)) {
                auto& vec = derived_id_remaps[(uint32_t)PatchFileId::ADMIN_POLYGONS];
                for (uint32_t i = 0; i < vec.size(); i++)
                    if (vec[i] != 0xFFFFFFFF) ad_rm[i] = vec[i];
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_ae = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_ae, ad_rm);
            write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
            write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            std::cerr << "  Rebuilt admin_entries" << (have_admin ? "" : " + admin_cells") << std::endl;
        }
    }

    // Apply buffered corrections: entry files first, then cell files
    // Entry corrections must be applied before geo_cells rebuild.
    std::vector<PendingCorrection> cell_corrections;
    for (auto& corr : pending_corrections) {
        if (corr.file_id == (uint32_t)PatchFileId::GEO_CELLS ||
            corr.file_id == (uint32_t)PatchFileId::ADMIN_CELLS) {
            cell_corrections.push_back(std::move(corr));
            continue;
        }
        // Apply entry correction
        std::string derived_path = out_dir + "/" + corr.filename;
        std::string zst_path = tmpdir + "/corr_" + corr.filename + ".zst";
        std::string corrected_path = tmpdir + "/corr_" + corr.filename + ".out";
        write_file(zst_path, corr.delta);
        std::string cmd = "zstd --patch-from='" + derived_path + "' -d '" + zst_path +
                          "' -o '" + corrected_path + "' -f --long=31 --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0)
            std::cerr << "  " << corr.filename << ": entry correction FAILED" << std::endl;
        else {
            auto corrected = read_file(corrected_path);
            write_file(derived_path, corrected);
            std::cerr << "  " << corr.filename << ": corrected → " << corrected.size() << " bytes" << std::endl;
        }
        remove(zst_path.c_str()); remove(corrected_path.c_str());
    }
    pending_corrections.clear(); // entry corrections done

    // Now apply cell corrections (after geo_cells rebuild from corrected entries)
    for (auto& corr : cell_corrections) {
        std::string derived_path = out_dir + "/" + corr.filename;
        std::string zst_path = tmpdir + "/corr_" + corr.filename + ".zst";
        std::string corrected_path = tmpdir + "/corr_" + corr.filename + ".out";
        write_file(zst_path, corr.delta);
        std::string cmd = "zstd --patch-from='" + derived_path + "' -d '" + zst_path +
                          "' -o '" + corrected_path + "' -f --long=31 --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0) {
            std::cerr << "  " << corr.filename << ": correction FAILED" << std::endl;
        } else {
            auto corrected = read_file(corrected_path);
            write_file(derived_path, corrected);
            std::cerr << "  " << corr.filename << ": corrected → " << corrected.size() << " bytes" << std::endl;
        }
        remove(zst_path.c_str()); remove(corrected_path.c_str());
    }

    // Post-correction geo_cells rebuild from corrected entries
    // This produces a nearly-correct geo_cells (>99.99% accurate)
    // which is then fixed by the tiny geo_cells correction delta.
    {
        auto corr_se = read_file(out_dir + "/street_entries.bin");
        auto corr_ae = read_file(out_dir + "/addr_entries.bin");
        auto corr_ie = read_file(out_dir + "/interp_entries.bin");
        auto derived_geo = read_file(out_dir + "/geo_cells.bin");

        if (!derived_geo.empty() && !pending_corrections.empty()) {
            size_t n = derived_geo.size() / 20;
            uint32_t no_data = 0xFFFFFFFF;

            // Count entries in each corrected file
            auto count_entries = [](const std::vector<char>& ef) -> size_t {
                size_t count = 0, pos = 0;
                while (pos + 2 <= ef.size()) {
                    uint16_t c; memcpy(&c, ef.data() + pos, 2);
                    pos += 2 + c * 4;
                    count++;
                }
                return count;
            };

            // Build offset arrays by scanning corrected entries
            auto build_offsets = [](const std::vector<char>& ef) -> std::vector<uint32_t> {
                std::vector<uint32_t> offsets;
                uint32_t pos = 0;
                while (pos + 2 <= ef.size()) {
                    offsets.push_back(pos);
                    uint16_t c; memcpy(&c, ef.data() + pos, 2);
                    pos += 2 + c * 4;
                }
                return offsets;
            };

            auto s_offsets = build_offsets(corr_se);
            auto a_offsets = build_offsets(corr_ae);
            auto i_offsets = build_offsets(corr_ie);

            // Walk derived geo_cells, replace offsets with corrected values
            size_t si = 0, ai = 0, ii = 0;
            std::vector<char> new_geo(derived_geo.size());
            for (size_t c = 0; c < n; c++) {
                // Copy cell_id
                memcpy(new_geo.data() + c * 20, derived_geo.data() + c * 20, 8);

                uint32_t old_s, old_a, old_i;
                memcpy(&old_s, derived_geo.data() + c * 20 + 8, 4);
                memcpy(&old_a, derived_geo.data() + c * 20 + 12, 4);
                memcpy(&old_i, derived_geo.data() + c * 20 + 16, 4);

                uint32_t new_s = (old_s != no_data && si < s_offsets.size()) ? s_offsets[si++] : no_data;
                uint32_t new_a = (old_a != no_data && ai < a_offsets.size()) ? a_offsets[ai++] : no_data;
                uint32_t new_i = (old_i != no_data && ii < i_offsets.size()) ? i_offsets[ii++] : no_data;

                memcpy(new_geo.data() + c * 20 + 8, &new_s, 4);
                memcpy(new_geo.data() + c * 20 + 12, &new_a, 4);
                memcpy(new_geo.data() + c * 20 + 16, &new_i, 4);
            }
            write_file(out_dir + "/geo_cells.bin", new_geo);
            std::cerr << "  Rebuilt geo_cells from corrected entries" << std::endl;
        }
    }
    if (false) { // legacy rebuild code
        auto se = read_file(out_dir + "/street_entries.bin");
        auto ae = read_file(out_dir + "/addr_entries.bin");
        auto ie = read_file(out_dir + "/interp_entries.bin");
        auto existing_geo = read_file(out_dir + "/geo_cells.bin");

        if (!existing_geo.empty()) {
            // Parse the existing (derived) geo_cells for cell IDs, rebuild offsets from corrected entries
            size_t n = existing_geo.size() / 20;

            // Parse each entry file to build cell→offset maps
            // Walk entries sequentially, matching against geo_cells cell order
            auto build_offsets_from_geo = [](const std::vector<char>& geo, size_t off_pos,
                                              const std::vector<char>& old_entries,
                                              const std::vector<char>& new_entries) -> std::vector<char> {
                // The corrected entries file has the right content.
                // We need to map cell_ids to byte offsets in the corrected entries.
                // The cell order is the same as in the derived geo_cells.
                // We just need to recompute offsets.
                return new_entries; // entries are already correct, just need geo_cells offsets
            };

            // Actually: re-walk the derived geo_cells, find each cell's entries in the corrected files
            // Since the cell IDs are the same and entries are in the same order, we just recompute offsets.
            // The corrected entry files have entries in cell_id order (from the rebuild).
            // We can walk them to compute offsets.

            // For each entry type, build cell_id → offset by scanning the entry file
            auto scan_entries = [](const std::vector<char>& entries) -> std::vector<uint32_t> {
                // Returns a vector of entry start positions
                std::vector<uint32_t> positions;
                uint32_t pos = 0;
                while (pos + 2 <= entries.size()) {
                    positions.push_back(pos);
                    uint16_t count; memcpy(&count, entries.data() + pos, 2);
                    pos += 2 + count * 4;
                }
                return positions;
            };

            auto s_pos = scan_entries(se);
            auto a_pos = scan_entries(ae);
            auto i_pos = scan_entries(ie);

            // The derived geo_cells has cell_ids + offsets into derived entries.
            // The corrected entries have the same cell order but potentially different byte offsets.
            // Rebuild geo_cells with corrected offsets.
            uint32_t no_data = 0xFFFFFFFF;
            size_t si = 0, ai = 0, ii = 0;
            std::vector<char> new_geo;
            for (size_t c = 0; c < n; c++) {
                uint64_t cid; memcpy(&cid, existing_geo.data() + c * 20, 8);
                new_geo.insert(new_geo.end(), (char*)&cid, (char*)&cid + 8);

                // For each entry type, check if this cell has data
                uint32_t old_s, old_a, old_i;
                memcpy(&old_s, existing_geo.data() + c * 20 + 8, 4);
                memcpy(&old_a, existing_geo.data() + c * 20 + 12, 4);
                memcpy(&old_i, existing_geo.data() + c * 20 + 16, 4);

                uint32_t new_s = (old_s != no_data && si < s_pos.size()) ? s_pos[si++] : no_data;
                uint32_t new_a = (old_a != no_data && ai < a_pos.size()) ? a_pos[ai++] : no_data;
                uint32_t new_i = (old_i != no_data && ii < i_pos.size()) ? i_pos[ii++] : no_data;

                new_geo.insert(new_geo.end(), (char*)&new_s, (char*)&new_s + 4);
                new_geo.insert(new_geo.end(), (char*)&new_a, (char*)&new_a + 4);
                new_geo.insert(new_geo.end(), (char*)&new_i, (char*)&new_i + 4);
            }
            write_file(out_dir + "/geo_cells.bin", new_geo);
            std::cerr << "  Rebuilt geo_cells.bin from corrected entries (" << new_geo.size() << " bytes)" << std::endl;
        }
    }

    // Similarly for admin_cells from admin_entries
    {
        auto ae = read_file(out_dir + "/admin_entries.bin");
        auto existing_ac = read_file(out_dir + "/admin_cells.bin");
        if (!existing_ac.empty()) {
            size_t n = existing_ac.size() / 12;
            uint32_t no_data = 0xFFFFFFFF;
            std::vector<uint32_t> a_pos;
            uint32_t p = 0;
            while (p + 2 <= ae.size()) {
                a_pos.push_back(p);
                uint16_t count; memcpy(&count, ae.data() + p, 2);
                p += 2 + count * 4;
            }
            size_t ai = 0;
            std::vector<char> new_ac;
            for (size_t c = 0; c < n; c++) {
                uint64_t cid; memcpy(&cid, existing_ac.data() + c * 12, 8);
                uint32_t old_off; memcpy(&old_off, existing_ac.data() + c * 12 + 8, 4);
                new_ac.insert(new_ac.end(), (char*)&cid, (char*)&cid + 8);
                uint32_t new_off = (old_off != no_data && ai < a_pos.size()) ? a_pos[ai++] : no_data;
                new_ac.insert(new_ac.end(), (char*)&new_off, (char*)&new_off + 4);
            }
            write_file(out_dir + "/admin_cells.bin", new_ac);
            std::cerr << "  Rebuilt admin_cells.bin from corrected entries (" << new_ac.size() << " bytes)" << std::endl;
        }
    }

    std::string rm_cmd = "rm -rf '" + tmpdir + "'";
    system(rm_cmd.c_str());
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

-        if (!derived_geo.empty() && !pending_corrections.empty()) {

+        if (!derived_geo.empty() && !cell_corrections.empty()) {


Bash

cmake --build build -j$(nproc) --target geocoder-patch 2>&1 | tail -2 && sshpass -p 'elon-is-insane' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:~/geocoder-patch 2>&1 && sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
rm -rf /home/michtest/patched-final7
/home/michtest/geocoder-patch /home/michtest/det-new /home/michtest/test-final7.gcpatch -o /home/michtest/patched-final7 2>&1 | grep -E "(corrected|Rebuilt|FAIL)"
all_match=true
for f in /home/michtest/det-today/*.bin; do
    b=$(basename $f); p=/home/michtest/patched-final7/$b
    [ -f "$p" ] && cmp -s "$f" "$p" || { all_match=false; echo "FAIL: $(basename $f)"; }
done
$all_match && echo "ALL FILES MATCH" || echo "SOME DIFFER"
' 2>&1

Test with fixed condition


[100%] Linking CXX executable geocoder-patch
[100%] Built target geocoder-patch
Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
  Rebuilt 3 entry files + geo_cells
  Rebuilt admin_entries + admin_cells
  street_entries.bin: corrected → 586625094 bytes
  addr_entries.bin: corrected → 395061190 bytes
  interp_entries.bin: corrected → 159386 bytes
  admin_entries.bin: corrected → 6117948 bytes
  geo_cells.bin: correction FAILED
  admin_cells.bin: corrected → 1692000 bytes
  Rebuilt geo_cells from corrected entries
  Rebuilt admin_cells.bin from corrected entries (1692000 bytes)
FAIL: geo_cells.bin
SOME DIFFER


The rebuild runs now ("Rebuilt geo_cells from corrected entries") but the correction still fails. The correction was computed against the diff's rebuild (from new entries). The patch's rebuild uses the same algorithm. But the prefix for the correction is the rebuilt geo_cells, while the correction delta was computed from a different version.

Let me check: is the diff's `geo_from_new_entries` the same as the patch's post-correction rebuild? Both use: derived cell structure (cell IDs + has/hasn't flags) + new/corrected entry files → scan offsets → assign to cells. If the algorithm is identical and the inputs are the same, the outputs must match.

The diff's algorithm is inline in the diff tool. The patch's is in the post-correction block. Let me verify they're identical...

Actually, the correction fails because the delta is 396 MiB — it's still using the old correction delta (from the previous test run). The patch file `test-final7.gcpatch` was generated from the PREVIOUS diff tool version. Let me regenerate:


sshpass -p 'elon-is-insane' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" /root/traccar-geocoder/builder/build/geocoder-diff michtest@100.113.88.121:~/geocoder-diff 2>&1 && sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
rm -f /home/michtest/test-final8.gcpatch
/home/michtest/geocoder-diff /home/michtest/det-new /home/michtest/det-today -o /home/michtest/test-final8.gcpatch 2>&1 | grep -E "(seq=|correction|Compressed|full )"
echo "Patch: $(ls -lh /home/michtest/test-final8.gcpatch | awk "{print \$5}")"
rm -rf /home/michtest/patched-final8
/home/michtest/geocoder-patch /home/michtest/det-new /home/michtest/test-final8.gcpatch -o /home/michtest/patched-final8 2>&1 | grep -E "(corrected|Rebuilt|FAIL)"
all_match=true
for f in /home/michtest/det-today/*.bin; do
    b=$(basename $f); p=/home/michtest/patched-final8/$b
    [ -f "$p" ] && cmp -s "$f" "$p" || { all_match=false; echo "FAIL: $(basename $f)"; }
done
$all_match && echo "ALL FILES MATCH" || echo "SOME DIFFER"
' 2>&1

Full test with new diff + new patch


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
  strings.bin: full 82576146 bytes
  addr_points.bin: seq=517114 fixups=0 (0.04%)
  street_ways.bin: seq=356397 fixups=19453346 (0.15%)
  street_nodes.bin: seq=32369106 fixups=0 (2.24%)
  interp_ways.bin: seq=5 fixups=0 (0.00%)
  interp_nodes.bin: seq=5 fixups=0 (0.00%)
  admin_polygons.bin: seq=18345 fixups=449366 (0.17%)
  admin_vertices.bin: seq=1890025 fixups=0 (0.65%)
  street_entries.bin: correction 633971 bytes (0.11%)
  addr_entries.bin: correction 508075 bytes (0.13%)
  interp_entries.bin: correction 39 bytes (0.02%)
  admin_entries.bin: correction 79772 bytes (1.30%)
  geo_cells.bin: correction 396349117 bytes (23.67%)
  admin_cells.bin: correction 500248 bytes (29.57%)
Compressed patch: 473569475 bytes (451 MiB)
Patch: 452M
  Rebuilt 3 entry files + geo_cells
  Rebuilt admin_entries + admin_cells
  street_entries.bin: corrected → 586625094 bytes
  addr_entries.bin: corrected → 395061190 bytes
  interp_entries.bin: corrected → 159386 bytes
  admin_entries.bin: corrected → 6117948 bytes
  geo_cells.bin: correction FAILED
  admin_cells.bin: corrected → 1692000 bytes
  Rebuilt geo_cells from corrected entries
  Rebuilt admin_cells.bin from corrected entries (1692000 bytes)
FAIL: geo_cells.bin
SOME DIFFER


geo_cells correction still 396 MiB. The diff's `rebuild_geo_from_entries` and the patch's post-correction rebuild produce different outputs, so the correction delta is massive.

Let me write that document first as requested, then come back to this.


git add tools/geocoder_diff.cpp tools/geocoder_patch.cpp tools/patch_format.h && git commit -m "$(cat <<'EOF'
wip: geo_cells correction approach — diff/patch rebuild mismatch

Entry corrections work perfectly (all entry files MATCH).
geo_cells correction still fails because diff and patch tool rebuilds
produce different output from the same inputs.

The entry-corrected rebuild approach is correct in principle — just
needs the rebuild algorithm to be byte-identical between tools.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
EOF
)"

Commit WIP


[main d3467f5] wip: geo_cells correction approach — diff/patch rebuild mismatch
 2 files changed, 128 insertions(+), 5 deletions(-)


Now let me write the documentation:


Write

# Geocoder Incremental Patch System

## Overview

A system for producing small daily patch files so users can update their geocoder
index (~7 GiB for Europe) without re-downloading the full dataset. Users apply
patches in sequence: day1 + patch_1→2 → day2, day2 + patch_2→3 → day3, etc.

## Architecture

```
build-index (deterministic) → geocoder-diff old/ new/ → patch.gcpatch
geocoder-patch old/ patch.gcpatch → new/  (byte-identical to fresh build)
```

### Tools
- **build-index**: Modified to produce deterministic output (canonical sort order)
- **geocoder-diff**: Compares two deterministic builds, produces .gcpatch file
- **geocoder-patch**: Applies .gcpatch to old build, produces new build

### Key Design Decisions
1. **Deterministic builds**: Same PBF → byte-identical output (verified)
2. **Custom patch format**: Merge sequences at record level, not raw binary diff
3. **String remapping**: Old string offsets mapped to new (alphabetical sort differs between builds)
4. **zstd transport compression**: Whole patch compressed for download

## Current State (2026-03-27)

### What Works
- Deterministic builds: verified byte-identical across runs
- Data files: all 8 data files produce byte-identical output after patching
- Entry files: corrected via zstd deltas against derived rebuild
- admin_cells: corrected via zstd delta
- **13 of 14 files match** — only geo_cells.bin remains

### Patch Size: 2-Day Europe Gap (Mar 25 → Mar 27)

| Component | Uncompressed | In Patch |
|-----------|-------------|----------|
| String remap (5M entries) | 40 MiB | ~12 MiB |
| Way offset fixups (19.4M) | 155 MiB | ~43 MiB |
| Admin offset fixups (449K) | 3.6 MiB | ~1 MiB |
| strings.bin (full replacement) | 83 MiB | ~15 MiB |
| addr_points merge (0.04%) | 517 KB | <1 MiB |
| street_ways merge (0.15%) | 356 KB | <1 MiB |
| street_nodes merge (2.24%) | 32 MiB | ~3 MiB |
| admin_vertices merge (0.65%) | 1.9 MiB | <1 MiB |
| admin_polygons merge (0.17%) | 18 KB | <1 MiB |
| interp files | <1 KB | <1 MiB |
| Cell changes (9K added, 2K removed) | 85 KB | <1 MiB |
| Entry corrections (4 files) | 1.2 MiB | <1 MiB |
| **geo_cells.bin correction** | **396 MiB** | **~370 MiB** |
| admin_cells correction | 500 KB | <1 MiB |
| **Total** | | **~452 MiB** |

Without geo_cells: **~78 MiB**. Estimated 1-day gap: **~40 MiB**.

## Patch Format (.gcpatch, version 2)

Whole file is zstd-compressed for transport. Uncompressed structure:

```
Header: "GCPATCH\0" (8) + version (u32) + flags (u32)

Section: String Remap
  marker: 0xFFFFFFFE (u32)
  count: u32
  entries: [(old_offset: u32, new_offset: u32)] × count

Section: Per-file merge sequences
  file_id: u32 (PatchFileId enum, 0-13)
  stride: u32 (record size, 0=full replacement, 0xFE=correction)
  old_size: u64, new_size: u64
  n_fixups: u32
  fixups: [(record_index: u32, new_offset_value: u32)] × n_fixups
  seq_size: u64
  ops: MATCH_RUN(count) | INSERT_RUN(count, data) | DELETE_RUN(count)

Section: Cell Changes
  marker: 0xFFFFFFFB/0xFFFFFFFA (geo/admin)
  n_added: u32, n_removed: u32
  added_cells: [cell_id: u64] × n_added
  removed_cells: [cell_id: u64] × n_removed

Section: Correction deltas (stride=0xFE)
  file_id: u32, stride: 0xFE
  derived_size: u64, new_size: u64
  n_fixups: u32 (always 0)
  delta_size: u64
  delta: zstd frame (derived → new via --patch-from)

End marker: 0xFFFFFFFF (u32)
```

## How Patching Works

### Diff Tool (geocoder-diff)
1. Build string remap (old string pool → new string pool offsets)
2. For each data file:
   a. Apply string remap to old file
   b. For ways/interps/admin: hash-match records by content, fix node_offset/vertex_offset
   c. Build merge sequence (MATCH/INSERT/DELETE at record stride)
   d. Record offset fixups for matched records
3. Derive ID remaps by replaying merge sequences (same as patch tool)
4. Rebuild entries from derived ID remaps + cell changes
5. Compute entry corrections: zstd(derived_entries → new_entries)
6. Compute cell index corrections: zstd(derived_cells → new_cells)
7. Package everything, compress with zstd

### Patch Tool (geocoder-patch)
1. Decompress zstd transport layer
2. Read string remap, detect strides
3. For each merge section:
   a. Apply string remap to old file
   b. Apply offset fixups (node_offset/vertex_offset)
   c. Replay merge sequence → output file
   d. Track old→new ID mapping from MATCH ops
4. Read cell changes (added/removed cells)
5. Rebuild entry files from derived ID remaps + cell changes
6. Rebuild admin files similarly
7. Apply entry corrections (zstd deltas)
8. Rebuild geo_cells from corrected entries
9. Apply cell index corrections (zstd deltas)

## Approaches Tried

### 1. Raw zstd --patch-from (baseline)
**Result**: 1.9 GiB patches (27% of 7 GiB). String offsets and record IDs differ
between independent builds, causing cascading byte differences in every file.

### 2. Canonical ordering in builder
Added deterministic sorting (string pool alphabetical, records by content).
**Result**: Same PBF → byte-identical output. But patches between different PBFs
still 1.7 GiB because string offsets cascade.

### 3. String remap + zstd --patch-from
Apply string offset remapping before binary diff.
**Result**: Data files drop to 0.2-0.7%. But entry/cell files still 24-79%.
Total: ~941 MiB.

### 4. Record ID remap + entry rebuild
Match records by content, remap IDs in entry files.
**Result**: Entry files drop to 0.02-0.07%. Total with compressed tables: ~626 MiB.
But remap tables are 240 MiB compressed (187 MiB for 86M addr pairs).

### 5. Custom merge sequences (current approach)
Walk sorted records in parallel, emit MATCH/INSERT/DELETE ops at record level.
Hash-indexed matching for data files. Sequential scan for coordinate files.
**Result**: Data files at 0.04-2.24%. Merge sequences implicitly encode ID remaps.
Eliminates explicit ID remap tables. Total: ~78 MiB without entry/cell files.

### 6. Entry corrections
Include zstd deltas: derived_entries → new_entries. Tiny corrections (~1 MiB).
**Result**: All entry files match. geo_cells.bin remains at 396 MiB because derived
geo_cells has cascading offset differences.

## Remaining Work

### geo_cells.bin (the last bottleneck)
- **Problem**: 1.6 GiB file with 84M × 20-byte records. Each record has 12 bytes of
  entry offsets that cascade when any cell's entries change.
- **Current**: 396 MiB correction delta (derived offsets → correct offsets)
- **Root cause**: Derived geo_cells is built from pre-correction entries. After corrections,
  entries are correct, but we need to recompute offsets. The rebuild algorithm in diff
  and patch tools must produce byte-identical output.
- **Potential fix**: Share exact rebuild algorithm, or include only the ~9K cells whose
  has/hasn't flags differ (~100 KB of correction data).

### Sequential patching test
- Have 3 Europe builds: det-A (Mar 21), det-new (Mar 25), det-today (Mar 27)
- Need: patch(det-A, p12) → result2, patch(result2, p23) → result3, verify result3 == det-today
- Blocked by: geo_cells.bin mismatch

### Code cleanup
- Remove prototype files (full_remap_test.cpp, remap_test.cpp)
- Remove dead code blocks (if (false) sections)
- Remove geocoder_canonicalize.cpp (superseded by build_index determinism)
- Clean up the diff/patch tools (currently ~700 lines each, could be ~400)

## File Inventory

### Modified
- `src/build_index.cpp` — deterministic ordering (sort strings, records, nodes, vertices)
- `CMakeLists.txt` — new build targets

### New (committed)
- `tools/geocoder_diff.cpp` — diff tool (~700 lines)
- `tools/geocoder_patch.cpp` — patch tool (~500 lines)
- `tools/patch_format.h` — shared format definitions + rebuild functions (~400 lines)
- `tools/geocoder_canonicalize.cpp` — standalone canonicalize (testing tool)

### Prototypes (uncommitted, can be removed)
- `tools/full_remap_test.cpp` — proved remap approach
- `tools/remap_test.cpp` — string-only remap test

## Performance

### Build Time Overhead
- Deterministic sorting: ~100s (string sort 5s, addr sort 55s, way sort 16s)
- Total build: ~12 min + 100s = ~14 min (+13%)

### Diff Time
- 2-day Europe gap: ~5 min (dominated by merge building + zstd compression)

### Patch Application Time
- ~2 min (dominated by merge replay + entry reconstruction + zstd decompression)

## Three Europe Builds Available for Testing
- `det-A`: March 21 PBF (europe.osm.pbf, 34.0 GiB)
- `det-new`: March 25 PBF (europe-new.osm.pbf, 34.1 GiB)
- `det-today`: March 27 PBF (europe-today.osm.pbf, 34.1 GiB)

Node 3: 32 cores, 174 GiB RAM, files at /home/michtest/


File created successfully at: /root/traccar-geocoder/builder/PATCHING.md

# Geocoder Incremental Patch System

## Overview

A system for producing small daily patch files so users can update their geocoder
index (~7 GiB for Europe) without re-downloading the full dataset. Users apply
patches in sequence: day1 + patch_1→2 → day2, day2 + patch_2→3 → day3, etc.

## Architecture

```
build-index (deterministic) → geocoder-diff old/ new/ → patch.gcpatch
geocoder-patch old/ patch.gcpatch → new/  (byte-identical to fresh build)
```

### Tools
- **build-index**: Modified to produce deterministic output (canonical sort order)
- **geocoder-diff**: Compares two deterministic builds, produces .gcpatch file
- **geocoder-patch**: Applies .gcpatch to old build, produces new build

### Key Design Decisions
1. **Deterministic builds**: Same PBF → byte-identical output (verified)
2. **Custom patch format**: Merge sequences at record level, not raw binary diff
3. **String remapping**: Old string offsets mapped to new (alphabetical sort differs between builds)
4. **zstd transport compression**: Whole patch compressed for download

## Current State (2026-03-27)

### What Works
- Deterministic builds: verified byte-identical across runs
- Data files: all 8 data files produce byte-identical output after patching
- Entry files: corrected via zstd deltas against derived rebuild
- admin_cells: corrected via zstd delta
- **13 of 14 files match** — only geo_cells.bin remains

### Patch Size: 2-Day Europe Gap (Mar 25 → Mar 27)

| Component | Uncompressed | In Patch |
|-----------|-------------|----------|
| String remap (5M entries) | 40 MiB | ~12 MiB |
| Way offset fixups (19.4M) | 155 MiB | ~43 MiB |
| Admin offset fixups (449K) | 3.6 MiB | ~1 MiB |
| strings.bin (full replacement) | 83 MiB | ~15 MiB |
| addr_points merge (0.04%) | 517 KB | <1 MiB |
| street_ways merge (0.15%) | 356 KB | <1 MiB |
| street_nodes merge (2.24%) | 32 MiB | ~3 MiB |
| admin_vertices merge (0.65%) | 1.9 MiB | <1 MiB |
| admin_polygons merge (0.17%) | 18 KB | <1 MiB |
| interp files | <1 KB | <1 MiB |
| Cell changes (9K added, 2K removed) | 85 KB | <1 MiB |
| Entry corrections (4 files) | 1.2 MiB | <1 MiB |
| **geo_cells.bin correction** | **396 MiB** | **~370 MiB** |
| admin_cells correction | 500 KB | <1 MiB |
| **Total** | | **~452 MiB** |

Without geo_cells: **~78 MiB**. Estimated 1-day gap: **~40 MiB**.

## Patch Format (.gcpatch, version 2)

Whole file is zstd-compressed for transport. Uncompressed structure:

```
Header: "GCPATCH\0" (8) + version (u32) + flags (u32)

Section: String Remap
  marker: 0xFFFFFFFE (u32)
  count: u32
  entries: [(old_offset: u32, new_offset: u32)] × count

Section: Per-file merge sequences
  file_id: u32 (PatchFileId enum, 0-13)
  stride: u32 (record size, 0=full replacement, 0xFE=correction)
  old_size: u64, new_size: u64
  n_fixups: u32
  fixups: [(record_index: u32, new_offset_value: u32)] × n_fixups
  seq_size: u64
  ops: MATCH_RUN(count) | INSERT_RUN(count, data) | DELETE_RUN(count)

Section: Cell Changes
  marker: 0xFFFFFFFB/0xFFFFFFFA (geo/admin)
  n_added: u32, n_removed: u32
  added_cells: [cell_id: u64] × n_added
  removed_cells: [cell_id: u64] × n_removed

Section: Correction deltas (stride=0xFE)
  file_id: u32, stride: 0xFE
  derived_size: u64, new_size: u64
  n_fixups: u32 (always 0)
  delta_size: u64
  delta: zstd frame (derived → new via --patch-from)

End marker: 0xFFFFFFFF (u32)
```

## How Patching Works

### Diff Tool (geocoder-diff)
1. Build string remap (old string pool → new string pool offsets)
2. For each data file:
   a. Apply string remap to old file
   b. For ways/interps/admin: hash-match records by content, fix node_offset/vertex_offset
   c. Build merge sequence (MATCH/INSERT/DELETE at record stride)
   d. Record offset fixups for matched records
3. Derive ID remaps by replaying merge sequences (same as patch tool)
4. Rebuild entries from derived ID remaps + cell changes
5. Compute entry corrections: zstd(derived_entries → new_entries)
6. Compute cell index corrections: zstd(derived_cells → new_cells)
7. Package everything, compress with zstd

### Patch Tool (geocoder-patch)
1. Decompress zstd transport layer
2. Read string remap, detect strides
3. For each merge section:
   a. Apply string remap to old file
   b. Apply offset fixups (node_offset/vertex_offset)
   c. Replay merge sequence → output file
   d. Track old→new ID mapping from MATCH ops
4. Read cell changes (added/removed cells)
5. Rebuild entry files from derived ID remaps + cell changes
6. Rebuild admin files similarly
7. Apply entry corrections (zstd deltas)
8. Rebuild geo_cells from corrected entries
9. Apply cell index corrections (zstd deltas)

## Approaches Tried

### 1. Raw zstd --patch-from (baseline)
**Result**: 1.9 GiB patches (27% of 7 GiB). String offsets and record IDs differ
between independent builds, causing cascading byte differences in every file.

### 2. Canonical ordering in builder
Added deterministic sorting (string pool alphabetical, records by content).
**Result**: Same PBF → byte-identical output. But patches between different PBFs
still 1.7 GiB because string offsets cascade.

### 3. String remap + zstd --patch-from
Apply string offset remapping before binary diff.
**Result**: Data files drop to 0.2-0.7%. But entry/cell files still 24-79%.
Total: ~941 MiB.

### 4. Record ID remap + entry rebuild
Match records by content, remap IDs in entry files.
**Result**: Entry files drop to 0.02-0.07%. Total with compressed tables: ~626 MiB.
But remap tables are 240 MiB compressed (187 MiB for 86M addr pairs).

### 5. Custom merge sequences (current approach)
Walk sorted records in parallel, emit MATCH/INSERT/DELETE ops at record level.
Hash-indexed matching for data files. Sequential scan for coordinate files.
**Result**: Data files at 0.04-2.24%. Merge sequences implicitly encode ID remaps.
Eliminates explicit ID remap tables. Total: ~78 MiB without entry/cell files.

### 6. Entry corrections
Include zstd deltas: derived_entries → new_entries. Tiny corrections (~1 MiB).
**Result**: All entry files match. geo_cells.bin remains at 396 MiB because derived
geo_cells has cascading offset differences.

## Remaining Work

### geo_cells.bin (the last bottleneck)
- **Problem**: 1.6 GiB file with 84M × 20-byte records. Each record has 12 bytes of
  entry offsets that cascade when any cell's entries change.
- **Current**: 396 MiB correction delta (derived offsets → correct offsets)
- **Root cause**: Derived geo_cells is built from pre-correction entries. After corrections,
  entries are correct, but we need to recompute offsets. The rebuild algorithm in diff
  and patch tools must produce byte-identical output.
- **Potential fix**: Share exact rebuild algorithm, or include only the ~9K cells whose
  has/hasn't flags differ (~100 KB of correction data).

### Sequential patching test
- Have 3 Europe builds: det-A (Mar 21), det-new (Mar 25), det-today (Mar 27)
- Need: patch(det-A, p12) → result2, patch(result2, p23) → result3, verify result3 == det-today
- Blocked by: geo_cells.bin mismatch

### Code cleanup
- Remove prototype files (full_remap_test.cpp, remap_test.cpp)
- Remove dead code blocks (if (false) sections)
- Remove geocoder_canonicalize.cpp (superseded by build_index determinism)
- Clean up the diff/patch tools (currently ~700 lines each, could be ~400)

## File Inventory

### Modified
- `src/build_index.cpp` — deterministic ordering (sort strings, records, nodes, vertices)
- `CMakeLists.txt` — new build targets

### New (committed)
- `tools/geocoder_diff.cpp` — diff tool (~700 lines)
- `tools/geocoder_patch.cpp` — patch tool (~500 lines)
- `tools/patch_format.h` — shared format definitions + rebuild functions (~400 lines)
- `tools/geocoder_canonicalize.cpp` — standalone canonicalize (testing tool)

### Prototypes (uncommitted, can be removed)
- `tools/full_remap_test.cpp` — proved remap approach
- `tools/remap_test.cpp` — string-only remap test

## Performance

### Build Time Overhead
- Deterministic sorting: ~100s (string sort 5s, addr sort 55s, way sort 16s)
- Total build: ~12 min + 100s = ~14 min (+13%)

### Diff Time
- 2-day Europe gap: ~5 min (dominated by merge building + zstd compression)

### Patch Application Time
- ~2 min (dominated by merge replay + entry reconstruction + zstd decompression)

## Three Europe Builds Available for Testing
- `det-A`: March 21 PBF (europe.osm.pbf, 34.0 GiB)
- `det-new`: March 25 PBF (europe-new.osm.pbf, 34.1 GiB)
- `det-today`: March 27 PBF (europe-today.osm.pbf, 34.1 GiB)

Node 3: 32 cores, 174 GiB RAM, files at /home/michtest/


Bash

git add PATCHING.md && git commit -m "$(cat <<'EOF'
docs: add PATCHING.md with full system documentation

Comprehensive log of the incremental patch system:
- Architecture and pipeline
- Current state (13/14 files matching, 452 MiB patches)
- Patch format specification
- All approaches tried and their results
- Remaining work (geo_cells bottleneck, sequential testing)
- Performance numbers and file inventory

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
EOF
)"

Commit PATCHING.md


[main 8742e81] docs: add PATCHING.md with full system documentation
 1 file changed, 213 insertions(+)
 create mode 100644 builder/PATCHING.md


Done. The document is at `builder/PATCHING.md`. It covers everything — current state, what works, what we've tried, what remains, the patch format spec, and performance numbers.

The TL;DR: **13 of 14 files match, 78 MiB without geo_cells, 452 MiB with. The one remaining issue is geo_cells.bin where the diff and patch tool rebuilds don't produce byte-identical output from the same inputs.** Once that's fixed, we're at ~80 MiB for a 2-day gap (~40 MiB per day).


<task-notification>
<task-id>bsmixvfvl</task-id>
<tool-use-id>toolu_01XnYCuQoNZUfkHhv61sajbd</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bsmixvfvl.output</output-file>
<status>completed</status>
<summary>Background command "Measure geo_cells standalone compression" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bsmixvfvl.output


Already processed — that was 261 MiB standalone compression for geo_cells.bin.

Now let me fix the geo_cells issue. The core problem: the diff and patch tools both rebuild geo_cells from entries using the same algorithm, but produce different output because they're in different source files with potentially different compilation/linking.

Actually, the real issue is simpler — they use the same `rebuild_geo_from_remap` shared function, but the diff's geo_cells correction is computed against a rebuild that uses NEW entry files directly, while the patch's rebuild uses the CORRECTED entry files (which should be identical to new, but the rebuild happens at a different point in the pipeline).

Let me just verify: are the corrected entries byte-identical to new entries? If yes, the rebuilds should match.


sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
# Check if corrected entries == new entries
for f in street_entries.bin addr_entries.bin interp_entries.bin; do
    cmp -s /home/michtest/patched-final8/$f /home/michtest/det-today/$f && echo "$f: MATCH" || echo "$f: DIFFER"
done
# Also check the rebuilt geo_cells vs new
cmp -s /home/michtest/patched-final8/geo_cells.bin /home/michtest/det-today/geo_cells.bin && echo "geo_cells: MATCH" || {
    s1=$(stat -c%s /home/michtest/patched-final8/geo_cells.bin)
    s2=$(stat -c%s /home/michtest/det-today/geo_cells.bin)
    echo "geo_cells: DIFFER ($s1 vs $s2)"
}
' 2>&1

Check if corrected entries match new


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
street_entries.bin: MATCH
addr_entries.bin: MATCH
interp_entries.bin: MATCH
geo_cells: DIFFER (1674442300 vs 1674442300)


Entries MATCH perfectly. geo_cells same size but different content. The rebuild from corrected entries produces wrong offsets. Both the diff and patch use the same rebuild algorithm — but one uses the new entries directly and the other uses corrected entries (which are byte-identical). So the rebuild inputs are the same. The only other input is the derived geo_cells (for cell IDs + has/hasn't flags).

The diff uses `derived.geo_cells_data` (built from `rebuild_geo_from_remap` using hash-based ID remap). The patch uses the same function's output. These should be identical since they use the same shared function.

Wait — the diff uses the hash-based ID remap (from `derive_remap`), while the patch uses the merge-derived ID remap. These are subtly different (the ~50 unmatched records). So the derived geo_cells has different has/hasn't flags between diff and patch.

That's the bug. The diff's correction is computed against a rebuild with hash-based ID remap flags. The patch rebuilds with merge-based ID remap flags. For the ~50 cells that differ, the offset assignment goes wrong.

The fix is simple: make the diff use the SAME rebuild as the patch (merge-derived, not hash-based). The diff already has `derived` from `derive_remap`. The correction should use THIS derived version, not a separate `geo_from_new_entries` rebuild.

Let me update the diff: the geo_cells correction should be `derived.geo_cells_data` (with corrected entry offsets) → new geo_cells. The "corrected entry offsets" version is what the patch produces after step 4 (rebuild from corrected entries). Since the patch's rebuild uses the same `derived.geo_cells_data` for cell flags, both tools produce the same intermediate.

The issue is that the diff currently rebuilds geo_cells from new entries (line `rebuild_geo_from_entries`), while the patch rebuilds from corrected entries. These should be the same, but the rebuild function also uses the `derived.geo_cells_data` has/hasn't flags, and the diff's `derived` uses hash-based remaps while the patch's uses merge-based remaps.

**The fix**: make the diff's geo_cells correction use the MERGE-DERIVED `derived.geo_cells_data` as the prefix for the correction delta. The patch will produce the same merge-derived version, then apply the correction to get the exact new version.


   660→            size_t n = derived_geo.size() / 20, si = 0, ai = 0, ii = 0;
   661→            uint32_t no_data = 0xFFFFFFFF;
   662→            std::vector<char> result(derived_geo.size());
   663→            for (size_t c = 0; c < n; c++) {
   664→                memcpy(result.data() + c*20, derived_geo.data() + c*20, 8);
   665→                uint32_t os, oa, oi;
   666→                memcpy(&os, derived_geo.data()+c*20+8, 4); memcpy(&oa, derived_geo.data()+c*20+12, 4);
   667→                memcpy(&oi, derived_geo.data()+c*20+16, 4);
   668→                uint32_t ns = (os != no_data && si < so.size()) ? so[si++] : no_data;
   669→                uint32_t na = (oa != no_data && ai < ao.size()) ? ao[ai++] : no_data;
   670→                uint32_t ni = (oi != no_data && ii < io.size()) ? io[ii++] : no_data;
   671→                memcpy(result.data()+c*20+8, &ns, 4); memcpy(result.data()+c*20+12, &na, 4);
   672→                memcpy(result.data()+c*20+16, &ni, 4);
   673→            }
   674→            return result;

            size_t n = derived_geo.size() / 20, si = 0, ai = 0, ii = 0;
            uint32_t no_data = 0xFFFFFFFF;
            std::vector<char> result(derived_geo.size());
            for (size_t c = 0; c < n; c++) {
                memcpy(result.data() + c*20, derived_geo.data() + c*20, 8);
                uint32_t os, oa, oi;
                memcpy(&os, derived_geo.data()+c*20+8, 4); memcpy(&oa, derived_geo.data()+c*20+12, 4);
                memcpy(&oi, derived_geo.data()+c*20+16, 4);
                uint32_t ns = (os != no_data && si < so.size()) ? so[si++] : no_data;
                uint32_t na = (oa != no_data && ai < ao.size()) ? ao[ai++] : no_data;
                uint32_t ni = (oi != no_data && ii < io.size()) ? io[ii++] : no_data;
                memcpy(result.data()+c*20+8, &ns, 4); memcpy(result.data()+c*20+12, &na, 4);
                memcpy(result.data()+c*20+16, &ni, 4);
            }
            return result;


The diff's `rebuild_geo_from_entries` takes `derived_geo` (from the hash-based rebuild) as the cell template. The patch's rebuild takes the merge-derived `geo.geo_cells_data` as the template. These templates have different has/hasn't flags for ~50 cells.

The fix: in the diff, use the merge-derived `derived.geo_cells_data` as the template instead of the hash-based one. Both tools then use the same template.


   675→        };
   676→
   677→        // For geo_cells: rebuild from new entries + derived cell structure
   678→        auto geo_from_new_entries = rebuild_geo_from_entries(derived.geo_cells_data,
   679→            new_dir + "/street_entries.bin", new_dir + "/addr_entries.bin", new_dir + "/interp_entries.bin");
   680→
   681→        for (auto& [c_fid, c_label, c_data, c_fname] : std::vector<std::tuple<PatchFileId, std::string, std::vector<char>*, std::string>>{
   682→            {PatchFileId::GEO_CELLS, "geo_cells", &geo_from_new_entries, "geo_cells.bin"},
   683→            {PatchFileId::ADMIN_CELLS, "admin_cells", &admin_derived.admin_cells_data, "admin_cells.bin"}})
   684→        {

        };

        // For geo_cells: rebuild from new entries + derived cell structure
        auto geo_from_new_entries = rebuild_geo_from_entries(derived.geo_cells_data,
            new_dir + "/street_entries.bin", new_dir + "/addr_entries.bin", new_dir + "/interp_entries.bin");

        for (auto& [c_fid, c_label, c_data, c_fname] : std::vector<std::tuple<PatchFileId, std::string, std::vector<char>*, std::string>>{
            {PatchFileId::GEO_CELLS, "geo_cells", &geo_from_new_entries, "geo_cells.bin"},
            {PatchFileId::ADMIN_CELLS, "admin_cells", &admin_derived.admin_cells_data, "admin_cells.bin"}})
        {


The diff currently uses `derived.geo_cells_data` (hash-based rebuild template) + new entries. The patch uses `geo.geo_cells_data` (merge-derived template) + corrected entries.

`derived` = hash-based rebuild (from `rebuild_geo_from_remap` with hash-based IDs)
`geo` (on patch side) = merge-derived rebuild (from `rebuild_geo_from_remap` with merge-derived IDs)

These templates differ because different ID remaps → different has/hasn't flags. To make them match, the diff should use the MERGE-DERIVED template. The diff already has `derived` (from the `derive_remap` section). Let me check:


No matches found


grep -n "rebuild_geo_from_remap" /root/traccar-geocoder/builder/tools/geocoder_diff.cpp

Find rebuild calls


602:        auto derived = rebuild_geo_from_remap(old_geo, old_se2, old_ae2, old_ie2,


Line 602: `derived` is built from `w_rm_d, a_rm_d, i_rm_d` (the merge-derived remaps). Good — the diff already uses merge-derived remaps for `derived`. And `rebuild_geo_from_entries` on line 678 uses `derived.geo_cells_data` as the template.

So the diff uses: merge-derived template + new entries → `geo_from_new_entries`.
The patch uses: merge-derived template + corrected entries.

Since corrected entries == new entries, these should be IDENTICAL. Why do they differ?

Let me check: does the `geo_added`/`geo_removed` differ? The diff passes cell changes to `rebuild_geo_from_remap`. The patch also gets cell changes from the patch file. Let me verify both get the same cell changes:

The diff computes: 8984 added, 1641 removed (from the output). The patch reads the same values from the patch file. Both pass them to `rebuild_geo_from_remap`. The shared function adds new cells and removes old cells.

After the shared rebuild, both have the same `geo_cells_data` template. Then both rebuild offsets from the same entries (new == corrected). So they MUST produce the same output.

Unless the rebuild on the patch side happens differently. Let me check the exact code path on the patch side:

The patch's post-correction rebuild reads corrected entry files from `out_dir`, then walks `derived_geo` (from the earlier reconstruction), assigns offsets. The diff does the same with `derived.geo_cells_data` and new entry files.

Wait — the `derived_geo` on the patch side is from the RECONSTRUCTION step (before corrections). The reconstruction writes geo_cells with wrong offsets (from pre-correction entries). Then corrections fix entries. The post-correction rebuild reads the corrected entries and the ORIGINAL derived geo_cells (with wrong offsets but correct has/hasn't flags).

But the has/hasn't flags in the derived geo_cells ARE correct (from the merge-derived rebuild). The reconstruction step uses `rebuild_geo_from_remap` which produces correct flags. So both diff and patch have the same has/hasn't flags.

The entries are the same (corrected == new). The flags are the same (merge-derived). The rebuild algorithm is the same (inline code that scans entries and assigns offsets). The output MUST be the same.

Unless there's a subtle bug in the inline code. Let me compare the two implementations byte by byte...

Actually, let me just test: save the diff's `geo_from_new_entries` to a file and compare with the patch's post-correction rebuild:


sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 'python3 -c "
import struct

# Load the derived geo_cells (from patch reconstruction)
derived = open(\"/home/michtest/patched-final8/geo_cells.bin\", \"rb\").read()
new_gc = open(\"/home/michtest/det-today/geo_cells.bin\", \"rb\").read()

# Load corrected entries
se = open(\"/home/michtest/patched-final8/street_entries.bin\", \"rb\").read()
ae = open(\"/home/michtest/patched-final8/addr_entries.bin\", \"rb\").read()
ie = open(\"/home/michtest/patched-final8/interp_entries.bin\", \"rb\").read()

# Rebuild offsets from corrected entries
def scan_entries(data):
    offs = []
    pos = 0
    while pos + 2 <= len(data):
        offs.append(pos)
        count = struct.unpack_from(\"<H\", data, pos)[0]
        pos += 2 + count * 4
    return offs

so = scan_entries(se)
ao = scan_entries(ae)
io = scan_entries(ie)

n = len(derived) // 20
si = ai = ii = 0
NO_DATA = 0xFFFFFFFF
rebuilt = bytearray(len(derived))
diffs = 0

for c in range(n):
    # Copy cell_id
    rebuilt[c*20:c*20+8] = derived[c*20:c*20+8]
    old_s = struct.unpack_from(\"<I\", derived, c*20+8)[0]
    old_a = struct.unpack_from(\"<I\", derived, c*20+12)[0]
    old_i = struct.unpack_from(\"<I\", derived, c*20+16)[0]

    ns = so[si] if (old_s != NO_DATA and si < len(so)) else NO_DATA
    if old_s != NO_DATA and si < len(so): si += 1
    na = ao[ai] if (old_a != NO_DATA and ai < len(ao)) else NO_DATA
    if old_a != NO_DATA and ai < len(ao): ai += 1
    ni = io[ii] if (old_i != NO_DATA and ii < len(io)) else NO_DATA
    if old_i != NO_DATA and ii < len(io): ii += 1

    struct.pack_into(\"<III\", rebuilt, c*20+8, ns, na, ni)

    if rebuilt[c*20:(c+1)*20] != new_gc[c*20:(c+1)*20]:
        diffs += 1
        if diffs <= 3:
            r_vals = struct.unpack_from(\"<QI I I\", bytes(rebuilt), c*20)
            n_vals = struct.unpack_from(\"<QI I I\", new_gc, c*20)
            print(f\"Cell {c}: rebuilt={r_vals[1:]}, new={n_vals[1:]}\")

print(f\"Total diffs: {diffs}/{n}\")
print(f\"Entry counts: street={len(so)} addr={len(ao)} interp={len(io)}\")
print(f\"Scan end: si={si} ai={ai} ii={ii}\")
"' 2>&1

Verify rebuild algorithm in Python


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Cell 19526: rebuilt=(4294967295, 4294967295, 4294967295), new=(138220, 4294967295, 4294967295)
Cell 19527: rebuilt=(138220, 21406, 4294967295), new=(138226, 21406, 4294967295)
Cell 19528: rebuilt=(138226, 21412, 4294967295), new=(138240, 21412, 4294967295)
Total diffs: 83694115/83722115
Entry counts: street=76031121 addr=25526329 interp=17285
Scan end: si=76023644 ai=25522286 ii=17285


The Python test shows 83.7M cells differ (99.97%). The entry scan consumed: si=76,023,644 of 76,031,121 street entries — 7,477 entries left unconsumed. This means the derived geo_cells has 7,477 MORE cells flagged as "has street data" than actual entries exist. Those extra cells get NO_DATA (because si runs out of entries), but the new geo_cells has valid offsets for them.

The has/hasn't flags in the derived geo_cells are wrong for these ~7K cells. This causes the offset assignment to desync, cascading to all subsequent cells.

The user's question about alternate approaches is spot on. Let me address that.

The current approach keeps hitting the same wall: the entry files reference records by sequential index, and any mismatch in the index mapping cascades through everything.

Your idea about hash-based IDs is interesting. Instead of sequential indices, records could be identified by a content hash. This would make cell entries stable across builds regardless of record ordering. But it requires changing the server's lookup — currently it does `ways[id]` (direct array indexing), which would need to become a hash table lookup.

However, there's a simpler solution for the geo_cells problem specifically. Rather than trying to get the rebuild perfect, I can just **include geo_cells as a merge sequence**. Each 20-byte record has a unique cell_id (first 8 bytes). With hash-based matching on cell_id, matched cells get their new offset bytes from INSERT (12 bytes per changed cell). Only ~9K cells are truly new/removed.

For 84M cells where 99.97% of cell_ids are shared: 84M MATCH ops (copying cell_ids) + 84M × 12 bytes of new offset data (since offsets differ for all cells). That's still 1 GiB.

But if I match on the FULL 20-byte record, only the ~9K truly changed cells need INSERT. The other 84M match byte-for-byte IF the offsets are correct. The offsets are correct when the entries are correct. But at this point in the pipeline, entries ARE correct (after corrections).

Actually, the real fix is much simpler than I've been making it. Let me just **rebuild geo_cells correctly from the corrected entries**. The Python test showed the problem: the has/hasn't flags are wrong for ~7K cells. These are cells where the merge-derived ID remap lost records, so the rebuilt entries have fewer entries than new.

Since the corrected entries ARE byte-identical to new, I just need a correct way to determine which cells have data. The simplest approach: **count the entries in the corrected files and compare with the derived geo_cells flags**. If the count doesn't match, something's wrong.

But the fundamental issue is: I can't determine which cells have data without the geo_cells mapping. And I can't build geo_cells without knowing which cells have data.

The pragmatic fix for now: include geo_cells as a full-replacement section. At 261 MiB compressed, the total patch would be ~339 MiB for 2 days (~170 MiB per day). Not ideal but functional.

Or: let me try a completely different approach for geo_cells. Since it's a sorted array of 20-byte records with unique cell_ids, I can merge it like any other data file. The cell_ids are deterministic (same lat/lng → same S2 cell), so matched cells appear in the same order. The 12 offset bytes differ, but the merge would encode them as INSERT data.

Let me try: merge geo_cells at 20-byte stride. Since cell_ids match (8 bytes) but offsets differ (12 bytes), every record is a DELETE+INSERT pair (no MATCH). But the INSERT data is just the new 20-byte records. With 84M records: 84M × (5+5+20) = 2.5 GiB of merge data. Worse than full replacement.

Better approach for the user's suggestion: include a flag in the patch format that says "use full replacement for these files." Then we get a working system at ~340 MiB. We can optimize geo_cells later as a separate effort.

Let me switch back to full-replacement geo_cells (which was verified working at 337 MiB), commit, and focus on sequential patching validation:


Edit

// Cell index corrections against the entry-corrected rebuild:

        // Cell index files: full replacement (correction approach doesn't work
        // because the derived geo_cells has wrong has/hasn't flags for ~7K cells,
        // causing offset assignment desync)
        for (const auto& cell_file : std::vector<std::pair<PatchFileId, std::string>>{
            {PatchFileId::GEO_CELLS, "geo_cells.bin"},
            {PatchFileId::ADMIN_CELLS, "admin_cells.bin"}})
        {
            auto data = read_file(new_dir + "/" + cell_file.second);
            uint32_t f = static_cast<uint32_t>(cell_file.first), st_v = 0;
            uint64_t os = 0, ns = data.size();
            uint32_t nfix = 0; uint64_t ss = data.size();
            wval(patch, &f, 4); wval(patch, &st_v, 4);
            wval(patch, &os, 8); wval(patch, &ns, 8);
            wval(patch, &nfix, 4); wval(patch, &ss, 8);
            patch.insert(patch.end(), data.begin(), data.end());
            std::cerr << "  " << cell_file.second << ": full " << data.size() << " bytes" << std::endl;
        }

        if (false) // disabled: correction approach doesn't produce matching rebuilds
        // Cell index corrections against the entry-corrected rebuild: