        std::cerr << "Entry pipeline (proven rebuild approach)..." << std::endl;

        auto get_rm = [&](PatchFileId fid) -> const std::vector<uint32_t>& {
            static std::vector<uint32_t> empty;
            auto it = id_remaps.find((uint32_t)fid); return it != id_remaps.end() ? it->second : empty;
        };
        auto& w_rm = get_rm(PatchFileId::STREET_WAYS);

// geocoder-patch v4: Low-memory streaming patch application.
//
// Key design: mmap old files (zero copy), stream output via fwrite (never accumulate),
// process entry pipeline cell-by-cell. Target: <1 GiB peak RSS for planet.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <fcntl.h>
#include <iomanip>
#include <iostream>
#include <malloc.h>
#include <string>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unordered_map>
#include <unordered_set>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

// --- Helpers ---
static double now_ms() {
    struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
    return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6;
}
static size_t get_rss_mb() {
    FILE* f = fopen("/proc/self/statm", "r");
    if (!f) return 0;
    size_t dummy, rss; fscanf(f, "%zu %zu", &dummy, &rss); fclose(f);
    return rss * 4096 / (1024*1024);
}
static void log_phase(const char* label, double start) {
    std::cerr << "  [" << std::fixed << std::setprecision(1) << (now_ms() - start) / 1000.0
              << "s, " << get_rss_mb() << " MiB] " << label << std::endl;
}

// mmap a file read-only. Returns {pointer, size}. Caller must munmap.
struct MappedFile { const char* data; size_t size; };
static MappedFile mmap_file(const std::string& path) {
    int fd = open(path.c_str(), O_RDONLY);
    if (fd < 0) return {nullptr, 0};
    struct stat st; fstat(fd, &st);
    size_t sz = st.st_size;
    if (sz == 0) { close(fd); return {nullptr, 0}; }
    void* p = mmap(nullptr, sz, PROT_READ, MAP_PRIVATE, fd, 0);
    close(fd);
    if (p == MAP_FAILED) return {nullptr, 0};
    return {static_cast<const char*>(p), sz};
}
static void unmap_file(MappedFile& f) {
    if (f.data) { munmap(const_cast<char*>(f.data), f.size); f.data = nullptr; f.size = 0; }
}

// Detect stride from file size
static size_t detect_stride(const std::string& path, std::initializer_list<size_t> candidates) {
    struct stat st; if (stat(path.c_str(), &st) != 0) return *candidates.begin();
    for (size_t s : candidates) if (st.st_size % s == 0 && st.st_size > 0) return s;
    return *candidates.begin();
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);
    double t_start = now_ms();

    // --- Phase 1: Decompress + mmap patch ---
    std::string raw_path = tmpdir + "/patch.raw";
    {
        std::string cmd = "zstd -d '" + patch_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0) { std::cerr << "Failed to decompress" << std::endl; return 1; }
    }
    MappedFile patch_map = mmap_file(raw_path);
    if (!patch_map.data) { std::cerr << "Failed to mmap patch" << std::endl; return 1; }
    remove(raw_path.c_str());
    const char* P = patch_map.data;
    size_t patch_size = patch_map.size;
    std::cerr << "Patch: " << patch_size << " bytes" << std::endl;
    log_phase("Decompress", t_start);

    size_t pos = 0;
    auto ru32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, P+pos, 4); pos += 4; return v; };
    auto ru64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, P+pos, 8); pos += 8; return v; };

    // Header
    if (memcmp(P, GCPATCH_MAGIC, 8) != 0) { std::cerr << "Bad magic" << std::endl; return 1; }
    pos = 8;
    uint32_t ver = ru32(); if (ver != 2) { std::cerr << "Bad version" << std::endl; return 1; }
    ru32(); // flags

    // --- Phase 2: String rebuild ---
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t marker = ru32();
        if (marker == 0xFFFFFFF7) {
            uint32_t n_added = ru32(), n_deleted = ru32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ru32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());

            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
            std::vector<std::pair<uint32_t, std::string>> old_strs;
            size_t sp = 0;
            while (sp < old_pool.size) {
                old_strs.push_back({(uint32_t)sp, std::string(old_pool.data + sp)});
                sp += strlen(old_pool.data + sp) + 1;
            }
            std::vector<std::string> merged;
            for (size_t i = 0; i < old_strs.size(); i++)
                if (!del_set.count(i)) merged.push_back(old_strs[i].second);
            for (auto& s : added) merged.push_back(s);
            std::sort(merged.begin(), merged.end());

            // Write new pool and build remap
            FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
            std::unordered_map<std::string, uint32_t> new_offs;
            uint32_t wpos = 0;
            for (auto& s : merged) {
                new_offs[s] = wpos;
                fwrite(s.c_str(), 1, s.size() + 1, fp);
                wpos += s.size() + 1;
            }
            fclose(fp);
            for (auto& [old_off, s] : old_strs) {
                auto it = new_offs.find(s);
                if (it != new_offs.end() && it->second != old_off) str_remap[old_off] = it->second;
            }
            unmap_file(old_pool);
            std::cerr << "  Strings: +" << n_added << " -" << n_deleted << " → " << merged.size()
                      << ", " << str_remap.size() << " remapped" << std::endl;
            marker = ru32();
        }
        if (marker == 0xFFFFFFFE) { uint32_t c = ru32(); for (uint32_t i = 0; i < c; i++) { uint32_t a = ru32(), b = ru32(); str_remap[a] = b; } }
        else pos -= 4;
    }
    log_phase("Strings", t_start);

    // Detect strides
    size_t way_stride = detect_stride(cur_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride(cur_dir + "/interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect_stride(cur_dir + "/admin_polygons.bin", {24, 20, 19});

    // String remap applier (works on mutable copy)
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t fid, size_t stride) {
        if (str_remap.empty()) return;
        std::vector<size_t> offs;
        if (fid == (uint32_t)PatchFileId::ADDR_POINTS) offs = {8, 12};
        else if (fid == (uint32_t)PatchFileId::STREET_WAYS) offs = {(stride == 12) ? 8ul : 5ul};
        else if (fid == (uint32_t)PatchFileId::INTERP_WAYS) offs = {(stride >= 20) ? 8ul : 5ul};
        else if (fid == (uint32_t)PatchFileId::ADMIN_POLYGONS) offs = {8};
        else return;
        for (size_t i = 0; i + stride <= data.size(); i += stride)
            for (size_t o : offs) { uint32_t v; memcpy(&v, data.data()+i+o, 4); auto it = str_remap.find(v); if (it != str_remap.end()) memcpy(data.data()+i+o, &it->second, 4); }
    };

    // --- Phase 3: Merge replays (streaming output) ---
    // ID remaps: file_id → vector<uint32_t> where remap[old_idx] = new_idx
    std::unordered_map<uint32_t, std::vector<uint32_t>> id_remaps;
    std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;
    std::unordered_map<uint64_t, uint8_t> flag_corrections;
    struct CellCorr { uint64_t cell_id; std::vector<uint32_t> ids; };
    std::unordered_map<uint32_t, std::vector<CellCorr>> entry_corrections;

    while (pos < patch_size) {
        uint32_t file_id = ru32();
        if (file_id == 0xFFFFFFFF) break;

        // --- Metadata sections ---
        if (file_id == 0xFFFFFFF7) {
            // String diff in main loop — process it (may be the only occurrence)
            uint32_t n_added = ru32(), n_deleted = ru32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ru32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());
            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
            std::vector<std::pair<uint32_t, std::string>> old_strs;
            size_t sp = 0;
            while (sp < old_pool.size) { old_strs.push_back({(uint32_t)sp, std::string(old_pool.data+sp)}); sp += strlen(old_pool.data+sp)+1; }
            std::vector<std::string> merged;
            for (size_t i = 0; i < old_strs.size(); i++) if (!del_set.count(i)) merged.push_back(old_strs[i].second);
            for (auto& s : added) merged.push_back(s);
            std::sort(merged.begin(), merged.end());
            FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
            std::unordered_map<std::string, uint32_t> new_offs;
            uint32_t wpos = 0;
            for (auto& s : merged) { new_offs[s] = wpos; fwrite(s.c_str(), 1, s.size()+1, fp); wpos += s.size()+1; }
            fclose(fp);
            for (auto& [old_off, s] : old_strs) { auto it = new_offs.find(s); if (it != new_offs.end() && it->second != old_off) str_remap[old_off] = it->second; }
            unmap_file(old_pool);
            std::cerr << "  Strings (loop): +" << n_added << " -" << n_deleted << " → " << merged.size() << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_GEO_MARKER) {
            uint32_t na = ru32(), nr = ru32();
            geo_added.resize(na); geo_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { memcpy(&geo_added[i], P+pos, 8); pos += 8; }
            for (uint32_t i = 0; i < nr; i++) { memcpy(&geo_removed[i], P+pos, 8); pos += 8; }
            std::cerr << "  Geo cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_ADMIN_MARKER) {
            uint32_t na = ru32(), nr = ru32();
            admin_added.resize(na); admin_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { memcpy(&admin_added[i], P+pos, 8); pos += 8; }
            for (uint32_t i = 0; i < nr; i++) { memcpy(&admin_removed[i], P+pos, 8); pos += 8; }
            std::cerr << "  Admin cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_FLAGS_MARKER) {
            uint32_t c = ru32();
            for (uint32_t i = 0; i < c; i++) { uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8; flag_corrections[cid] = P[pos++]; }
            std::cerr << "  Flag corrections: " << c << std::endl;
            continue;
        }
        if (file_id == SECONDARY_REMAP_MARKER) {
            uint32_t nf = ru32();
            for (uint32_t f = 0; f < nf; f++) {
                uint32_t fid = ru32(), np = ru32();
                auto& rm = id_remaps[fid];
                for (uint32_t i = 0; i < np; i++) { uint32_t o = ru32(), n = ru32(); if (o < rm.size()) rm[o] = n; }
                std::cerr << "  Secondary remap " << fid << ": " << np << " pairs" << std::endl;
            }
            continue;
        }
        if (file_id == ENTRY_CORRECTION_MARKER) {
            uint32_t fid = ru32(), c = ru32();
            auto& list = entry_corrections[fid];
            for (uint32_t i = 0; i < c; i++) {
                uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8;
                uint16_t ec; memcpy(&ec, P+pos, 2); pos += 2;
                std::vector<uint32_t> ids(ec);
                if (ec > 0) { memcpy(ids.data(), P+pos, ec*4); pos += ec*4; }
                list.push_back({cid, std::move(ids)});
            }
            std::cerr << "  Entry corrections " << fid << ": " << c << std::endl;
            continue;
        }

        // --- Merge sequence replay (streaming) ---
        uint32_t stride = ru32();
        uint64_t old_size = ru64(), new_size = ru64();
        if (file_id >= (uint32_t)PatchFileId::COUNT) { std::cerr << "Unknown file " << file_id << std::endl; break; }
        const char* fname = patch_file_names[file_id];

        if (stride == 0) {
            // Full replacement — write directly from patch mmap
            uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64();
            FILE* fp = fopen((out_dir + "/" + fname).c_str(), "wb");
            fwrite(P+pos, 1, ds, fp); fclose(fp);
            pos += ds;
            std::cerr << "  " << fname << ": full replace " << ds << " bytes" << std::endl;
            continue;
        }
        if (stride == 0xFE) { uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64(); pos += ds; continue; }

        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        // Load old file into mutable buffer (need to apply remap + fixups)
        // For large files (nodes/vertices), this is the peak memory per file.
        // Files are processed sequentially, so only one is in memory at a time.
        auto old_data = read_file(cur_dir + "/" + std::string(fname));

        // Zero padding
        if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24)
            for (size_t i = 0; i + actual_stride <= old_data.size(); i += actual_stride) memset(old_data.data()+i+13, 0, 3);
        if (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24)
            for (size_t i = 0; i + actual_stride <= old_data.size(); i += actual_stride) { memset(old_data.data()+i+5, 0, 3); memset(old_data.data()+i+21, 0, 3); }

        apply_str_remap(old_data, file_id, actual_stride);

        // Apply fixups
        uint32_t n_fixups = ru32();
        if (n_fixups > 0) {
            uint32_t delta_size = ru32();
            uint32_t pi = 0, pv = 0; size_t fp = pos;
            for (uint32_t i = 0; i < n_fixups; i++) {
                uint32_t idx = pi + read_varint(P, fp);
                uint32_t val = pv + read_varint(P, fp);
                if ((size_t)idx * actual_stride + 4 <= old_data.size())
                    memcpy(old_data.data() + (size_t)idx * actual_stride, &val, 4);
                pi = idx; pv = val;
            }
            pos = fp;
        }

        // Replay merge sequence — stream output directly to file
        uint64_t seq_size = ru64();
        size_t seq_end = pos + seq_size;
        FILE* outf = fopen((out_dir + "/" + fname).c_str(), "wb");

        // Track ID remap (only for data files that need entry reconstruction)
        bool track = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
                      file_id == (uint32_t)PatchFileId::STREET_WAYS ||
                      file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
                      file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
        std::vector<uint32_t> id_map;
        if (track) id_map.assign(old_data.size() / actual_stride, 0xFFFFFFFF);

        size_t old_rec = 0, new_rec = 0, old_bytes = 0, written = 0;
        while (pos < seq_end) {
            uint8_t op = P[pos++];
            uint32_t count; memcpy(&count, P+pos, 4); pos += 4;
            if (op == OP_MATCH_RUN) {
                size_t bytes = count * actual_stride;
                fwrite(old_data.data() + old_bytes, 1, bytes, outf);
                written += bytes;
                if (track) for (uint32_t k = 0; k < count; k++) if (old_rec+k < id_map.size()) id_map[old_rec+k] = new_rec+k;
                old_rec += count; new_rec += count; old_bytes += bytes;
            } else if (op == OP_INSERT_RUN) {
                size_t bytes = count * actual_stride;
                fwrite(P+pos, 1, bytes, outf);
                written += bytes; pos += bytes; new_rec += count;
            } else if (op == OP_DELETE_RUN) {
                old_rec += count; old_bytes += count * actual_stride;
            }
        }
        fclose(outf);

        if (track) {
            id_remaps[file_id] = std::move(id_map);
            size_t matched = 0;
            for (auto v : id_remaps[file_id]) if (v != 0xFFFFFFFF) matched++;
            std::cerr << "  " << fname << ": " << written << " bytes (mapped " << matched << "/" << id_remaps[file_id].size() << ")" << std::endl;
        } else {
            std::cerr << "  " << fname << ": " << written << " bytes" << std::endl;
        }

        // Free old data immediately — don't let glibc hold the pages
        { std::vector<char>().swap(old_data); }
        malloc_trim(0);
    }
    log_phase("Merge replays", t_start);

    // Free string remap (no longer needed)
    { std::unordered_map<uint32_t,uint32_t>().swap(str_remap); }
    malloc_trim(0);

    // --- Phase 4: Streaming entry pipeline ---
    // Walk old geo_cells, remap IDs, apply corrections, write output directly to files.
    {
        double t_entry = now_ms();
        std::cerr << "Entry pipeline (proven rebuild approach)..." << std::endl;

        auto& w_rm = get_rm(PatchFileId::STREET_WAYS);
        auto& a_rm = get_rm(PatchFileId::ADDR_POINTS);
        auto& i_rm = get_rm(PatchFileId::INTERP_WAYS);

        // Rebuild entries using proven rebuild_geo_from_remap_vec
        auto old_geo_data = read_file(cur_dir + "/geo_cells.bin");
        auto old_se = read_file(cur_dir + "/street_entries.bin");
        auto old_ae = read_file(cur_dir + "/addr_entries.bin");
        auto old_ie = read_file(cur_dir + "/interp_entries.bin");
        auto geo = rebuild_geo_from_remap_vec(old_geo_data, old_se, old_ae, old_ie,
                                                w_rm, a_rm, i_rm, geo_added, geo_removed);
        // Free inputs
        { std::vector<char>().swap(old_se); std::vector<char>().swap(old_ae); std::vector<char>().swap(old_ie); }
        log_phase("  Geo rebuild", t_entry);

        // Apply corrections sequentially (one file at a time to limit memory)
        // Build correction maps
        auto apply_corrections = [&](uint32_t fid, const std::string& fname,
                                      std::vector<char>& derived_entries,
                                      const std::vector<char>& derived_cells, size_t cell_stride,
                                      size_t off_pos) {
            auto ecit = entry_corrections.find(fid);
            if (ecit == entry_corrections.end() || ecit->second.empty()) {
                write_file(out_dir + "/" + fname, derived_entries);
                return;
            }
            std::unordered_map<uint64_t, const std::vector<uint32_t>*> cm;
            for (auto& c : ecit->second) cm[c.cell_id] = &c.ids;

            // Walk cells, apply corrections, write output via FILE*
            size_t n = derived_cells.size() / cell_stride;
            FILE* fe = fopen((out_dir + "/" + fname).c_str(), "wb");
            FILE* fc_out = nullptr;
            std::vector<char> new_cells;
            uint32_t no_data_v = 0xFFFFFFFF;
            uint32_t fe_pos = 0;

            for (size_t i = 0; i < n; i++) {
                uint64_t cid; memcpy(&cid, derived_cells.data() + i * cell_stride, 8);
                uint32_t off; memcpy(&off, derived_cells.data() + i * cell_stride + off_pos, 4);

                auto cit = cm.find(cid);
                if (cit != cm.end()) {
                    // Use correction data
                    uint32_t new_off = cit->second->empty() ? no_data_v : fe_pos;
                    if (!cit->second->empty()) {
                        uint16_t c = cit->second->size();
                        fwrite(&c, 2, 1, fe);
                        fwrite(cit->second->data(), 4, c, fe);
                        fe_pos += 2 + c * 4;
                    }
                    // Update offset in cells data (in memory)
                    memcpy(const_cast<char*>(derived_cells.data()) + i * cell_stride + off_pos, &new_off, 4);
                } else if (off != no_data_v) {
                    // Copy original entry data
                    if (off + 2 <= derived_entries.size()) {
                        uint16_t c; memcpy(&c, derived_entries.data() + off, 2);
                        uint32_t new_off = fe_pos;
                        fwrite(derived_entries.data() + off, 1, 2 + c * 4, fe);
                        fe_pos += 2 + c * 4;
                        memcpy(const_cast<char*>(derived_cells.data()) + i * cell_stride + off_pos, &new_off, 4);
                    }
                }
            }
            fclose(fe);
            std::cerr << "  " << fname << ": " << cm.size() << " corrections applied" << std::endl;
        };

        // Apply corrections for each geo entry type
        apply_corrections((uint32_t)PatchFileId::STREET_ENTRIES, "street_entries.bin",
                         geo.street_entries_data, geo.geo_cells_data, 20, 8);
        apply_corrections((uint32_t)PatchFileId::ADDR_ENTRIES, "addr_entries.bin",
                         geo.addr_entries_data, geo.geo_cells_data, 20, 12);
        apply_corrections((uint32_t)PatchFileId::INTERP_ENTRIES, "interp_entries.bin",
                         geo.interp_entries_data, geo.geo_cells_data, 20, 16);

        // Write the corrected geo_cells (offsets updated in-place by apply_corrections)
        // Rebuild offsets using flag corrections
        {
            size_t n = geo.geo_cells_data.size() / 20;
            FILE* fg = fopen((out_dir + "/geo_cells.bin").c_str(), "wb");
            // Scan corrected entry files to build offset arrays
            auto scan_offsets = [](const std::string& path) -> std::vector<uint32_t> {
                auto data = read_file(path);
                std::vector<uint32_t> offsets;
                uint32_t pos = 0;
                while (pos + 2 <= data.size()) {
                    offsets.push_back(pos);
                    uint16_t c; memcpy(&c, data.data() + pos, 2);
                    pos += 2 + c * 4;
                }
                return offsets;
            };
            auto s_offs = scan_offsets(out_dir + "/street_entries.bin");
            auto a_offs = scan_offsets(out_dir + "/addr_entries.bin");
            auto i_offs = scan_offsets(out_dir + "/interp_entries.bin");

            uint32_t no_data_v = 0xFFFFFFFF;
            size_t si = 0, ai = 0, ii = 0;
            for (size_t c = 0; c < n; c++) {
                uint64_t cid; memcpy(&cid, geo.geo_cells_data.data() + c*20, 8);
                uint32_t old_s, old_a, old_i;
                memcpy(&old_s, geo.geo_cells_data.data()+c*20+8, 4);
                memcpy(&old_a, geo.geo_cells_data.data()+c*20+12, 4);
                memcpy(&old_i, geo.geo_cells_data.data()+c*20+16, 4);
                bool hs = (old_s != no_data_v), ha = (old_a != no_data_v), hi = (old_i != no_data_v);
                auto fc = flag_corrections.find(cid);
                if (fc != flag_corrections.end()) { hs = fc->second & 1; ha = fc->second & 2; hi = fc->second & 4; }
                uint32_t ns = (hs && si < s_offs.size()) ? s_offs[si++] : no_data_v;
                uint32_t na = (ha && ai < a_offs.size()) ? a_offs[ai++] : no_data_v;
                uint32_t ni = (hi && ii < i_offs.size()) ? i_offs[ii++] : no_data_v;
                fwrite(&cid, 8, 1, fg); fwrite(&ns, 4, 1, fg); fwrite(&na, 4, 1, fg); fwrite(&ni, 4, 1, fg);
            }
            fclose(fg);
        }
        geo = RebuiltGeo{}; // free
        malloc_trim(0);
        log_phase("  Geo entries + corrections", t_entry);

        // Admin: small, use existing rebuild + corrections
        {
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            auto ait = id_remaps.find((uint32_t)PatchFileId::ADMIN_POLYGONS);
            if (ait != id_remaps.end()) {
                for (uint32_t i = 0; i < ait->second.size(); i++)
                    if (ait->second[i] != 0xFFFFFFFF) ad_rm[i] = ait->second[i];
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_adme = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_adme, ad_rm, admin_added, admin_removed);

            auto ecit = entry_corrections.find((uint32_t)PatchFileId::ADMIN_ENTRIES);
            if (ecit != entry_corrections.end()) {
                std::unordered_map<uint64_t, const std::vector<uint32_t>*> ac_corr;
                for (auto& c : ecit->second) ac_corr[c.cell_id] = &c.ids;
                size_t n = admin.admin_cells_data.size() / 12;
                FILE* fac = fopen((out_dir + "/admin_cells.bin").c_str(), "wb");
                FILE* fae = fopen((out_dir + "/admin_entries.bin").c_str(), "wb");
                uint32_t ae_wpos = 0;
                for (size_t i = 0; i < n; i++) {
                    uint64_t cid; memcpy(&cid, admin.admin_cells_data.data()+i*12, 8);
                    fwrite(&cid, 8, 1, fac);
                    auto cit = ac_corr.find(cid);
                    if (cit != ac_corr.end()) {
                        uint32_t off = cit->second->empty() ? no_data : ae_wpos;
                        fwrite(&off, 4, 1, fac);
                        if (!cit->second->empty()) {
                            uint16_t c = cit->second->size();
                            fwrite(&c, 2, 1, fae); fwrite(cit->second->data(), 4, c, fae);
                            ae_wpos += 2 + c*4;
                        }
                    } else {
                        uint32_t old_off; memcpy(&old_off, admin.admin_cells_data.data()+i*12+8, 4);
                        if (old_off != no_data && old_off+2 <= admin.admin_entries_data.size()) {
                            uint32_t new_off = ae_wpos; fwrite(&new_off, 4, 1, fac);
                            uint16_t c; memcpy(&c, admin.admin_entries_data.data()+old_off, 2);
                            fwrite(admin.admin_entries_data.data()+old_off, 1, 2+c*4, fae);
                            ae_wpos += 2+c*4;
                        } else {
                            fwrite(&no_data, 4, 1, fac);
                        }
                    }
                }
                fclose(fac); fclose(fae);
                std::cerr << "  Admin: " << n << " cells, " << ac_corr.size() << " corrections" << std::endl;
            } else {
                write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
                write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            }
            log_phase("  Admin", t_entry);
        }
    }

    // Cleanup
    unmap_file(patch_map);
    { std::string cmd = "rm -rf '" + tmpdir + "'"; system(cmd.c_str()); }
    log_phase("Total", t_start);
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

+        auto get_rm = [&](PatchFileId fid) -> const std::vector<uint32_t>& {

+            static std::vector<uint32_t> empty;

+            auto it = id_remaps.find((uint32_t)fid); return it != id_remaps.end() ? it->second : empty;


For `no_data`, it's used in the admin section. Let me add it:


403:            uint32_t no_data_v = 0xFFFFFFFF;
413:                    uint32_t new_off = cit->second->empty() ? no_data_v : fe_pos;
422:                } else if (off != no_data_v) {
466:            uint32_t no_data_v = 0xFFFFFFFF;
474:                bool hs = (old_s != no_data_v), ha = (old_a != no_data_v), hi = (old_i != no_data_v);
477:                uint32_t ns = (hs && si < s_offs.size()) ? s_offs[si++] : no_data_v;
478:                uint32_t na = (ha && ai < a_offs.size()) ? a_offs[ai++] : no_data_v;
479:                uint32_t ni = (hi && ii < i_offs.size()) ? i_offs[ii++] : no_data_v;
513:                        uint32_t off = cit->second->empty() ? no_data : ae_wpos;
522:                        if (old_off != no_data && old_off+2 <= admin.admin_entries_data.size()) {
528:                            fwrite(&no_data, 4, 1, fac);


Lines 513, 522, 528 use `no_data` without declaring it. Let me add it before the admin section:


Edit

        // Admin: small, use existing rebuild + corrections
        {
            std::unordered_map<uint32_t,uint32_t> ad_rm;

        // Admin: small, use existing rebuild + corrections
        {
            uint32_t no_data = 0xFFFFFFFF;
            std::unordered_map<uint32_t,uint32_t> ad_rm;


The file /root/traccar-geocoder/builder/tools/geocoder_patch.cpp has been updated successfully.

        // Admin: small, use existing rebuild + corrections
        {
            uint32_t no_data = 0xFFFFFFFF;
            std::unordered_map<uint32_t,uint32_t> ad_rm;

// geocoder-patch v4: Low-memory streaming patch application.
//
// Key design: mmap old files (zero copy), stream output via fwrite (never accumulate),
// process entry pipeline cell-by-cell. Target: <1 GiB peak RSS for planet.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <fcntl.h>
#include <iomanip>
#include <iostream>
#include <malloc.h>
#include <string>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unordered_map>
#include <unordered_set>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

// --- Helpers ---
static double now_ms() {
    struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
    return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6;
}
static size_t get_rss_mb() {
    FILE* f = fopen("/proc/self/statm", "r");
    if (!f) return 0;
    size_t dummy, rss; fscanf(f, "%zu %zu", &dummy, &rss); fclose(f);
    return rss * 4096 / (1024*1024);
}
static void log_phase(const char* label, double start) {
    std::cerr << "  [" << std::fixed << std::setprecision(1) << (now_ms() - start) / 1000.0
              << "s, " << get_rss_mb() << " MiB] " << label << std::endl;
}

// mmap a file read-only. Returns {pointer, size}. Caller must munmap.
struct MappedFile { const char* data; size_t size; };
static MappedFile mmap_file(const std::string& path) {
    int fd = open(path.c_str(), O_RDONLY);
    if (fd < 0) return {nullptr, 0};
    struct stat st; fstat(fd, &st);
    size_t sz = st.st_size;
    if (sz == 0) { close(fd); return {nullptr, 0}; }
    void* p = mmap(nullptr, sz, PROT_READ, MAP_PRIVATE, fd, 0);
    close(fd);
    if (p == MAP_FAILED) return {nullptr, 0};
    return {static_cast<const char*>(p), sz};
}
static void unmap_file(MappedFile& f) {
    if (f.data) { munmap(const_cast<char*>(f.data), f.size); f.data = nullptr; f.size = 0; }
}

// Detect stride from file size
static size_t detect_stride(const std::string& path, std::initializer_list<size_t> candidates) {
    struct stat st; if (stat(path.c_str(), &st) != 0) return *candidates.begin();
    for (size_t s : candidates) if (st.st_size % s == 0 && st.st_size > 0) return s;
    return *candidates.begin();
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);
    double t_start = now_ms();

    // --- Phase 1: Decompress + mmap patch ---
    std::string raw_path = tmpdir + "/patch.raw";
    {
        std::string cmd = "zstd -d '" + patch_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0) { std::cerr << "Failed to decompress" << std::endl; return 1; }
    }
    MappedFile patch_map = mmap_file(raw_path);
    if (!patch_map.data) { std::cerr << "Failed to mmap patch" << std::endl; return 1; }
    remove(raw_path.c_str());
    const char* P = patch_map.data;
    size_t patch_size = patch_map.size;
    std::cerr << "Patch: " << patch_size << " bytes" << std::endl;
    log_phase("Decompress", t_start);

    size_t pos = 0;
    auto ru32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, P+pos, 4); pos += 4; return v; };
    auto ru64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, P+pos, 8); pos += 8; return v; };

    // Header
    if (memcmp(P, GCPATCH_MAGIC, 8) != 0) { std::cerr << "Bad magic" << std::endl; return 1; }
    pos = 8;
    uint32_t ver = ru32(); if (ver != 2) { std::cerr << "Bad version" << std::endl; return 1; }
    ru32(); // flags

    // --- Phase 2: String rebuild ---
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t marker = ru32();
        if (marker == 0xFFFFFFF7) {
            uint32_t n_added = ru32(), n_deleted = ru32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ru32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());

            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
            std::vector<std::pair<uint32_t, std::string>> old_strs;
            size_t sp = 0;
            while (sp < old_pool.size) {
                old_strs.push_back({(uint32_t)sp, std::string(old_pool.data + sp)});
                sp += strlen(old_pool.data + sp) + 1;
            }
            std::vector<std::string> merged;
            for (size_t i = 0; i < old_strs.size(); i++)
                if (!del_set.count(i)) merged.push_back(old_strs[i].second);
            for (auto& s : added) merged.push_back(s);
            std::sort(merged.begin(), merged.end());

            // Write new pool and build remap
            FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
            std::unordered_map<std::string, uint32_t> new_offs;
            uint32_t wpos = 0;
            for (auto& s : merged) {
                new_offs[s] = wpos;
                fwrite(s.c_str(), 1, s.size() + 1, fp);
                wpos += s.size() + 1;
            }
            fclose(fp);
            for (auto& [old_off, s] : old_strs) {
                auto it = new_offs.find(s);
                if (it != new_offs.end() && it->second != old_off) str_remap[old_off] = it->second;
            }
            unmap_file(old_pool);
            std::cerr << "  Strings: +" << n_added << " -" << n_deleted << " → " << merged.size()
                      << ", " << str_remap.size() << " remapped" << std::endl;
            marker = ru32();
        }
        if (marker == 0xFFFFFFFE) { uint32_t c = ru32(); for (uint32_t i = 0; i < c; i++) { uint32_t a = ru32(), b = ru32(); str_remap[a] = b; } }
        else pos -= 4;
    }
    log_phase("Strings", t_start);

    // Detect strides
    size_t way_stride = detect_stride(cur_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride(cur_dir + "/interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect_stride(cur_dir + "/admin_polygons.bin", {24, 20, 19});

    // String remap applier (works on mutable copy)
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t fid, size_t stride) {
        if (str_remap.empty()) return;
        std::vector<size_t> offs;
        if (fid == (uint32_t)PatchFileId::ADDR_POINTS) offs = {8, 12};
        else if (fid == (uint32_t)PatchFileId::STREET_WAYS) offs = {(stride == 12) ? 8ul : 5ul};
        else if (fid == (uint32_t)PatchFileId::INTERP_WAYS) offs = {(stride >= 20) ? 8ul : 5ul};
        else if (fid == (uint32_t)PatchFileId::ADMIN_POLYGONS) offs = {8};
        else return;
        for (size_t i = 0; i + stride <= data.size(); i += stride)
            for (size_t o : offs) { uint32_t v; memcpy(&v, data.data()+i+o, 4); auto it = str_remap.find(v); if (it != str_remap.end()) memcpy(data.data()+i+o, &it->second, 4); }
    };

    // --- Phase 3: Merge replays (streaming output) ---
    // ID remaps: file_id → vector<uint32_t> where remap[old_idx] = new_idx
    std::unordered_map<uint32_t, std::vector<uint32_t>> id_remaps;
    std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;
    std::unordered_map<uint64_t, uint8_t> flag_corrections;
    struct CellCorr { uint64_t cell_id; std::vector<uint32_t> ids; };
    std::unordered_map<uint32_t, std::vector<CellCorr>> entry_corrections;

    while (pos < patch_size) {
        uint32_t file_id = ru32();
        if (file_id == 0xFFFFFFFF) break;

        // --- Metadata sections ---
        if (file_id == 0xFFFFFFF7) {
            // String diff in main loop — process it (may be the only occurrence)
            uint32_t n_added = ru32(), n_deleted = ru32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ru32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());
            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
            std::vector<std::pair<uint32_t, std::string>> old_strs;
            size_t sp = 0;
            while (sp < old_pool.size) { old_strs.push_back({(uint32_t)sp, std::string(old_pool.data+sp)}); sp += strlen(old_pool.data+sp)+1; }
            std::vector<std::string> merged;
            for (size_t i = 0; i < old_strs.size(); i++) if (!del_set.count(i)) merged.push_back(old_strs[i].second);
            for (auto& s : added) merged.push_back(s);
            std::sort(merged.begin(), merged.end());
            FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
            std::unordered_map<std::string, uint32_t> new_offs;
            uint32_t wpos = 0;
            for (auto& s : merged) { new_offs[s] = wpos; fwrite(s.c_str(), 1, s.size()+1, fp); wpos += s.size()+1; }
            fclose(fp);
            for (auto& [old_off, s] : old_strs) { auto it = new_offs.find(s); if (it != new_offs.end() && it->second != old_off) str_remap[old_off] = it->second; }
            unmap_file(old_pool);
            std::cerr << "  Strings (loop): +" << n_added << " -" << n_deleted << " → " << merged.size() << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_GEO_MARKER) {
            uint32_t na = ru32(), nr = ru32();
            geo_added.resize(na); geo_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { memcpy(&geo_added[i], P+pos, 8); pos += 8; }
            for (uint32_t i = 0; i < nr; i++) { memcpy(&geo_removed[i], P+pos, 8); pos += 8; }
            std::cerr << "  Geo cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_ADMIN_MARKER) {
            uint32_t na = ru32(), nr = ru32();
            admin_added.resize(na); admin_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { memcpy(&admin_added[i], P+pos, 8); pos += 8; }
            for (uint32_t i = 0; i < nr; i++) { memcpy(&admin_removed[i], P+pos, 8); pos += 8; }
            std::cerr << "  Admin cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_FLAGS_MARKER) {
            uint32_t c = ru32();
            for (uint32_t i = 0; i < c; i++) { uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8; flag_corrections[cid] = P[pos++]; }
            std::cerr << "  Flag corrections: " << c << std::endl;
            continue;
        }
        if (file_id == SECONDARY_REMAP_MARKER) {
            uint32_t nf = ru32();
            for (uint32_t f = 0; f < nf; f++) {
                uint32_t fid = ru32(), np = ru32();
                auto& rm = id_remaps[fid];
                for (uint32_t i = 0; i < np; i++) { uint32_t o = ru32(), n = ru32(); if (o < rm.size()) rm[o] = n; }
                std::cerr << "  Secondary remap " << fid << ": " << np << " pairs" << std::endl;
            }
            continue;
        }
        if (file_id == ENTRY_CORRECTION_MARKER) {
            uint32_t fid = ru32(), c = ru32();
            auto& list = entry_corrections[fid];
            for (uint32_t i = 0; i < c; i++) {
                uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8;
                uint16_t ec; memcpy(&ec, P+pos, 2); pos += 2;
                std::vector<uint32_t> ids(ec);
                if (ec > 0) { memcpy(ids.data(), P+pos, ec*4); pos += ec*4; }
                list.push_back({cid, std::move(ids)});
            }
            std::cerr << "  Entry corrections " << fid << ": " << c << std::endl;
            continue;
        }

        // --- Merge sequence replay (streaming) ---
        uint32_t stride = ru32();
        uint64_t old_size = ru64(), new_size = ru64();
        if (file_id >= (uint32_t)PatchFileId::COUNT) { std::cerr << "Unknown file " << file_id << std::endl; break; }
        const char* fname = patch_file_names[file_id];

        if (stride == 0) {
            // Full replacement — write directly from patch mmap
            uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64();
            FILE* fp = fopen((out_dir + "/" + fname).c_str(), "wb");
            fwrite(P+pos, 1, ds, fp); fclose(fp);
            pos += ds;
            std::cerr << "  " << fname << ": full replace " << ds << " bytes" << std::endl;
            continue;
        }
        if (stride == 0xFE) { uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64(); pos += ds; continue; }

        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        // Load old file into mutable buffer (need to apply remap + fixups)
        // For large files (nodes/vertices), this is the peak memory per file.
        // Files are processed sequentially, so only one is in memory at a time.
        auto old_data = read_file(cur_dir + "/" + std::string(fname));

        // Zero padding
        if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24)
            for (size_t i = 0; i + actual_stride <= old_data.size(); i += actual_stride) memset(old_data.data()+i+13, 0, 3);
        if (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24)
            for (size_t i = 0; i + actual_stride <= old_data.size(); i += actual_stride) { memset(old_data.data()+i+5, 0, 3); memset(old_data.data()+i+21, 0, 3); }

        apply_str_remap(old_data, file_id, actual_stride);

        // Apply fixups
        uint32_t n_fixups = ru32();
        if (n_fixups > 0) {
            uint32_t delta_size = ru32();
            uint32_t pi = 0, pv = 0; size_t fp = pos;
            for (uint32_t i = 0; i < n_fixups; i++) {
                uint32_t idx = pi + read_varint(P, fp);
                uint32_t val = pv + read_varint(P, fp);
                if ((size_t)idx * actual_stride + 4 <= old_data.size())
                    memcpy(old_data.data() + (size_t)idx * actual_stride, &val, 4);
                pi = idx; pv = val;
            }
            pos = fp;
        }

        // Replay merge sequence — stream output directly to file
        uint64_t seq_size = ru64();
        size_t seq_end = pos + seq_size;
        FILE* outf = fopen((out_dir + "/" + fname).c_str(), "wb");

        // Track ID remap (only for data files that need entry reconstruction)
        bool track = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
                      file_id == (uint32_t)PatchFileId::STREET_WAYS ||
                      file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
                      file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
        std::vector<uint32_t> id_map;
        if (track) id_map.assign(old_data.size() / actual_stride, 0xFFFFFFFF);

        size_t old_rec = 0, new_rec = 0, old_bytes = 0, written = 0;
        while (pos < seq_end) {
            uint8_t op = P[pos++];
            uint32_t count; memcpy(&count, P+pos, 4); pos += 4;
            if (op == OP_MATCH_RUN) {
                size_t bytes = count * actual_stride;
                fwrite(old_data.data() + old_bytes, 1, bytes, outf);
                written += bytes;
                if (track) for (uint32_t k = 0; k < count; k++) if (old_rec+k < id_map.size()) id_map[old_rec+k] = new_rec+k;
                old_rec += count; new_rec += count; old_bytes += bytes;
            } else if (op == OP_INSERT_RUN) {
                size_t bytes = count * actual_stride;
                fwrite(P+pos, 1, bytes, outf);
                written += bytes; pos += bytes; new_rec += count;
            } else if (op == OP_DELETE_RUN) {
                old_rec += count; old_bytes += count * actual_stride;
            }
        }
        fclose(outf);

        if (track) {
            id_remaps[file_id] = std::move(id_map);
            size_t matched = 0;
            for (auto v : id_remaps[file_id]) if (v != 0xFFFFFFFF) matched++;
            std::cerr << "  " << fname << ": " << written << " bytes (mapped " << matched << "/" << id_remaps[file_id].size() << ")" << std::endl;
        } else {
            std::cerr << "  " << fname << ": " << written << " bytes" << std::endl;
        }

        // Free old data immediately — don't let glibc hold the pages
        { std::vector<char>().swap(old_data); }
        malloc_trim(0);
    }
    log_phase("Merge replays", t_start);

    // Free string remap (no longer needed)
    { std::unordered_map<uint32_t,uint32_t>().swap(str_remap); }
    malloc_trim(0);

    // --- Phase 4: Streaming entry pipeline ---
    // Walk old geo_cells, remap IDs, apply corrections, write output directly to files.
    {
        double t_entry = now_ms();
        std::cerr << "Entry pipeline (proven rebuild approach)..." << std::endl;

        auto get_rm = [&](PatchFileId fid) -> const std::vector<uint32_t>& {
            static std::vector<uint32_t> empty;
            auto it = id_remaps.find((uint32_t)fid); return it != id_remaps.end() ? it->second : empty;
        };
        auto& w_rm = get_rm(PatchFileId::STREET_WAYS);
        auto& a_rm = get_rm(PatchFileId::ADDR_POINTS);
        auto& i_rm = get_rm(PatchFileId::INTERP_WAYS);

        // Rebuild entries using proven rebuild_geo_from_remap_vec
        auto old_geo_data = read_file(cur_dir + "/geo_cells.bin");
        auto old_se = read_file(cur_dir + "/street_entries.bin");
        auto old_ae = read_file(cur_dir + "/addr_entries.bin");
        auto old_ie = read_file(cur_dir + "/interp_entries.bin");
        auto geo = rebuild_geo_from_remap_vec(old_geo_data, old_se, old_ae, old_ie,
                                                w_rm, a_rm, i_rm, geo_added, geo_removed);
        // Free inputs
        { std::vector<char>().swap(old_se); std::vector<char>().swap(old_ae); std::vector<char>().swap(old_ie); }
        log_phase("  Geo rebuild", t_entry);

        // Apply corrections sequentially (one file at a time to limit memory)
        // Build correction maps
        auto apply_corrections = [&](uint32_t fid, const std::string& fname,
                                      std::vector<char>& derived_entries,
                                      const std::vector<char>& derived_cells, size_t cell_stride,
                                      size_t off_pos) {
            auto ecit = entry_corrections.find(fid);
            if (ecit == entry_corrections.end() || ecit->second.empty()) {
                write_file(out_dir + "/" + fname, derived_entries);
                return;
            }
            std::unordered_map<uint64_t, const std::vector<uint32_t>*> cm;
            for (auto& c : ecit->second) cm[c.cell_id] = &c.ids;

            // Walk cells, apply corrections, write output via FILE*
            size_t n = derived_cells.size() / cell_stride;
            FILE* fe = fopen((out_dir + "/" + fname).c_str(), "wb");
            FILE* fc_out = nullptr;
            std::vector<char> new_cells;
            uint32_t no_data_v = 0xFFFFFFFF;
            uint32_t fe_pos = 0;

            for (size_t i = 0; i < n; i++) {
                uint64_t cid; memcpy(&cid, derived_cells.data() + i * cell_stride, 8);
                uint32_t off; memcpy(&off, derived_cells.data() + i * cell_stride + off_pos, 4);

                auto cit = cm.find(cid);
                if (cit != cm.end()) {
                    // Use correction data
                    uint32_t new_off = cit->second->empty() ? no_data_v : fe_pos;
                    if (!cit->second->empty()) {
                        uint16_t c = cit->second->size();
                        fwrite(&c, 2, 1, fe);
                        fwrite(cit->second->data(), 4, c, fe);
                        fe_pos += 2 + c * 4;
                    }
                    // Update offset in cells data (in memory)
                    memcpy(const_cast<char*>(derived_cells.data()) + i * cell_stride + off_pos, &new_off, 4);
                } else if (off != no_data_v) {
                    // Copy original entry data
                    if (off + 2 <= derived_entries.size()) {
                        uint16_t c; memcpy(&c, derived_entries.data() + off, 2);
                        uint32_t new_off = fe_pos;
                        fwrite(derived_entries.data() + off, 1, 2 + c * 4, fe);
                        fe_pos += 2 + c * 4;
                        memcpy(const_cast<char*>(derived_cells.data()) + i * cell_stride + off_pos, &new_off, 4);
                    }
                }
            }
            fclose(fe);
            std::cerr << "  " << fname << ": " << cm.size() << " corrections applied" << std::endl;
        };

        // Apply corrections for each geo entry type
        apply_corrections((uint32_t)PatchFileId::STREET_ENTRIES, "street_entries.bin",
                         geo.street_entries_data, geo.geo_cells_data, 20, 8);
        apply_corrections((uint32_t)PatchFileId::ADDR_ENTRIES, "addr_entries.bin",
                         geo.addr_entries_data, geo.geo_cells_data, 20, 12);
        apply_corrections((uint32_t)PatchFileId::INTERP_ENTRIES, "interp_entries.bin",
                         geo.interp_entries_data, geo.geo_cells_data, 20, 16);

        // Write the corrected geo_cells (offsets updated in-place by apply_corrections)
        // Rebuild offsets using flag corrections
        {
            size_t n = geo.geo_cells_data.size() / 20;
            FILE* fg = fopen((out_dir + "/geo_cells.bin").c_str(), "wb");
            // Scan corrected entry files to build offset arrays
            auto scan_offsets = [](const std::string& path) -> std::vector<uint32_t> {
                auto data = read_file(path);
                std::vector<uint32_t> offsets;
                uint32_t pos = 0;
                while (pos + 2 <= data.size()) {
                    offsets.push_back(pos);
                    uint16_t c; memcpy(&c, data.data() + pos, 2);
                    pos += 2 + c * 4;
                }
                return offsets;
            };
            auto s_offs = scan_offsets(out_dir + "/street_entries.bin");
            auto a_offs = scan_offsets(out_dir + "/addr_entries.bin");
            auto i_offs = scan_offsets(out_dir + "/interp_entries.bin");

            uint32_t no_data_v = 0xFFFFFFFF;
            size_t si = 0, ai = 0, ii = 0;
            for (size_t c = 0; c < n; c++) {
                uint64_t cid; memcpy(&cid, geo.geo_cells_data.data() + c*20, 8);
                uint32_t old_s, old_a, old_i;
                memcpy(&old_s, geo.geo_cells_data.data()+c*20+8, 4);
                memcpy(&old_a, geo.geo_cells_data.data()+c*20+12, 4);
                memcpy(&old_i, geo.geo_cells_data.data()+c*20+16, 4);
                bool hs = (old_s != no_data_v), ha = (old_a != no_data_v), hi = (old_i != no_data_v);
                auto fc = flag_corrections.find(cid);
                if (fc != flag_corrections.end()) { hs = fc->second & 1; ha = fc->second & 2; hi = fc->second & 4; }
                uint32_t ns = (hs && si < s_offs.size()) ? s_offs[si++] : no_data_v;
                uint32_t na = (ha && ai < a_offs.size()) ? a_offs[ai++] : no_data_v;
                uint32_t ni = (hi && ii < i_offs.size()) ? i_offs[ii++] : no_data_v;
                fwrite(&cid, 8, 1, fg); fwrite(&ns, 4, 1, fg); fwrite(&na, 4, 1, fg); fwrite(&ni, 4, 1, fg);
            }
            fclose(fg);
        }
        geo = RebuiltGeo{}; // free
        malloc_trim(0);
        log_phase("  Geo entries + corrections", t_entry);

        // Admin: small, use existing rebuild + corrections
        {
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            auto ait = id_remaps.find((uint32_t)PatchFileId::ADMIN_POLYGONS);
            if (ait != id_remaps.end()) {
                for (uint32_t i = 0; i < ait->second.size(); i++)
                    if (ait->second[i] != 0xFFFFFFFF) ad_rm[i] = ait->second[i];
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_adme = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_adme, ad_rm, admin_added, admin_removed);

            auto ecit = entry_corrections.find((uint32_t)PatchFileId::ADMIN_ENTRIES);
            if (ecit != entry_corrections.end()) {
                std::unordered_map<uint64_t, const std::vector<uint32_t>*> ac_corr;
                for (auto& c : ecit->second) ac_corr[c.cell_id] = &c.ids;
                size_t n = admin.admin_cells_data.size() / 12;
                FILE* fac = fopen((out_dir + "/admin_cells.bin").c_str(), "wb");
                FILE* fae = fopen((out_dir + "/admin_entries.bin").c_str(), "wb");
                uint32_t ae_wpos = 0;
                for (size_t i = 0; i < n; i++) {
                    uint64_t cid; memcpy(&cid, admin.admin_cells_data.data()+i*12, 8);
                    fwrite(&cid, 8, 1, fac);
                    auto cit = ac_corr.find(cid);
                    if (cit != ac_corr.end()) {
                        uint32_t off = cit->second->empty() ? no_data : ae_wpos;
                        fwrite(&off, 4, 1, fac);
                        if (!cit->second->empty()) {
                            uint16_t c = cit->second->size();
                            fwrite(&c, 2, 1, fae); fwrite(cit->second->data(), 4, c, fae);
                            ae_wpos += 2 + c*4;
                        }
                    } else {
                        uint32_t old_off; memcpy(&old_off, admin.admin_cells_data.data()+i*12+8, 4);
                        if (old_off != no_data && old_off+2 <= admin.admin_entries_data.size()) {
                            uint32_t new_off = ae_wpos; fwrite(&new_off, 4, 1, fac);
                            uint16_t c; memcpy(&c, admin.admin_entries_data.data()+old_off, 2);
                            fwrite(admin.admin_entries_data.data()+old_off, 1, 2+c*4, fae);
                            ae_wpos += 2+c*4;
                        } else {
                            fwrite(&no_data, 4, 1, fac);
                        }
                    }
                }
                fclose(fac); fclose(fae);
                std::cerr << "  Admin: " << n << " cells, " << ac_corr.size() << " corrections" << std::endl;
            } else {
                write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
                write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            }
            log_phase("  Admin", t_entry);
        }
    }

    // Cleanup
    unmap_file(patch_map);
    { std::string cmd = "rm -rf '" + tmpdir + "'"; system(cmd.c_str()); }
    log_phase("Total", t_start);
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

+            uint32_t no_data = 0xFFFFFFFF;


Bash

make -j$(nproc) geocoder-patch 2>&1 | tail -3

Build


[ 50%] Building CXX object CMakeFiles/geocoder-patch.dir/tools/geocoder_patch.cpp.o
[100%] Linking CXX executable geocoder-patch
[100%] Built target geocoder-patch


Quick Europe test:


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "killall -9 geocoder-patch 2>/dev/null; rm -f /home/michtest/geocoder-patch; rm -rf /home/michtest/patched-*/" 2>&1 && sshpass -p 'elon-is-insane' scp -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:/home/michtest/ 2>&1 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 'cd /home/michtest && rm -rf patched-test/ && time ./geocoder-patch det-A/ validated-patches/europe-mar21-to-mar27-v2.gcpatch -o patched-test/ 2>&1 | grep -E "\\[.*MiB\\]|real" && for f in det-today/*; do b=$(basename $f); cmp -s patched-test/$b det-today/$b || echo "MISMATCH: $b"; done && echo "Europe: ALL MATCH" && rm -rf patched-test/' 2>&1

Europe test with proven approach


  [0.1s, 3 MiB] Decompress
  [0.1s, 3 MiB] Strings
  [12.4s, 841 MiB] Merge replays
  [76.8s, 11058 MiB]   Geo rebuild
  [87.0s, 2213 MiB]   Geo entries + corrections
  [87.1s, 2277 MiB]   Admin
  [100.2s, 514 MiB] Total

real	1m40.245s
user	1m34.101s
sys	0m5.934s
MISMATCH: manifest.json
Europe: ALL MATCH


**Europe: ALL MATCH, 100s, 514 MiB final, 11 GiB peak during rebuild.** The rebuild is the memory spike (11 GiB) but it drops back to 2.2 GiB after. Now planet:


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 'cd /home/michtest && rm -rf patched-planet-stream/ && nohup bash -c "time ./geocoder-patch planet-B/ validated-patches/planet-mar16-to-mar23-v2.gcpatch -o patched-planet-stream/ 2>&1 && echo ---VERIFY--- && for f in planet-C/*; do b=\$(basename \$f); cmp -s patched-planet-stream/\$b planet-C/\$b || echo \"MISMATCH: \$b\"; done && echo All_checked" > planet-patch-final.log 2>&1 &' 2>&1 && echo "Started"

Start planet test


Command running in background with ID: bwgz53uuq. Output is being written to: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bwgz53uuq.output


sleep 60 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "grep '\\[.*MiB\\]' /home/michtest/planet-patch-final.log && ps aux | grep 'geocoder-patch planet' | grep -v grep | awk '{printf \"RSS: %.0f GiB\\n\", \$6/1024/1024}'" 2>&1

Check 1 min


  [0.3s, 3 MiB] Decompress
  [0.3s, 3 MiB] Strings
  [25.9s, 1816 MiB] Merge replays
RSS: 0 GiB
RSS: 0 GiB
RSS: 60 GiB


In rebuild phase at 60 GiB. This is the known memory spike from `rebuild_geo_from_remap_vec`. Let me wait for completion:


sleep 300 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "grep '\\[.*MiB\\]' /home/michtest/planet-patch-final.log && ps aux | grep 'geocoder-patch planet' | grep -v grep | awk '{printf \"RSS: %.0f GiB\\n\", \$6/1024/1024}'" 2>&1

Check 6 min


  [0.3s, 3 MiB] Decompress
  [0.3s, 3 MiB] Strings
  [25.9s, 1816 MiB] Merge replays
  [323.0s, 38382 MiB]   Geo rebuild
  [375.1s, 7459 MiB]   Geo entries + corrections
  [375.6s, 7947 MiB]   Admin
  [403.4s, 1390 MiB] Total


**Done!** 403s (6.7 min), 38 GiB peak during rebuild, 1.4 GiB final. Let me check the verification:


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "grep -E 'MISMATCH|All_checked|real' /home/michtest/planet-patch-final.log" 2>&1

Check verification


real	6m43.411s
MISMATCH: manifest.json
All_checked


**ALL 14 data files MATCH! Planet patch in 6m43s with 38 GiB peak memory.**

| Metric | Before optimization | Now |
|--------|--------------------|----|
| **Europe time** | 4m23s | **1m40s** |
| **Planet time** | 20m10s | **6m43s** (3x faster) |
| **Planet peak RSS** | 63 GiB | **38 GiB** |
| **Planet final RSS** | 38 GiB | **1.4 GiB** |
| **Merge replay** (planet) | 37s | **26s** |
| **Entry pipeline** (planet) | 19m30s | **6m10s** |

The 38 GiB peak is from `rebuild_geo_from_remap_vec` which builds per-cell entry vectors for 322M cells. The streaming fwrite approach for corrections saves significant memory vs the old 4-pass approach (63 GiB → 38 GiB).

To reach the <500 MiB target, we'd need to eliminate the `rebuild_geo_from_remap_vec` materialization entirely and implement the streaming approach correctly. The streaming approach has a subtle bug at planet scale that I wasn't able to resolve in this session. This should be investigated in a follow-up.


Updated task #23 status


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "rm -rf /home/michtest/patched-planet-stream/ /home/michtest/patched-test/" 2>&1

Clean up


(Bash completed with no output)


Current state summary:

**Diff tool (patch creation):**
- Europe: 3m40s (was 8m08s, 2.2x faster)
- Planet: 15m22s (was 36m24s, 2.4x faster)
- All 14 files verified MATCH on both

**Patch tool (patch application):**
- Europe: 1m40s, 11 GiB peak, 514 MiB final
- Planet: 6m43s, 38 GiB peak, 1.4 GiB final
- All 14 files verified MATCH on both

**Remaining work for <500 MiB target:**
- The `rebuild_geo_from_remap_vec` function materializes 322M per-cell entry vectors = 38 GiB peak
- A true streaming approach (process one cell at a time) would fix this but has a subtle bug at planet scale
- The bug causes entry file sizes to explode ~10x between cells 105M-110M
- Needs deeper investigation to find root cause


<task-notification>
<task-id>bdz68g3au</task-id>
<tool-use-id>toolu_0169YQLpZTcEGFGsbpfuuKtU</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bdz68g3au.output</output-file>
<status>completed</status>
<summary>Background command "Start planet patch" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bdz68g3au.output


<task-notification>
<task-id>bdelkjy98</task-id>
<tool-use-id>toolu_01UjwinUNty2fUb1adCbEdNy</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bdelkjy98.output</output-file>
<status>completed</status>
<summary>Background command "Start planet test" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bdelkjy98.output


<task-notification>
<task-id>bf3wpsytv</task-id>
<tool-use-id>toolu_01AjkXewzvbm2dR5ALDAzH77</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bf3wpsytv.output</output-file>
<status>completed</status>
<summary>Background command "Start fresh planet test" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bf3wpsytv.output


<task-notification>
<task-id>bohdr7dy2</task-id>
<tool-use-id>toolu_01EuUyzHAD6WeEHmSYjUfJ2R</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bohdr7dy2.output</output-file>
<status>completed</status>
<summary>Background command "Start full planet run" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bohdr7dy2.output


<task-notification>
<task-id>b7w7h80ie</task-id>
<tool-use-id>toolu_01Vf5VmAJ6CLagSMGmqEC5JZ</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/b7w7h80ie.output</output-file>
<status>completed</status>
<summary>Background command "Start clean planet run" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/b7w7h80ie.output


<task-notification>
<task-id>bfu4mz9ap</task-id>
<tool-use-id>toolu_0179Y8S6bTahEh1pJTSAXY2f</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bfu4mz9ap.output</output-file>
<status>completed</status>
<summary>Background command "Clean deploy and run" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bfu4mz9ap.output


<task-notification>
<task-id>bfdehaqa0</task-id>
<tool-use-id>toolu_011WY3pedtCANZDDbdF8mjmt</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bfdehaqa0.output</output-file>
<status>completed</status>
<summary>Background command "Check 20 min" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bfdehaqa0.output


<task-notification>
<task-id>be54zngj4</task-id>
<tool-use-id>toolu_01Ny7oaiDBaa6moTcuKhHgB8</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/be54zngj4.output</output-file>
<status>completed</status>
<summary>Background command "Final clean planet run" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/be54zngj4.output


<task-notification>
<task-id>bdwgrdonf</task-id>
<tool-use-id>toolu_01LmKLDJr8EHwNtVH1h4vtXD</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bdwgrdonf.output</output-file>
<status>completed</status>
<summary>Background command "Check 30 min" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bdwgrdonf.output


<task-notification>
<task-id>b8s1rc0fl</task-id>
<tool-use-id>toolu_01QTZv6cLcNombyXGGunGuRQ</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/b8s1rc0fl.output</output-file>
<status>completed</status>
<summary>Background command "Planet test with ftell" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/b8s1rc0fl.output


<task-notification>
<task-id>bhmojrh95</task-id>
<tool-use-id>toolu_01MtxCuSKoqmR5gVnaggWYNr</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bhmojrh95.output</output-file>
<status>completed</status>
<summary>Background command "Start clean planet run with ftell fix" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bhmojrh95.output


<task-notification>
<task-id>btuw5jrqt</task-id>
<tool-use-id>toolu_01XQq2GC92rnUFrFBky5MBFV</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/btuw5jrqt.output</output-file>
<status>failed</status>
<summary>Background command "Check 10 min" failed with exit code 1</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/btuw5jrqt.output


<task-notification>
<task-id>b5saodjw9</task-id>
<tool-use-id>toolu_01SFF4XAqFxkmUgZzLhk5KDa</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/b5saodjw9.output</output-file>
<status>completed</status>
<summary>Background command "Start and monitor" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/b5saodjw9.output


<task-notification>
<task-id>b10wdjd12</task-id>
<tool-use-id>toolu_01SYZyHGzJjr6EUETG6HHDmH</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/b10wdjd12.output</output-file>
<status>completed</status>
<summary>Background command "Trace the explosion point" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/b10wdjd12.output


<task-notification>
<task-id>bwgz53uuq</task-id>
<tool-use-id>toolu_01P9fMimJMH13fPsgiF9NxeJ</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bwgz53uuq.output</output-file>
<status>completed</status>
<summary>Background command "Start planet test" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bwgz53uuq.output


All background tasks completed. The key results:

**Planet patch: 6m43s, all 14 files MATCH.** Peak 38 GiB during rebuild, 1.4 GiB final.

The streaming approach that would get us to <500 MiB has a bug at planet scale where entry file sizes explode ~10x between cells 105M-110M. The proven `rebuild_geo_from_remap_vec` approach works correctly but materializes 322M per-cell vectors causing the 38 GiB peak.

For now we have a working, verified, significantly faster pipeline:
- **Diff: 15m** (was 36m)
- **Patch: 6m43s** (was 20m)
- **Total: ~22m** (was ~57m, 2.6x faster)

The <500 MiB memory target needs the streaming bug fixed in a follow-up session. The root cause is likely in how the streaming walk interacts with the mmap'd old entry files at the boundary around cell 107M on planet.


Lets commit this but then work towards the streaming approach again


<tool_use_error>Unknown skill: commit</tool_use_error>


Show working tree status


On branch main
Your branch is up to date with 'origin/main'.

Changes not staged for commit:
  (use "git add <file>..." to update what will be committed)
  (use "git restore <file>..." to discard changes in working directory)
	modified:   builder/PATCHING.md
	modified:   builder/tools/geocoder_diff.cpp
	modified:   builder/tools/geocoder_patch.cpp
	modified:   builder/tools/patch_format.h
	modified:   builder/tools/simplification_map.html

Untracked files:
  (use "git add <file>..." to include in what will be committed)
	builder/build-presplit/
	builder/test/deep_compare.cpp
	builder/test/monaco_osmium_reference.txt
	builder/test/name_overlap.py
	builder/tools/comparison.geojson
	builder/tools/simplification_viewer.html

no changes added to commit (use "git add" and/or "git commit -a")


Show diff stats


 builder/PATCHING.md                   |  153 +++--
 builder/tools/geocoder_diff.cpp       | 1119 +++++++++++++++++----------------
 builder/tools/geocoder_patch.cpp      |  994 +++++++++++------------------
 builder/tools/patch_format.h          |   83 +++
 builder/tools/simplification_map.html |   13 +-
 5 files changed, 1141 insertions(+), 1221 deletions(-)


Recent commits


bd7f977 feat: planet 7-day: 77 MiB (was 160 MiB) — delta fixups verified
f8c79f6 docs: Europe 34 MiB (~5.7 MiB/day), delta fixups done
9988933 feat: delta-encoded fixups — Europe 34 MiB (was 68 MiB)
7898d84 feat: planet sequential PASS with optimized diff + dedup (160 MiB/week)
a85979f chore: code cleanup — remove prototype files and dead code


git add builder/tools/geocoder_diff.cpp builder/tools/geocoder_patch.cpp builder/tools/patch_format.h builder/PATCHING.md && git commit -m "$(cat <<'EOF'
perf: parallelize diff tool, optimize patch tool, add secondary matching

Diff tool (geocoder-diff):
- Parallel merge sequence building (4 thread groups)
- Secondary ID matching for modified records (reduces cell corrections)
- Sorted cell diff (vector vs hash set, 64s → 3s)
- ID remaps computed in parallel merge groups (eliminates sequential derivation)
- zstd -19 -T0 for multithreaded compression
- Fix admin polygon country_code hash (was reading padding byte)
- Per-phase timing instrumentation

Patch tool (geocoder-patch):
- mmap for decompressed patch (zero copy)
- Streaming fwrite for merge replay output (vs vector accumulation)
- malloc_trim between phases to reclaim heap
- Streaming entry corrections via fwrite
- rebuild_geo_from_remap_vec for entry reconstruction
- Legacy dead code removed

Shared (patch_format.h):
- Vector-based rebuild_geo_from_remap_vec (O(1) lookups vs hash map)
- Secondary remap marker (0xFFFFFFF6)

Results (planet, verified all 14 files MATCH):
- Diff: 36m → 15m (2.4x faster)
- Patch: 20m → 6m43s (3x faster)
- Patch peak RSS: 63 GiB → 38 GiB

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
EOF
)"

Commit performance improvements


[main e0d0778] perf: parallelize diff tool, optimize patch tool, add secondary matching
 4 files changed, 1132 insertions(+), 1217 deletions(-)


Push to remote


To github.com:zp-forks/traccar-geocoder.git
   bd7f977..e0d0778  main -> main


Pushed. Now let me investigate and fix the streaming approach bug. The key finding: entry file sizes explode ~10x between cells 105M-110M on planet. The streaming approach worked perfectly on Europe (83M cells) and for the first 105M cells on planet.

Let me re-examine the bug. The ftell-reported sizes were correct at 105M cells, then jumped at 110M. The only code path that writes entries is `write_ids` which calls `fwrite` and returns `ftell()`. Since `ftell()` returns `long` (64-bit on 64-bit Linux), there's no overflow.

Let me look at the code that was producing the bloated output more carefully. The streaming version uses `MappedFile` for old entries while the working version uses `read_file()`. Let me test if that's the difference — use `read_file()` in the streaming approach:


   360→    // Walk old geo_cells, remap IDs, apply corrections, write output directly to files.
   361→    {
   362→        double t_entry = now_ms();
   363→        std::cerr << "Entry pipeline (proven rebuild approach)..." << std::endl;
   364→
   365→        auto get_rm = [&](PatchFileId fid) -> const std::vector<uint32_t>& {
   366→            static std::vector<uint32_t> empty;
   367→            auto it = id_remaps.find((uint32_t)fid); return it != id_remaps.end() ? it->second : empty;
   368→        };
   369→        auto& w_rm = get_rm(PatchFileId::STREET_WAYS);
   370→        auto& a_rm = get_rm(PatchFileId::ADDR_POINTS);
   371→        auto& i_rm = get_rm(PatchFileId::INTERP_WAYS);
   372→
   373→        // Rebuild entries using proven rebuild_geo_from_remap_vec
   374→        auto old_geo_data = read_file(cur_dir + "/geo_cells.bin");
   375→        auto old_se = read_file(cur_dir + "/street_entries.bin");
   376→        auto old_ae = read_file(cur_dir + "/addr_entries.bin");
   377→        auto old_ie = read_file(cur_dir + "/interp_entries.bin");
   378→        auto geo = rebuild_geo_from_remap_vec(old_geo_data, old_se, old_ae, old_ie,
   379→                                                w_rm, a_rm, i_rm, geo_added, geo_removed);
   380→        // Free inputs
   381→        { std::vector<char>().swap(old_se); std::vector<char>().swap(old_ae); std::vector<char>().swap(old_ie); }
   382→        log_phase("  Geo rebuild", t_entry);
   383→
   384→        // Apply corrections sequentially (one file at a time to limit memory)
   385→        // Build correction maps
   386→        auto apply_corrections = [&](uint32_t fid, const std::string& fname,
   387→                                      std::vector<char>& derived_entries,
   388→                                      const std::vector<char>& derived_cells, size_t cell_stride,
   389→                                      size_t off_pos) {

    // Walk old geo_cells, remap IDs, apply corrections, write output directly to files.
    {
        double t_entry = now_ms();
        std::cerr << "Entry pipeline (proven rebuild approach)..." << std::endl;

        auto get_rm = [&](PatchFileId fid) -> const std::vector<uint32_t>& {
            static std::vector<uint32_t> empty;
            auto it = id_remaps.find((uint32_t)fid); return it != id_remaps.end() ? it->second : empty;
        };
        auto& w_rm = get_rm(PatchFileId::STREET_WAYS);
        auto& a_rm = get_rm(PatchFileId::ADDR_POINTS);
        auto& i_rm = get_rm(PatchFileId::INTERP_WAYS);

        // Rebuild entries using proven rebuild_geo_from_remap_vec
        auto old_geo_data = read_file(cur_dir + "/geo_cells.bin");
        auto old_se = read_file(cur_dir + "/street_entries.bin");
        auto old_ae = read_file(cur_dir + "/addr_entries.bin");
        auto old_ie = read_file(cur_dir + "/interp_entries.bin");
        auto geo = rebuild_geo_from_remap_vec(old_geo_data, old_se, old_ae, old_ie,
                                                w_rm, a_rm, i_rm, geo_added, geo_removed);
        // Free inputs
        { std::vector<char>().swap(old_se); std::vector<char>().swap(old_ae); std::vector<char>().swap(old_ie); }
        log_phase("  Geo rebuild", t_entry);

        // Apply corrections sequentially (one file at a time to limit memory)
        // Build correction maps
        auto apply_corrections = [&](uint32_t fid, const std::string& fname,
                                      std::vector<char>& derived_entries,
                                      const std::vector<char>& derived_cells, size_t cell_stride,
                                      size_t off_pos) {


Let me write a clean streaming version that:
1. Uses `read_file()` instead of mmap for old entries (to rule out mmap as the issue)
2. Uses `uint64_t` for file positions (to rule out overflow)
3. Doesn't use the shared `id_buf` trick (to rule out buffer aliasing)
4. Processes one entry type at a time (to isolate the bug)

I'll put it alongside the proven approach so I can compare output:


Edit

        std::cerr << "Entry pipeline (proven rebuild approach)..." << std::endl;

        auto get_rm = [&](PatchFileId fid) -> const std::vector<uint32_t>& {
            static std::vector<uint32_t> empty;
            auto it = id_remaps.find((uint32_t)fid); return it != id_remaps.end() ? it->second : empty;
        };
        auto& w_rm = get_rm(PatchFileId::STREET_WAYS);
        auto& a_rm = get_rm(PatchFileId::ADDR_POINTS);
        auto& i_rm = get_rm(PatchFileId::INTERP_WAYS);

        // Rebuild entries using proven rebuild_geo_from_remap_vec
        auto old_geo_data = read_file(cur_dir + "/geo_cells.bin");
        auto old_se = read_file(cur_dir + "/street_entries.bin");
        auto old_ae = read_file(cur_dir + "/addr_entries.bin");
        auto old_ie = read_file(cur_dir + "/interp_entries.bin");
        auto geo = rebuild_geo_from_remap_vec(old_geo_data, old_se, old_ae, old_ie,
                                                w_rm, a_rm, i_rm, geo_added, geo_removed);
        // Free inputs
        { std::vector<char>().swap(old_se); std::vector<char>().swap(old_ae); std::vector<char>().swap(old_ie); }
        log_phase("  Geo rebuild", t_entry);

        std::cerr << "Entry pipeline (streaming, one type at a time)..." << std::endl;

        auto get_rm = [&](PatchFileId fid) -> const std::vector<uint32_t>& {
            static std::vector<uint32_t> empty;
            auto it = id_remaps.find((uint32_t)fid); return it != id_remaps.end() ? it->second : empty;
        };

        // Load old geo_cells
        auto old_geo_data = read_file(cur_dir + "/geo_cells.bin");
        size_t n_old_cells = old_geo_data.size() / 20;

        // Build derived cell list (old - removed + added, sorted)
        std::unordered_set<uint64_t> rm_set(geo_removed.begin(), geo_removed.end());
        struct CellRef { uint64_t cid; int32_t old_idx; };
        std::vector<CellRef> cells;
        cells.reserve(n_old_cells + geo_added.size());
        for (size_t i = 0; i < n_old_cells; i++) {
            uint64_t cid; memcpy(&cid, old_geo_data.data() + i * 20, 8);
            if (!rm_set.count(cid)) cells.push_back({cid, (int32_t)i});
        }
        { std::unordered_set<uint64_t>().swap(rm_set); } // free
        for (uint64_t cid : geo_added) cells.push_back({cid, -1});
        std::sort(cells.begin(), cells.end(), [](const CellRef& a, const CellRef& b) { return a.cid < b.cid; });
        log_phase("  Cell list built", t_entry);

        // Process each entry type SEPARATELY to keep memory low
        // For each type: load old entries → walk cells → remap + correct → write via fwrite
        // geo_cells offsets are recorded per-type and assembled at the end

        // Per-cell output offsets for geo_cells (0xFFFFFFFF = no data)
        std::vector<uint32_t> se_offsets(cells.size(), 0xFFFFFFFF);
        std::vector<uint32_t> ae_offsets(cells.size(), 0xFFFFFFFF);
        std::vector<uint32_t> ie_offsets(cells.size(), 0xFFFFFFFF);

        auto process_entry_type = [&](PatchFileId data_fid, PatchFileId entry_fid,
                                       const std::string& entry_fname, size_t geo_off_pos,
                                       std::vector<uint32_t>& out_offsets) {
            auto& rm = get_rm(data_fid);
            auto old_entries = read_file(cur_dir + "/" + entry_fname);

            // Build correction map for this type
            std::unordered_map<uint64_t, const std::vector<uint32_t>*> corr_map;
            auto ecit = entry_corrections.find((uint32_t)entry_fid);
            if (ecit != entry_corrections.end())
                for (auto& c : ecit->second) corr_map[c.cell_id] = &c.ids;

            FILE* outf = fopen((out_dir + "/" + entry_fname).c_str(), "wb");
            std::vector<uint32_t> ids; // reusable per-cell buffer
            ids.reserve(256);

            for (size_t ci = 0; ci < cells.size(); ci++) {
                uint64_t cid = cells[ci].cid;
                int32_t oi = cells[ci].old_idx;

                // Determine if this cell has entries for this type
                bool has_entry = false;
                if (oi >= 0) {
                    uint32_t off; memcpy(&off, old_geo_data.data() + oi * 20 + geo_off_pos, 4);
                    has_entry = (off != 0xFFFFFFFF);
                }
                // Flag corrections can change has_entry
                auto fc = flag_corrections.find(cid);
                if (fc != flag_corrections.end()) {
                    uint8_t bit = (geo_off_pos == 8) ? 1 : (geo_off_pos == 12) ? 2 : 4;
                    has_entry = (fc->second & bit) != 0;
                }
                // Corrections can introduce entries
                if (corr_map.count(cid)) has_entry = true;

                if (!has_entry) continue;

                // Check for correction
                auto cit = corr_map.find(cid);
                if (cit != corr_map.end()) {
                    // Write correction data directly
                    if (!cit->second->empty()) {
                        out_offsets[ci] = (uint32_t)ftell(outf);
                        uint16_t count = cit->second->size();
                        fwrite(&count, 2, 1, outf);
                        fwrite(cit->second->data(), 4, count, outf);
                    }
                    continue;
                }

                // No correction: read old entries, remap, write
                if (oi < 0) continue; // added cell, no old data
                uint32_t off; memcpy(&off, old_geo_data.data() + oi * 20 + geo_off_pos, 4);
                if (off == 0xFFFFFFFF || off + 2 > old_entries.size()) continue;
                uint16_t count; memcpy(&count, old_entries.data() + off, 2);
                if (count == 0 || off + 2 + count * 4 > old_entries.size()) continue;

                ids.resize(count);
                memcpy(ids.data(), old_entries.data() + off + 2, count * 4);
                // Remap
                for (auto& id : ids)
                    if (id < rm.size() && rm[id] != 0xFFFFFFFF) id = rm[id];
                std::sort(ids.begin(), ids.end());

                out_offsets[ci] = (uint32_t)ftell(outf);
                fwrite(&count, 2, 1, outf);
                fwrite(ids.data(), 4, count, outf);
            }
            fclose(outf);
        };

        process_entry_type(PatchFileId::STREET_WAYS, PatchFileId::STREET_ENTRIES,
                          "street_entries.bin", 8, se_offsets);
        log_phase("  street_entries", t_entry);
        malloc_trim(0);

        process_entry_type(PatchFileId::ADDR_POINTS, PatchFileId::ADDR_ENTRIES,
                          "addr_entries.bin", 12, ae_offsets);
        log_phase("  addr_entries", t_entry);
        malloc_trim(0);

        process_entry_type(PatchFileId::INTERP_WAYS, PatchFileId::INTERP_ENTRIES,
                          "interp_entries.bin", 16, ie_offsets);
        log_phase("  interp_entries", t_entry);

        // Write geo_cells using the recorded offsets
        {
            FILE* fg = fopen((out_dir + "/geo_cells.bin").c_str(), "wb");
            for (size_t ci = 0; ci < cells.size(); ci++) {
                fwrite(&cells[ci].cid, 8, 1, fg);
                fwrite(&se_offsets[ci], 4, 1, fg);
                fwrite(&ae_offsets[ci], 4, 1, fg);
                fwrite(&ie_offsets[ci], 4, 1, fg);
            }
            fclose(fg);
        }
        log_phase("  geo_cells written", t_entry);