            if (cell_count == 1) {
                // Count how many old cells have each flag
                size_t os = 0, oa = 0, oi_c = 0;
                for (size_t ci = 0; ci < old_geo.size / 20; ci++) {
                    uint32_t sv, av, iv;
                    memcpy(&sv, old_geo.data + ci*20+8, 4); memcpy(&av, old_geo.data + ci*20+12, 4); memcpy(&iv, old_geo.data + ci*20+16, 4);
                    if (sv != no_data) os++; if (av != no_data) oa++; if (iv != no_data) oi_c++;
                }
                std::cerr << "    DEBUG: old cells=" << old_geo.size/20 << " has_street=" << os << " has_addr=" << oa << " has_interp=" << oi_c << std::endl;
            }

// geocoder-patch v4: Low-memory streaming patch application.
//
// Key design: mmap old files (zero copy), stream output via fwrite (never accumulate),
// process entry pipeline cell-by-cell. Target: <1 GiB peak RSS for planet.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <fcntl.h>
#include <iomanip>
#include <iostream>
#include <malloc.h>
#include <string>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unordered_map>
#include <unordered_set>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

// --- Helpers ---
static double now_ms() {
    struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
    return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6;
}
static size_t get_rss_mb() {
    FILE* f = fopen("/proc/self/statm", "r");
    if (!f) return 0;
    size_t dummy, rss; fscanf(f, "%zu %zu", &dummy, &rss); fclose(f);
    return rss * 4096 / (1024*1024);
}
static void log_phase(const char* label, double start) {
    std::cerr << "  [" << std::fixed << std::setprecision(1) << (now_ms() - start) / 1000.0
              << "s, " << get_rss_mb() << " MiB] " << label << std::endl;
}

// mmap a file read-only. Returns {pointer, size}. Caller must munmap.
struct MappedFile { const char* data; size_t size; };
static MappedFile mmap_file(const std::string& path) {
    int fd = open(path.c_str(), O_RDONLY);
    if (fd < 0) return {nullptr, 0};
    struct stat st; fstat(fd, &st);
    size_t sz = st.st_size;
    if (sz == 0) { close(fd); return {nullptr, 0}; }
    void* p = mmap(nullptr, sz, PROT_READ, MAP_PRIVATE, fd, 0);
    close(fd);
    if (p == MAP_FAILED) return {nullptr, 0};
    return {static_cast<const char*>(p), sz};
}
static void unmap_file(MappedFile& f) {
    if (f.data) { munmap(const_cast<char*>(f.data), f.size); f.data = nullptr; f.size = 0; }
}

// Detect stride from file size
static size_t detect_stride(const std::string& path, std::initializer_list<size_t> candidates) {
    struct stat st; if (stat(path.c_str(), &st) != 0) return *candidates.begin();
    for (size_t s : candidates) if (st.st_size % s == 0 && st.st_size > 0) return s;
    return *candidates.begin();
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);
    double t_start = now_ms();

    // --- Phase 1: Decompress + mmap patch ---
    std::string raw_path = tmpdir + "/patch.raw";
    {
        std::string cmd = "zstd -d '" + patch_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0) { std::cerr << "Failed to decompress" << std::endl; return 1; }
    }
    MappedFile patch_map = mmap_file(raw_path);
    if (!patch_map.data) { std::cerr << "Failed to mmap patch" << std::endl; return 1; }
    remove(raw_path.c_str());
    const char* P = patch_map.data;
    size_t patch_size = patch_map.size;
    std::cerr << "Patch: " << patch_size << " bytes" << std::endl;
    log_phase("Decompress", t_start);

    size_t pos = 0;
    auto ru32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, P+pos, 4); pos += 4; return v; };
    auto ru64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, P+pos, 8); pos += 8; return v; };

    // Header
    if (memcmp(P, GCPATCH_MAGIC, 8) != 0) { std::cerr << "Bad magic" << std::endl; return 1; }
    pos = 8;
    uint32_t ver = ru32(); if (ver != 2) { std::cerr << "Bad version" << std::endl; return 1; }
    ru32(); // flags

    // --- Phase 2: String rebuild ---
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t marker = ru32();
        if (marker == 0xFFFFFFF7) {
            uint32_t n_added = ru32(), n_deleted = ru32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ru32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());

            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
            std::vector<std::pair<uint32_t, std::string>> old_strs;
            size_t sp = 0;
            while (sp < old_pool.size) {
                old_strs.push_back({(uint32_t)sp, std::string(old_pool.data + sp)});
                sp += strlen(old_pool.data + sp) + 1;
            }
            std::vector<std::string> merged;
            for (size_t i = 0; i < old_strs.size(); i++)
                if (!del_set.count(i)) merged.push_back(old_strs[i].second);
            for (auto& s : added) merged.push_back(s);
            std::sort(merged.begin(), merged.end());

            // Write new pool and build remap
            FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
            std::unordered_map<std::string, uint32_t> new_offs;
            uint32_t wpos = 0;
            for (auto& s : merged) {
                new_offs[s] = wpos;
                fwrite(s.c_str(), 1, s.size() + 1, fp);
                wpos += s.size() + 1;
            }
            fclose(fp);
            for (auto& [old_off, s] : old_strs) {
                auto it = new_offs.find(s);
                if (it != new_offs.end() && it->second != old_off) str_remap[old_off] = it->second;
            }
            unmap_file(old_pool);
            std::cerr << "  Strings: +" << n_added << " -" << n_deleted << " → " << merged.size()
                      << ", " << str_remap.size() << " remapped" << std::endl;
            marker = ru32();
        }
        if (marker == 0xFFFFFFFE) { uint32_t c = ru32(); for (uint32_t i = 0; i < c; i++) { uint32_t a = ru32(), b = ru32(); str_remap[a] = b; } }
        else pos -= 4;
    }
    log_phase("Strings", t_start);

    // Detect strides
    size_t way_stride = detect_stride(cur_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride(cur_dir + "/interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect_stride(cur_dir + "/admin_polygons.bin", {24, 20, 19});

    // String remap applier (works on mutable copy)
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t fid, size_t stride) {
        if (str_remap.empty()) return;
        std::vector<size_t> offs;
        if (fid == (uint32_t)PatchFileId::ADDR_POINTS) offs = {8, 12};
        else if (fid == (uint32_t)PatchFileId::STREET_WAYS) offs = {(stride == 12) ? 8ul : 5ul};
        else if (fid == (uint32_t)PatchFileId::INTERP_WAYS) offs = {(stride >= 20) ? 8ul : 5ul};
        else if (fid == (uint32_t)PatchFileId::ADMIN_POLYGONS) offs = {8};
        else return;
        for (size_t i = 0; i + stride <= data.size(); i += stride)
            for (size_t o : offs) { uint32_t v; memcpy(&v, data.data()+i+o, 4); auto it = str_remap.find(v); if (it != str_remap.end()) memcpy(data.data()+i+o, &it->second, 4); }
    };

    // --- Phase 3: Merge replays (streaming output) ---
    // ID remaps: file_id → vector<uint32_t> where remap[old_idx] = new_idx
    std::unordered_map<uint32_t, std::vector<uint32_t>> id_remaps;
    std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;
    std::unordered_map<uint64_t, uint8_t> flag_corrections;
    struct CellCorr { uint64_t cell_id; std::vector<uint32_t> ids; };
    std::unordered_map<uint32_t, std::vector<CellCorr>> entry_corrections;

    while (pos < patch_size) {
        uint32_t file_id = ru32();
        if (file_id == 0xFFFFFFFF) break;

        // --- Metadata sections ---
        if (file_id == 0xFFFFFFF7) {
            // String diff in main loop — process it (may be the only occurrence)
            uint32_t n_added = ru32(), n_deleted = ru32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ru32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());
            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
            std::vector<std::pair<uint32_t, std::string>> old_strs;
            size_t sp = 0;
            while (sp < old_pool.size) { old_strs.push_back({(uint32_t)sp, std::string(old_pool.data+sp)}); sp += strlen(old_pool.data+sp)+1; }
            std::vector<std::string> merged;
            for (size_t i = 0; i < old_strs.size(); i++) if (!del_set.count(i)) merged.push_back(old_strs[i].second);
            for (auto& s : added) merged.push_back(s);
            std::sort(merged.begin(), merged.end());
            FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
            std::unordered_map<std::string, uint32_t> new_offs;
            uint32_t wpos = 0;
            for (auto& s : merged) { new_offs[s] = wpos; fwrite(s.c_str(), 1, s.size()+1, fp); wpos += s.size()+1; }
            fclose(fp);
            for (auto& [old_off, s] : old_strs) { auto it = new_offs.find(s); if (it != new_offs.end() && it->second != old_off) str_remap[old_off] = it->second; }
            unmap_file(old_pool);
            std::cerr << "  Strings (loop): +" << n_added << " -" << n_deleted << " → " << merged.size() << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_GEO_MARKER) {
            uint32_t na = ru32(), nr = ru32();
            geo_added.resize(na); geo_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { memcpy(&geo_added[i], P+pos, 8); pos += 8; }
            for (uint32_t i = 0; i < nr; i++) { memcpy(&geo_removed[i], P+pos, 8); pos += 8; }
            std::cerr << "  Geo cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_ADMIN_MARKER) {
            uint32_t na = ru32(), nr = ru32();
            admin_added.resize(na); admin_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { memcpy(&admin_added[i], P+pos, 8); pos += 8; }
            for (uint32_t i = 0; i < nr; i++) { memcpy(&admin_removed[i], P+pos, 8); pos += 8; }
            std::cerr << "  Admin cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_FLAGS_MARKER) {
            uint32_t c = ru32();
            for (uint32_t i = 0; i < c; i++) { uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8; flag_corrections[cid] = P[pos++]; }
            std::cerr << "  Flag corrections: " << c << std::endl;
            continue;
        }
        if (file_id == SECONDARY_REMAP_MARKER) {
            uint32_t nf = ru32();
            for (uint32_t f = 0; f < nf; f++) {
                uint32_t fid = ru32(), np = ru32();
                auto& rm = id_remaps[fid];
                for (uint32_t i = 0; i < np; i++) { uint32_t o = ru32(), n = ru32(); if (o < rm.size()) rm[o] = n; }
                std::cerr << "  Secondary remap " << fid << ": " << np << " pairs" << std::endl;
            }
            continue;
        }
        if (file_id == ENTRY_CORRECTION_MARKER) {
            uint32_t fid = ru32(), c = ru32();
            auto& list = entry_corrections[fid];
            for (uint32_t i = 0; i < c; i++) {
                uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8;
                uint16_t ec; memcpy(&ec, P+pos, 2); pos += 2;
                std::vector<uint32_t> ids(ec);
                if (ec > 0) { memcpy(ids.data(), P+pos, ec*4); pos += ec*4; }
                list.push_back({cid, std::move(ids)});
            }
            std::cerr << "  Entry corrections " << fid << ": " << c << std::endl;
            continue;
        }

        // --- Merge sequence replay (streaming) ---
        uint32_t stride = ru32();
        uint64_t old_size = ru64(), new_size = ru64();
        if (file_id >= (uint32_t)PatchFileId::COUNT) { std::cerr << "Unknown file " << file_id << std::endl; break; }
        const char* fname = patch_file_names[file_id];

        if (stride == 0) {
            // Full replacement — write directly from patch mmap
            uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64();
            FILE* fp = fopen((out_dir + "/" + fname).c_str(), "wb");
            fwrite(P+pos, 1, ds, fp); fclose(fp);
            pos += ds;
            std::cerr << "  " << fname << ": full replace " << ds << " bytes" << std::endl;
            continue;
        }
        if (stride == 0xFE) { uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64(); pos += ds; continue; }

        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        // Load old file into mutable buffer (need to apply remap + fixups)
        // For large files (nodes/vertices), this is the peak memory per file.
        // Files are processed sequentially, so only one is in memory at a time.
        auto old_data = read_file(cur_dir + "/" + std::string(fname));

        // Zero padding
        if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24)
            for (size_t i = 0; i + actual_stride <= old_data.size(); i += actual_stride) memset(old_data.data()+i+13, 0, 3);
        if (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24)
            for (size_t i = 0; i + actual_stride <= old_data.size(); i += actual_stride) { memset(old_data.data()+i+5, 0, 3); memset(old_data.data()+i+21, 0, 3); }

        apply_str_remap(old_data, file_id, actual_stride);

        // Apply fixups
        uint32_t n_fixups = ru32();
        if (n_fixups > 0) {
            uint32_t delta_size = ru32();
            uint32_t pi = 0, pv = 0; size_t fp = pos;
            for (uint32_t i = 0; i < n_fixups; i++) {
                uint32_t idx = pi + read_varint(P, fp);
                uint32_t val = pv + read_varint(P, fp);
                if ((size_t)idx * actual_stride + 4 <= old_data.size())
                    memcpy(old_data.data() + (size_t)idx * actual_stride, &val, 4);
                pi = idx; pv = val;
            }
            pos = fp;
        }

        // Replay merge sequence — stream output directly to file
        uint64_t seq_size = ru64();
        size_t seq_end = pos + seq_size;
        FILE* outf = fopen((out_dir + "/" + fname).c_str(), "wb");

        // Track ID remap (only for data files that need entry reconstruction)
        bool track = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
                      file_id == (uint32_t)PatchFileId::STREET_WAYS ||
                      file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
                      file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
        std::vector<uint32_t> id_map;
        if (track) id_map.assign(old_data.size() / actual_stride, 0xFFFFFFFF);

        size_t old_rec = 0, new_rec = 0, old_bytes = 0, written = 0;
        while (pos < seq_end) {
            uint8_t op = P[pos++];
            uint32_t count; memcpy(&count, P+pos, 4); pos += 4;
            if (op == OP_MATCH_RUN) {
                size_t bytes = count * actual_stride;
                fwrite(old_data.data() + old_bytes, 1, bytes, outf);
                written += bytes;
                if (track) for (uint32_t k = 0; k < count; k++) if (old_rec+k < id_map.size()) id_map[old_rec+k] = new_rec+k;
                old_rec += count; new_rec += count; old_bytes += bytes;
            } else if (op == OP_INSERT_RUN) {
                size_t bytes = count * actual_stride;
                fwrite(P+pos, 1, bytes, outf);
                written += bytes; pos += bytes; new_rec += count;
            } else if (op == OP_DELETE_RUN) {
                old_rec += count; old_bytes += count * actual_stride;
            }
        }
        fclose(outf);

        if (track) {
            id_remaps[file_id] = std::move(id_map);
            size_t matched = 0;
            for (auto v : id_remaps[file_id]) if (v != 0xFFFFFFFF) matched++;
            std::cerr << "  " << fname << ": " << written << " bytes (mapped " << matched << "/" << id_remaps[file_id].size() << ")" << std::endl;
        } else {
            std::cerr << "  " << fname << ": " << written << " bytes" << std::endl;
        }

        // Free old data immediately — don't let glibc hold the pages
        { std::vector<char>().swap(old_data); }
        malloc_trim(0);
    }
    log_phase("Merge replays", t_start);

    // Free string remap (no longer needed)
    { std::unordered_map<uint32_t,uint32_t>().swap(str_remap); }
    malloc_trim(0);

    // --- Phase 4: Streaming entry pipeline ---
    // Walk old geo_cells, remap IDs, apply corrections, write output directly to files.
    {
        double t_entry = now_ms();
        std::cerr << "Entry pipeline..." << std::endl;

        auto get_rm = [&](PatchFileId fid) -> const std::vector<uint32_t>& {
            static std::vector<uint32_t> empty;
            auto it = id_remaps.find((uint32_t)fid); return it != id_remaps.end() ? it->second : empty;
        };
        auto& w_rm = get_rm(PatchFileId::STREET_WAYS);
        auto& a_rm = get_rm(PatchFileId::ADDR_POINTS);
        auto& i_rm = get_rm(PatchFileId::INTERP_WAYS);

        // Build correction lookup maps
        std::unordered_map<uint64_t, const std::vector<uint32_t>*> cs, ca, ci;
        for (auto& [fid, list] : entry_corrections) {
            auto* m = (fid == (uint32_t)PatchFileId::STREET_ENTRIES) ? &cs :
                      (fid == (uint32_t)PatchFileId::ADDR_ENTRIES) ? &ca :
                      (fid == (uint32_t)PatchFileId::INTERP_ENTRIES) ? &ci : nullptr;
            if (m) for (auto& c : list) (*m)[c.cell_id] = &c.ids;
        }

        // mmap old files (zero-copy read)
        MappedFile old_geo = mmap_file(cur_dir + "/geo_cells.bin");
        MappedFile old_se = mmap_file(cur_dir + "/street_entries.bin");
        MappedFile old_ae = mmap_file(cur_dir + "/addr_entries.bin");
        MappedFile old_ie = mmap_file(cur_dir + "/interp_entries.bin");

        // Build derived cell list
        std::unordered_set<uint64_t> rm_set(geo_removed.begin(), geo_removed.end());
        struct CR { uint64_t cid; int32_t idx; };
        std::vector<CR> cells;
        cells.reserve(old_geo.size / 20 + geo_added.size());
        for (size_t i = 0; i < old_geo.size / 20; i++) {
            uint64_t cid; memcpy(&cid, old_geo.data + i*20, 8);
            if (!rm_set.count(cid)) cells.push_back({cid, (int32_t)i});
        }
        for (uint64_t cid : geo_added) cells.push_back({cid, -1});
        std::sort(cells.begin(), cells.end(), [](const CR& a, const CR& b) { return a.cid < b.cid; });
        log_phase("  Cell list", t_entry);

        // Open output files for streaming write
        FILE* f_geo = fopen((out_dir + "/geo_cells.bin").c_str(), "wb");
        FILE* f_se = fopen((out_dir + "/street_entries.bin").c_str(), "wb");
        FILE* f_ae = fopen((out_dir + "/addr_entries.bin").c_str(), "wb");
        FILE* f_ie = fopen((out_dir + "/interp_entries.bin").c_str(), "wb");
        uint32_t se_pos = 0, ae_pos = 0, ie_pos = 0;
        uint32_t no_data = 0xFFFFFFFF;

        // Reusable buffer
        std::vector<uint32_t> id_buf;
        id_buf.reserve(256);

        auto parse_ids = [](const MappedFile& f, uint32_t off, std::vector<uint32_t>& buf) {
            buf.clear();
            if (off == 0xFFFFFFFF || off + 2 > f.size) return;
            uint16_t c; memcpy(&c, f.data+off, 2);
            if (c > 10000 || off+2+(size_t)c*4 > f.size) return; // sanity: no cell has >10K entries
            buf.resize(c); memcpy(buf.data(), f.data+off+2, c*4);
        };
        auto remap_buf = [](std::vector<uint32_t>& buf, const std::vector<uint32_t>& rm) {
            for (auto& id : buf) if (id < rm.size() && rm[id] != 0xFFFFFFFF) id = rm[id];
            std::sort(buf.begin(), buf.end());
        };
        auto write_ids = [](FILE* f, const uint32_t* ids, size_t n, uint32_t& fpos) -> uint32_t {
            if (n == 0) return 0xFFFFFFFF;
            uint32_t off = fpos;
            uint16_t c = (uint16_t)n;
            fwrite(&c, 2, 1, f); fwrite(ids, 4, n, f);
            fpos += 2 + n*4;
            return off;
        };

        size_t sc = 0, ac = 0, ic = 0;
        size_t cell_count = 0;
        for (auto& [cid, oi] : cells) {
            if (++cell_count % 5000000 == 0) {
                std::cerr << "    " << cell_count << "/" << cells.size() << " cells, se=" << se_pos/1024/1024 << "M ae=" << ae_pos/1024/1024 << "M ie=" << ie_pos/1024/1024 << "M rss=" << get_rss_mb() << "M" << std::endl;
            }
            // Process one entry type at a time using shared id_buf
            auto process = [&](const MappedFile& old_e, size_t geo_off,
                               const std::vector<uint32_t>& rm,
                               const std::unordered_map<uint64_t, const std::vector<uint32_t>*>& corr,
                               FILE* outf, uint32_t& fpos, size_t& cc) -> uint32_t {
                auto cit = corr.find(cid);
                if (cit != corr.end()) { cc++; return write_ids(outf, cit->second->data(), cit->second->size(), fpos); }
                if (oi < 0) return no_data;
                uint32_t off; memcpy(&off, old_geo.data + oi*20 + geo_off, 4);
                parse_ids(old_e, off, id_buf);
                if (id_buf.empty()) return no_data;
                remap_buf(id_buf, rm);
                return write_ids(outf, id_buf.data(), id_buf.size(), fpos);
            };

            // Determine flags
            bool hs = false, ha = false, hi = false;
            if (oi >= 0) {
                uint32_t s, a, ip; memcpy(&s, old_geo.data+oi*20+8, 4); memcpy(&a, old_geo.data+oi*20+12, 4); memcpy(&ip, old_geo.data+oi*20+16, 4);
                hs = (s != no_data); ha = (a != no_data); hi = (ip != no_data);
            }
            auto fc = flag_corrections.find(cid);
            if (fc != flag_corrections.end()) { hs = fc->second & 1; ha = fc->second & 2; hi = fc->second & 4; }
            if (cs.count(cid)) hs = true;
            if (ca.count(cid)) ha = true;
            if (ci.count(cid)) hi = true;

            uint32_t so = hs ? process(old_se, 8, w_rm, cs, f_se, se_pos, sc) : no_data;
            uint32_t ao = ha ? process(old_ae, 12, a_rm, ca, f_ae, ae_pos, ac) : no_data;
            uint32_t io = hi ? process(old_ie, 16, i_rm, ci, f_ie, ie_pos, ic) : no_data;
            if (cell_count <= 3) {
                std::cerr << "    DEBUG cell " << cell_count << ": cid=" << cid << " oi=" << oi
                          << " hs=" << hs << " ha=" << ha << " hi=" << hi
                          << " se_pos=" << se_pos << " ae_pos=" << ae_pos << " ie_pos=" << ie_pos << std::endl;
            }

            fwrite(&cid, 8, 1, f_geo); fwrite(&so, 4, 1, f_geo); fwrite(&ao, 4, 1, f_geo); fwrite(&io, 4, 1, f_geo);
        }
        fclose(f_geo); fclose(f_se); fclose(f_ae); fclose(f_ie);
        unmap_file(old_geo); unmap_file(old_se); unmap_file(old_ae); unmap_file(old_ie);
        std::cerr << "  Geo: " << cells.size() << " cells, corrections: " << sc << " st, " << ac << " ad, " << ic << " in" << std::endl;
        log_phase("  Geo entries", t_entry);

        // Admin: small, use existing rebuild + corrections
        {
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            auto ait = id_remaps.find((uint32_t)PatchFileId::ADMIN_POLYGONS);
            if (ait != id_remaps.end()) {
                for (uint32_t i = 0; i < ait->second.size(); i++)
                    if (ait->second[i] != 0xFFFFFFFF) ad_rm[i] = ait->second[i];
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_adme = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_adme, ad_rm, admin_added, admin_removed);

            auto ecit = entry_corrections.find((uint32_t)PatchFileId::ADMIN_ENTRIES);
            if (ecit != entry_corrections.end()) {
                std::unordered_map<uint64_t, const std::vector<uint32_t>*> ac_corr;
                for (auto& c : ecit->second) ac_corr[c.cell_id] = &c.ids;
                size_t n = admin.admin_cells_data.size() / 12;
                FILE* fac = fopen((out_dir + "/admin_cells.bin").c_str(), "wb");
                FILE* fae = fopen((out_dir + "/admin_entries.bin").c_str(), "wb");
                uint32_t ae_wpos = 0;
                for (size_t i = 0; i < n; i++) {
                    uint64_t cid; memcpy(&cid, admin.admin_cells_data.data()+i*12, 8);
                    fwrite(&cid, 8, 1, fac);
                    auto cit = ac_corr.find(cid);
                    if (cit != ac_corr.end()) {
                        uint32_t off = cit->second->empty() ? no_data : ae_wpos;
                        fwrite(&off, 4, 1, fac);
                        if (!cit->second->empty()) {
                            uint16_t c = cit->second->size();
                            fwrite(&c, 2, 1, fae); fwrite(cit->second->data(), 4, c, fae);
                            ae_wpos += 2 + c*4;
                        }
                    } else {
                        uint32_t old_off; memcpy(&old_off, admin.admin_cells_data.data()+i*12+8, 4);
                        if (old_off != no_data && old_off+2 <= admin.admin_entries_data.size()) {
                            uint32_t new_off = ae_wpos; fwrite(&new_off, 4, 1, fac);
                            uint16_t c; memcpy(&c, admin.admin_entries_data.data()+old_off, 2);
                            fwrite(admin.admin_entries_data.data()+old_off, 1, 2+c*4, fae);
                            ae_wpos += 2+c*4;
                        } else {
                            fwrite(&no_data, 4, 1, fac);
                        }
                    }
                }
                fclose(fac); fclose(fae);
                std::cerr << "  Admin: " << n << " cells, " << ac_corr.size() << " corrections" << std::endl;
            } else {
                write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
                write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            }
            log_phase("  Admin", t_entry);
        }
    }

    // Cleanup
    unmap_file(patch_map);
    { std::string cmd = "rm -rf '" + tmpdir + "'"; system(cmd.c_str()); }
    log_phase("Total", t_start);
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

-                std::cerr << "    DEBUG cell " << cell_count << ": cid=" << cid << " oi=" << oi

-                          << " se_pos=" << se_pos << " ae_pos=" << ae_pos << " ie_pos=" << ie_pos << std::endl;

+                // Count how many old cells have each flag

+                size_t os = 0, oa = 0, oi_c = 0;

+                for (size_t ci = 0; ci < old_geo.size / 20; ci++) {

+                    uint32_t sv, av, iv;

+                    memcpy(&sv, old_geo.data + ci*20+8, 4); memcpy(&av, old_geo.data + ci*20+12, 4); memcpy(&iv, old_geo.data + ci*20+16, 4);

+                    if (sv != no_data) os++; if (av != no_data) oa++; if (iv != no_data) oi_c++;

+                std::cerr << "    DEBUG: old cells=" << old_geo.size/20 << " has_street=" << os << " has_addr=" << oa << " has_interp=" << oi_c << std::endl;


Bash

make -j$(nproc) geocoder-patch 2>&1 | tail -3 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "kill -9 \$(ps aux | grep 'geocoder-patch' | grep -v grep | awk '{print \$2}') 2>/dev/null; rm -f /home/michtest/geocoder-patch; rm -rf /home/michtest/patched-planet-stream/" 2>&1 && sshpass -p 'elon-is-insane' scp -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:/home/michtest/ 2>&1 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 'cd /home/michtest && rm -rf patched-planet-stream/ && ./geocoder-patch planet-B/ validated-patches/planet-mar16-to-mar23-v2.gcpatch -o patched-planet-stream/ 2>&1 | head -35 && kill -9 $(ps aux | grep "geocoder-patch planet" | grep -v grep | awk "{print \$2}") 2>/dev/null; rm -rf patched-planet-stream/' 2>&1

Run with flag count debug


[ 50%] Building CXX object CMakeFiles/geocoder-patch.dir/tools/geocoder_patch.cpp.o
[100%] Linking CXX executable geocoder-patch
[100%] Built target geocoder-patch
Patch: 423813763 bytes
  [0.3s, 3 MiB] Decompress
  [0.3s, 3 MiB] Strings
  Strings (loop): +10703 -4486 → 11894530
  addr_points.bin: 2497261184 bytes (mapped 155813360/155902060)
  street_ways.bin: 574853820 bytes (mapped 47686691/47864210)
  street_nodes.bin: 3966274352 bytes
  interp_ways.bin: 1749240 bytes (mapped 72656/72839)
  interp_nodes.bin: 3484432 bytes
  admin_polygons.bin: 22763904 bytes (mapped 942932/947905)
  admin_vertices.bin: 717223152 bytes
  Geo cells: +183728 -45783
  Admin cells: +4 -0
  Secondary remap 1: 55091 pairs
  Secondary remap 3: 58838 pairs
  Secondary remap 4: 88 pairs
  Secondary remap 6: 4603 pairs
  Entry corrections 9: 1588748
  Entry corrections 10: 95288
  Entry corrections 11: 256
  Entry corrections 13: 4556
  Flag corrections: 224324
  [27.3s, 1816 MiB] Merge replays
Entry pipeline...
  [14.2s, 12469 MiB]   Cell list
    DEBUG: old cells=322588816 has_street=306918478 has_addr=48969395 has_interp=146493
    5000000/322726761 cells, se=39M ae=4M ie=0M rss=12512M
    10000000/322726761 cells, se=78M ae=12M ie=0M rss=12559M
    15000000/322726761 cells, se=111M ae=17M ie=0M rss=12598M
    20000000/322726761 cells, se=147M ae=27M ie=0M rss=12644M
    25000000/322726761 cells, se=186M ae=39M ie=0M rss=12695M
    30000000/322726761 cells, se=221M ae=40M ie=0M rss=12731M
    35000000/322726761 cells, se=254M ae=41M ie=0M rss=12765M
    40000000/322726761 cells, se=292M ae=43M ie=0M rss=12804M
    45000000/322726761 cells, se=329M ae=79M ie=0M rss=12877M


Key data:
- **322M old cells**: 306M have streets (95%), 49M have addr (15%), 147K have interp
- Entry growth rate: street_entries = 39M per 5M cells = 7.8 bytes/cell avg. For 322M total = 2.5 GiB (ref is 2.1 GiB — close!)
- addr_entries = 4M per 5M cells initially, then jumps at some points. For 49M addr cells total: 49M × ~14 bytes = 686 MiB. That's close to the ref 706 MiB.

So the output sizes ARE roughly correct! The 11 GiB I saw earlier was from the STALE old process, not the new one. The new run's growth rates match the reference. Let me let it complete:


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 'cd /home/michtest && rm -rf patched-planet-stream/ && nohup bash -c "time ./geocoder-patch planet-B/ validated-patches/planet-mar16-to-mar23-v2.gcpatch -o patched-planet-stream/ 2>&1 && echo ---VERIFY--- && for f in planet-C/*; do b=\$(basename \$f); cmp -s patched-planet-stream/\$b planet-C/\$b || echo \"MISMATCH: \$b\"; done && echo All_checked" > planet-patch-v5.log 2>&1 &' 2>&1 && echo "Started"

Start clean planet run


Command running in background with ID: b7w7h80ie. Output is being written to: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/b7w7h80ie.output


sleep 60 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "grep '\\[.*MiB\\]' /home/michtest/planet-patch-v5.log && ps aux | grep 'geocoder-patch planet' | grep -v grep | awk '{printf \"RSS: %.0f MiB\\n\", \$6/1024}'" 2>&1

Check 1 min


  [0.3s, 3 MiB] Decompress
  [0.3s, 3 MiB] Strings
  [26.2s, 1816 MiB] Merge replays
  [14.3s, 12469 MiB]   Cell list
RSS: 2 MiB
RSS: 3 MiB
RSS: 13446 MiB


Good. 13.4 GiB, stable during entry walk. Let me check at reasonable intervals:


sleep 300 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "grep 'cells/' /home/michtest/planet-patch-v5.log | tail -3 && ps aux | grep 'geocoder-patch planet' | grep -v grep | awk '{printf \"RSS: %.0f MiB TIME: %s\\n\", \$6/1024, \$10}'" 2>&1

Check 6 min


RSS: 2 MiB TIME: 0:00
RSS: 3 MiB TIME: 0:00
RSS: 13482 MiB TIME: 8:17


No progress output yet — still processing first 5M cells at 8:17 CPU. That's very slow — 5M cells should take ~10s at Europe's rate.

Actually wait — Europe had 83M cells and took 26s for the entry walk. Planet has 322M cells = 3.9x more. At the same rate: 3.9 × 26 = 101s. But at 8 min CPU, it's 5x slower than expected. The mmap page faults on larger files cause more cache misses.


sleep 300 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "grep 'cells/' /home/michtest/planet-patch-v5.log | tail -3 && ls -lh /home/michtest/patched-planet-stream/street_entries.bin /home/michtest/patched-planet-stream/addr_entries.bin 2>/dev/null" 2>&1

Check 11 min


-rw-rw-r-- 1 michtest michtest 62G Mar 30 05:19 /home/michtest/patched-planet-stream/addr_entries.bin
-rw-rw-r-- 1 michtest michtest 44G Mar 30 05:19 /home/michtest/patched-planet-stream/street_entries.bin


62 GiB addr_entries and 44 GiB street_entries!! The bug IS real — not from the old process. The entry files are growing way too fast.

But from the debug output, the growth rate was:
- 5M cells: se=39M, ae=4M — that's 7.8 bytes/cell and 0.8 bytes/cell
- Expected final: se=2.5 GiB, ae=0.26 GiB

But 62 GiB for addr means ~200 bytes per cell for ALL 322M cells. But only 49M cells have addr entries. So either:
1. All 322M cells are getting addr entries written (even those without)
2. The entries per cell are massive

Let me check: the `process_entry` returns `write_ids(...)` which writes to the file. If `has_a` is false, it should skip. But the `ha` flag is determined by the old geo_cells offset. For cells where the old offset is `0xFFFFFFFF`, `ha` is false and we skip. For the 49M cells with addr, we write ~14 bytes avg = 686 MiB.

Unless — the remap is corrupting the entries. When we remap IDs, if an old ID maps to a very large new ID, and there are MANY IDs per cell, the output is larger. But the remap shouldn't change the COUNT of IDs, just their values.

Wait — I just realized: the `parse_ids` sanity check limits count to 10000. But valid cells can have more than 10000 entries? No, that's unreasonable. But even 10000 IDs per cell × 4 bytes = 40 KB. For 322M cells that's 12.9 TB. Something is really wrong.

Let me add a stronger check and log the first anomalous write:


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "kill -9 \$(ps aux | grep 'geocoder-patch' | grep -v grep | awk '{print \$2}') 2>/dev/null; rm -rf /home/michtest/patched-planet-stream/" 2>&1

Kill and clean


(Bash completed with no output)


Edit

        auto write_ids = [](FILE* f, const uint32_t* ids, size_t n, uint32_t& fpos) -> uint32_t {
            if (n == 0) return 0xFFFFFFFF;
            uint32_t off = fpos;
            uint16_t c = (uint16_t)n;
            fwrite(&c, 2, 1, f); fwrite(ids, 4, n, f);
            fpos += 2 + n*4;
            return off;
        };

        auto write_ids = [](FILE* f, const uint32_t* ids, size_t n, uint32_t& fpos) -> uint32_t {
            if (n == 0) return 0xFFFFFFFF;
            uint32_t off = fpos;
            uint16_t c = (uint16_t)n;
            fwrite(&c, 2, 1, f); fwrite(ids, 4, n, f);
            uint64_t new_fpos = (uint64_t)fpos + 2 + n*4;
            fpos = (uint32_t)new_fpos; // Note: wraps at 4GiB
            return off;
        };
        size_t total_writes_s = 0, total_writes_a = 0, total_writes_i = 0;


The file /root/traccar-geocoder/builder/tools/geocoder_patch.cpp has been updated successfully.

        auto write_ids = [](FILE* f, const uint32_t* ids, size_t n, uint32_t& fpos) -> uint32_t {
            if (n == 0) return 0xFFFFFFFF;
            uint32_t off = fpos;
            uint16_t c = (uint16_t)n;
            fwrite(&c, 2, 1, f); fwrite(ids, 4, n, f);
            uint64_t new_fpos = (uint64_t)fpos + 2 + n*4;
            fpos = (uint32_t)new_fpos; // Note: wraps at 4GiB
            return off;
        };
        size_t total_writes_s = 0, total_writes_a = 0, total_writes_i = 0;

// geocoder-patch v4: Low-memory streaming patch application.
//
// Key design: mmap old files (zero copy), stream output via fwrite (never accumulate),
// process entry pipeline cell-by-cell. Target: <1 GiB peak RSS for planet.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <fcntl.h>
#include <iomanip>
#include <iostream>
#include <malloc.h>
#include <string>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unordered_map>
#include <unordered_set>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

// --- Helpers ---
static double now_ms() {
    struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
    return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6;
}
static size_t get_rss_mb() {
    FILE* f = fopen("/proc/self/statm", "r");
    if (!f) return 0;
    size_t dummy, rss; fscanf(f, "%zu %zu", &dummy, &rss); fclose(f);
    return rss * 4096 / (1024*1024);
}
static void log_phase(const char* label, double start) {
    std::cerr << "  [" << std::fixed << std::setprecision(1) << (now_ms() - start) / 1000.0
              << "s, " << get_rss_mb() << " MiB] " << label << std::endl;
}

// mmap a file read-only. Returns {pointer, size}. Caller must munmap.
struct MappedFile { const char* data; size_t size; };
static MappedFile mmap_file(const std::string& path) {
    int fd = open(path.c_str(), O_RDONLY);
    if (fd < 0) return {nullptr, 0};
    struct stat st; fstat(fd, &st);
    size_t sz = st.st_size;
    if (sz == 0) { close(fd); return {nullptr, 0}; }
    void* p = mmap(nullptr, sz, PROT_READ, MAP_PRIVATE, fd, 0);
    close(fd);
    if (p == MAP_FAILED) return {nullptr, 0};
    return {static_cast<const char*>(p), sz};
}
static void unmap_file(MappedFile& f) {
    if (f.data) { munmap(const_cast<char*>(f.data), f.size); f.data = nullptr; f.size = 0; }
}

// Detect stride from file size
static size_t detect_stride(const std::string& path, std::initializer_list<size_t> candidates) {
    struct stat st; if (stat(path.c_str(), &st) != 0) return *candidates.begin();
    for (size_t s : candidates) if (st.st_size % s == 0 && st.st_size > 0) return s;
    return *candidates.begin();
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);
    double t_start = now_ms();

    // --- Phase 1: Decompress + mmap patch ---
    std::string raw_path = tmpdir + "/patch.raw";
    {
        std::string cmd = "zstd -d '" + patch_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0) { std::cerr << "Failed to decompress" << std::endl; return 1; }
    }
    MappedFile patch_map = mmap_file(raw_path);
    if (!patch_map.data) { std::cerr << "Failed to mmap patch" << std::endl; return 1; }
    remove(raw_path.c_str());
    const char* P = patch_map.data;
    size_t patch_size = patch_map.size;
    std::cerr << "Patch: " << patch_size << " bytes" << std::endl;
    log_phase("Decompress", t_start);

    size_t pos = 0;
    auto ru32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, P+pos, 4); pos += 4; return v; };
    auto ru64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, P+pos, 8); pos += 8; return v; };

    // Header
    if (memcmp(P, GCPATCH_MAGIC, 8) != 0) { std::cerr << "Bad magic" << std::endl; return 1; }
    pos = 8;
    uint32_t ver = ru32(); if (ver != 2) { std::cerr << "Bad version" << std::endl; return 1; }
    ru32(); // flags

    // --- Phase 2: String rebuild ---
    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        uint32_t marker = ru32();
        if (marker == 0xFFFFFFF7) {
            uint32_t n_added = ru32(), n_deleted = ru32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ru32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());

            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
            std::vector<std::pair<uint32_t, std::string>> old_strs;
            size_t sp = 0;
            while (sp < old_pool.size) {
                old_strs.push_back({(uint32_t)sp, std::string(old_pool.data + sp)});
                sp += strlen(old_pool.data + sp) + 1;
            }
            std::vector<std::string> merged;
            for (size_t i = 0; i < old_strs.size(); i++)
                if (!del_set.count(i)) merged.push_back(old_strs[i].second);
            for (auto& s : added) merged.push_back(s);
            std::sort(merged.begin(), merged.end());

            // Write new pool and build remap
            FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
            std::unordered_map<std::string, uint32_t> new_offs;
            uint32_t wpos = 0;
            for (auto& s : merged) {
                new_offs[s] = wpos;
                fwrite(s.c_str(), 1, s.size() + 1, fp);
                wpos += s.size() + 1;
            }
            fclose(fp);
            for (auto& [old_off, s] : old_strs) {
                auto it = new_offs.find(s);
                if (it != new_offs.end() && it->second != old_off) str_remap[old_off] = it->second;
            }
            unmap_file(old_pool);
            std::cerr << "  Strings: +" << n_added << " -" << n_deleted << " → " << merged.size()
                      << ", " << str_remap.size() << " remapped" << std::endl;
            marker = ru32();
        }
        if (marker == 0xFFFFFFFE) { uint32_t c = ru32(); for (uint32_t i = 0; i < c; i++) { uint32_t a = ru32(), b = ru32(); str_remap[a] = b; } }
        else pos -= 4;
    }
    log_phase("Strings", t_start);

    // Detect strides
    size_t way_stride = detect_stride(cur_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride(cur_dir + "/interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect_stride(cur_dir + "/admin_polygons.bin", {24, 20, 19});

    // String remap applier (works on mutable copy)
    auto apply_str_remap = [&](std::vector<char>& data, uint32_t fid, size_t stride) {
        if (str_remap.empty()) return;
        std::vector<size_t> offs;
        if (fid == (uint32_t)PatchFileId::ADDR_POINTS) offs = {8, 12};
        else if (fid == (uint32_t)PatchFileId::STREET_WAYS) offs = {(stride == 12) ? 8ul : 5ul};
        else if (fid == (uint32_t)PatchFileId::INTERP_WAYS) offs = {(stride >= 20) ? 8ul : 5ul};
        else if (fid == (uint32_t)PatchFileId::ADMIN_POLYGONS) offs = {8};
        else return;
        for (size_t i = 0; i + stride <= data.size(); i += stride)
            for (size_t o : offs) { uint32_t v; memcpy(&v, data.data()+i+o, 4); auto it = str_remap.find(v); if (it != str_remap.end()) memcpy(data.data()+i+o, &it->second, 4); }
    };

    // --- Phase 3: Merge replays (streaming output) ---
    // ID remaps: file_id → vector<uint32_t> where remap[old_idx] = new_idx
    std::unordered_map<uint32_t, std::vector<uint32_t>> id_remaps;
    std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;
    std::unordered_map<uint64_t, uint8_t> flag_corrections;
    struct CellCorr { uint64_t cell_id; std::vector<uint32_t> ids; };
    std::unordered_map<uint32_t, std::vector<CellCorr>> entry_corrections;

    while (pos < patch_size) {
        uint32_t file_id = ru32();
        if (file_id == 0xFFFFFFFF) break;

        // --- Metadata sections ---
        if (file_id == 0xFFFFFFF7) {
            // String diff in main loop — process it (may be the only occurrence)
            uint32_t n_added = ru32(), n_deleted = ru32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ru32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());
            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
            std::vector<std::pair<uint32_t, std::string>> old_strs;
            size_t sp = 0;
            while (sp < old_pool.size) { old_strs.push_back({(uint32_t)sp, std::string(old_pool.data+sp)}); sp += strlen(old_pool.data+sp)+1; }
            std::vector<std::string> merged;
            for (size_t i = 0; i < old_strs.size(); i++) if (!del_set.count(i)) merged.push_back(old_strs[i].second);
            for (auto& s : added) merged.push_back(s);
            std::sort(merged.begin(), merged.end());
            FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
            std::unordered_map<std::string, uint32_t> new_offs;
            uint32_t wpos = 0;
            for (auto& s : merged) { new_offs[s] = wpos; fwrite(s.c_str(), 1, s.size()+1, fp); wpos += s.size()+1; }
            fclose(fp);
            for (auto& [old_off, s] : old_strs) { auto it = new_offs.find(s); if (it != new_offs.end() && it->second != old_off) str_remap[old_off] = it->second; }
            unmap_file(old_pool);
            std::cerr << "  Strings (loop): +" << n_added << " -" << n_deleted << " → " << merged.size() << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_GEO_MARKER) {
            uint32_t na = ru32(), nr = ru32();
            geo_added.resize(na); geo_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { memcpy(&geo_added[i], P+pos, 8); pos += 8; }
            for (uint32_t i = 0; i < nr; i++) { memcpy(&geo_removed[i], P+pos, 8); pos += 8; }
            std::cerr << "  Geo cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_ADMIN_MARKER) {
            uint32_t na = ru32(), nr = ru32();
            admin_added.resize(na); admin_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { memcpy(&admin_added[i], P+pos, 8); pos += 8; }
            for (uint32_t i = 0; i < nr; i++) { memcpy(&admin_removed[i], P+pos, 8); pos += 8; }
            std::cerr << "  Admin cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_FLAGS_MARKER) {
            uint32_t c = ru32();
            for (uint32_t i = 0; i < c; i++) { uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8; flag_corrections[cid] = P[pos++]; }
            std::cerr << "  Flag corrections: " << c << std::endl;
            continue;
        }
        if (file_id == SECONDARY_REMAP_MARKER) {
            uint32_t nf = ru32();
            for (uint32_t f = 0; f < nf; f++) {
                uint32_t fid = ru32(), np = ru32();
                auto& rm = id_remaps[fid];
                for (uint32_t i = 0; i < np; i++) { uint32_t o = ru32(), n = ru32(); if (o < rm.size()) rm[o] = n; }
                std::cerr << "  Secondary remap " << fid << ": " << np << " pairs" << std::endl;
            }
            continue;
        }
        if (file_id == ENTRY_CORRECTION_MARKER) {
            uint32_t fid = ru32(), c = ru32();
            auto& list = entry_corrections[fid];
            for (uint32_t i = 0; i < c; i++) {
                uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8;
                uint16_t ec; memcpy(&ec, P+pos, 2); pos += 2;
                std::vector<uint32_t> ids(ec);
                if (ec > 0) { memcpy(ids.data(), P+pos, ec*4); pos += ec*4; }
                list.push_back({cid, std::move(ids)});
            }
            std::cerr << "  Entry corrections " << fid << ": " << c << std::endl;
            continue;
        }

        // --- Merge sequence replay (streaming) ---
        uint32_t stride = ru32();
        uint64_t old_size = ru64(), new_size = ru64();
        if (file_id >= (uint32_t)PatchFileId::COUNT) { std::cerr << "Unknown file " << file_id << std::endl; break; }
        const char* fname = patch_file_names[file_id];

        if (stride == 0) {
            // Full replacement — write directly from patch mmap
            uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64();
            FILE* fp = fopen((out_dir + "/" + fname).c_str(), "wb");
            fwrite(P+pos, 1, ds, fp); fclose(fp);
            pos += ds;
            std::cerr << "  " << fname << ": full replace " << ds << " bytes" << std::endl;
            continue;
        }
        if (stride == 0xFE) { uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64(); pos += ds; continue; }

        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        // Load old file into mutable buffer (need to apply remap + fixups)
        // For large files (nodes/vertices), this is the peak memory per file.
        // Files are processed sequentially, so only one is in memory at a time.
        auto old_data = read_file(cur_dir + "/" + std::string(fname));

        // Zero padding
        if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24)
            for (size_t i = 0; i + actual_stride <= old_data.size(); i += actual_stride) memset(old_data.data()+i+13, 0, 3);
        if (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24)
            for (size_t i = 0; i + actual_stride <= old_data.size(); i += actual_stride) { memset(old_data.data()+i+5, 0, 3); memset(old_data.data()+i+21, 0, 3); }

        apply_str_remap(old_data, file_id, actual_stride);

        // Apply fixups
        uint32_t n_fixups = ru32();
        if (n_fixups > 0) {
            uint32_t delta_size = ru32();
            uint32_t pi = 0, pv = 0; size_t fp = pos;
            for (uint32_t i = 0; i < n_fixups; i++) {
                uint32_t idx = pi + read_varint(P, fp);
                uint32_t val = pv + read_varint(P, fp);
                if ((size_t)idx * actual_stride + 4 <= old_data.size())
                    memcpy(old_data.data() + (size_t)idx * actual_stride, &val, 4);
                pi = idx; pv = val;
            }
            pos = fp;
        }

        // Replay merge sequence — stream output directly to file
        uint64_t seq_size = ru64();
        size_t seq_end = pos + seq_size;
        FILE* outf = fopen((out_dir + "/" + fname).c_str(), "wb");

        // Track ID remap (only for data files that need entry reconstruction)
        bool track = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
                      file_id == (uint32_t)PatchFileId::STREET_WAYS ||
                      file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
                      file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
        std::vector<uint32_t> id_map;
        if (track) id_map.assign(old_data.size() / actual_stride, 0xFFFFFFFF);

        size_t old_rec = 0, new_rec = 0, old_bytes = 0, written = 0;
        while (pos < seq_end) {
            uint8_t op = P[pos++];
            uint32_t count; memcpy(&count, P+pos, 4); pos += 4;
            if (op == OP_MATCH_RUN) {
                size_t bytes = count * actual_stride;
                fwrite(old_data.data() + old_bytes, 1, bytes, outf);
                written += bytes;
                if (track) for (uint32_t k = 0; k < count; k++) if (old_rec+k < id_map.size()) id_map[old_rec+k] = new_rec+k;
                old_rec += count; new_rec += count; old_bytes += bytes;
            } else if (op == OP_INSERT_RUN) {
                size_t bytes = count * actual_stride;
                fwrite(P+pos, 1, bytes, outf);
                written += bytes; pos += bytes; new_rec += count;
            } else if (op == OP_DELETE_RUN) {
                old_rec += count; old_bytes += count * actual_stride;
            }
        }
        fclose(outf);

        if (track) {
            id_remaps[file_id] = std::move(id_map);
            size_t matched = 0;
            for (auto v : id_remaps[file_id]) if (v != 0xFFFFFFFF) matched++;
            std::cerr << "  " << fname << ": " << written << " bytes (mapped " << matched << "/" << id_remaps[file_id].size() << ")" << std::endl;
        } else {
            std::cerr << "  " << fname << ": " << written << " bytes" << std::endl;
        }

        // Free old data immediately — don't let glibc hold the pages
        { std::vector<char>().swap(old_data); }
        malloc_trim(0);
    }
    log_phase("Merge replays", t_start);

    // Free string remap (no longer needed)
    { std::unordered_map<uint32_t,uint32_t>().swap(str_remap); }
    malloc_trim(0);

    // --- Phase 4: Streaming entry pipeline ---
    // Walk old geo_cells, remap IDs, apply corrections, write output directly to files.
    {
        double t_entry = now_ms();
        std::cerr << "Entry pipeline..." << std::endl;

        auto get_rm = [&](PatchFileId fid) -> const std::vector<uint32_t>& {
            static std::vector<uint32_t> empty;
            auto it = id_remaps.find((uint32_t)fid); return it != id_remaps.end() ? it->second : empty;
        };
        auto& w_rm = get_rm(PatchFileId::STREET_WAYS);
        auto& a_rm = get_rm(PatchFileId::ADDR_POINTS);
        auto& i_rm = get_rm(PatchFileId::INTERP_WAYS);

        // Build correction lookup maps
        std::unordered_map<uint64_t, const std::vector<uint32_t>*> cs, ca, ci;
        for (auto& [fid, list] : entry_corrections) {
            auto* m = (fid == (uint32_t)PatchFileId::STREET_ENTRIES) ? &cs :
                      (fid == (uint32_t)PatchFileId::ADDR_ENTRIES) ? &ca :
                      (fid == (uint32_t)PatchFileId::INTERP_ENTRIES) ? &ci : nullptr;
            if (m) for (auto& c : list) (*m)[c.cell_id] = &c.ids;
        }

        // mmap old files (zero-copy read)
        MappedFile old_geo = mmap_file(cur_dir + "/geo_cells.bin");
        MappedFile old_se = mmap_file(cur_dir + "/street_entries.bin");
        MappedFile old_ae = mmap_file(cur_dir + "/addr_entries.bin");
        MappedFile old_ie = mmap_file(cur_dir + "/interp_entries.bin");

        // Build derived cell list
        std::unordered_set<uint64_t> rm_set(geo_removed.begin(), geo_removed.end());
        struct CR { uint64_t cid; int32_t idx; };
        std::vector<CR> cells;
        cells.reserve(old_geo.size / 20 + geo_added.size());
        for (size_t i = 0; i < old_geo.size / 20; i++) {
            uint64_t cid; memcpy(&cid, old_geo.data + i*20, 8);
            if (!rm_set.count(cid)) cells.push_back({cid, (int32_t)i});
        }
        for (uint64_t cid : geo_added) cells.push_back({cid, -1});
        std::sort(cells.begin(), cells.end(), [](const CR& a, const CR& b) { return a.cid < b.cid; });
        log_phase("  Cell list", t_entry);

        // Open output files for streaming write
        FILE* f_geo = fopen((out_dir + "/geo_cells.bin").c_str(), "wb");
        FILE* f_se = fopen((out_dir + "/street_entries.bin").c_str(), "wb");
        FILE* f_ae = fopen((out_dir + "/addr_entries.bin").c_str(), "wb");
        FILE* f_ie = fopen((out_dir + "/interp_entries.bin").c_str(), "wb");
        uint32_t se_pos = 0, ae_pos = 0, ie_pos = 0;
        uint32_t no_data = 0xFFFFFFFF;

        // Reusable buffer
        std::vector<uint32_t> id_buf;
        id_buf.reserve(256);

        auto parse_ids = [](const MappedFile& f, uint32_t off, std::vector<uint32_t>& buf) {
            buf.clear();
            if (off == 0xFFFFFFFF || off + 2 > f.size) return;
            uint16_t c; memcpy(&c, f.data+off, 2);
            if (c > 10000 || off+2+(size_t)c*4 > f.size) return; // sanity: no cell has >10K entries
            buf.resize(c); memcpy(buf.data(), f.data+off+2, c*4);
        };
        auto remap_buf = [](std::vector<uint32_t>& buf, const std::vector<uint32_t>& rm) {
            for (auto& id : buf) if (id < rm.size() && rm[id] != 0xFFFFFFFF) id = rm[id];
            std::sort(buf.begin(), buf.end());
        };
        auto write_ids = [](FILE* f, const uint32_t* ids, size_t n, uint32_t& fpos) -> uint32_t {
            if (n == 0) return 0xFFFFFFFF;
            uint32_t off = fpos;
            uint16_t c = (uint16_t)n;
            fwrite(&c, 2, 1, f); fwrite(ids, 4, n, f);
            fpos += 2 + n*4;
            return off;
        };

        size_t sc = 0, ac = 0, ic = 0;
        size_t cell_count = 0;
        for (auto& [cid, oi] : cells) {
            if (++cell_count % 5000000 == 0) {
                std::cerr << "    " << cell_count << "/" << cells.size() << " cells, se=" << se_pos/1024/1024 << "M ae=" << ae_pos/1024/1024 << "M ie=" << ie_pos/1024/1024 << "M rss=" << get_rss_mb() << "M" << std::endl;
            }
            // Process one entry type at a time using shared id_buf
            auto process = [&](const MappedFile& old_e, size_t geo_off,
                               const std::vector<uint32_t>& rm,
                               const std::unordered_map<uint64_t, const std::vector<uint32_t>*>& corr,
                               FILE* outf, uint32_t& fpos, size_t& cc) -> uint32_t {
                auto cit = corr.find(cid);
                if (cit != corr.end()) { cc++; return write_ids(outf, cit->second->data(), cit->second->size(), fpos); }
                if (oi < 0) return no_data;
                uint32_t off; memcpy(&off, old_geo.data + oi*20 + geo_off, 4);
                parse_ids(old_e, off, id_buf);
                if (id_buf.empty()) return no_data;
                remap_buf(id_buf, rm);
                return write_ids(outf, id_buf.data(), id_buf.size(), fpos);
            };

            // Determine flags
            bool hs = false, ha = false, hi = false;
            if (oi >= 0) {
                uint32_t s, a, ip; memcpy(&s, old_geo.data+oi*20+8, 4); memcpy(&a, old_geo.data+oi*20+12, 4); memcpy(&ip, old_geo.data+oi*20+16, 4);
                hs = (s != no_data); ha = (a != no_data); hi = (ip != no_data);
            }
            auto fc = flag_corrections.find(cid);
            if (fc != flag_corrections.end()) { hs = fc->second & 1; ha = fc->second & 2; hi = fc->second & 4; }
            if (cs.count(cid)) hs = true;
            if (ca.count(cid)) ha = true;
            if (ci.count(cid)) hi = true;

            uint32_t so = hs ? process(old_se, 8, w_rm, cs, f_se, se_pos, sc) : no_data;
            uint32_t ao = ha ? process(old_ae, 12, a_rm, ca, f_ae, ae_pos, ac) : no_data;
            uint32_t io = hi ? process(old_ie, 16, i_rm, ci, f_ie, ie_pos, ic) : no_data;
            if (cell_count == 1) {
                // Count how many old cells have each flag
                size_t os = 0, oa = 0, oi_c = 0;
                for (size_t ci = 0; ci < old_geo.size / 20; ci++) {
                    uint32_t sv, av, iv;
                    memcpy(&sv, old_geo.data + ci*20+8, 4); memcpy(&av, old_geo.data + ci*20+12, 4); memcpy(&iv, old_geo.data + ci*20+16, 4);
                    if (sv != no_data) os++; if (av != no_data) oa++; if (iv != no_data) oi_c++;
                }
                std::cerr << "    DEBUG: old cells=" << old_geo.size/20 << " has_street=" << os << " has_addr=" << oa << " has_interp=" << oi_c << std::endl;
            }

            fwrite(&cid, 8, 1, f_geo); fwrite(&so, 4, 1, f_geo); fwrite(&ao, 4, 1, f_geo); fwrite(&io, 4, 1, f_geo);
        }
        fclose(f_geo); fclose(f_se); fclose(f_ae); fclose(f_ie);
        unmap_file(old_geo); unmap_file(old_se); unmap_file(old_ae); unmap_file(old_ie);
        std::cerr << "  Geo: " << cells.size() << " cells, corrections: " << sc << " st, " << ac << " ad, " << ic << " in" << std::endl;
        log_phase("  Geo entries", t_entry);

        // Admin: small, use existing rebuild + corrections
        {
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            auto ait = id_remaps.find((uint32_t)PatchFileId::ADMIN_POLYGONS);
            if (ait != id_remaps.end()) {
                for (uint32_t i = 0; i < ait->second.size(); i++)
                    if (ait->second[i] != 0xFFFFFFFF) ad_rm[i] = ait->second[i];
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_adme = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_adme, ad_rm, admin_added, admin_removed);

            auto ecit = entry_corrections.find((uint32_t)PatchFileId::ADMIN_ENTRIES);
            if (ecit != entry_corrections.end()) {
                std::unordered_map<uint64_t, const std::vector<uint32_t>*> ac_corr;
                for (auto& c : ecit->second) ac_corr[c.cell_id] = &c.ids;
                size_t n = admin.admin_cells_data.size() / 12;
                FILE* fac = fopen((out_dir + "/admin_cells.bin").c_str(), "wb");
                FILE* fae = fopen((out_dir + "/admin_entries.bin").c_str(), "wb");
                uint32_t ae_wpos = 0;
                for (size_t i = 0; i < n; i++) {
                    uint64_t cid; memcpy(&cid, admin.admin_cells_data.data()+i*12, 8);
                    fwrite(&cid, 8, 1, fac);
                    auto cit = ac_corr.find(cid);
                    if (cit != ac_corr.end()) {
                        uint32_t off = cit->second->empty() ? no_data : ae_wpos;
                        fwrite(&off, 4, 1, fac);
                        if (!cit->second->empty()) {
                            uint16_t c = cit->second->size();
                            fwrite(&c, 2, 1, fae); fwrite(cit->second->data(), 4, c, fae);
                            ae_wpos += 2 + c*4;
                        }
                    } else {
                        uint32_t old_off; memcpy(&old_off, admin.admin_cells_data.data()+i*12+8, 4);
                        if (old_off != no_data && old_off+2 <= admin.admin_entries_data.size()) {
                            uint32_t new_off = ae_wpos; fwrite(&new_off, 4, 1, fac);
                            uint16_t c; memcpy(&c, admin.admin_entries_data.data()+old_off, 2);
                            fwrite(admin.admin_entries_data.data()+old_off, 1, 2+c*4, fae);
                            ae_wpos += 2+c*4;
                        } else {
                            fwrite(&no_data, 4, 1, fac);
                        }
                    }
                }
                fclose(fac); fclose(fae);
                std::cerr << "  Admin: " << n << " cells, " << ac_corr.size() << " corrections" << std::endl;
            } else {
                write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
                write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            }
            log_phase("  Admin", t_entry);
        }
    }

    // Cleanup
    unmap_file(patch_map);
    { std::string cmd = "rm -rf '" + tmpdir + "'"; system(cmd.c_str()); }
    log_phase("Total", t_start);
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

+            uint64_t new_fpos = (uint64_t)fpos + 2 + n*4;

+            fpos = (uint32_t)new_fpos; // Note: wraps at 4GiB

+        size_t total_writes_s = 0, total_writes_a = 0, total_writes_i = 0;


Actually, the real problem might be simpler: the `has_a` check reads from old_geo USING `oi` as the index. But `oi` comes from the cell list which has been SORTED. After sorting, `oi` still refers to the original position in old_geo. But the remap lookup also uses `oi`. So `parse_ids(old_ae, off, id_buf)` reads `off` from `old_geo.data + oi*20 + 12`. If `off` is not `0xFFFFFFFF`, it reads from old_ae at that offset.

This should work... unless `old_ae` (mmap'd) is referencing the wrong file. Let me check the mmap order:


No matches found


48:static MappedFile mmap_file(const std::string& path) {
87:    MappedFile patch_map = mmap_file(raw_path);
117:            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
193:            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
383:        MappedFile old_geo = mmap_file(cur_dir + "/geo_cells.bin");
384:        MappedFile old_se = mmap_file(cur_dir + "/street_entries.bin");
385:        MappedFile old_ae = mmap_file(cur_dir + "/addr_entries.bin");
386:        MappedFile old_ie = mmap_file(cur_dir + "/interp_entries.bin");


The mmap'd files at lines 383-386 are correct. Let me check — maybe the issue is that `process_entry` calls `parse_ids(old_ae, ...)` which reads from the addr entries file, but the old addr file has been MODIFIED during the merge replay! The merge replay wrote the NEW addr_points.bin to `out_dir + "/addr_points.bin"`, but the OLD addr entries at `cur_dir + "/addr_entries.bin"` should be unchanged.

Wait — the merge replay doesn't touch entry files. It only replays data files (addr_points, street_ways, etc.). The entry files (addr_entries, street_entries) are NEVER part of the merge — they're reconstructed. So the old entry files at `cur_dir/` are untouched. The mmap should be fine.

Let me look for a different bug. The `id_buf` is shared — but it's used correctly (cleared before each use). Let me check if the `parse_ids` function has a bug with the `off` value reading. For planet, the entry files are 700 MiB - 2.1 GiB. The `off` field in geo_cells is `uint32_t`, so max 4 GiB. The street entries at 2.1 GiB is fine.

Wait — I bet I found it! The `has_a` check: `if (corr_ae.count(cid)) has_a = true;`. This sets `has_a` to true for ANY cell with an addr correction — even cells that SHOULDN'T have addr entries. The correction might be replacing an existing entry, but for new cells (added) that have no old addr data, the `process_entry` function calls `parse_ids(old_ae, off, id_buf)` where `off` comes from the OLD geo_cells. But for a cell with `oi >= 0`, the off is valid. For `oi < 0` (added cell), it returns `no_data`.

Actually, re-reading the process_entry lambda:
```cpp
auto cit = corr.find(cid);
if (cit != corr.end()) { cc++; return write_ids(outf, cit->second->data(), cit->second->size(), fpos); }
if (oi < 0) return no_data;
```

If there's a correction, it writes the correction data. If no correction and cell is added, returns no_data. If no correction and cell exists in old, it reads old entries and remaps.

The issue is subtle: the `has_a` is set to true by the correction check, causing `process_entry` to be called. The correction data is the CORRECT new entry IDs. But `cit->second->size()` could be 0 (empty correction = no entries for this cell), in which case `write_ids` returns `0xFFFFFFFF`. But the correction check already handles `has_a = true` for empty corrections...

Actually wait — I'm looking at the wrong thing. Let me think about what's different between Europe and planet. Europe has 83M cells and WORKED. Planet has 322M cells. The addr_entries grew to 62 GiB for planet but was correct for Europe.

The only difference that matters at this scale is numeric: `fpos` is `uint32_t`. For street_entries at 2.1 GiB on planet, `fpos` reaches ~2.1 GiB, which fits in uint32_t. But the addr_entries are only 700 MiB, also fine. The geo_cells at 6.1 GiB, however, would overflow... but `geo_cells` offsets are written directly (not via `write_ids`).

Hmm, let me look at the geo_cells output: `out_geo` uses `fwrite` directly in the loop for each cell's 20 bytes. That's 322M × 20 = 6.4 GiB written directly — no `fpos` tracking needed.

Let me run on a small subset to find the exact bug. Actually, let me just look at what the output sizes ARE after the progress counters:

From the debug: at 45M cells (14% through):
- se=329M → projected 2.35 GiB (ref: 2.1 GiB — close)
- ae=79M → projected 564 MiB (ref: 706 MiB — close)
- ie=0M → projected 0 (ref: 1.4 MiB — tiny)

These are CORRECT! The growth rates match reference. The 62 GiB I saw was from the OLD stale process (which I thought I killed but didn't). This run should produce correct output!

Let me verify by re-running and monitoring the file sizes from THIS process only:


Bash

sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "ps aux | grep geocoder-patch | grep -v grep | awk '{print \$2, \$6/1024/1024, \$10}'" 2>&1

Check running processes


No processes running — the most recent one also got killed (probably when I killed the old one, I also killed the new one). Let me restart cleanly with NO stale processes:


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "killall -9 geocoder-patch 2>/dev/null; rm -rf /home/michtest/patched-planet-stream/ && df -h / && free -h | head -2" 2>&1

Clean start


Filesystem      Size  Used Avail Use% Mounted on
/dev/vda1       934G  276G  611G  32% /
               total        used        free      shared  buff/cache   available
Mem:           174Gi       1.8Gi       155Gi        92Ki        19Gi       173Gi


611 GiB free, 173 GiB available memory. Clean state. Build and deploy fresh:


make -j$(nproc) geocoder-patch 2>&1 | tail -3 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "rm -f /home/michtest/geocoder-patch" 2>&1 && sshpass -p 'elon-is-insane' scp -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:/home/michtest/ 2>&1 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 'cd /home/michtest && rm -rf patched-planet-stream/ && nohup bash -c "time ./geocoder-patch planet-B/ validated-patches/planet-mar16-to-mar23-v2.gcpatch -o patched-planet-stream/ 2>&1 && echo ---VERIFY--- && for f in planet-C/*; do b=\$(basename \$f); cmp -s patched-planet-stream/\$b planet-C/\$b || echo \"MISMATCH: \$b\"; done && echo All_checked" > planet-patch-final.log 2>&1 &' 2>&1 && echo "Started"

Clean deploy and run


Command running in background with ID: bfu4mz9ap. Output is being written to: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bfu4mz9ap.output


sleep 60 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "grep '\\[.*MiB\\]' /home/michtest/planet-patch-final.log && echo '---files---' && ls -lh /home/michtest/patched-planet-stream/street_entries.bin /home/michtest/patched-planet-stream/addr_entries.bin /home/michtest/patched-planet-stream/geo_cells.bin 2>/dev/null && echo '---procs---' && ps aux | grep 'geocoder-patch' | grep -v grep | wc -l" 2>&1

Check 1 min


  [0.3s, 3 MiB] Decompress
  [0.3s, 3 MiB] Strings
  [30.1s, 1816 MiB] Merge replays
  [14.3s, 12469 MiB]   Cell list
---files---
-rw-rw-r-- 1 michtest michtest 9.6G Mar 30 05:24 /home/michtest/patched-planet-stream/addr_entries.bin
-rw-rw-r-- 1 michtest michtest 2.2G Mar 30 05:24 /home/michtest/patched-planet-stream/geo_cells.bin
-rw-rw-r-- 1 michtest michtest 6.7G Mar 30 05:24 /home/michtest/patched-planet-stream/street_entries.bin
---procs---
3


9.6 GiB addr_entries at 1 min — this IS the new process (only 3 processes, clean start). The entry files are too large. The growth rate from the debug output was normal (39M per 5M cells for streets) but the ACTUAL files on disk are 6.7 GiB for streets. Let me do the math:

- 1 min into entry walk ≈ ~15-20M cells processed (at Europe rate)
- street_entries at 6.7 GiB / 20M cells = 335 bytes per cell

But the debug showed 7.8 bytes/cell for streets. Something doesn't add up. Unless the file is growing MUCH faster than the progress counter reports.

Wait — let me check if there's an old process still writing from the previous run:


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "fuser /home/michtest/patched-planet-stream/street_entries.bin 2>/dev/null && echo '---all fds---' && ls -l /proc/\$(ps aux | grep 'geocoder-patch planet' | grep -v grep | head -1 | awk '{print \$2}')/fd/ 2>/dev/null | grep patched" 2>&1

Check who's writing to the file


Exit code 127


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "ls -l /proc/\$(pgrep -f 'geocoder-patch planet' | tail -1)/fd/ 2>/dev/null | grep -E 'entries|geo_cells'" 2>&1

Check file descriptors


Exit code 1


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "ps aux | grep geocoder-patch | grep -v grep" 2>&1

List all processes


michtest  174367  0.0  0.0   7076  1924 ?        S    05:21   0:00 bash -c cd /home/michtest && rm -rf patched-planet-stream/ && nohup bash -c "time ./geocoder-patch planet-B/ validated-patches/planet-mar16-to-mar23-v2.gcpatch -o patched-planet-stream/ 2>&1 && echo ---VERIFY--- && for f in planet-C/*; do b=\$(basename \$f); cmp -s patched-planet-stream/\$b planet-C/\$b || echo \"MISMATCH: \$b\"; done && echo All_checked" > planet-patch-final.log 2>&1 &
michtest  174369  0.0  0.0   7076  3496 ?        S    05:21   0:00 bash -c time ./geocoder-patch planet-B/ validated-patches/planet-mar16-to-mar23-v2.gcpatch -o patched-planet-stream/ 2>&1 && echo ---VERIFY--- && for f in planet-C/*; do b=$(basename $f); cmp -s patched-planet-stream/$b planet-C/$b || echo "MISMATCH: $b"; done && echo All_checked
michtest  174370 99.7  7.5 17079524 13769212 ?   R    05:21   3:30 ./geocoder-patch planet-B/ validated-patches/planet-mar16-to-mar23-v2.gcpatch -o patched-planet-stream/


Only 1 geocoder-patch process (PID 174370), 13.5 GiB RSS. Only 3:30 CPU time. The file sizes are growing from THIS process.

So the bug is real: 6.7 GiB street_entries in 3.5 min is too fast. Let me check if the `fwrite` is writing more than expected. Maybe the `write_ids` `fpos` tracking is wrong, but the actual fwrite writes are correct...

Actually wait — I just realized the bug. The `parse_ids` reads from the mmap'd old_ae file. But the old entry file at `cur_dir/addr_entries.bin` might have been OVERWRITTEN by the merge replay! The merge replay writes to `out_dir/addr_entries.bin`, but... no, the entry files aren't part of the merge. Only data files (addr_points, street_ways, etc.) are merged.

Unless... the `cur_dir` and `out_dir` are the SAME directory! If the user runs `geocoder-patch planet-B/ patch.gcpatch -o planet-B/`, the output overwrites the input. But in my test, `cur_dir` is `planet-B/` and `out_dir` is `patched-planet-stream/`. Different.

OK, I need to actually look at what's being written. Let me add a check for the first cell's addr entry:

Actually, let me just compare the first few bytes of the output against the reference:


sleep 60 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "xxd /home/michtest/patched-planet-stream/addr_entries.bin 2>/dev/null | head -5 && echo --- && xxd /home/michtest/planet-C/addr_entries.bin 2>/dev/null | head -5 && echo ---size--- && ls -l /home/michtest/patched-planet-stream/addr_entries.bin 2>/dev/null | awk '{print \$5}'" 2>&1

Compare first bytes of addr_entries


---
---size---
20728426496


xxd output is empty — file is too large or still being written. 20.7 GiB after 2 more min. Let me kill and investigate with a small test. Let me run on the Europe dataset but print the first few writes:


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "kill -9 \$(ps aux | grep 'geocoder-patch' | grep -v grep | awk '{print \$2}') 2>/dev/null; rm -rf /home/michtest/patched-planet-stream/" 2>&1

Kill and clean


(Bash completed with no output)


The Europe test matched correctly. The planet test has the same code. The difference is scale: 322M cells vs 83M. Let me add a diagnostic that logs the actual bytes written vs expected:


Edit

            if (++cell_count % 5000000 == 0) {
                // Also check actual file sizes vs tracked positions
                long se_actual = ftell(f_se), ae_actual = ftell(f_ae), ie_actual = ftell(f_ie);
                std::cerr << "    " << cell_count << "/" << cells.size()
                          << " se_pos=" << se_pos << " se_ftell=" << se_actual
                          << " ae_pos=" << ae_pos << " ae_ftell=" << ae_actual
                          << " rss=" << get_rss_mb() << "M" << std::endl;
            }