The file /root/traccar-geocoder/builder/tools/geocoder_patch.cpp has been updated successfully.

// geocoder-patch v4: Low-memory streaming patch application.
//
// Key design: mmap old files (zero copy), stream output via fwrite (never accumulate),
// process entry pipeline cell-by-cell. Target: <1 GiB peak RSS for planet.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <fcntl.h>
#include <iomanip>
#include <iostream>
#include <malloc.h>
#include <string>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unordered_map>
#include <unordered_set>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

// Streaming zstd decompressor — reads compressed file, provides sequential access
// to decompressed data via a sliding buffer. No temp file needed.
#include <zstd.h>
class PatchStream {
    FILE* input_;
    ZSTD_DCtx* dctx_;
    std::vector<char> in_buf_, out_buf_;
    size_t out_start_ = 0, out_end_ = 0; // valid range in out_buf_
    size_t global_pos_ = 0; // position of out_buf_[0] in decompressed stream
    size_t total_decompressed_ = 0;
    bool input_done_ = false;
    static constexpr size_t IN_BUF_SIZE = 256 * 1024;
    static constexpr size_t OUT_BUF_SIZE = 4 * 1024 * 1024;
    ZSTD_inBuffer zin_{};

    void decompress_more() {
        // Compact: move unread data to front
        if (out_start_ > 0) {
            size_t remain = out_end_ - out_start_;
            if (remain > 0) memmove(out_buf_.data(), out_buf_.data() + out_start_, remain);
            global_pos_ += out_start_;
            out_end_ = remain;
            out_start_ = 0;
        }
        // Ensure output buffer has room
        if (out_buf_.size() - out_end_ < OUT_BUF_SIZE)
            out_buf_.resize(out_end_ + OUT_BUF_SIZE);

        while (!input_done_ && out_end_ - out_start_ < OUT_BUF_SIZE) {
            if (zin_.pos >= zin_.size) {
                size_t n = fread(in_buf_.data(), 1, IN_BUF_SIZE, input_);
                if (n == 0) { input_done_ = true; break; }
                zin_ = {in_buf_.data(), n, 0};
            }
            ZSTD_outBuffer zout = {out_buf_.data() + out_end_, out_buf_.size() - out_end_, 0};
            size_t ret = ZSTD_decompressStream(dctx_, &zout, &zin_);
            if (ZSTD_isError(ret)) { std::cerr << "zstd error: " << ZSTD_getErrorName(ret) << std::endl; break; }
            out_end_ += zout.pos;
            total_decompressed_ = global_pos_ + out_end_;
        }
    }

public:
    bool open(const std::string& path) {
        input_ = fopen(path.c_str(), "rb");
        if (!input_) return false;
        dctx_ = ZSTD_createDCtx();
        in_buf_.resize(IN_BUF_SIZE);
        out_buf_.resize(OUT_BUF_SIZE * 2);
        decompress_more();
        return true;
    }
    ~PatchStream() { if (dctx_) ZSTD_freeDCtx(dctx_); if (input_) fclose(input_); }

    // Ensure at least n bytes are available at current read position
    void ensure(size_t n) {
        while (out_end_ - out_start_ < n && !input_done_) decompress_more();
    }

    // Read n bytes at current position into dst, advance
    void read(void* dst, size_t n) {
        ensure(n);
        memcpy(dst, out_buf_.data() + out_start_, n);
        out_start_ += n;
    }
    uint32_t read_u32() { uint32_t v; read(&v, 4); return v; }
    uint64_t read_u64() { uint64_t v; read(&v, 8); return v; }
    uint8_t read_u8() { ensure(1); return (uint8_t)out_buf_[out_start_++]; }

    // Get pointer to n bytes at current position (valid until next ensure/read)
    const char* peek(size_t n) { ensure(n); return out_buf_.data() + out_start_; }
    void skip(size_t n) { ensure(n); out_start_ += n; }

    // Write n bytes from current position directly to a FILE*, advance
    void write_to(FILE* f, size_t n) {
        while (n > 0) {
            ensure(std::min(n, (size_t)OUT_BUF_SIZE));
            size_t avail = out_end_ - out_start_;
            size_t chunk = std::min(n, avail);
            fwrite(out_buf_.data() + out_start_, 1, chunk, f);
            out_start_ += chunk;
            n -= chunk;
        }
    }

    // Read a null-terminated string at current position
    std::string read_string() {
        std::string s;
        while (true) {
            ensure(1);
            char c = out_buf_[out_start_++];
            if (c == '\0') break;
            s += c;
        }
        return s;
    }

    size_t position() const { return global_pos_ + out_start_; }
    size_t decompressed_size() const { return total_decompressed_; }
};

// --- Helpers ---
static double now_ms() {
    struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
    return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6;
}
static size_t get_rss_mb() {
    FILE* f = fopen("/proc/self/statm", "r");
    if (!f) return 0;
    size_t dummy, rss; fscanf(f, "%zu %zu", &dummy, &rss); fclose(f);
    return rss * 4096 / (1024*1024);
}
static void log_phase(const char* label, double start) {
    std::cerr << "  [" << std::fixed << std::setprecision(1) << (now_ms() - start) / 1000.0
              << "s, " << get_rss_mb() << " MiB] " << label << std::endl;
}

// mmap a file read-only. Returns {pointer, size}. Caller must munmap.
struct MappedFile { const char* data; size_t size; };
static MappedFile mmap_file(const std::string& path) {
    int fd = open(path.c_str(), O_RDONLY);
    if (fd < 0) return {nullptr, 0};
    struct stat st; fstat(fd, &st);
    size_t sz = st.st_size;
    if (sz == 0) { close(fd); return {nullptr, 0}; }
    void* p = mmap(nullptr, sz, PROT_READ, MAP_PRIVATE, fd, 0);
    close(fd);
    if (p == MAP_FAILED) return {nullptr, 0};
    return {static_cast<const char*>(p), sz};
}
static void unmap_file(MappedFile& f) {
    if (f.data) { munmap(const_cast<char*>(f.data), f.size); f.data = nullptr; f.size = 0; }
}

// Detect stride from file size
static size_t detect_stride(const std::string& path, std::initializer_list<size_t> candidates) {
    struct stat st; if (stat(path.c_str(), &st) != 0) return *candidates.begin();
    for (size_t s : candidates) if (st.st_size % s == 0 && st.st_size > 0) return s;
    return *candidates.begin();
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);
    double t_start = now_ms();

    // --- Phase 1: Streaming zstd decompression (no temp file) ---
    PatchStream ps;
    if (!ps.open(patch_path)) { std::cerr << "Failed to open patch" << std::endl; return 1; }
    std::cerr << "Streaming patch decompression..." << std::endl;
    log_phase("Open", t_start);

    // Header
    { char magic[8]; ps.read(magic, 8);
      if (memcmp(magic, GCPATCH_MAGIC, 8) != 0) { std::cerr << "Bad magic" << std::endl; return 1; } }
    uint32_t ver = ps.read_u32(); if (ver != 2) { std::cerr << "Bad version" << std::endl; return 1; }
    ps.read_u32(); // flags

    // --- Phase 2: String rebuild ---
    // Sorted vector remap: (old_offset, new_offset) pairs sorted by old_offset.
    // Uses 96 MiB for 12M entries vs 672 MiB for unordered_map.
    std::vector<std::pair<uint32_t, uint32_t>> str_remap_vec;
    {
        uint32_t marker = ps.read_u32();
        if (marker == 0xFFFFFFF7) {
            uint32_t n_added = ps.read_u32(), n_deleted = ps.read_u32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ps.read_u32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());

            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");

            // Phase A: merge old (minus deleted) + added strings, write directly to file.
            // Both old pool and added are sorted alphabetically → merge-write with no sort.
            // Memory: only the added strings vector (~small) + del_set (~small).
            {
                FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
                size_t sp = 0; uint32_t idx = 0; size_t ai = 0;
                // Sort added strings (they should already be sorted from diff tool, but ensure)
                std::sort(added.begin(), added.end());
                while (sp < old_pool.size || ai < added.size()) {
                    const char* old_s = nullptr;
                    // Skip deleted strings
                    while (sp < old_pool.size) {
                        if (!del_set.count(idx)) { old_s = old_pool.data + sp; break; }
                        sp += strlen(old_pool.data + sp) + 1; idx++;
                    }
                    const char* add_s = ai < added.size() ? added[ai].c_str() : nullptr;
                    // Merge: pick the smaller string
                    if (old_s && add_s) {
                        int c = strcmp(old_s, add_s);
                        if (c <= 0) {
                            size_t l = strlen(old_s) + 1; fwrite(old_s, 1, l, fp);
                            sp += l; idx++;
                            if (c == 0) ai++; // duplicate — skip added
                        } else {
                            size_t l = added[ai].size() + 1; fwrite(add_s, 1, l, fp);
                            ai++;
                        }
                    } else if (old_s) {
                        size_t l = strlen(old_s) + 1; fwrite(old_s, 1, l, fp);
                        sp += l; idx++;
                    } else if (add_s) {
                        size_t l = added[ai].size() + 1; fwrite(add_s, 1, l, fp);
                        ai++;
                    } else break;
                }
                fclose(fp);
            }
            { std::vector<std::string>().swap(added); std::unordered_set<uint32_t>().swap(del_set);
              std::vector<uint32_t>().swap(del_idx); }
            malloc_trim(0);

            // Phase B: build remap by merge-walking old pool vs new pool (both sorted, both mmap'd)
            MappedFile new_pool = mmap_file(out_dir + "/strings.bin");
            {
                // Walk both pools simultaneously — no vectors needed, just cursors
                size_t old_pos = 0, new_pos = 0;
                while (old_pos < old_pool.size && new_pos < new_pool.size) {
                    const char* os = old_pool.data + old_pos;
                    const char* ns = new_pool.data + new_pos;
                    int c = strcmp(os, ns);
                    if (c == 0) {
                        if (old_pos != new_pos) // offset changed
                            str_remap_vec.push_back({(uint32_t)old_pos, (uint32_t)new_pos});
                        old_pos += strlen(os) + 1;
                        new_pos += strlen(ns) + 1;
                    } else if (c < 0) {
                        old_pos += strlen(os) + 1; // deleted string
                    } else {
                        new_pos += strlen(ns) + 1; // added string
                    }
                }
            }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
            unmap_file(old_pool);
            unmap_file(new_pool);
            std::cerr << "  Strings: +" << n_added << " -" << n_deleted
                      << ", " << str_remap_vec.size() << " remapped (" << str_remap_vec.size() * 8 / 1024 / 1024 << " MiB)" << std::endl;
            marker = ps.read_u32();
        }
        if (marker == 0xFFFFFFFE) {
            uint32_t c = ps.read_u32();
            for (uint32_t i = 0; i < c; i++) { uint32_t a = ps.read_u32(), b = ps.read_u32(); str_remap_vec.push_back({a, b}); }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
        } else pos -= 4;
    }
    // Release patch pages read so far (string section)
    madvise(const_cast<char*>(patch_map.data), pos, MADV_DONTNEED);
    log_phase("Strings", t_start);

    // Detect strides
    size_t way_stride = detect_stride(cur_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride(cur_dir + "/interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect_stride(cur_dir + "/admin_polygons.bin", {24, 20, 19});

    // String remap lookup via sorted vector + binary search
    auto str_remap_lookup = [&](uint32_t old_off) -> uint32_t {
        auto it = std::lower_bound(str_remap_vec.begin(), str_remap_vec.end(),
            std::make_pair(old_off, (uint32_t)0));
        if (it != str_remap_vec.end() && it->first == old_off) return it->second;
        return old_off; // no remap
    };

    // --- Phase 3: Scan sections + parallel merge replays ---
    // Collect merge section info for parallel replay, process metadata inline
    struct MergeSection {
        uint32_t file_id, stride; size_t actual_stride;
        uint64_t old_size, new_size;
        uint32_t n_fixups; size_t fixup_data_pos;
        size_t seq_pos; uint64_t seq_size;
        bool needs_remap, needs_padding;
    };
    std::vector<MergeSection> merge_sections;
    std::unordered_map<uint32_t, std::vector<uint32_t>> id_remaps;
    std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;
    std::unordered_map<uint64_t, uint8_t> flag_corrections;
    struct CellCorr { uint64_t cell_id; std::vector<uint32_t> ids; };
    std::unordered_map<uint32_t, std::vector<CellCorr>> entry_corrections;

    while (pos < patch_size) {
        uint32_t file_id = ps.read_u32();
        if (file_id == 0xFFFFFFFF) break;

        // --- Metadata sections ---
        if (file_id == 0xFFFFFFF7) {
            // String diff in main loop — same memory-efficient approach as Phase 2
            uint32_t n_added = ps.read_u32(), n_deleted = ps.read_u32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ps.read_u32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());
            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
            struct OS { uint32_t idx; uint32_t off; const char* str; };
            std::vector<OS> old_strs;
            { size_t sp = 0; uint32_t ix = 0;
              while (sp < old_pool.size) { old_strs.push_back({ix++, (uint32_t)sp, old_pool.data+sp}); sp += strlen(old_pool.data+sp)+1; }
            }
            std::vector<const char*> merged;
            for (auto& os : old_strs) if (!del_set.count(os.idx)) merged.push_back(os.str);
            for (auto& s : added) merged.push_back(s.c_str());
            std::sort(merged.begin(), merged.end(), [](const char* a, const char* b) { return strcmp(a,b) < 0; });
            FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
            uint32_t wpos = 0;
            for (const char* s : merged) { size_t l = strlen(s)+1; fwrite(s, 1, l, fp); wpos += l; }
            fclose(fp);
            // Build remap via merge-walk
            MappedFile new_pool = mmap_file(out_dir + "/strings.bin");
            std::vector<std::pair<const char*, uint32_t>> new_strs;
            { size_t sp = 0;
              while (sp < new_pool.size) { new_strs.push_back({new_pool.data+sp, (uint32_t)sp}); sp += strlen(new_pool.data+sp)+1; }
            }
            size_t oi2 = 0, ni2 = 0;
            while (oi2 < old_strs.size() && ni2 < new_strs.size()) {
                int c = strcmp(old_strs[oi2].str, new_strs[ni2].first);
                if (c == 0) {
                    if (old_strs[oi2].off != new_strs[ni2].second)
                        str_remap_vec.push_back({old_strs[oi2].off, new_strs[ni2].second});
                    oi2++; ni2++;
                } else if (c < 0) { oi2++; } else { ni2++; }
            }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
            unmap_file(old_pool); unmap_file(new_pool);
            std::cerr << "  Strings (loop): +" << n_added << " -" << n_deleted << " → " << merged.size()
                      << ", " << str_remap_vec.size() << " remapped" << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_GEO_MARKER) {
            uint32_t na = ps.read_u32(), nr = ps.read_u32();
            geo_added.resize(na); geo_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { ps.read(&geo_added[i], 8); }
            for (uint32_t i = 0; i < nr; i++) { ps.read(&geo_removed[i], 8); }
            std::cerr << "  Geo cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_ADMIN_MARKER) {
            uint32_t na = ps.read_u32(), nr = ps.read_u32();
            admin_added.resize(na); admin_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { ps.read(&admin_added[i], 8); }
            for (uint32_t i = 0; i < nr; i++) { ps.read(&admin_removed[i], 8); }
            std::cerr << "  Admin cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_FLAGS_MARKER) {
            uint32_t c = ps.read_u32();
            for (uint32_t i = 0; i < c; i++) { uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8; flag_corrections[cid] = P[pos++]; }
            std::cerr << "  Flag corrections: " << c << std::endl;
            continue;
        }
        if (file_id == SECONDARY_REMAP_MARKER) {
            // Apply secondary remaps directly to the temp remap files on disk
            uint32_t nf = ps.read_u32();
            for (uint32_t f = 0; f < nf; f++) {
                uint32_t fid = ps.read_u32(), np = ps.read_u32();
                std::string remap_path = tmpdir + "/remap_" + std::to_string(fid) + ".bin";
                // Open remap file for read-write
                int rfd = open(remap_path.c_str(), O_RDWR);
                if (rfd >= 0) {
                    struct stat st; fstat(rfd, &st);
                    size_t remap_size = st.st_size;
                    uint32_t* remap_data = static_cast<uint32_t*>(
                        mmap(nullptr, remap_size, PROT_READ | PROT_WRITE, MAP_SHARED, rfd, 0));
                    size_t remap_count = remap_size / 4;
                    for (uint32_t i = 0; i < np; i++) {
                        uint32_t o = ps.read_u32(), n = ps.read_u32();
                        if (o < remap_count) remap_data[o] = n + 1; // +1 encoding
                    }
                    munmap(remap_data, remap_size);
                    close(rfd);
                } else {
                    // Skip data if file doesn't exist
                    for (uint32_t i = 0; i < np; i++) { ps.read_u32(); ps.read_u32(); }
                }
                std::cerr << "  Secondary remap " << fid << ": " << np << " pairs" << std::endl;
            }
            continue;
        }
        if (file_id == ENTRY_CORRECTION_MARKER) {
            uint32_t fid = ps.read_u32(), c = ps.read_u32();
            auto& list = entry_corrections[fid];
            for (uint32_t i = 0; i < c; i++) {
                uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8;
                uint16_t ec; memcpy(&ec, P+pos, 2); pos += 2;
                std::vector<uint32_t> ids(ec);
                if (ec > 0) { memcpy(ids.data(), P+pos, ec*4); pos += ec*4; }
                list.push_back({cid, std::move(ids)});
            }
            std::cerr << "  Entry corrections " << fid << ": " << c << std::endl;
            continue;
        }

        // --- Collect merge section info (replay happens in parallel below) ---
        uint32_t stride = ps.read_u32();
        uint64_t old_size = ps.read_u64(), new_size = ps.read_u64();
        if (file_id >= (uint32_t)PatchFileId::COUNT) { std::cerr << "Unknown file " << file_id << std::endl; break; }
        const char* fname = patch_file_names[file_id];

        if (stride == 0) {
            uint32_t nf = ps.read_u32(); (void)nf; uint64_t ds = ps.read_u64();
            FILE* fp = fopen((out_dir + "/" + fname).c_str(), "wb");
            fwrite(P+pos, 1, ds, fp); fclose(fp);
            pos += ds;
            std::cerr << "  " << fname << ": full replace " << ds << " bytes" << std::endl;
            continue;
        }
        if (stride == 0xFE) { uint32_t nf = ps.read_u32(); (void)nf; uint64_t ds = ps.read_u64(); pos += ds; continue; }

        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        bool needs_remap = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
                            file_id == (uint32_t)PatchFileId::STREET_WAYS ||
                            file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
                            file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
        bool needs_padding = (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24) ||
                             (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24);

        uint32_t n_fixups = ps.read_u32();
        size_t fixup_data_pos = pos;
        if (n_fixups > 0) {
            uint32_t delta_size = ps.read_u32();
            fixup_data_pos = pos;
            pos += delta_size;
        }
        // Lazy fixup decoder state (reads from patch mmap on demand)
        struct FixupDecoder {
            const char* data; size_t read_pos;
            uint32_t prev_idx = 0, prev_val = 0;
            uint32_t cur_idx = UINT32_MAX, cur_val = 0;
            uint32_t remaining;
            void init(const char* d, size_t p, uint32_t count) {
                data = d; read_pos = p; remaining = count;
                if (remaining > 0) advance();
            }
            void advance() {
                if (remaining == 0) { cur_idx = UINT32_MAX; return; }
                cur_idx = prev_idx + read_varint(data, read_pos);
                cur_val = prev_val + read_varint(data, read_pos);
                prev_idx = cur_idx; prev_val = cur_val;
                remaining--;
            }
            // Advance past all fixups with index < target
            bool lookup(uint32_t target_idx, uint32_t& out_val) {
                while (cur_idx < target_idx && (remaining > 0 || cur_idx != UINT32_MAX)) advance();
                if (cur_idx == target_idx) { out_val = cur_val; advance(); return true; }
                return false;
            }
        } fixup_dec;
        fixup_dec.init(P, fixup_data_pos, n_fixups);

        // Get string remap field offsets for this file type
        std::vector<size_t> remap_offs;
        if (!str_remap_vec.empty() && needs_remap) {
            if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) remap_offs = {8, 12};
            else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) remap_offs = {(actual_stride == 12) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) remap_offs = {(actual_stride >= 20) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) remap_offs = {8};
        }

        // mmap old file read-only (zero allocation)
        MappedFile old_mmap = mmap_file(cur_dir + "/" + std::string(fname));
        madvise(const_cast<char*>(old_mmap.data), old_mmap.size, MADV_SEQUENTIAL);
        size_t n_old_records = old_mmap.size / actual_stride;

        // Replay merge sequence — stream output, apply remap/fixups per-record inline
        uint64_t seq_size = ps.read_u64();
        size_t seq_end = pos + seq_size;
        FILE* outf = fopen((out_dir + "/" + fname).c_str(), "wb");

        bool track = needs_remap; // track ID remap only for data files
        // File-backed id_map: create temp file, fill with 0xFF, mmap read-write
        int remap_fd = -1;
        uint32_t* id_map_ptr = nullptr;
        std::string remap_path;
        if (track) {
            remap_path = tmpdir + "/remap_" + std::to_string(file_id) + ".bin";
            remap_fd = open(remap_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644);
            size_t remap_bytes = n_old_records * 4;
            ftruncate(remap_fd, remap_bytes); // zero-filled by OS (sparse)
            id_map_ptr = static_cast<uint32_t*>(
                mmap(nullptr, remap_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, remap_fd, 0));
            // File is zero-filled. We store new_index+1 so that 0 means "unmapped".
            // The entry pipeline reads these and subtracts 1.
        }

        // Per-record buffer for applying remap/fixups inline (avoid copying entire file)
        std::vector<char> rec_buf(actual_stride);

        size_t old_rec = 0, new_rec = 0, old_bytes = 0, written = 0;
        while (pos < seq_end) {
            uint8_t op = P[pos++];
            uint32_t count; memcpy(&count, P+pos, 4); pos += 4;
            if (op == OP_MATCH_RUN) {
                for (uint32_t k = 0; k < count; k++) {
                    size_t rec_off = old_bytes + k * actual_stride;
                    if (rec_off + actual_stride > old_mmap.size) break;

                    bool modified = false;
                    // Check if this record needs any modification
                    if (!remap_offs.empty() || needs_padding) modified = true;
                    // Check fixup using lazy decoder (reads from patch mmap, zero allocation)
                    uint32_t fixup_val = 0;
                    bool has_fixup = fixup_dec.lookup(old_rec + k, fixup_val);
                    if (has_fixup) modified = true;

                    if (modified) {
                        memcpy(rec_buf.data(), old_mmap.data + rec_off, actual_stride);
                        // Apply padding zeroing
                        if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24)
                            memset(rec_buf.data() + 13, 0, 3);
                        if (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24) {
                            memset(rec_buf.data() + 5, 0, 3);
                            memset(rec_buf.data() + 21, 0, 3);
                        }
                        // Apply string remap
                        for (size_t off : remap_offs) {
                            uint32_t v; memcpy(&v, rec_buf.data() + off, 4);
                            uint32_t nv = str_remap_lookup(v);
                            if (nv != v) memcpy(rec_buf.data() + off, &nv, 4);
                        }
                        // Apply fixup (node_offset/vertex_offset at byte 0)
                        if (has_fixup)
                            memcpy(rec_buf.data(), &fixup_val, 4);
                        fwrite(rec_buf.data(), 1, actual_stride, outf);
                    } else {
                        // No modification needed — write directly from mmap
                        fwrite(old_mmap.data + rec_off, 1, actual_stride, outf);
                    }
                    if (track && old_rec + k < n_old_records) id_map_ptr[old_rec + k] = (uint32_t)(new_rec + k) + 1; // +1 so 0 means unmapped
                }
                written += count * actual_stride;
                old_rec += count; new_rec += count; old_bytes += count * actual_stride;
            } else if (op == OP_INSERT_RUN) {
                size_t bytes = count * actual_stride;
                fwrite(P+pos, 1, bytes, outf);
                written += bytes; pos += bytes; new_rec += count;
            } else if (op == OP_DELETE_RUN) {
                old_rec += count; old_bytes += count * actual_stride;
            }
        }
        fclose(outf);
        unmap_file(old_mmap);

        if (track) {
            // Sync and munmap the file-backed remap (data is already on disk)
            size_t remap_bytes = n_old_records * 4;
            msync(id_map_ptr, remap_bytes, MS_SYNC);
            munmap(id_map_ptr, remap_bytes);
            close(remap_fd);
            id_map_ptr = nullptr;
            id_remaps[file_id] = {}; // empty placeholder
            std::cerr << "  " << fname << ": " << written << " bytes (remap " << n_old_records << " records to disk)" << std::endl;
        } else {
            std::cerr << "  " << fname << ": " << written << " bytes" << std::endl;
        }
        // Release patch pages used by this section
        madvise(const_cast<char*>(patch_map.data), pos, MADV_DONTNEED);
        malloc_trim(0);
    }
    log_phase("Merge replays", t_start);

    // Free string remap + release all processed patch pages
    { std::vector<std::pair<uint32_t,uint32_t>>().swap(str_remap_vec); }
    madvise(const_cast<char*>(patch_map.data), pos, MADV_DONTNEED);
    malloc_trim(0);

    // --- Phase 4: Streaming entry pipeline ---
    // Walk old geo_cells, remap IDs, apply corrections, write output directly to files.
    {
        double t_entry = now_ms();
        std::cerr << "Entry pipeline (lean streaming)..." << std::endl;

        // mmap remap files from disk (only pages accessed are in RSS)
        auto mmap_remap = [&](PatchFileId fid) -> MappedFile {
            std::string path = tmpdir + "/remap_" + std::to_string((uint32_t)fid) + ".bin";
            auto m = mmap_file(path);
            if (m.data) madvise(const_cast<char*>(m.data), m.size, MADV_SEQUENTIAL);
            return m;
        };
        // Helper: look up remap value from mmap'd file
        // Remap values stored as new_index+1 (0 means unmapped)
        struct RemapRef {
            const uint32_t* data;
            size_t count;
            uint32_t operator[](size_t i) const {
                if (i >= count) return 0xFFFFFFFF;
                uint32_t v = data[i];
                return v == 0 ? 0xFFFFFFFF : v - 1;
            }
            size_t size() const { return count; }
        };
        MappedFile m_w_rm = mmap_remap(PatchFileId::STREET_WAYS);
        MappedFile m_a_rm = mmap_remap(PatchFileId::ADDR_POINTS);
        MappedFile m_i_rm = mmap_remap(PatchFileId::INTERP_WAYS);
        RemapRef w_rm = {(const uint32_t*)m_w_rm.data, m_w_rm.size / 4};
        RemapRef a_rm = {(const uint32_t*)m_a_rm.data, m_a_rm.size / 4};
        RemapRef i_rm = {(const uint32_t*)m_i_rm.data, m_i_rm.size / 4};

        // mmap old files (zero RSS until pages are accessed, then only working set)
        MappedFile m_geo = mmap_file(cur_dir + "/geo_cells.bin");
        MappedFile m_se = mmap_file(cur_dir + "/street_entries.bin");
        MappedFile m_ae = mmap_file(cur_dir + "/addr_entries.bin");
        MappedFile m_ie = mmap_file(cur_dir + "/interp_entries.bin");
        madvise(const_cast<char*>(m_se.data), m_se.size, MADV_SEQUENTIAL);
        madvise(const_cast<char*>(m_ae.data), m_ae.size, MADV_SEQUENTIAL);
        madvise(const_cast<char*>(m_ie.data), m_ie.size, MADV_SEQUENTIAL);
        size_t n_old = m_geo.size / 20;

        // Build sorted correction vectors + removed set (better cache locality than hash maps)
        std::unordered_set<uint64_t> rm_set(geo_removed.begin(), geo_removed.end());
        struct CorrEntry { uint64_t cid; const std::vector<uint32_t>* ids; };
        std::vector<CorrEntry> cs, ca, ci_map;
        for (auto& [fid, list] : entry_corrections) {
            auto* v = (fid == (uint32_t)PatchFileId::STREET_ENTRIES) ? &cs :
                      (fid == (uint32_t)PatchFileId::ADDR_ENTRIES) ? &ca :
                      (fid == (uint32_t)PatchFileId::INTERP_ENTRIES) ? &ci_map : nullptr;
            if (v) for (auto& c : list) v->push_back({c.cell_id, &c.ids});
        }
        auto corr_cmp = [](const CorrEntry& a, const CorrEntry& b) { return a.cid < b.cid; };
        std::sort(cs.begin(), cs.end(), corr_cmp);
        std::sort(ca.begin(), ca.end(), corr_cmp);
        std::sort(ci_map.begin(), ci_map.end(), corr_cmp);
        // Binary search helper
        auto corr_find = [](const std::vector<CorrEntry>& v, uint64_t cid) -> const std::vector<uint32_t>* {
            auto it = std::lower_bound(v.begin(), v.end(), CorrEntry{cid, nullptr},
                [](const CorrEntry& a, const CorrEntry& b) { return a.cid < b.cid; });
            return (it != v.end() && it->cid == cid) ? it->ids : nullptr;
        };
        // Added cells sorted
        std::sort(geo_added.begin(), geo_added.end());
        log_phase("  Setup", t_entry);

        // Open 4 output files
        FILE* f_geo = fopen((out_dir + "/geo_cells.bin").c_str(), "wb");
        FILE* f_se = fopen((out_dir + "/street_entries.bin").c_str(), "wb");
        FILE* f_ae = fopen((out_dir + "/addr_entries.bin").c_str(), "wb");
        FILE* f_ie = fopen((out_dir + "/interp_entries.bin").c_str(), "wb");
        constexpr uint32_t NO = 0xFFFFFFFF;

        // Reusable buffer (one per entry type to avoid aliasing issues)
        std::vector<uint32_t> buf;
        buf.reserve(4096);

        // Helper: parse IDs from mmap'd entry file
        auto parse = [&buf](const MappedFile& f, uint32_t off) {
            buf.clear();
            if (off == NO || off + 2 > f.size) return;
            uint16_t c; memcpy(&c, f.data + off, 2);
            if (off + 2 + (size_t)c * 4 > f.size) return;
            buf.resize(c);
            memcpy(buf.data(), f.data + off + 2, c * 4);
        };
        auto remap = [](std::vector<uint32_t>& ids, const RemapRef& rm) {
            constexpr uint32_t NO2 = 0xFFFFFFFF;
            for (auto& id : ids) if (id < rm.size() && rm[id] != NO2) id = rm[id];
            std::sort(ids.begin(), ids.end());
        };
        // Write entry and return offset, or NO if empty
        auto emit = [&NO](FILE* f, const uint32_t* ids, size_t n) -> uint32_t {
            if (n == 0) return NO;
            uint32_t off = (uint32_t)ftell(f);
            uint16_t c = (uint16_t)n;
            fwrite(&c, 2, 1, f); fwrite(ids, 4, n, f);
            return off;
        };

        // Merge-walk: old cells + added cells in sorted order
        // Both are sorted by cell_id. Merge them, skip removed.
        size_t old_i = 0, add_i = 0;
        size_t cells_written = 0;

        auto process_cell = [&](uint64_t cid, int32_t oi) {
            // For each entry type: check correction → remap old → write
            auto do_entry = [&](const MappedFile& old_e, size_t geo_off, const RemapRef& rm,
                                const std::vector<CorrEntry>& corr,
                                FILE* outf, uint8_t flag_bit) -> uint32_t {
                // Check flag
                bool has = false;
                if (oi >= 0 && (size_t)oi * 20 + geo_off + 4 <= m_geo.size) {
                    uint32_t off; memcpy(&off, m_geo.data + oi * 20 + geo_off, 4);
                    has = (off != NO);
                } else if (oi >= 0) {
                    std::cerr << "OOB: oi=" << oi << " geo_off=" << geo_off << " m_geo.size=" << m_geo.size << std::endl;
                    return NO;
                }
                auto fc = flag_corrections.find(cid);
                if (fc != flag_corrections.end()) has = (fc->second & flag_bit) != 0;
                auto* corr_ids = corr_find(corr, cid);
                if (corr_ids) has = true;
                if (!has) return NO;

                if (corr_ids) return emit(outf, corr_ids->data(), corr_ids->size());

                if (oi < 0) return NO;
                if ((size_t)oi * 20 + geo_off + 4 > m_geo.size) return NO;
                uint32_t off; memcpy(&off, m_geo.data + oi * 20 + geo_off, 4);
                parse(old_e, off);
                if (buf.empty()) return NO;
                remap(buf, rm);
                return emit(outf, buf.data(), buf.size());
            };

            uint32_t so = do_entry(m_se, 8, w_rm, cs, f_se, 1);
            uint32_t ao = do_entry(m_ae, 12, a_rm, ca, f_ae, 2);
            uint32_t io = do_entry(m_ie, 16, i_rm, ci_map, f_ie, 4);
            fwrite(&cid, 8, 1, f_geo); fwrite(&so, 4, 1, f_geo); fwrite(&ao, 4, 1, f_geo); fwrite(&io, 4, 1, f_geo);
            cells_written++;
        };

        while (old_i < n_old || add_i < geo_added.size()) {
            uint64_t old_cid = UINT64_MAX, add_cid = UINT64_MAX;
            if (old_i < n_old) memcpy(&old_cid, m_geo.data + old_i * 20, 8);
            if (add_i < geo_added.size()) add_cid = geo_added[add_i];

            if (old_cid <= add_cid) {
                if (!rm_set.count(old_cid))
                    process_cell(old_cid, (int32_t)old_i);
                old_i++;
                if (old_cid == add_cid) add_i++; // skip duplicate add
            } else {
                process_cell(add_cid, -1);
                add_i++;
            }

            if (cells_written % 10000000 == 0 && cells_written > 0) {
                std::cerr << "    " << cells_written << " cells, se=" << ftell(f_se)/1024/1024
                          << "M ae=" << ftell(f_ae)/1024/1024 << "M rss=" << get_rss_mb() << "M" << std::endl;
            }
        }

        fclose(f_geo); fclose(f_se); fclose(f_ae); fclose(f_ie);
        unmap_file(m_geo); unmap_file(m_se); unmap_file(m_ae); unmap_file(m_ie);
        unmap_file(m_w_rm); unmap_file(m_a_rm); unmap_file(m_i_rm);
        std::cerr << "  Geo: " << cells_written << " cells written" << std::endl;
        malloc_trim(0);
        log_phase("  Geo entries complete", t_entry);

        // Admin: small, use existing rebuild + corrections
        {
            uint32_t no_data = 0xFFFFFFFF;
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            MappedFile m_ad_rm = mmap_remap(PatchFileId::ADMIN_POLYGONS);
            if (m_ad_rm.data) {
                const uint32_t* ad_vec = (const uint32_t*)m_ad_rm.data;
                size_t ad_count = m_ad_rm.size / 4;
                for (uint32_t i = 0; i < ad_count; i++)
                    if (ad_vec[i] != 0) ad_rm[i] = ad_vec[i] - 1; // decode +1 encoding
                unmap_file(m_ad_rm);
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_adme = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_adme, ad_rm, admin_added, admin_removed);

            auto ecit = entry_corrections.find((uint32_t)PatchFileId::ADMIN_ENTRIES);
            if (ecit != entry_corrections.end()) {
                std::unordered_map<uint64_t, const std::vector<uint32_t>*> ac_corr;
                for (auto& c : ecit->second) ac_corr[c.cell_id] = &c.ids;
                size_t n = admin.admin_cells_data.size() / 12;
                FILE* fac = fopen((out_dir + "/admin_cells.bin").c_str(), "wb");
                FILE* fae = fopen((out_dir + "/admin_entries.bin").c_str(), "wb");
                uint32_t ae_wpos = 0;
                for (size_t i = 0; i < n; i++) {
                    uint64_t cid; memcpy(&cid, admin.admin_cells_data.data()+i*12, 8);
                    fwrite(&cid, 8, 1, fac);
                    auto cit = ac_corr.find(cid);
                    if (cit != ac_corr.end()) {
                        uint32_t off = cit->second->empty() ? no_data : ae_wpos;
                        fwrite(&off, 4, 1, fac);
                        if (!cit->second->empty()) {
                            uint16_t c = cit->second->size();
                            fwrite(&c, 2, 1, fae); fwrite(cit->second->data(), 4, c, fae);
                            ae_wpos += 2 + c*4;
                        }
                    } else {
                        uint32_t old_off; memcpy(&old_off, admin.admin_cells_data.data()+i*12+8, 4);
                        if (old_off != no_data && old_off+2 <= admin.admin_entries_data.size()) {
                            uint32_t new_off = ae_wpos; fwrite(&new_off, 4, 1, fac);
                            uint16_t c; memcpy(&c, admin.admin_entries_data.data()+old_off, 2);
                            fwrite(admin.admin_entries_data.data()+old_off, 1, 2+c*4, fae);
                            ae_wpos += 2+c*4;
                        } else {
                            fwrite(&no_data, 4, 1, fac);
                        }
                    }
                }
                fclose(fac); fclose(fae);
                std::cerr << "  Admin: " << n << " cells, " << ac_corr.size() << " corrections" << std::endl;
            } else {
                write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
                write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            }
            log_phase("  Admin", t_entry);
        }
    }

    // Cleanup
    unmap_file(patch_map);
    remove(raw_path.c_str());
    { std::string cmd = "rm -rf '" + tmpdir + "'"; system(cmd.c_str()); }
    log_phase("Total", t_start);
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

-            for (uint32_t i = 0; i < c; i++) { uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8; flag_corrections[cid] = P[pos++]; }

+            for (uint32_t i = 0; i < c; i++) { uint64_t cid; ps.read(&cid, 8); flag_corrections[cid] = ps.read_u8(); }


Now the entry corrections reading:


Edit

                uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8;
                uint16_t ec; memcpy(&ec, P+pos, 2); pos += 2;
                std::vector<uint32_t> ids(ec);
                if (ec > 0) { memcpy(ids.data(), P+pos, ec*4); pos += ec*4; }

                uint64_t cid; ps.read(&cid, 8);
                uint16_t ec; ps.read(&ec, 2);
                std::vector<uint32_t> ids(ec);
                if (ec > 0) ps.read(ids.data(), ec*4);


                uint64_t cid; ps.read(&cid, 8);
                uint16_t ec; ps.read(&ec, 2);
                std::vector<uint32_t> ids(ec);
                if (ec > 0) ps.read(ids.data(), ec*4);

// geocoder-patch v4: Low-memory streaming patch application.
//
// Key design: mmap old files (zero copy), stream output via fwrite (never accumulate),
// process entry pipeline cell-by-cell. Target: <1 GiB peak RSS for planet.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <fcntl.h>
#include <iomanip>
#include <iostream>
#include <malloc.h>
#include <string>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unordered_map>
#include <unordered_set>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

// Streaming zstd decompressor — reads compressed file, provides sequential access
// to decompressed data via a sliding buffer. No temp file needed.
#include <zstd.h>
class PatchStream {
    FILE* input_;
    ZSTD_DCtx* dctx_;
    std::vector<char> in_buf_, out_buf_;
    size_t out_start_ = 0, out_end_ = 0; // valid range in out_buf_
    size_t global_pos_ = 0; // position of out_buf_[0] in decompressed stream
    size_t total_decompressed_ = 0;
    bool input_done_ = false;
    static constexpr size_t IN_BUF_SIZE = 256 * 1024;
    static constexpr size_t OUT_BUF_SIZE = 4 * 1024 * 1024;
    ZSTD_inBuffer zin_{};

    void decompress_more() {
        // Compact: move unread data to front
        if (out_start_ > 0) {
            size_t remain = out_end_ - out_start_;
            if (remain > 0) memmove(out_buf_.data(), out_buf_.data() + out_start_, remain);
            global_pos_ += out_start_;
            out_end_ = remain;
            out_start_ = 0;
        }
        // Ensure output buffer has room
        if (out_buf_.size() - out_end_ < OUT_BUF_SIZE)
            out_buf_.resize(out_end_ + OUT_BUF_SIZE);

        while (!input_done_ && out_end_ - out_start_ < OUT_BUF_SIZE) {
            if (zin_.pos >= zin_.size) {
                size_t n = fread(in_buf_.data(), 1, IN_BUF_SIZE, input_);
                if (n == 0) { input_done_ = true; break; }
                zin_ = {in_buf_.data(), n, 0};
            }
            ZSTD_outBuffer zout = {out_buf_.data() + out_end_, out_buf_.size() - out_end_, 0};
            size_t ret = ZSTD_decompressStream(dctx_, &zout, &zin_);
            if (ZSTD_isError(ret)) { std::cerr << "zstd error: " << ZSTD_getErrorName(ret) << std::endl; break; }
            out_end_ += zout.pos;
            total_decompressed_ = global_pos_ + out_end_;
        }
    }

public:
    bool open(const std::string& path) {
        input_ = fopen(path.c_str(), "rb");
        if (!input_) return false;
        dctx_ = ZSTD_createDCtx();
        in_buf_.resize(IN_BUF_SIZE);
        out_buf_.resize(OUT_BUF_SIZE * 2);
        decompress_more();
        return true;
    }
    ~PatchStream() { if (dctx_) ZSTD_freeDCtx(dctx_); if (input_) fclose(input_); }

    // Ensure at least n bytes are available at current read position
    void ensure(size_t n) {
        while (out_end_ - out_start_ < n && !input_done_) decompress_more();
    }

    // Read n bytes at current position into dst, advance
    void read(void* dst, size_t n) {
        ensure(n);
        memcpy(dst, out_buf_.data() + out_start_, n);
        out_start_ += n;
    }
    uint32_t read_u32() { uint32_t v; read(&v, 4); return v; }
    uint64_t read_u64() { uint64_t v; read(&v, 8); return v; }
    uint8_t read_u8() { ensure(1); return (uint8_t)out_buf_[out_start_++]; }

    // Get pointer to n bytes at current position (valid until next ensure/read)
    const char* peek(size_t n) { ensure(n); return out_buf_.data() + out_start_; }
    void skip(size_t n) { ensure(n); out_start_ += n; }

    // Write n bytes from current position directly to a FILE*, advance
    void write_to(FILE* f, size_t n) {
        while (n > 0) {
            ensure(std::min(n, (size_t)OUT_BUF_SIZE));
            size_t avail = out_end_ - out_start_;
            size_t chunk = std::min(n, avail);
            fwrite(out_buf_.data() + out_start_, 1, chunk, f);
            out_start_ += chunk;
            n -= chunk;
        }
    }

    // Read a null-terminated string at current position
    std::string read_string() {
        std::string s;
        while (true) {
            ensure(1);
            char c = out_buf_[out_start_++];
            if (c == '\0') break;
            s += c;
        }
        return s;
    }

    size_t position() const { return global_pos_ + out_start_; }
    size_t decompressed_size() const { return total_decompressed_; }
};

// --- Helpers ---
static double now_ms() {
    struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
    return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6;
}
static size_t get_rss_mb() {
    FILE* f = fopen("/proc/self/statm", "r");
    if (!f) return 0;
    size_t dummy, rss; fscanf(f, "%zu %zu", &dummy, &rss); fclose(f);
    return rss * 4096 / (1024*1024);
}
static void log_phase(const char* label, double start) {
    std::cerr << "  [" << std::fixed << std::setprecision(1) << (now_ms() - start) / 1000.0
              << "s, " << get_rss_mb() << " MiB] " << label << std::endl;
}

// mmap a file read-only. Returns {pointer, size}. Caller must munmap.
struct MappedFile { const char* data; size_t size; };
static MappedFile mmap_file(const std::string& path) {
    int fd = open(path.c_str(), O_RDONLY);
    if (fd < 0) return {nullptr, 0};
    struct stat st; fstat(fd, &st);
    size_t sz = st.st_size;
    if (sz == 0) { close(fd); return {nullptr, 0}; }
    void* p = mmap(nullptr, sz, PROT_READ, MAP_PRIVATE, fd, 0);
    close(fd);
    if (p == MAP_FAILED) return {nullptr, 0};
    return {static_cast<const char*>(p), sz};
}
static void unmap_file(MappedFile& f) {
    if (f.data) { munmap(const_cast<char*>(f.data), f.size); f.data = nullptr; f.size = 0; }
}

// Detect stride from file size
static size_t detect_stride(const std::string& path, std::initializer_list<size_t> candidates) {
    struct stat st; if (stat(path.c_str(), &st) != 0) return *candidates.begin();
    for (size_t s : candidates) if (st.st_size % s == 0 && st.st_size > 0) return s;
    return *candidates.begin();
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);
    double t_start = now_ms();

    // --- Phase 1: Streaming zstd decompression (no temp file) ---
    PatchStream ps;
    if (!ps.open(patch_path)) { std::cerr << "Failed to open patch" << std::endl; return 1; }
    std::cerr << "Streaming patch decompression..." << std::endl;
    log_phase("Open", t_start);

    // Header
    { char magic[8]; ps.read(magic, 8);
      if (memcmp(magic, GCPATCH_MAGIC, 8) != 0) { std::cerr << "Bad magic" << std::endl; return 1; } }
    uint32_t ver = ps.read_u32(); if (ver != 2) { std::cerr << "Bad version" << std::endl; return 1; }
    ps.read_u32(); // flags

    // --- Phase 2: String rebuild ---
    // Sorted vector remap: (old_offset, new_offset) pairs sorted by old_offset.
    // Uses 96 MiB for 12M entries vs 672 MiB for unordered_map.
    std::vector<std::pair<uint32_t, uint32_t>> str_remap_vec;
    {
        uint32_t marker = ps.read_u32();
        if (marker == 0xFFFFFFF7) {
            uint32_t n_added = ps.read_u32(), n_deleted = ps.read_u32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ps.read_u32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());

            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");

            // Phase A: merge old (minus deleted) + added strings, write directly to file.
            // Both old pool and added are sorted alphabetically → merge-write with no sort.
            // Memory: only the added strings vector (~small) + del_set (~small).
            {
                FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
                size_t sp = 0; uint32_t idx = 0; size_t ai = 0;
                // Sort added strings (they should already be sorted from diff tool, but ensure)
                std::sort(added.begin(), added.end());
                while (sp < old_pool.size || ai < added.size()) {
                    const char* old_s = nullptr;
                    // Skip deleted strings
                    while (sp < old_pool.size) {
                        if (!del_set.count(idx)) { old_s = old_pool.data + sp; break; }
                        sp += strlen(old_pool.data + sp) + 1; idx++;
                    }
                    const char* add_s = ai < added.size() ? added[ai].c_str() : nullptr;
                    // Merge: pick the smaller string
                    if (old_s && add_s) {
                        int c = strcmp(old_s, add_s);
                        if (c <= 0) {
                            size_t l = strlen(old_s) + 1; fwrite(old_s, 1, l, fp);
                            sp += l; idx++;
                            if (c == 0) ai++; // duplicate — skip added
                        } else {
                            size_t l = added[ai].size() + 1; fwrite(add_s, 1, l, fp);
                            ai++;
                        }
                    } else if (old_s) {
                        size_t l = strlen(old_s) + 1; fwrite(old_s, 1, l, fp);
                        sp += l; idx++;
                    } else if (add_s) {
                        size_t l = added[ai].size() + 1; fwrite(add_s, 1, l, fp);
                        ai++;
                    } else break;
                }
                fclose(fp);
            }
            { std::vector<std::string>().swap(added); std::unordered_set<uint32_t>().swap(del_set);
              std::vector<uint32_t>().swap(del_idx); }
            malloc_trim(0);

            // Phase B: build remap by merge-walking old pool vs new pool (both sorted, both mmap'd)
            MappedFile new_pool = mmap_file(out_dir + "/strings.bin");
            {
                // Walk both pools simultaneously — no vectors needed, just cursors
                size_t old_pos = 0, new_pos = 0;
                while (old_pos < old_pool.size && new_pos < new_pool.size) {
                    const char* os = old_pool.data + old_pos;
                    const char* ns = new_pool.data + new_pos;
                    int c = strcmp(os, ns);
                    if (c == 0) {
                        if (old_pos != new_pos) // offset changed
                            str_remap_vec.push_back({(uint32_t)old_pos, (uint32_t)new_pos});
                        old_pos += strlen(os) + 1;
                        new_pos += strlen(ns) + 1;
                    } else if (c < 0) {
                        old_pos += strlen(os) + 1; // deleted string
                    } else {
                        new_pos += strlen(ns) + 1; // added string
                    }
                }
            }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
            unmap_file(old_pool);
            unmap_file(new_pool);
            std::cerr << "  Strings: +" << n_added << " -" << n_deleted
                      << ", " << str_remap_vec.size() << " remapped (" << str_remap_vec.size() * 8 / 1024 / 1024 << " MiB)" << std::endl;
            marker = ps.read_u32();
        }
        if (marker == 0xFFFFFFFE) {
            uint32_t c = ps.read_u32();
            for (uint32_t i = 0; i < c; i++) { uint32_t a = ps.read_u32(), b = ps.read_u32(); str_remap_vec.push_back({a, b}); }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
        } else pos -= 4;
    }
    // Release patch pages read so far (string section)
    madvise(const_cast<char*>(patch_map.data), pos, MADV_DONTNEED);
    log_phase("Strings", t_start);

    // Detect strides
    size_t way_stride = detect_stride(cur_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride(cur_dir + "/interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect_stride(cur_dir + "/admin_polygons.bin", {24, 20, 19});

    // String remap lookup via sorted vector + binary search
    auto str_remap_lookup = [&](uint32_t old_off) -> uint32_t {
        auto it = std::lower_bound(str_remap_vec.begin(), str_remap_vec.end(),
            std::make_pair(old_off, (uint32_t)0));
        if (it != str_remap_vec.end() && it->first == old_off) return it->second;
        return old_off; // no remap
    };

    // --- Phase 3: Scan sections + parallel merge replays ---
    // Collect merge section info for parallel replay, process metadata inline
    struct MergeSection {
        uint32_t file_id, stride; size_t actual_stride;
        uint64_t old_size, new_size;
        uint32_t n_fixups; size_t fixup_data_pos;
        size_t seq_pos; uint64_t seq_size;
        bool needs_remap, needs_padding;
    };
    std::vector<MergeSection> merge_sections;
    std::unordered_map<uint32_t, std::vector<uint32_t>> id_remaps;
    std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;
    std::unordered_map<uint64_t, uint8_t> flag_corrections;
    struct CellCorr { uint64_t cell_id; std::vector<uint32_t> ids; };
    std::unordered_map<uint32_t, std::vector<CellCorr>> entry_corrections;

    while (pos < patch_size) {
        uint32_t file_id = ps.read_u32();
        if (file_id == 0xFFFFFFFF) break;

        // --- Metadata sections ---
        if (file_id == 0xFFFFFFF7) {
            // String diff in main loop — same memory-efficient approach as Phase 2
            uint32_t n_added = ps.read_u32(), n_deleted = ps.read_u32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ps.read_u32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());
            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
            struct OS { uint32_t idx; uint32_t off; const char* str; };
            std::vector<OS> old_strs;
            { size_t sp = 0; uint32_t ix = 0;
              while (sp < old_pool.size) { old_strs.push_back({ix++, (uint32_t)sp, old_pool.data+sp}); sp += strlen(old_pool.data+sp)+1; }
            }
            std::vector<const char*> merged;
            for (auto& os : old_strs) if (!del_set.count(os.idx)) merged.push_back(os.str);
            for (auto& s : added) merged.push_back(s.c_str());
            std::sort(merged.begin(), merged.end(), [](const char* a, const char* b) { return strcmp(a,b) < 0; });
            FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
            uint32_t wpos = 0;
            for (const char* s : merged) { size_t l = strlen(s)+1; fwrite(s, 1, l, fp); wpos += l; }
            fclose(fp);
            // Build remap via merge-walk
            MappedFile new_pool = mmap_file(out_dir + "/strings.bin");
            std::vector<std::pair<const char*, uint32_t>> new_strs;
            { size_t sp = 0;
              while (sp < new_pool.size) { new_strs.push_back({new_pool.data+sp, (uint32_t)sp}); sp += strlen(new_pool.data+sp)+1; }
            }
            size_t oi2 = 0, ni2 = 0;
            while (oi2 < old_strs.size() && ni2 < new_strs.size()) {
                int c = strcmp(old_strs[oi2].str, new_strs[ni2].first);
                if (c == 0) {
                    if (old_strs[oi2].off != new_strs[ni2].second)
                        str_remap_vec.push_back({old_strs[oi2].off, new_strs[ni2].second});
                    oi2++; ni2++;
                } else if (c < 0) { oi2++; } else { ni2++; }
            }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
            unmap_file(old_pool); unmap_file(new_pool);
            std::cerr << "  Strings (loop): +" << n_added << " -" << n_deleted << " → " << merged.size()
                      << ", " << str_remap_vec.size() << " remapped" << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_GEO_MARKER) {
            uint32_t na = ps.read_u32(), nr = ps.read_u32();
            geo_added.resize(na); geo_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { ps.read(&geo_added[i], 8); }
            for (uint32_t i = 0; i < nr; i++) { ps.read(&geo_removed[i], 8); }
            std::cerr << "  Geo cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_ADMIN_MARKER) {
            uint32_t na = ps.read_u32(), nr = ps.read_u32();
            admin_added.resize(na); admin_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { ps.read(&admin_added[i], 8); }
            for (uint32_t i = 0; i < nr; i++) { ps.read(&admin_removed[i], 8); }
            std::cerr << "  Admin cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_FLAGS_MARKER) {
            uint32_t c = ps.read_u32();
            for (uint32_t i = 0; i < c; i++) { uint64_t cid; ps.read(&cid, 8); flag_corrections[cid] = ps.read_u8(); }
            std::cerr << "  Flag corrections: " << c << std::endl;
            continue;
        }
        if (file_id == SECONDARY_REMAP_MARKER) {
            // Apply secondary remaps directly to the temp remap files on disk
            uint32_t nf = ps.read_u32();
            for (uint32_t f = 0; f < nf; f++) {
                uint32_t fid = ps.read_u32(), np = ps.read_u32();
                std::string remap_path = tmpdir + "/remap_" + std::to_string(fid) + ".bin";
                // Open remap file for read-write
                int rfd = open(remap_path.c_str(), O_RDWR);
                if (rfd >= 0) {
                    struct stat st; fstat(rfd, &st);
                    size_t remap_size = st.st_size;
                    uint32_t* remap_data = static_cast<uint32_t*>(
                        mmap(nullptr, remap_size, PROT_READ | PROT_WRITE, MAP_SHARED, rfd, 0));
                    size_t remap_count = remap_size / 4;
                    for (uint32_t i = 0; i < np; i++) {
                        uint32_t o = ps.read_u32(), n = ps.read_u32();
                        if (o < remap_count) remap_data[o] = n + 1; // +1 encoding
                    }
                    munmap(remap_data, remap_size);
                    close(rfd);
                } else {
                    // Skip data if file doesn't exist
                    for (uint32_t i = 0; i < np; i++) { ps.read_u32(); ps.read_u32(); }
                }
                std::cerr << "  Secondary remap " << fid << ": " << np << " pairs" << std::endl;
            }
            continue;
        }
        if (file_id == ENTRY_CORRECTION_MARKER) {
            uint32_t fid = ps.read_u32(), c = ps.read_u32();
            auto& list = entry_corrections[fid];
            for (uint32_t i = 0; i < c; i++) {
                uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8;
                uint16_t ec; memcpy(&ec, P+pos, 2); pos += 2;
                std::vector<uint32_t> ids(ec);
                if (ec > 0) { memcpy(ids.data(), P+pos, ec*4); pos += ec*4; }
                list.push_back({cid, std::move(ids)});
            }
            std::cerr << "  Entry corrections " << fid << ": " << c << std::endl;
            continue;
        }

        // --- Collect merge section info (replay happens in parallel below) ---
        uint32_t stride = ps.read_u32();
        uint64_t old_size = ps.read_u64(), new_size = ps.read_u64();
        if (file_id >= (uint32_t)PatchFileId::COUNT) { std::cerr << "Unknown file " << file_id << std::endl; break; }
        const char* fname = patch_file_names[file_id];

        if (stride == 0) {
            uint32_t nf = ps.read_u32(); (void)nf; uint64_t ds = ps.read_u64();
            FILE* fp = fopen((out_dir + "/" + fname).c_str(), "wb");
            fwrite(P+pos, 1, ds, fp); fclose(fp);
            pos += ds;
            std::cerr << "  " << fname << ": full replace " << ds << " bytes" << std::endl;
            continue;
        }
        if (stride == 0xFE) { uint32_t nf = ps.read_u32(); (void)nf; uint64_t ds = ps.read_u64(); pos += ds; continue; }

        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        bool needs_remap = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
                            file_id == (uint32_t)PatchFileId::STREET_WAYS ||
                            file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
                            file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
        bool needs_padding = (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24) ||
                             (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24);

        uint32_t n_fixups = ps.read_u32();
        size_t fixup_data_pos = pos;
        if (n_fixups > 0) {
            uint32_t delta_size = ps.read_u32();
            fixup_data_pos = pos;
            pos += delta_size;
        }
        // Lazy fixup decoder state (reads from patch mmap on demand)
        struct FixupDecoder {
            const char* data; size_t read_pos;
            uint32_t prev_idx = 0, prev_val = 0;
            uint32_t cur_idx = UINT32_MAX, cur_val = 0;
            uint32_t remaining;
            void init(const char* d, size_t p, uint32_t count) {
                data = d; read_pos = p; remaining = count;
                if (remaining > 0) advance();
            }
            void advance() {
                if (remaining == 0) { cur_idx = UINT32_MAX; return; }
                cur_idx = prev_idx + read_varint(data, read_pos);
                cur_val = prev_val + read_varint(data, read_pos);
                prev_idx = cur_idx; prev_val = cur_val;
                remaining--;
            }
            // Advance past all fixups with index < target
            bool lookup(uint32_t target_idx, uint32_t& out_val) {
                while (cur_idx < target_idx && (remaining > 0 || cur_idx != UINT32_MAX)) advance();
                if (cur_idx == target_idx) { out_val = cur_val; advance(); return true; }
                return false;
            }
        } fixup_dec;
        fixup_dec.init(P, fixup_data_pos, n_fixups);

        // Get string remap field offsets for this file type
        std::vector<size_t> remap_offs;
        if (!str_remap_vec.empty() && needs_remap) {
            if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) remap_offs = {8, 12};
            else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) remap_offs = {(actual_stride == 12) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) remap_offs = {(actual_stride >= 20) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) remap_offs = {8};
        }

        // mmap old file read-only (zero allocation)
        MappedFile old_mmap = mmap_file(cur_dir + "/" + std::string(fname));
        madvise(const_cast<char*>(old_mmap.data), old_mmap.size, MADV_SEQUENTIAL);
        size_t n_old_records = old_mmap.size / actual_stride;

        // Replay merge sequence — stream output, apply remap/fixups per-record inline
        uint64_t seq_size = ps.read_u64();
        size_t seq_end = pos + seq_size;
        FILE* outf = fopen((out_dir + "/" + fname).c_str(), "wb");

        bool track = needs_remap; // track ID remap only for data files
        // File-backed id_map: create temp file, fill with 0xFF, mmap read-write
        int remap_fd = -1;
        uint32_t* id_map_ptr = nullptr;
        std::string remap_path;
        if (track) {
            remap_path = tmpdir + "/remap_" + std::to_string(file_id) + ".bin";
            remap_fd = open(remap_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644);
            size_t remap_bytes = n_old_records * 4;
            ftruncate(remap_fd, remap_bytes); // zero-filled by OS (sparse)
            id_map_ptr = static_cast<uint32_t*>(
                mmap(nullptr, remap_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, remap_fd, 0));
            // File is zero-filled. We store new_index+1 so that 0 means "unmapped".
            // The entry pipeline reads these and subtracts 1.
        }

        // Per-record buffer for applying remap/fixups inline (avoid copying entire file)
        std::vector<char> rec_buf(actual_stride);

        size_t old_rec = 0, new_rec = 0, old_bytes = 0, written = 0;
        while (pos < seq_end) {
            uint8_t op = P[pos++];
            uint32_t count; memcpy(&count, P+pos, 4); pos += 4;
            if (op == OP_MATCH_RUN) {
                for (uint32_t k = 0; k < count; k++) {
                    size_t rec_off = old_bytes + k * actual_stride;
                    if (rec_off + actual_stride > old_mmap.size) break;

                    bool modified = false;
                    // Check if this record needs any modification
                    if (!remap_offs.empty() || needs_padding) modified = true;
                    // Check fixup using lazy decoder (reads from patch mmap, zero allocation)
                    uint32_t fixup_val = 0;
                    bool has_fixup = fixup_dec.lookup(old_rec + k, fixup_val);
                    if (has_fixup) modified = true;

                    if (modified) {
                        memcpy(rec_buf.data(), old_mmap.data + rec_off, actual_stride);
                        // Apply padding zeroing
                        if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24)
                            memset(rec_buf.data() + 13, 0, 3);
                        if (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24) {
                            memset(rec_buf.data() + 5, 0, 3);
                            memset(rec_buf.data() + 21, 0, 3);
                        }
                        // Apply string remap
                        for (size_t off : remap_offs) {
                            uint32_t v; memcpy(&v, rec_buf.data() + off, 4);
                            uint32_t nv = str_remap_lookup(v);
                            if (nv != v) memcpy(rec_buf.data() + off, &nv, 4);
                        }
                        // Apply fixup (node_offset/vertex_offset at byte 0)
                        if (has_fixup)
                            memcpy(rec_buf.data(), &fixup_val, 4);
                        fwrite(rec_buf.data(), 1, actual_stride, outf);
                    } else {
                        // No modification needed — write directly from mmap
                        fwrite(old_mmap.data + rec_off, 1, actual_stride, outf);
                    }
                    if (track && old_rec + k < n_old_records) id_map_ptr[old_rec + k] = (uint32_t)(new_rec + k) + 1; // +1 so 0 means unmapped
                }
                written += count * actual_stride;
                old_rec += count; new_rec += count; old_bytes += count * actual_stride;
            } else if (op == OP_INSERT_RUN) {
                size_t bytes = count * actual_stride;
                fwrite(P+pos, 1, bytes, outf);
                written += bytes; pos += bytes; new_rec += count;
            } else if (op == OP_DELETE_RUN) {
                old_rec += count; old_bytes += count * actual_stride;
            }
        }
        fclose(outf);
        unmap_file(old_mmap);

        if (track) {
            // Sync and munmap the file-backed remap (data is already on disk)
            size_t remap_bytes = n_old_records * 4;
            msync(id_map_ptr, remap_bytes, MS_SYNC);
            munmap(id_map_ptr, remap_bytes);
            close(remap_fd);
            id_map_ptr = nullptr;
            id_remaps[file_id] = {}; // empty placeholder
            std::cerr << "  " << fname << ": " << written << " bytes (remap " << n_old_records << " records to disk)" << std::endl;
        } else {
            std::cerr << "  " << fname << ": " << written << " bytes" << std::endl;
        }
        // Release patch pages used by this section
        madvise(const_cast<char*>(patch_map.data), pos, MADV_DONTNEED);
        malloc_trim(0);
    }
    log_phase("Merge replays", t_start);

    // Free string remap + release all processed patch pages
    { std::vector<std::pair<uint32_t,uint32_t>>().swap(str_remap_vec); }
    madvise(const_cast<char*>(patch_map.data), pos, MADV_DONTNEED);
    malloc_trim(0);

    // --- Phase 4: Streaming entry pipeline ---
    // Walk old geo_cells, remap IDs, apply corrections, write output directly to files.
    {
        double t_entry = now_ms();
        std::cerr << "Entry pipeline (lean streaming)..." << std::endl;

        // mmap remap files from disk (only pages accessed are in RSS)
        auto mmap_remap = [&](PatchFileId fid) -> MappedFile {
            std::string path = tmpdir + "/remap_" + std::to_string((uint32_t)fid) + ".bin";
            auto m = mmap_file(path);
            if (m.data) madvise(const_cast<char*>(m.data), m.size, MADV_SEQUENTIAL);
            return m;
        };
        // Helper: look up remap value from mmap'd file
        // Remap values stored as new_index+1 (0 means unmapped)
        struct RemapRef {
            const uint32_t* data;
            size_t count;
            uint32_t operator[](size_t i) const {
                if (i >= count) return 0xFFFFFFFF;
                uint32_t v = data[i];
                return v == 0 ? 0xFFFFFFFF : v - 1;
            }
            size_t size() const { return count; }
        };
        MappedFile m_w_rm = mmap_remap(PatchFileId::STREET_WAYS);
        MappedFile m_a_rm = mmap_remap(PatchFileId::ADDR_POINTS);
        MappedFile m_i_rm = mmap_remap(PatchFileId::INTERP_WAYS);
        RemapRef w_rm = {(const uint32_t*)m_w_rm.data, m_w_rm.size / 4};
        RemapRef a_rm = {(const uint32_t*)m_a_rm.data, m_a_rm.size / 4};
        RemapRef i_rm = {(const uint32_t*)m_i_rm.data, m_i_rm.size / 4};

        // mmap old files (zero RSS until pages are accessed, then only working set)
        MappedFile m_geo = mmap_file(cur_dir + "/geo_cells.bin");
        MappedFile m_se = mmap_file(cur_dir + "/street_entries.bin");
        MappedFile m_ae = mmap_file(cur_dir + "/addr_entries.bin");
        MappedFile m_ie = mmap_file(cur_dir + "/interp_entries.bin");
        madvise(const_cast<char*>(m_se.data), m_se.size, MADV_SEQUENTIAL);
        madvise(const_cast<char*>(m_ae.data), m_ae.size, MADV_SEQUENTIAL);
        madvise(const_cast<char*>(m_ie.data), m_ie.size, MADV_SEQUENTIAL);
        size_t n_old = m_geo.size / 20;

        // Build sorted correction vectors + removed set (better cache locality than hash maps)
        std::unordered_set<uint64_t> rm_set(geo_removed.begin(), geo_removed.end());
        struct CorrEntry { uint64_t cid; const std::vector<uint32_t>* ids; };
        std::vector<CorrEntry> cs, ca, ci_map;
        for (auto& [fid, list] : entry_corrections) {
            auto* v = (fid == (uint32_t)PatchFileId::STREET_ENTRIES) ? &cs :
                      (fid == (uint32_t)PatchFileId::ADDR_ENTRIES) ? &ca :
                      (fid == (uint32_t)PatchFileId::INTERP_ENTRIES) ? &ci_map : nullptr;
            if (v) for (auto& c : list) v->push_back({c.cell_id, &c.ids});
        }
        auto corr_cmp = [](const CorrEntry& a, const CorrEntry& b) { return a.cid < b.cid; };
        std::sort(cs.begin(), cs.end(), corr_cmp);
        std::sort(ca.begin(), ca.end(), corr_cmp);
        std::sort(ci_map.begin(), ci_map.end(), corr_cmp);
        // Binary search helper
        auto corr_find = [](const std::vector<CorrEntry>& v, uint64_t cid) -> const std::vector<uint32_t>* {
            auto it = std::lower_bound(v.begin(), v.end(), CorrEntry{cid, nullptr},
                [](const CorrEntry& a, const CorrEntry& b) { return a.cid < b.cid; });
            return (it != v.end() && it->cid == cid) ? it->ids : nullptr;
        };
        // Added cells sorted
        std::sort(geo_added.begin(), geo_added.end());
        log_phase("  Setup", t_entry);

        // Open 4 output files
        FILE* f_geo = fopen((out_dir + "/geo_cells.bin").c_str(), "wb");
        FILE* f_se = fopen((out_dir + "/street_entries.bin").c_str(), "wb");
        FILE* f_ae = fopen((out_dir + "/addr_entries.bin").c_str(), "wb");
        FILE* f_ie = fopen((out_dir + "/interp_entries.bin").c_str(), "wb");
        constexpr uint32_t NO = 0xFFFFFFFF;

        // Reusable buffer (one per entry type to avoid aliasing issues)
        std::vector<uint32_t> buf;
        buf.reserve(4096);

        // Helper: parse IDs from mmap'd entry file
        auto parse = [&buf](const MappedFile& f, uint32_t off) {
            buf.clear();
            if (off == NO || off + 2 > f.size) return;
            uint16_t c; memcpy(&c, f.data + off, 2);
            if (off + 2 + (size_t)c * 4 > f.size) return;
            buf.resize(c);
            memcpy(buf.data(), f.data + off + 2, c * 4);
        };
        auto remap = [](std::vector<uint32_t>& ids, const RemapRef& rm) {
            constexpr uint32_t NO2 = 0xFFFFFFFF;
            for (auto& id : ids) if (id < rm.size() && rm[id] != NO2) id = rm[id];
            std::sort(ids.begin(), ids.end());
        };
        // Write entry and return offset, or NO if empty
        auto emit = [&NO](FILE* f, const uint32_t* ids, size_t n) -> uint32_t {
            if (n == 0) return NO;
            uint32_t off = (uint32_t)ftell(f);
            uint16_t c = (uint16_t)n;
            fwrite(&c, 2, 1, f); fwrite(ids, 4, n, f);
            return off;
        };

        // Merge-walk: old cells + added cells in sorted order
        // Both are sorted by cell_id. Merge them, skip removed.
        size_t old_i = 0, add_i = 0;
        size_t cells_written = 0;

        auto process_cell = [&](uint64_t cid, int32_t oi) {
            // For each entry type: check correction → remap old → write
            auto do_entry = [&](const MappedFile& old_e, size_t geo_off, const RemapRef& rm,
                                const std::vector<CorrEntry>& corr,
                                FILE* outf, uint8_t flag_bit) -> uint32_t {
                // Check flag
                bool has = false;
                if (oi >= 0 && (size_t)oi * 20 + geo_off + 4 <= m_geo.size) {
                    uint32_t off; memcpy(&off, m_geo.data + oi * 20 + geo_off, 4);
                    has = (off != NO);
                } else if (oi >= 0) {
                    std::cerr << "OOB: oi=" << oi << " geo_off=" << geo_off << " m_geo.size=" << m_geo.size << std::endl;
                    return NO;
                }
                auto fc = flag_corrections.find(cid);
                if (fc != flag_corrections.end()) has = (fc->second & flag_bit) != 0;
                auto* corr_ids = corr_find(corr, cid);
                if (corr_ids) has = true;
                if (!has) return NO;

                if (corr_ids) return emit(outf, corr_ids->data(), corr_ids->size());

                if (oi < 0) return NO;
                if ((size_t)oi * 20 + geo_off + 4 > m_geo.size) return NO;
                uint32_t off; memcpy(&off, m_geo.data + oi * 20 + geo_off, 4);
                parse(old_e, off);
                if (buf.empty()) return NO;
                remap(buf, rm);
                return emit(outf, buf.data(), buf.size());
            };

            uint32_t so = do_entry(m_se, 8, w_rm, cs, f_se, 1);
            uint32_t ao = do_entry(m_ae, 12, a_rm, ca, f_ae, 2);
            uint32_t io = do_entry(m_ie, 16, i_rm, ci_map, f_ie, 4);
            fwrite(&cid, 8, 1, f_geo); fwrite(&so, 4, 1, f_geo); fwrite(&ao, 4, 1, f_geo); fwrite(&io, 4, 1, f_geo);
            cells_written++;
        };

        while (old_i < n_old || add_i < geo_added.size()) {
            uint64_t old_cid = UINT64_MAX, add_cid = UINT64_MAX;
            if (old_i < n_old) memcpy(&old_cid, m_geo.data + old_i * 20, 8);
            if (add_i < geo_added.size()) add_cid = geo_added[add_i];

            if (old_cid <= add_cid) {
                if (!rm_set.count(old_cid))
                    process_cell(old_cid, (int32_t)old_i);
                old_i++;
                if (old_cid == add_cid) add_i++; // skip duplicate add
            } else {
                process_cell(add_cid, -1);
                add_i++;
            }

            if (cells_written % 10000000 == 0 && cells_written > 0) {
                std::cerr << "    " << cells_written << " cells, se=" << ftell(f_se)/1024/1024
                          << "M ae=" << ftell(f_ae)/1024/1024 << "M rss=" << get_rss_mb() << "M" << std::endl;
            }
        }

        fclose(f_geo); fclose(f_se); fclose(f_ae); fclose(f_ie);
        unmap_file(m_geo); unmap_file(m_se); unmap_file(m_ae); unmap_file(m_ie);
        unmap_file(m_w_rm); unmap_file(m_a_rm); unmap_file(m_i_rm);
        std::cerr << "  Geo: " << cells_written << " cells written" << std::endl;
        malloc_trim(0);
        log_phase("  Geo entries complete", t_entry);

        // Admin: small, use existing rebuild + corrections
        {
            uint32_t no_data = 0xFFFFFFFF;
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            MappedFile m_ad_rm = mmap_remap(PatchFileId::ADMIN_POLYGONS);
            if (m_ad_rm.data) {
                const uint32_t* ad_vec = (const uint32_t*)m_ad_rm.data;
                size_t ad_count = m_ad_rm.size / 4;
                for (uint32_t i = 0; i < ad_count; i++)
                    if (ad_vec[i] != 0) ad_rm[i] = ad_vec[i] - 1; // decode +1 encoding
                unmap_file(m_ad_rm);
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_adme = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_adme, ad_rm, admin_added, admin_removed);

            auto ecit = entry_corrections.find((uint32_t)PatchFileId::ADMIN_ENTRIES);
            if (ecit != entry_corrections.end()) {
                std::unordered_map<uint64_t, const std::vector<uint32_t>*> ac_corr;
                for (auto& c : ecit->second) ac_corr[c.cell_id] = &c.ids;
                size_t n = admin.admin_cells_data.size() / 12;
                FILE* fac = fopen((out_dir + "/admin_cells.bin").c_str(), "wb");
                FILE* fae = fopen((out_dir + "/admin_entries.bin").c_str(), "wb");
                uint32_t ae_wpos = 0;
                for (size_t i = 0; i < n; i++) {
                    uint64_t cid; memcpy(&cid, admin.admin_cells_data.data()+i*12, 8);
                    fwrite(&cid, 8, 1, fac);
                    auto cit = ac_corr.find(cid);
                    if (cit != ac_corr.end()) {
                        uint32_t off = cit->second->empty() ? no_data : ae_wpos;
                        fwrite(&off, 4, 1, fac);
                        if (!cit->second->empty()) {
                            uint16_t c = cit->second->size();
                            fwrite(&c, 2, 1, fae); fwrite(cit->second->data(), 4, c, fae);
                            ae_wpos += 2 + c*4;
                        }
                    } else {
                        uint32_t old_off; memcpy(&old_off, admin.admin_cells_data.data()+i*12+8, 4);
                        if (old_off != no_data && old_off+2 <= admin.admin_entries_data.size()) {
                            uint32_t new_off = ae_wpos; fwrite(&new_off, 4, 1, fac);
                            uint16_t c; memcpy(&c, admin.admin_entries_data.data()+old_off, 2);
                            fwrite(admin.admin_entries_data.data()+old_off, 1, 2+c*4, fae);
                            ae_wpos += 2+c*4;
                        } else {
                            fwrite(&no_data, 4, 1, fac);
                        }
                    }
                }
                fclose(fac); fclose(fae);
                std::cerr << "  Admin: " << n << " cells, " << ac_corr.size() << " corrections" << std::endl;
            } else {
                write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
                write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            }
            log_phase("  Admin", t_entry);
        }
    }

    // Cleanup
    unmap_file(patch_map);
    remove(raw_path.c_str());
    { std::string cmd = "rm -rf '" + tmpdir + "'"; system(cmd.c_str()); }
    log_phase("Total", t_start);
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

-                uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8;

-                uint16_t ec; memcpy(&ec, P+pos, 2); pos += 2;

+                uint64_t cid; ps.read(&cid, 8);

+                uint16_t ec; ps.read(&ec, 2);

-                if (ec > 0) { memcpy(ids.data(), P+pos, ec*4); pos += ec*4; }

+                if (ec > 0) ps.read(ids.data(), ec*4);


String reading with `P+pos` and `strlen`:


{ const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }