                        for (size_t off : remap_offs) {
                            uint32_t v; memcpy(&v, rec_buf.data() + off, 4);
                            uint32_t nv = str_cursor.lookup(v);
                            if (nv != v) memcpy(rec_buf.data() + off, &nv, 4);
                        }

// geocoder-patch v4: Low-memory streaming patch application.
//
// Key design: mmap old files (zero copy), stream output via fwrite (never accumulate),
// process entry pipeline cell-by-cell. Target: <1 GiB peak RSS for planet.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <fcntl.h>
#include <iomanip>
#include <iostream>
#include <malloc.h>
#include <string>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unordered_map>
#include <unordered_set>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

// Streaming zstd decompressor — reads compressed file, provides sequential access
// to decompressed data via a sliding buffer. No temp file needed.
#include <zstd.h>
class PatchStream {
    FILE* input_;
    ZSTD_DCtx* dctx_;
    std::vector<char> in_buf_, out_buf_;
    size_t out_start_ = 0, out_end_ = 0; // valid range in out_buf_
    size_t global_pos_ = 0; // position of out_buf_[0] in decompressed stream
    size_t total_decompressed_ = 0;
    bool input_done_ = false;
    static constexpr size_t IN_BUF_SIZE = 256 * 1024;
    static constexpr size_t OUT_BUF_SIZE = 4 * 1024 * 1024;
    ZSTD_inBuffer zin_{};

    void decompress_more() {
        // Compact: move unread data to front
        if (out_start_ > 0) {
            size_t remain = out_end_ - out_start_;
            if (remain > 0) memmove(out_buf_.data(), out_buf_.data() + out_start_, remain);
            global_pos_ += out_start_;
            out_end_ = remain;
            out_start_ = 0;
        }
        // Ensure output buffer has room
        if (out_buf_.size() - out_end_ < OUT_BUF_SIZE)
            out_buf_.resize(out_end_ + OUT_BUF_SIZE);

        while (!input_done_ && out_end_ - out_start_ < OUT_BUF_SIZE) {
            if (zin_.pos >= zin_.size) {
                size_t n = fread(in_buf_.data(), 1, IN_BUF_SIZE, input_);
                if (n == 0) { input_done_ = true; break; }
                zin_ = {in_buf_.data(), n, 0};
            }
            ZSTD_outBuffer zout = {out_buf_.data() + out_end_, out_buf_.size() - out_end_, 0};
            size_t ret = ZSTD_decompressStream(dctx_, &zout, &zin_);
            if (ZSTD_isError(ret)) { std::cerr << "zstd error: " << ZSTD_getErrorName(ret) << std::endl; break; }
            out_end_ += zout.pos;
            total_decompressed_ = global_pos_ + out_end_;
        }
    }

public:
    bool open(const std::string& path) {
        input_ = fopen(path.c_str(), "rb");
        if (!input_) return false;
        dctx_ = ZSTD_createDCtx();
        in_buf_.resize(IN_BUF_SIZE);
        out_buf_.resize(OUT_BUF_SIZE * 2);
        decompress_more();
        return true;
    }
    ~PatchStream() { if (dctx_) ZSTD_freeDCtx(dctx_); if (input_) fclose(input_); }

    // Ensure at least n bytes are available at current read position
    void ensure(size_t n) {
        while (out_end_ - out_start_ < n && !input_done_) decompress_more();
    }

    // Read n bytes at current position into dst, advance
    void read(void* dst, size_t n) {
        ensure(n);
        memcpy(dst, out_buf_.data() + out_start_, n);
        out_start_ += n;
    }
    uint32_t read_u32() { uint32_t v; read(&v, 4); return v; }
    uint64_t read_u64() { uint64_t v; read(&v, 8); return v; }
    uint8_t read_u8() { ensure(1); return (uint8_t)out_buf_[out_start_++]; }

    // Get pointer to n bytes at current position (valid until next ensure/read)
    const char* peek(size_t n) { ensure(n); return out_buf_.data() + out_start_; }
    void skip(size_t n) { ensure(n); out_start_ += n; }

    // Write n bytes from current position directly to a FILE*, advance
    void write_to(FILE* f, size_t n) {
        while (n > 0) {
            ensure(std::min(n, (size_t)OUT_BUF_SIZE));
            size_t avail = out_end_ - out_start_;
            size_t chunk = std::min(n, avail);
            fwrite(out_buf_.data() + out_start_, 1, chunk, f);
            out_start_ += chunk;
            n -= chunk;
        }
    }

    // Read a null-terminated string at current position
    std::string read_string() {
        std::string s;
        while (true) {
            ensure(1);
            char c = out_buf_[out_start_++];
            if (c == '\0') break;
            s += c;
        }
        return s;
    }

    size_t position() const { return global_pos_ + out_start_; }
    size_t decompressed_size() const { return total_decompressed_; }
};

// --- Helpers ---
static double now_ms() {
    struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
    return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6;
}
static size_t get_rss_mb() {
    FILE* f = fopen("/proc/self/statm", "r");
    if (!f) return 0;
    size_t dummy, rss; fscanf(f, "%zu %zu", &dummy, &rss); fclose(f);
    return rss * 4096 / (1024*1024);
}
static void log_phase(const char* label, double start) {
    std::cerr << "  [" << std::fixed << std::setprecision(1) << (now_ms() - start) / 1000.0
              << "s, " << get_rss_mb() << " MiB] " << label << std::endl;
}

// mmap a file read-only. Returns {pointer, size}. Caller must munmap.
struct MappedFile { const char* data; size_t size; };
static MappedFile mmap_file(const std::string& path) {
    int fd = open(path.c_str(), O_RDONLY);
    if (fd < 0) return {nullptr, 0};
    struct stat st; fstat(fd, &st);
    size_t sz = st.st_size;
    if (sz == 0) { close(fd); return {nullptr, 0}; }
    void* p = mmap(nullptr, sz, PROT_READ, MAP_PRIVATE, fd, 0);
    close(fd);
    if (p == MAP_FAILED) return {nullptr, 0};
    return {static_cast<const char*>(p), sz};
}
static void unmap_file(MappedFile& f) {
    if (f.data) { munmap(const_cast<char*>(f.data), f.size); f.data = nullptr; f.size = 0; }
}

// Detect stride from file size
static size_t detect_stride(const std::string& path, std::initializer_list<size_t> candidates) {
    struct stat st; if (stat(path.c_str(), &st) != 0) return *candidates.begin();
    for (size_t s : candidates) if (st.st_size % s == 0 && st.st_size > 0) return s;
    return *candidates.begin();
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);
    double t_start = now_ms();

    // --- Phase 1: Streaming zstd decompression (no temp file) ---
    PatchStream ps;
    if (!ps.open(patch_path)) { std::cerr << "Failed to open patch" << std::endl; return 1; }
    std::cerr << "Streaming patch decompression..." << std::endl;
    log_phase("Open", t_start);

    // Header
    { char magic[8]; ps.read(magic, 8);
      if (memcmp(magic, GCPATCH_MAGIC, 8) != 0) { std::cerr << "Bad magic" << std::endl; return 1; } }
    uint32_t ver = ps.read_u32(); if (ver != 2) { std::cerr << "Bad version" << std::endl; return 1; }
    ps.read_u32(); // flags

    // --- Phase 2: String rebuild ---
    // Sorted vector remap: (old_offset, new_offset) pairs sorted by old_offset.
    // Uses 96 MiB for 12M entries vs 672 MiB for unordered_map.
    std::vector<std::pair<uint32_t, uint32_t>> str_remap_vec;
    uint32_t first_section_marker = 0;
    {
        uint32_t marker = ps.read_u32();
        if (marker == 0xFFFFFFF7) {
            uint32_t n_added = ps.read_u32(), n_deleted = ps.read_u32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) added.push_back(ps.read_string());
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ps.read_u32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());

            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");

            // Phase A: merge old (minus deleted) + added strings, write directly to file.
            // Both old pool and added are sorted alphabetically → merge-write with no sort.
            // Memory: only the added strings vector (~small) + del_set (~small).
            {
                FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
                size_t sp = 0; uint32_t idx = 0; size_t ai = 0;
                // Sort added strings (they should already be sorted from diff tool, but ensure)
                std::sort(added.begin(), added.end());
                while (sp < old_pool.size || ai < added.size()) {
                    const char* old_s = nullptr;
                    // Skip deleted strings
                    while (sp < old_pool.size) {
                        if (!del_set.count(idx)) { old_s = old_pool.data + sp; break; }
                        sp += strlen(old_pool.data + sp) + 1; idx++;
                    }
                    const char* add_s = ai < added.size() ? added[ai].c_str() : nullptr;
                    // Merge: pick the smaller string
                    if (old_s && add_s) {
                        int c = strcmp(old_s, add_s);
                        if (c <= 0) {
                            size_t l = strlen(old_s) + 1; fwrite(old_s, 1, l, fp);
                            sp += l; idx++;
                            if (c == 0) ai++; // duplicate — skip added
                        } else {
                            size_t l = added[ai].size() + 1; fwrite(add_s, 1, l, fp);
                            ai++;
                        }
                    } else if (old_s) {
                        size_t l = strlen(old_s) + 1; fwrite(old_s, 1, l, fp);
                        sp += l; idx++;
                    } else if (add_s) {
                        size_t l = added[ai].size() + 1; fwrite(add_s, 1, l, fp);
                        ai++;
                    } else break;
                }
                fclose(fp);
            }
            { std::vector<std::string>().swap(added); std::unordered_set<uint32_t>().swap(del_set);
              std::vector<uint32_t>().swap(del_idx); }
            malloc_trim(0);

            // Phase B: build remap by merge-walking old pool vs new pool (both sorted, both mmap'd)
            MappedFile new_pool = mmap_file(out_dir + "/strings.bin");
            {
                // Walk both pools simultaneously — no vectors needed, just cursors
                size_t old_pos = 0, new_pos = 0;
                while (old_pos < old_pool.size && new_pos < new_pool.size) {
                    const char* os = old_pool.data + old_pos;
                    const char* ns = new_pool.data + new_pos;
                    int c = strcmp(os, ns);
                    if (c == 0) {
                        if (old_pos != new_pos) // offset changed
                            str_remap_vec.push_back({(uint32_t)old_pos, (uint32_t)new_pos});
                        old_pos += strlen(os) + 1;
                        new_pos += strlen(ns) + 1;
                    } else if (c < 0) {
                        old_pos += strlen(os) + 1; // deleted string
                    } else {
                        new_pos += strlen(ns) + 1; // added string
                    }
                }
            }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
            unmap_file(old_pool);
            unmap_file(new_pool);
            std::cerr << "  Strings: +" << n_added << " -" << n_deleted
                      << ", " << str_remap_vec.size() << " remapped (" << str_remap_vec.size() * 8 / 1024 / 1024 << " MiB)" << std::endl;
            marker = ps.read_u32();
        }
        if (marker == 0xFFFFFFFE) {
            uint32_t c = ps.read_u32();
            for (uint32_t i = 0; i < c; i++) { uint32_t a = ps.read_u32(), b = ps.read_u32(); str_remap_vec.push_back({a, b}); }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
        } else {
            first_section_marker = marker; // not a remap section — pass to main loop
        }
    }
    log_phase("Strings", t_start);

    // Detect strides
    size_t way_stride = detect_stride(cur_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride(cur_dir + "/interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect_stride(cur_dir + "/admin_polygons.bin", {24, 20, 19});

    // String remap lookup — binary search for random access, cursor for sequential
    auto str_remap_lookup = [&](uint32_t old_off) -> uint32_t {
        auto it = std::lower_bound(str_remap_vec.begin(), str_remap_vec.end(),
            std::make_pair(old_off, (uint32_t)0));
        if (it != str_remap_vec.end() && it->first == old_off) return it->second;
        return old_off;
    };
    // Sequential cursor for batch remap during merge replay (O(1) amortized)
    struct StrRemapCursor {
        const std::vector<std::pair<uint32_t,uint32_t>>* vec;
        size_t idx = 0;
        void reset() { idx = 0; }
        uint32_t lookup(uint32_t old_off) {
            while (idx < vec->size() && (*vec)[idx].first < old_off) idx++;
            if (idx < vec->size() && (*vec)[idx].first == old_off) { return (*vec)[idx++].second; }
            return old_off;
        }
    };

    // --- Phase 3: Scan sections + parallel merge replays ---
    // Collect merge section info for parallel replay, process metadata inline
    struct MergeSection {
        uint32_t file_id, stride; size_t actual_stride;
        uint64_t old_size, new_size;
        uint32_t n_fixups; size_t fixup_data_pos;
        size_t seq_pos; uint64_t seq_size;
        bool needs_remap, needs_padding;
    };
    std::vector<MergeSection> merge_sections;
    std::unordered_map<uint32_t, std::vector<uint32_t>> id_remaps;
    std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;
    std::unordered_map<uint64_t, uint8_t> flag_corrections;
    struct CellCorr { uint64_t cell_id; std::vector<uint32_t> ids; };
    std::unordered_map<uint32_t, std::vector<CellCorr>> entry_corrections;

    while (true) {
        uint32_t file_id;
        if (first_section_marker != 0) { file_id = first_section_marker; first_section_marker = 0; }
        else file_id = ps.read_u32();
        if (file_id == 0xFFFFFFFF) break;

        // --- Metadata sections ---
        if (file_id == 0xFFFFFFF7) {
            // String diff in main loop — same memory-efficient approach as Phase 2
            uint32_t n_added = ps.read_u32(), n_deleted = ps.read_u32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) added.push_back(ps.read_string());
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ps.read_u32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());
            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
            struct OS { uint32_t idx; uint32_t off; const char* str; };
            std::vector<OS> old_strs;
            { size_t sp = 0; uint32_t ix = 0;
              while (sp < old_pool.size) { old_strs.push_back({ix++, (uint32_t)sp, old_pool.data+sp}); sp += strlen(old_pool.data+sp)+1; }
            }
            std::vector<const char*> merged;
            for (auto& os : old_strs) if (!del_set.count(os.idx)) merged.push_back(os.str);
            for (auto& s : added) merged.push_back(s.c_str());
            std::sort(merged.begin(), merged.end(), [](const char* a, const char* b) { return strcmp(a,b) < 0; });
            FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
            uint32_t wpos = 0;
            for (const char* s : merged) { size_t l = strlen(s)+1; fwrite(s, 1, l, fp); wpos += l; }
            fclose(fp);
            // Build remap via merge-walk
            MappedFile new_pool = mmap_file(out_dir + "/strings.bin");
            std::vector<std::pair<const char*, uint32_t>> new_strs;
            { size_t sp = 0;
              while (sp < new_pool.size) { new_strs.push_back({new_pool.data+sp, (uint32_t)sp}); sp += strlen(new_pool.data+sp)+1; }
            }
            size_t oi2 = 0, ni2 = 0;
            while (oi2 < old_strs.size() && ni2 < new_strs.size()) {
                int c = strcmp(old_strs[oi2].str, new_strs[ni2].first);
                if (c == 0) {
                    if (old_strs[oi2].off != new_strs[ni2].second)
                        str_remap_vec.push_back({old_strs[oi2].off, new_strs[ni2].second});
                    oi2++; ni2++;
                } else if (c < 0) { oi2++; } else { ni2++; }
            }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
            unmap_file(old_pool); unmap_file(new_pool);
            std::cerr << "  Strings (loop): +" << n_added << " -" << n_deleted << " → " << merged.size()
                      << ", " << str_remap_vec.size() << " remapped" << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_GEO_MARKER) {
            uint32_t na = ps.read_u32(), nr = ps.read_u32();
            geo_added.resize(na); geo_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { ps.read(&geo_added[i], 8); }
            for (uint32_t i = 0; i < nr; i++) { ps.read(&geo_removed[i], 8); }
            std::cerr << "  Geo cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_ADMIN_MARKER) {
            uint32_t na = ps.read_u32(), nr = ps.read_u32();
            admin_added.resize(na); admin_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { ps.read(&admin_added[i], 8); }
            for (uint32_t i = 0; i < nr; i++) { ps.read(&admin_removed[i], 8); }
            std::cerr << "  Admin cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_FLAGS_MARKER) {
            uint32_t c = ps.read_u32();
            for (uint32_t i = 0; i < c; i++) { uint64_t cid; ps.read(&cid, 8); flag_corrections[cid] = ps.read_u8(); }
            std::cerr << "  Flag corrections: " << c << std::endl;
            continue;
        }
        if (file_id == SECONDARY_REMAP_MARKER) {
            // Apply secondary remaps directly to the temp remap files on disk
            uint32_t nf = ps.read_u32();
            for (uint32_t f = 0; f < nf; f++) {
                uint32_t fid = ps.read_u32(), np = ps.read_u32();
                std::string remap_path = tmpdir + "/remap_" + std::to_string(fid) + ".bin";
                // Open remap file for read-write
                int rfd = open(remap_path.c_str(), O_RDWR);
                if (rfd >= 0) {
                    struct stat st; fstat(rfd, &st);
                    size_t remap_size = st.st_size;
                    uint32_t* remap_data = static_cast<uint32_t*>(
                        mmap(nullptr, remap_size, PROT_READ | PROT_WRITE, MAP_SHARED, rfd, 0));
                    size_t remap_count = remap_size / 4;
                    for (uint32_t i = 0; i < np; i++) {
                        uint32_t o = ps.read_u32(), n = ps.read_u32();
                        if (o < remap_count) remap_data[o] = n + 1; // +1 encoding
                    }
                    munmap(remap_data, remap_size);
                    close(rfd);
                } else {
                    // Skip data if file doesn't exist
                    for (uint32_t i = 0; i < np; i++) { ps.read_u32(); ps.read_u32(); }
                }
                std::cerr << "  Secondary remap " << fid << ": " << np << " pairs" << std::endl;
            }
            continue;
        }
        if (file_id == ENTRY_CORRECTION_MARKER) {
            uint32_t fid = ps.read_u32(), c = ps.read_u32();
            auto& list = entry_corrections[fid];
            for (uint32_t i = 0; i < c; i++) {
                uint64_t cid; ps.read(&cid, 8);
                uint16_t ec; ps.read(&ec, 2);
                std::vector<uint32_t> ids(ec);
                if (ec > 0) ps.read(ids.data(), ec*4);
                list.push_back({cid, std::move(ids)});
            }
            std::cerr << "  Entry corrections " << fid << ": " << c << std::endl;
            continue;
        }

        // --- Collect merge section info (replay happens in parallel below) ---
        uint32_t stride = ps.read_u32();
        uint64_t old_size = ps.read_u64(), new_size = ps.read_u64();
        if (file_id >= (uint32_t)PatchFileId::COUNT) { std::cerr << "Unknown file " << file_id << std::endl; break; }
        const char* fname = patch_file_names[file_id];

        if (stride == 0) {
            uint32_t nf = ps.read_u32(); (void)nf; uint64_t ds = ps.read_u64();
            FILE* fp = fopen((out_dir + "/" + fname).c_str(), "wb");
            ps.write_to(fp, ds); fclose(fp);
            std::cerr << "  " << fname << ": full replace " << ds << " bytes" << std::endl;
            continue;
        }
        if (stride == 0xFE) { uint32_t nf = ps.read_u32(); (void)nf; uint64_t ds = ps.read_u64(); ps.skip(ds); continue; }

        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        bool needs_remap = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
                            file_id == (uint32_t)PatchFileId::STREET_WAYS ||
                            file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
                            file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
        bool needs_padding = (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24) ||
                             (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24);

        uint32_t n_fixups = ps.read_u32();
        // Read fixup data into a small buffer (decoded lazily during merge replay)
        std::vector<char> fixup_buf;
        if (n_fixups > 0) {
            uint32_t delta_size = ps.read_u32();
            fixup_buf.resize(delta_size);
            ps.read(fixup_buf.data(), delta_size);
        }
        // Lazy fixup decoder state (reads from patch mmap on demand)
        struct FixupDecoder {
            const char* data; size_t read_pos;
            uint32_t prev_idx = 0, prev_val = 0;
            uint32_t cur_idx = UINT32_MAX, cur_val = 0;
            uint32_t remaining;
            void init(const char* d, size_t p, uint32_t count) {
                data = d; read_pos = p; remaining = count;
                if (remaining > 0) advance();
            }
            void advance() {
                if (remaining == 0) { cur_idx = UINT32_MAX; return; }
                cur_idx = prev_idx + read_varint(data, read_pos);
                cur_val = prev_val + read_varint(data, read_pos);
                prev_idx = cur_idx; prev_val = cur_val;
                remaining--;
            }
            // Advance past all fixups with index < target
            bool lookup(uint32_t target_idx, uint32_t& out_val) {
                while (cur_idx < target_idx && (remaining > 0 || cur_idx != UINT32_MAX)) advance();
                if (cur_idx == target_idx) { out_val = cur_val; advance(); return true; }
                return false;
            }
        } fixup_dec;
        fixup_dec.init(fixup_buf.data(), 0, n_fixups);

        // Get string remap field offsets for this file type
        std::vector<size_t> remap_offs;
        if (!str_remap_vec.empty() && needs_remap) {
            if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) remap_offs = {8, 12};
            else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) remap_offs = {(actual_stride == 12) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) remap_offs = {(actual_stride >= 20) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) remap_offs = {8};
        }

        // mmap old file read-only (zero allocation)
        MappedFile old_mmap = mmap_file(cur_dir + "/" + std::string(fname));
        madvise(const_cast<char*>(old_mmap.data), old_mmap.size, MADV_SEQUENTIAL);
        size_t n_old_records = old_mmap.size / actual_stride;

        // Replay merge sequence — stream output, apply remap/fixups per-record inline
        uint64_t seq_size = ps.read_u64();
        size_t seq_start_pos = ps.position();
        size_t seq_bytes_read = 0;
        FILE* outf = fopen((out_dir + "/" + fname).c_str(), "wb");

        bool track = needs_remap; // track ID remap only for data files
        // File-backed id_map: create temp file, fill with 0xFF, mmap read-write
        int remap_fd = -1;
        uint32_t* id_map_ptr = nullptr;
        std::string remap_path;
        if (track) {
            remap_path = tmpdir + "/remap_" + std::to_string(file_id) + ".bin";
            remap_fd = open(remap_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644);
            size_t remap_bytes = n_old_records * 4;
            ftruncate(remap_fd, remap_bytes); // zero-filled by OS (sparse)
            id_map_ptr = static_cast<uint32_t*>(
                mmap(nullptr, remap_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, remap_fd, 0));
            // File is zero-filled. We store new_index+1 so that 0 means "unmapped".
            // The entry pipeline reads these and subtracts 1.
        }

        // Per-record buffer for applying remap/fixups inline
        std::vector<char> rec_buf(actual_stride);
        // Sequential string remap cursor (O(1) amortized for sorted data)
        StrRemapCursor str_cursor;
        str_cursor.vec = &str_remap_vec;
        str_cursor.reset();

        size_t old_rec = 0, new_rec = 0, old_bytes = 0, written = 0;
        while (seq_bytes_read < seq_size) {
            uint8_t op = ps.read_u8(); seq_bytes_read++;
            uint32_t count = ps.read_u32(); seq_bytes_read += 4;
            if (op == OP_MATCH_RUN) {
                for (uint32_t k = 0; k < count; k++) {
                    size_t rec_off = old_bytes + k * actual_stride;
                    if (rec_off + actual_stride > old_mmap.size) break;

                    bool modified = false;
                    // Check if this record needs any modification
                    if (!remap_offs.empty() || needs_padding) modified = true;
                    // Check fixup using lazy decoder (reads from patch mmap, zero allocation)
                    uint32_t fixup_val = 0;
                    bool has_fixup = fixup_dec.lookup(old_rec + k, fixup_val);
                    if (has_fixup) modified = true;

                    if (modified) {
                        memcpy(rec_buf.data(), old_mmap.data + rec_off, actual_stride);
                        // Apply padding zeroing
                        if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24)
                            memset(rec_buf.data() + 13, 0, 3);
                        if (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24) {
                            memset(rec_buf.data() + 5, 0, 3);
                            memset(rec_buf.data() + 21, 0, 3);
                        }
                        // Apply string remap
                        for (size_t off : remap_offs) {
                            uint32_t v; memcpy(&v, rec_buf.data() + off, 4);
                            uint32_t nv = str_remap_lookup(v);
                            if (nv != v) memcpy(rec_buf.data() + off, &nv, 4);
                        }
                        // Apply fixup (node_offset/vertex_offset at byte 0)
                        if (has_fixup)
                            memcpy(rec_buf.data(), &fixup_val, 4);
                        fwrite(rec_buf.data(), 1, actual_stride, outf);
                    } else {
                        // No modification needed — write directly from mmap
                        fwrite(old_mmap.data + rec_off, 1, actual_stride, outf);
                    }
                    if (track && old_rec + k < n_old_records) id_map_ptr[old_rec + k] = (uint32_t)(new_rec + k) + 1; // +1 so 0 means unmapped
                }
                written += count * actual_stride;
                old_rec += count; new_rec += count; old_bytes += count * actual_stride;
            } else if (op == OP_INSERT_RUN) {
                size_t bytes = count * actual_stride;
                ps.write_to(outf, bytes);
                written += bytes; seq_bytes_read += bytes; new_rec += count;
            } else if (op == OP_DELETE_RUN) {
                old_rec += count; old_bytes += count * actual_stride;
            }
        }
        fclose(outf);
        unmap_file(old_mmap);

        if (track) {
            // Sync and munmap the file-backed remap (data is already on disk)
            size_t remap_bytes = n_old_records * 4;
            msync(id_map_ptr, remap_bytes, MS_SYNC);
            munmap(id_map_ptr, remap_bytes);
            close(remap_fd);
            id_map_ptr = nullptr;
            id_remaps[file_id] = {}; // empty placeholder
            std::cerr << "  " << fname << ": " << written << " bytes (remap " << n_old_records << " records to disk)" << std::endl;
        } else {
            std::cerr << "  " << fname << ": " << written << " bytes" << std::endl;
        }
        malloc_trim(0);
    }
    log_phase("Merge replays", t_start);

    // Free string remap
    { std::vector<std::pair<uint32_t,uint32_t>>().swap(str_remap_vec); }
    malloc_trim(0);

    // --- Phase 4: Streaming entry pipeline ---
    // Walk old geo_cells, remap IDs, apply corrections, write output directly to files.
    {
        double t_entry = now_ms();
        std::cerr << "Entry pipeline (lean streaming)..." << std::endl;

        // mmap remap files from disk (only pages accessed are in RSS)
        auto mmap_remap = [&](PatchFileId fid) -> MappedFile {
            std::string path = tmpdir + "/remap_" + std::to_string((uint32_t)fid) + ".bin";
            auto m = mmap_file(path);
            if (m.data) madvise(const_cast<char*>(m.data), m.size, MADV_SEQUENTIAL);
            return m;
        };
        // Helper: look up remap value from mmap'd file
        // Remap values stored as new_index+1 (0 means unmapped)
        struct RemapRef {
            const uint32_t* data;
            size_t count;
            uint32_t operator[](size_t i) const {
                if (i >= count) return 0xFFFFFFFF;
                uint32_t v = data[i];
                return v == 0 ? 0xFFFFFFFF : v - 1;
            }
            size_t size() const { return count; }
        };
        MappedFile m_w_rm = mmap_remap(PatchFileId::STREET_WAYS);
        MappedFile m_a_rm = mmap_remap(PatchFileId::ADDR_POINTS);
        MappedFile m_i_rm = mmap_remap(PatchFileId::INTERP_WAYS);
        RemapRef w_rm = {(const uint32_t*)m_w_rm.data, m_w_rm.size / 4};
        RemapRef a_rm = {(const uint32_t*)m_a_rm.data, m_a_rm.size / 4};
        RemapRef i_rm = {(const uint32_t*)m_i_rm.data, m_i_rm.size / 4};

        // mmap old files (zero RSS until pages are accessed, then only working set)
        MappedFile m_geo = mmap_file(cur_dir + "/geo_cells.bin");
        MappedFile m_se = mmap_file(cur_dir + "/street_entries.bin");
        MappedFile m_ae = mmap_file(cur_dir + "/addr_entries.bin");
        MappedFile m_ie = mmap_file(cur_dir + "/interp_entries.bin");
        madvise(const_cast<char*>(m_se.data), m_se.size, MADV_SEQUENTIAL);
        madvise(const_cast<char*>(m_ae.data), m_ae.size, MADV_SEQUENTIAL);
        madvise(const_cast<char*>(m_ie.data), m_ie.size, MADV_SEQUENTIAL);
        size_t n_old = m_geo.size / 20;

        // Build sorted correction vectors + removed set (better cache locality than hash maps)
        std::unordered_set<uint64_t> rm_set(geo_removed.begin(), geo_removed.end());
        struct CorrEntry { uint64_t cid; const std::vector<uint32_t>* ids; };
        std::vector<CorrEntry> cs, ca, ci_map;
        for (auto& [fid, list] : entry_corrections) {
            auto* v = (fid == (uint32_t)PatchFileId::STREET_ENTRIES) ? &cs :
                      (fid == (uint32_t)PatchFileId::ADDR_ENTRIES) ? &ca :
                      (fid == (uint32_t)PatchFileId::INTERP_ENTRIES) ? &ci_map : nullptr;
            if (v) for (auto& c : list) v->push_back({c.cell_id, &c.ids});
        }
        auto corr_cmp = [](const CorrEntry& a, const CorrEntry& b) { return a.cid < b.cid; };
        std::sort(cs.begin(), cs.end(), corr_cmp);
        std::sort(ca.begin(), ca.end(), corr_cmp);
        std::sort(ci_map.begin(), ci_map.end(), corr_cmp);
        // Binary search helper
        auto corr_find = [](const std::vector<CorrEntry>& v, uint64_t cid) -> const std::vector<uint32_t>* {
            auto it = std::lower_bound(v.begin(), v.end(), CorrEntry{cid, nullptr},
                [](const CorrEntry& a, const CorrEntry& b) { return a.cid < b.cid; });
            return (it != v.end() && it->cid == cid) ? it->ids : nullptr;
        };
        // Added cells sorted
        std::sort(geo_added.begin(), geo_added.end());
        log_phase("  Setup", t_entry);

        // Open 4 output files
        FILE* f_geo = fopen((out_dir + "/geo_cells.bin").c_str(), "wb");
        FILE* f_se = fopen((out_dir + "/street_entries.bin").c_str(), "wb");
        FILE* f_ae = fopen((out_dir + "/addr_entries.bin").c_str(), "wb");
        FILE* f_ie = fopen((out_dir + "/interp_entries.bin").c_str(), "wb");
        constexpr uint32_t NO = 0xFFFFFFFF;

        // Reusable buffer (one per entry type to avoid aliasing issues)
        std::vector<uint32_t> buf;
        buf.reserve(4096);

        // Helper: parse IDs from mmap'd entry file
        auto parse = [&buf](const MappedFile& f, uint32_t off) {
            buf.clear();
            if (off == NO || off + 2 > f.size) return;
            uint16_t c; memcpy(&c, f.data + off, 2);
            if (off + 2 + (size_t)c * 4 > f.size) return;
            buf.resize(c);
            memcpy(buf.data(), f.data + off + 2, c * 4);
        };
        auto remap = [](std::vector<uint32_t>& ids, const RemapRef& rm) {
            constexpr uint32_t NO2 = 0xFFFFFFFF;
            for (auto& id : ids) if (id < rm.size() && rm[id] != NO2) id = rm[id];
            std::sort(ids.begin(), ids.end());
        };
        // Write entry and return offset, or NO if empty
        auto emit = [&NO](FILE* f, const uint32_t* ids, size_t n) -> uint32_t {
            if (n == 0) return NO;
            uint32_t off = (uint32_t)ftell(f);
            uint16_t c = (uint16_t)n;
            fwrite(&c, 2, 1, f); fwrite(ids, 4, n, f);
            return off;
        };

        // Merge-walk: old cells + added cells in sorted order
        // Both are sorted by cell_id. Merge them, skip removed.
        size_t old_i = 0, add_i = 0;
        size_t cells_written = 0;

        auto process_cell = [&](uint64_t cid, int32_t oi) {
            // For each entry type: check correction → remap old → write
            auto do_entry = [&](const MappedFile& old_e, size_t geo_off, const RemapRef& rm,
                                const std::vector<CorrEntry>& corr,
                                FILE* outf, uint8_t flag_bit) -> uint32_t {
                // Check flag
                bool has = false;
                if (oi >= 0 && (size_t)oi * 20 + geo_off + 4 <= m_geo.size) {
                    uint32_t off; memcpy(&off, m_geo.data + oi * 20 + geo_off, 4);
                    has = (off != NO);
                } else if (oi >= 0) {
                    std::cerr << "OOB: oi=" << oi << " geo_off=" << geo_off << " m_geo.size=" << m_geo.size << std::endl;
                    return NO;
                }
                auto fc = flag_corrections.find(cid);
                if (fc != flag_corrections.end()) has = (fc->second & flag_bit) != 0;
                auto* corr_ids = corr_find(corr, cid);
                if (corr_ids) has = true;
                if (!has) return NO;

                if (corr_ids) return emit(outf, corr_ids->data(), corr_ids->size());

                if (oi < 0) return NO;
                if ((size_t)oi * 20 + geo_off + 4 > m_geo.size) return NO;
                uint32_t off; memcpy(&off, m_geo.data + oi * 20 + geo_off, 4);
                parse(old_e, off);
                if (buf.empty()) return NO;
                remap(buf, rm);
                return emit(outf, buf.data(), buf.size());
            };

            uint32_t so = do_entry(m_se, 8, w_rm, cs, f_se, 1);
            uint32_t ao = do_entry(m_ae, 12, a_rm, ca, f_ae, 2);
            uint32_t io = do_entry(m_ie, 16, i_rm, ci_map, f_ie, 4);
            fwrite(&cid, 8, 1, f_geo); fwrite(&so, 4, 1, f_geo); fwrite(&ao, 4, 1, f_geo); fwrite(&io, 4, 1, f_geo);
            cells_written++;
        };

        while (old_i < n_old || add_i < geo_added.size()) {
            uint64_t old_cid = UINT64_MAX, add_cid = UINT64_MAX;
            if (old_i < n_old) memcpy(&old_cid, m_geo.data + old_i * 20, 8);
            if (add_i < geo_added.size()) add_cid = geo_added[add_i];

            if (old_cid <= add_cid) {
                if (!rm_set.count(old_cid))
                    process_cell(old_cid, (int32_t)old_i);
                old_i++;
                if (old_cid == add_cid) add_i++; // skip duplicate add
            } else {
                process_cell(add_cid, -1);
                add_i++;
            }

            if (cells_written % 10000000 == 0 && cells_written > 0) {
                std::cerr << "    " << cells_written << " cells, se=" << ftell(f_se)/1024/1024
                          << "M ae=" << ftell(f_ae)/1024/1024 << "M rss=" << get_rss_mb() << "M" << std::endl;
            }
        }

        fclose(f_geo); fclose(f_se); fclose(f_ae); fclose(f_ie);
        unmap_file(m_geo); unmap_file(m_se); unmap_file(m_ae); unmap_file(m_ie);
        unmap_file(m_w_rm); unmap_file(m_a_rm); unmap_file(m_i_rm);
        std::cerr << "  Geo: " << cells_written << " cells written" << std::endl;
        malloc_trim(0);
        log_phase("  Geo entries complete", t_entry);

        // Admin: small, use existing rebuild + corrections
        {
            uint32_t no_data = 0xFFFFFFFF;
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            MappedFile m_ad_rm = mmap_remap(PatchFileId::ADMIN_POLYGONS);
            if (m_ad_rm.data) {
                const uint32_t* ad_vec = (const uint32_t*)m_ad_rm.data;
                size_t ad_count = m_ad_rm.size / 4;
                for (uint32_t i = 0; i < ad_count; i++)
                    if (ad_vec[i] != 0) ad_rm[i] = ad_vec[i] - 1; // decode +1 encoding
                unmap_file(m_ad_rm);
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_adme = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_adme, ad_rm, admin_added, admin_removed);

            auto ecit = entry_corrections.find((uint32_t)PatchFileId::ADMIN_ENTRIES);
            if (ecit != entry_corrections.end()) {
                std::unordered_map<uint64_t, const std::vector<uint32_t>*> ac_corr;
                for (auto& c : ecit->second) ac_corr[c.cell_id] = &c.ids;
                size_t n = admin.admin_cells_data.size() / 12;
                FILE* fac = fopen((out_dir + "/admin_cells.bin").c_str(), "wb");
                FILE* fae = fopen((out_dir + "/admin_entries.bin").c_str(), "wb");
                uint32_t ae_wpos = 0;
                for (size_t i = 0; i < n; i++) {
                    uint64_t cid; memcpy(&cid, admin.admin_cells_data.data()+i*12, 8);
                    fwrite(&cid, 8, 1, fac);
                    auto cit = ac_corr.find(cid);
                    if (cit != ac_corr.end()) {
                        uint32_t off = cit->second->empty() ? no_data : ae_wpos;
                        fwrite(&off, 4, 1, fac);
                        if (!cit->second->empty()) {
                            uint16_t c = cit->second->size();
                            fwrite(&c, 2, 1, fae); fwrite(cit->second->data(), 4, c, fae);
                            ae_wpos += 2 + c*4;
                        }
                    } else {
                        uint32_t old_off; memcpy(&old_off, admin.admin_cells_data.data()+i*12+8, 4);
                        if (old_off != no_data && old_off+2 <= admin.admin_entries_data.size()) {
                            uint32_t new_off = ae_wpos; fwrite(&new_off, 4, 1, fac);
                            uint16_t c; memcpy(&c, admin.admin_entries_data.data()+old_off, 2);
                            fwrite(admin.admin_entries_data.data()+old_off, 1, 2+c*4, fae);
                            ae_wpos += 2+c*4;
                        } else {
                            fwrite(&no_data, 4, 1, fac);
                        }
                    }
                }
                fclose(fac); fclose(fae);
                std::cerr << "  Admin: " << n << " cells, " << ac_corr.size() << " corrections" << std::endl;
            } else {
                write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
                write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            }
            log_phase("  Admin", t_entry);
        }
    }

    // Cleanup
    { std::string cmd = "rm -rf '" + tmpdir + "'"; system(cmd.c_str()); }
    log_phase("Total", t_start);
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

-                            uint32_t nv = str_remap_lookup(v);

+                            uint32_t nv = str_cursor.lookup(v);


Wait — the cursor approach assumes string offsets in records are monotonically increasing. But MATCH records are from the old file in order, and DELETE/INSERT interleave. After a DELETE+INSERT, the next MATCH starts from a different old record. The string offsets in old records ARE in order (sorted deterministically), but after a DELETE gap, the next MATCH's string offsets might be less than the cursor position.

Actually, the cursor tracks positions in the `str_remap_vec` (sorted by old_offset). The old records are also sorted by content. For MATCH ops, we process old records in order, so their string offsets are monotonically increasing. The cursor will work for MATCH records. For INSERT records (from patch data), we don't apply string remap. For DELETE records, we skip them — the cursor just stays where it is.

But there's a subtlety: after a DELETE, the next MATCH's old records might have string offsets that are LESS than the cursor's current position (if the deleted records had higher offsets). Actually no — old records are in order, and MATCH processes them in order. DELETE skips forward. The next MATCH continues from where DELETE left off. So string offsets always increase across MATCHes.

The cursor should work. Let me also reset the cursor at the start of each file's replay:


Bash

make -j$(nproc) geocoder-patch 2>&1 | tail -3

Build with cursor remap


[ 50%] Building CXX object CMakeFiles/geocoder-patch.dir/tools/geocoder_patch.cpp.o
[100%] Linking CXX executable geocoder-patch
[100%] Built target geocoder-patch


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "kill -9 \$(ps aux | grep geocoder-patch | grep -v grep | awk '{print \$2}') 2>/dev/null; rm -f /home/michtest/geocoder-patch; rm -rf /home/michtest/patched-test/" 2>&1 && sshpass -p 'elon-is-insane' scp -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:/home/michtest/ 2>&1 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 'cd /home/michtest && rm -rf patched-test/ && time ./geocoder-patch det-A/ validated-patches/europe-mar21-to-mar27-v2.gcpatch -o patched-test/ 2>&1 | grep -E "\\[.*MiB\\]|real" && for f in det-today/*; do b=$(basename $f); cmp -s patched-test/$b det-today/$b || echo "MISMATCH: $b"; done && echo "Europe: ALL MATCH" && rm -rf patched-test/' 2>&1

Europe test with cursor remap + streaming zstd


Command running in background with ID: bko59hdxj. Output is being written to: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bko59hdxj.output


sleep 60 && cat /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bko59hdxj.output 2>/dev/null

Check 1 min


(Bash completed with no output)


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "ls /home/michtest/patched-test/ 2>/dev/null | wc -l && ps aux | grep geocoder-patch | grep -v grep | awk '{printf \"RSS: %.0f MiB\\n\", \$6/1024}'" 2>&1

Quick check


2
RSS: 97 MiB


Still only 2 files at 1 min. The issue isn't the binary search — it's the streaming decompression overhead per read. The merge replay does millions of small reads (`read_u8`, `read_u32`) through the PatchStream, each requiring buffer boundary checks and potential decompress_more calls. The mmap approach is zero-overhead for these reads.

The problem: the PatchStream is fundamentally slower for many small reads compared to mmap. The decompression bandwidth is fine, but the function call overhead for each 4-byte read is killing performance.

The fix: buffer the merge sequence data. Read the entire merge sequence section into a local buffer at the start of each file's replay, then process from the buffer. This gives mmap-like performance for the hot loop:


Edit

        // Replay merge sequence — stream output, apply remap/fixups per-record inline
        uint64_t seq_size = ps.read_u64();
        size_t seq_start_pos = ps.position();
        size_t seq_bytes_read = 0;

        // Read entire merge sequence into local buffer (fast sequential decompression)
        uint64_t seq_size = ps.read_u64();
        std::vector<char> seq_buf(seq_size);
        ps.read(seq_buf.data(), seq_size);
        size_t seq_pos = 0;


The file /root/traccar-geocoder/builder/tools/geocoder_patch.cpp has been updated successfully.

        // Read entire merge sequence into local buffer (fast sequential decompression)
        uint64_t seq_size = ps.read_u64();
        std::vector<char> seq_buf(seq_size);
        ps.read(seq_buf.data(), seq_size);
        size_t seq_pos = 0;

// geocoder-patch v4: Low-memory streaming patch application.
//
// Key design: mmap old files (zero copy), stream output via fwrite (never accumulate),
// process entry pipeline cell-by-cell. Target: <1 GiB peak RSS for planet.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <fcntl.h>
#include <iomanip>
#include <iostream>
#include <malloc.h>
#include <string>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unordered_map>
#include <unordered_set>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

// Streaming zstd decompressor — reads compressed file, provides sequential access
// to decompressed data via a sliding buffer. No temp file needed.
#include <zstd.h>
class PatchStream {
    FILE* input_;
    ZSTD_DCtx* dctx_;
    std::vector<char> in_buf_, out_buf_;
    size_t out_start_ = 0, out_end_ = 0; // valid range in out_buf_
    size_t global_pos_ = 0; // position of out_buf_[0] in decompressed stream
    size_t total_decompressed_ = 0;
    bool input_done_ = false;
    static constexpr size_t IN_BUF_SIZE = 256 * 1024;
    static constexpr size_t OUT_BUF_SIZE = 4 * 1024 * 1024;
    ZSTD_inBuffer zin_{};

    void decompress_more() {
        // Compact: move unread data to front
        if (out_start_ > 0) {
            size_t remain = out_end_ - out_start_;
            if (remain > 0) memmove(out_buf_.data(), out_buf_.data() + out_start_, remain);
            global_pos_ += out_start_;
            out_end_ = remain;
            out_start_ = 0;
        }
        // Ensure output buffer has room
        if (out_buf_.size() - out_end_ < OUT_BUF_SIZE)
            out_buf_.resize(out_end_ + OUT_BUF_SIZE);

        while (!input_done_ && out_end_ - out_start_ < OUT_BUF_SIZE) {
            if (zin_.pos >= zin_.size) {
                size_t n = fread(in_buf_.data(), 1, IN_BUF_SIZE, input_);
                if (n == 0) { input_done_ = true; break; }
                zin_ = {in_buf_.data(), n, 0};
            }
            ZSTD_outBuffer zout = {out_buf_.data() + out_end_, out_buf_.size() - out_end_, 0};
            size_t ret = ZSTD_decompressStream(dctx_, &zout, &zin_);
            if (ZSTD_isError(ret)) { std::cerr << "zstd error: " << ZSTD_getErrorName(ret) << std::endl; break; }
            out_end_ += zout.pos;
            total_decompressed_ = global_pos_ + out_end_;
        }
    }

public:
    bool open(const std::string& path) {
        input_ = fopen(path.c_str(), "rb");
        if (!input_) return false;
        dctx_ = ZSTD_createDCtx();
        in_buf_.resize(IN_BUF_SIZE);
        out_buf_.resize(OUT_BUF_SIZE * 2);
        decompress_more();
        return true;
    }
    ~PatchStream() { if (dctx_) ZSTD_freeDCtx(dctx_); if (input_) fclose(input_); }

    // Ensure at least n bytes are available at current read position
    void ensure(size_t n) {
        while (out_end_ - out_start_ < n && !input_done_) decompress_more();
    }

    // Read n bytes at current position into dst, advance
    void read(void* dst, size_t n) {
        ensure(n);
        memcpy(dst, out_buf_.data() + out_start_, n);
        out_start_ += n;
    }
    uint32_t read_u32() { uint32_t v; read(&v, 4); return v; }
    uint64_t read_u64() { uint64_t v; read(&v, 8); return v; }
    uint8_t read_u8() { ensure(1); return (uint8_t)out_buf_[out_start_++]; }

    // Get pointer to n bytes at current position (valid until next ensure/read)
    const char* peek(size_t n) { ensure(n); return out_buf_.data() + out_start_; }
    void skip(size_t n) { ensure(n); out_start_ += n; }

    // Write n bytes from current position directly to a FILE*, advance
    void write_to(FILE* f, size_t n) {
        while (n > 0) {
            ensure(std::min(n, (size_t)OUT_BUF_SIZE));
            size_t avail = out_end_ - out_start_;
            size_t chunk = std::min(n, avail);
            fwrite(out_buf_.data() + out_start_, 1, chunk, f);
            out_start_ += chunk;
            n -= chunk;
        }
    }

    // Read a null-terminated string at current position
    std::string read_string() {
        std::string s;
        while (true) {
            ensure(1);
            char c = out_buf_[out_start_++];
            if (c == '\0') break;
            s += c;
        }
        return s;
    }

    size_t position() const { return global_pos_ + out_start_; }
    size_t decompressed_size() const { return total_decompressed_; }
};

// --- Helpers ---
static double now_ms() {
    struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
    return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6;
}
static size_t get_rss_mb() {
    FILE* f = fopen("/proc/self/statm", "r");
    if (!f) return 0;
    size_t dummy, rss; fscanf(f, "%zu %zu", &dummy, &rss); fclose(f);
    return rss * 4096 / (1024*1024);
}
static void log_phase(const char* label, double start) {
    std::cerr << "  [" << std::fixed << std::setprecision(1) << (now_ms() - start) / 1000.0
              << "s, " << get_rss_mb() << " MiB] " << label << std::endl;
}

// mmap a file read-only. Returns {pointer, size}. Caller must munmap.
struct MappedFile { const char* data; size_t size; };
static MappedFile mmap_file(const std::string& path) {
    int fd = open(path.c_str(), O_RDONLY);
    if (fd < 0) return {nullptr, 0};
    struct stat st; fstat(fd, &st);
    size_t sz = st.st_size;
    if (sz == 0) { close(fd); return {nullptr, 0}; }
    void* p = mmap(nullptr, sz, PROT_READ, MAP_PRIVATE, fd, 0);
    close(fd);
    if (p == MAP_FAILED) return {nullptr, 0};
    return {static_cast<const char*>(p), sz};
}
static void unmap_file(MappedFile& f) {
    if (f.data) { munmap(const_cast<char*>(f.data), f.size); f.data = nullptr; f.size = 0; }
}

// Detect stride from file size
static size_t detect_stride(const std::string& path, std::initializer_list<size_t> candidates) {
    struct stat st; if (stat(path.c_str(), &st) != 0) return *candidates.begin();
    for (size_t s : candidates) if (st.st_size % s == 0 && st.st_size > 0) return s;
    return *candidates.begin();
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);
    double t_start = now_ms();

    // --- Phase 1: Streaming zstd decompression (no temp file) ---
    PatchStream ps;
    if (!ps.open(patch_path)) { std::cerr << "Failed to open patch" << std::endl; return 1; }
    std::cerr << "Streaming patch decompression..." << std::endl;
    log_phase("Open", t_start);

    // Header
    { char magic[8]; ps.read(magic, 8);
      if (memcmp(magic, GCPATCH_MAGIC, 8) != 0) { std::cerr << "Bad magic" << std::endl; return 1; } }
    uint32_t ver = ps.read_u32(); if (ver != 2) { std::cerr << "Bad version" << std::endl; return 1; }
    ps.read_u32(); // flags

    // --- Phase 2: String rebuild ---
    // Sorted vector remap: (old_offset, new_offset) pairs sorted by old_offset.
    // Uses 96 MiB for 12M entries vs 672 MiB for unordered_map.
    std::vector<std::pair<uint32_t, uint32_t>> str_remap_vec;
    uint32_t first_section_marker = 0;
    {
        uint32_t marker = ps.read_u32();
        if (marker == 0xFFFFFFF7) {
            uint32_t n_added = ps.read_u32(), n_deleted = ps.read_u32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) added.push_back(ps.read_string());
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ps.read_u32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());

            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");

            // Phase A: merge old (minus deleted) + added strings, write directly to file.
            // Both old pool and added are sorted alphabetically → merge-write with no sort.
            // Memory: only the added strings vector (~small) + del_set (~small).
            {
                FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
                size_t sp = 0; uint32_t idx = 0; size_t ai = 0;
                // Sort added strings (they should already be sorted from diff tool, but ensure)
                std::sort(added.begin(), added.end());
                while (sp < old_pool.size || ai < added.size()) {
                    const char* old_s = nullptr;
                    // Skip deleted strings
                    while (sp < old_pool.size) {
                        if (!del_set.count(idx)) { old_s = old_pool.data + sp; break; }
                        sp += strlen(old_pool.data + sp) + 1; idx++;
                    }
                    const char* add_s = ai < added.size() ? added[ai].c_str() : nullptr;
                    // Merge: pick the smaller string
                    if (old_s && add_s) {
                        int c = strcmp(old_s, add_s);
                        if (c <= 0) {
                            size_t l = strlen(old_s) + 1; fwrite(old_s, 1, l, fp);
                            sp += l; idx++;
                            if (c == 0) ai++; // duplicate — skip added
                        } else {
                            size_t l = added[ai].size() + 1; fwrite(add_s, 1, l, fp);
                            ai++;
                        }
                    } else if (old_s) {
                        size_t l = strlen(old_s) + 1; fwrite(old_s, 1, l, fp);
                        sp += l; idx++;
                    } else if (add_s) {
                        size_t l = added[ai].size() + 1; fwrite(add_s, 1, l, fp);
                        ai++;
                    } else break;
                }
                fclose(fp);
            }
            { std::vector<std::string>().swap(added); std::unordered_set<uint32_t>().swap(del_set);
              std::vector<uint32_t>().swap(del_idx); }
            malloc_trim(0);

            // Phase B: build remap by merge-walking old pool vs new pool (both sorted, both mmap'd)
            MappedFile new_pool = mmap_file(out_dir + "/strings.bin");
            {
                // Walk both pools simultaneously — no vectors needed, just cursors
                size_t old_pos = 0, new_pos = 0;
                while (old_pos < old_pool.size && new_pos < new_pool.size) {
                    const char* os = old_pool.data + old_pos;
                    const char* ns = new_pool.data + new_pos;
                    int c = strcmp(os, ns);
                    if (c == 0) {
                        if (old_pos != new_pos) // offset changed
                            str_remap_vec.push_back({(uint32_t)old_pos, (uint32_t)new_pos});
                        old_pos += strlen(os) + 1;
                        new_pos += strlen(ns) + 1;
                    } else if (c < 0) {
                        old_pos += strlen(os) + 1; // deleted string
                    } else {
                        new_pos += strlen(ns) + 1; // added string
                    }
                }
            }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
            unmap_file(old_pool);
            unmap_file(new_pool);
            std::cerr << "  Strings: +" << n_added << " -" << n_deleted
                      << ", " << str_remap_vec.size() << " remapped (" << str_remap_vec.size() * 8 / 1024 / 1024 << " MiB)" << std::endl;
            marker = ps.read_u32();
        }
        if (marker == 0xFFFFFFFE) {
            uint32_t c = ps.read_u32();
            for (uint32_t i = 0; i < c; i++) { uint32_t a = ps.read_u32(), b = ps.read_u32(); str_remap_vec.push_back({a, b}); }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
        } else {
            first_section_marker = marker; // not a remap section — pass to main loop
        }
    }
    log_phase("Strings", t_start);

    // Detect strides
    size_t way_stride = detect_stride(cur_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride(cur_dir + "/interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect_stride(cur_dir + "/admin_polygons.bin", {24, 20, 19});

    // String remap lookup — binary search for random access, cursor for sequential
    auto str_remap_lookup = [&](uint32_t old_off) -> uint32_t {
        auto it = std::lower_bound(str_remap_vec.begin(), str_remap_vec.end(),
            std::make_pair(old_off, (uint32_t)0));
        if (it != str_remap_vec.end() && it->first == old_off) return it->second;
        return old_off;
    };
    // Sequential cursor for batch remap during merge replay (O(1) amortized)
    struct StrRemapCursor {
        const std::vector<std::pair<uint32_t,uint32_t>>* vec;
        size_t idx = 0;
        void reset() { idx = 0; }
        uint32_t lookup(uint32_t old_off) {
            while (idx < vec->size() && (*vec)[idx].first < old_off) idx++;
            if (idx < vec->size() && (*vec)[idx].first == old_off) { return (*vec)[idx++].second; }
            return old_off;
        }
    };

    // --- Phase 3: Scan sections + parallel merge replays ---
    // Collect merge section info for parallel replay, process metadata inline
    struct MergeSection {
        uint32_t file_id, stride; size_t actual_stride;
        uint64_t old_size, new_size;
        uint32_t n_fixups; size_t fixup_data_pos;
        size_t seq_pos; uint64_t seq_size;
        bool needs_remap, needs_padding;
    };
    std::vector<MergeSection> merge_sections;
    std::unordered_map<uint32_t, std::vector<uint32_t>> id_remaps;
    std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;
    std::unordered_map<uint64_t, uint8_t> flag_corrections;
    struct CellCorr { uint64_t cell_id; std::vector<uint32_t> ids; };
    std::unordered_map<uint32_t, std::vector<CellCorr>> entry_corrections;

    while (true) {
        uint32_t file_id;
        if (first_section_marker != 0) { file_id = first_section_marker; first_section_marker = 0; }
        else file_id = ps.read_u32();
        if (file_id == 0xFFFFFFFF) break;

        // --- Metadata sections ---
        if (file_id == 0xFFFFFFF7) {
            // String diff in main loop — same memory-efficient approach as Phase 2
            uint32_t n_added = ps.read_u32(), n_deleted = ps.read_u32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) added.push_back(ps.read_string());
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ps.read_u32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());
            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
            struct OS { uint32_t idx; uint32_t off; const char* str; };
            std::vector<OS> old_strs;
            { size_t sp = 0; uint32_t ix = 0;
              while (sp < old_pool.size) { old_strs.push_back({ix++, (uint32_t)sp, old_pool.data+sp}); sp += strlen(old_pool.data+sp)+1; }
            }
            std::vector<const char*> merged;
            for (auto& os : old_strs) if (!del_set.count(os.idx)) merged.push_back(os.str);
            for (auto& s : added) merged.push_back(s.c_str());
            std::sort(merged.begin(), merged.end(), [](const char* a, const char* b) { return strcmp(a,b) < 0; });
            FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
            uint32_t wpos = 0;
            for (const char* s : merged) { size_t l = strlen(s)+1; fwrite(s, 1, l, fp); wpos += l; }
            fclose(fp);
            // Build remap via merge-walk
            MappedFile new_pool = mmap_file(out_dir + "/strings.bin");
            std::vector<std::pair<const char*, uint32_t>> new_strs;
            { size_t sp = 0;
              while (sp < new_pool.size) { new_strs.push_back({new_pool.data+sp, (uint32_t)sp}); sp += strlen(new_pool.data+sp)+1; }
            }
            size_t oi2 = 0, ni2 = 0;
            while (oi2 < old_strs.size() && ni2 < new_strs.size()) {
                int c = strcmp(old_strs[oi2].str, new_strs[ni2].first);
                if (c == 0) {
                    if (old_strs[oi2].off != new_strs[ni2].second)
                        str_remap_vec.push_back({old_strs[oi2].off, new_strs[ni2].second});
                    oi2++; ni2++;
                } else if (c < 0) { oi2++; } else { ni2++; }
            }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
            unmap_file(old_pool); unmap_file(new_pool);
            std::cerr << "  Strings (loop): +" << n_added << " -" << n_deleted << " → " << merged.size()
                      << ", " << str_remap_vec.size() << " remapped" << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_GEO_MARKER) {
            uint32_t na = ps.read_u32(), nr = ps.read_u32();
            geo_added.resize(na); geo_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { ps.read(&geo_added[i], 8); }
            for (uint32_t i = 0; i < nr; i++) { ps.read(&geo_removed[i], 8); }
            std::cerr << "  Geo cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_ADMIN_MARKER) {
            uint32_t na = ps.read_u32(), nr = ps.read_u32();
            admin_added.resize(na); admin_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { ps.read(&admin_added[i], 8); }
            for (uint32_t i = 0; i < nr; i++) { ps.read(&admin_removed[i], 8); }
            std::cerr << "  Admin cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_FLAGS_MARKER) {
            uint32_t c = ps.read_u32();
            for (uint32_t i = 0; i < c; i++) { uint64_t cid; ps.read(&cid, 8); flag_corrections[cid] = ps.read_u8(); }
            std::cerr << "  Flag corrections: " << c << std::endl;
            continue;
        }
        if (file_id == SECONDARY_REMAP_MARKER) {
            // Apply secondary remaps directly to the temp remap files on disk
            uint32_t nf = ps.read_u32();
            for (uint32_t f = 0; f < nf; f++) {
                uint32_t fid = ps.read_u32(), np = ps.read_u32();
                std::string remap_path = tmpdir + "/remap_" + std::to_string(fid) + ".bin";
                // Open remap file for read-write
                int rfd = open(remap_path.c_str(), O_RDWR);
                if (rfd >= 0) {
                    struct stat st; fstat(rfd, &st);
                    size_t remap_size = st.st_size;
                    uint32_t* remap_data = static_cast<uint32_t*>(
                        mmap(nullptr, remap_size, PROT_READ | PROT_WRITE, MAP_SHARED, rfd, 0));
                    size_t remap_count = remap_size / 4;
                    for (uint32_t i = 0; i < np; i++) {
                        uint32_t o = ps.read_u32(), n = ps.read_u32();
                        if (o < remap_count) remap_data[o] = n + 1; // +1 encoding
                    }
                    munmap(remap_data, remap_size);
                    close(rfd);
                } else {
                    // Skip data if file doesn't exist
                    for (uint32_t i = 0; i < np; i++) { ps.read_u32(); ps.read_u32(); }
                }
                std::cerr << "  Secondary remap " << fid << ": " << np << " pairs" << std::endl;
            }
            continue;
        }
        if (file_id == ENTRY_CORRECTION_MARKER) {
            uint32_t fid = ps.read_u32(), c = ps.read_u32();
            auto& list = entry_corrections[fid];
            for (uint32_t i = 0; i < c; i++) {
                uint64_t cid; ps.read(&cid, 8);
                uint16_t ec; ps.read(&ec, 2);
                std::vector<uint32_t> ids(ec);
                if (ec > 0) ps.read(ids.data(), ec*4);
                list.push_back({cid, std::move(ids)});
            }
            std::cerr << "  Entry corrections " << fid << ": " << c << std::endl;
            continue;
        }

        // --- Collect merge section info (replay happens in parallel below) ---
        uint32_t stride = ps.read_u32();
        uint64_t old_size = ps.read_u64(), new_size = ps.read_u64();
        if (file_id >= (uint32_t)PatchFileId::COUNT) { std::cerr << "Unknown file " << file_id << std::endl; break; }
        const char* fname = patch_file_names[file_id];

        if (stride == 0) {
            uint32_t nf = ps.read_u32(); (void)nf; uint64_t ds = ps.read_u64();
            FILE* fp = fopen((out_dir + "/" + fname).c_str(), "wb");
            ps.write_to(fp, ds); fclose(fp);
            std::cerr << "  " << fname << ": full replace " << ds << " bytes" << std::endl;
            continue;
        }
        if (stride == 0xFE) { uint32_t nf = ps.read_u32(); (void)nf; uint64_t ds = ps.read_u64(); ps.skip(ds); continue; }

        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        bool needs_remap = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
                            file_id == (uint32_t)PatchFileId::STREET_WAYS ||
                            file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
                            file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
        bool needs_padding = (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24) ||
                             (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24);

        uint32_t n_fixups = ps.read_u32();
        // Read fixup data into a small buffer (decoded lazily during merge replay)
        std::vector<char> fixup_buf;
        if (n_fixups > 0) {
            uint32_t delta_size = ps.read_u32();
            fixup_buf.resize(delta_size);
            ps.read(fixup_buf.data(), delta_size);
        }
        // Lazy fixup decoder state (reads from patch mmap on demand)
        struct FixupDecoder {
            const char* data; size_t read_pos;
            uint32_t prev_idx = 0, prev_val = 0;
            uint32_t cur_idx = UINT32_MAX, cur_val = 0;
            uint32_t remaining;
            void init(const char* d, size_t p, uint32_t count) {
                data = d; read_pos = p; remaining = count;
                if (remaining > 0) advance();
            }
            void advance() {
                if (remaining == 0) { cur_idx = UINT32_MAX; return; }
                cur_idx = prev_idx + read_varint(data, read_pos);
                cur_val = prev_val + read_varint(data, read_pos);
                prev_idx = cur_idx; prev_val = cur_val;
                remaining--;
            }
            // Advance past all fixups with index < target
            bool lookup(uint32_t target_idx, uint32_t& out_val) {
                while (cur_idx < target_idx && (remaining > 0 || cur_idx != UINT32_MAX)) advance();
                if (cur_idx == target_idx) { out_val = cur_val; advance(); return true; }
                return false;
            }
        } fixup_dec;
        fixup_dec.init(fixup_buf.data(), 0, n_fixups);

        // Get string remap field offsets for this file type
        std::vector<size_t> remap_offs;
        if (!str_remap_vec.empty() && needs_remap) {
            if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) remap_offs = {8, 12};
            else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) remap_offs = {(actual_stride == 12) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) remap_offs = {(actual_stride >= 20) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) remap_offs = {8};
        }

        // mmap old file read-only (zero allocation)
        MappedFile old_mmap = mmap_file(cur_dir + "/" + std::string(fname));
        madvise(const_cast<char*>(old_mmap.data), old_mmap.size, MADV_SEQUENTIAL);
        size_t n_old_records = old_mmap.size / actual_stride;

        // Replay merge sequence — stream output, apply remap/fixups per-record inline
        uint64_t seq_size = ps.read_u64();
        size_t seq_start_pos = ps.position();
        size_t seq_bytes_read = 0;
        FILE* outf = fopen((out_dir + "/" + fname).c_str(), "wb");

        bool track = needs_remap; // track ID remap only for data files
        // File-backed id_map: create temp file, fill with 0xFF, mmap read-write
        int remap_fd = -1;
        uint32_t* id_map_ptr = nullptr;
        std::string remap_path;
        if (track) {
            remap_path = tmpdir + "/remap_" + std::to_string(file_id) + ".bin";
            remap_fd = open(remap_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644);
            size_t remap_bytes = n_old_records * 4;
            ftruncate(remap_fd, remap_bytes); // zero-filled by OS (sparse)
            id_map_ptr = static_cast<uint32_t*>(
                mmap(nullptr, remap_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, remap_fd, 0));
            // File is zero-filled. We store new_index+1 so that 0 means "unmapped".
            // The entry pipeline reads these and subtracts 1.
        }

        // Per-record buffer for applying remap/fixups inline
        std::vector<char> rec_buf(actual_stride);
        // Sequential string remap cursor (O(1) amortized for sorted data)
        StrRemapCursor str_cursor;
        str_cursor.vec = &str_remap_vec;
        str_cursor.reset();

        size_t old_rec = 0, new_rec = 0, old_bytes = 0, written = 0;
        while (seq_bytes_read < seq_size) {
            uint8_t op = ps.read_u8(); seq_bytes_read++;
            uint32_t count = ps.read_u32(); seq_bytes_read += 4;
            if (op == OP_MATCH_RUN) {
                for (uint32_t k = 0; k < count; k++) {
                    size_t rec_off = old_bytes + k * actual_stride;
                    if (rec_off + actual_stride > old_mmap.size) break;

                    bool modified = false;
                    // Check if this record needs any modification
                    if (!remap_offs.empty() || needs_padding) modified = true;
                    // Check fixup using lazy decoder (reads from patch mmap, zero allocation)
                    uint32_t fixup_val = 0;
                    bool has_fixup = fixup_dec.lookup(old_rec + k, fixup_val);
                    if (has_fixup) modified = true;

                    if (modified) {
                        memcpy(rec_buf.data(), old_mmap.data + rec_off, actual_stride);
                        // Apply padding zeroing
                        if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24)
                            memset(rec_buf.data() + 13, 0, 3);
                        if (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24) {
                            memset(rec_buf.data() + 5, 0, 3);
                            memset(rec_buf.data() + 21, 0, 3);
                        }
                        // Apply string remap
                        for (size_t off : remap_offs) {
                            uint32_t v; memcpy(&v, rec_buf.data() + off, 4);
                            uint32_t nv = str_cursor.lookup(v);
                            if (nv != v) memcpy(rec_buf.data() + off, &nv, 4);
                        }
                        // Apply fixup (node_offset/vertex_offset at byte 0)
                        if (has_fixup)
                            memcpy(rec_buf.data(), &fixup_val, 4);
                        fwrite(rec_buf.data(), 1, actual_stride, outf);
                    } else {
                        // No modification needed — write directly from mmap
                        fwrite(old_mmap.data + rec_off, 1, actual_stride, outf);
                    }
                    if (track && old_rec + k < n_old_records) id_map_ptr[old_rec + k] = (uint32_t)(new_rec + k) + 1; // +1 so 0 means unmapped
                }
                written += count * actual_stride;
                old_rec += count; new_rec += count; old_bytes += count * actual_stride;
            } else if (op == OP_INSERT_RUN) {
                size_t bytes = count * actual_stride;
                ps.write_to(outf, bytes);
                written += bytes; seq_bytes_read += bytes; new_rec += count;
            } else if (op == OP_DELETE_RUN) {
                old_rec += count; old_bytes += count * actual_stride;
            }
        }
        fclose(outf);
        unmap_file(old_mmap);

        if (track) {
            // Sync and munmap the file-backed remap (data is already on disk)
            size_t remap_bytes = n_old_records * 4;
            msync(id_map_ptr, remap_bytes, MS_SYNC);
            munmap(id_map_ptr, remap_bytes);
            close(remap_fd);
            id_map_ptr = nullptr;
            id_remaps[file_id] = {}; // empty placeholder
            std::cerr << "  " << fname << ": " << written << " bytes (remap " << n_old_records << " records to disk)" << std::endl;
        } else {
            std::cerr << "  " << fname << ": " << written << " bytes" << std::endl;
        }
        malloc_trim(0);
    }
    log_phase("Merge replays", t_start);

    // Free string remap
    { std::vector<std::pair<uint32_t,uint32_t>>().swap(str_remap_vec); }
    malloc_trim(0);

    // --- Phase 4: Streaming entry pipeline ---
    // Walk old geo_cells, remap IDs, apply corrections, write output directly to files.
    {
        double t_entry = now_ms();
        std::cerr << "Entry pipeline (lean streaming)..." << std::endl;

        // mmap remap files from disk (only pages accessed are in RSS)
        auto mmap_remap = [&](PatchFileId fid) -> MappedFile {
            std::string path = tmpdir + "/remap_" + std::to_string((uint32_t)fid) + ".bin";
            auto m = mmap_file(path);
            if (m.data) madvise(const_cast<char*>(m.data), m.size, MADV_SEQUENTIAL);
            return m;
        };
        // Helper: look up remap value from mmap'd file
        // Remap values stored as new_index+1 (0 means unmapped)
        struct RemapRef {
            const uint32_t* data;
            size_t count;
            uint32_t operator[](size_t i) const {
                if (i >= count) return 0xFFFFFFFF;
                uint32_t v = data[i];
                return v == 0 ? 0xFFFFFFFF : v - 1;
            }
            size_t size() const { return count; }
        };
        MappedFile m_w_rm = mmap_remap(PatchFileId::STREET_WAYS);
        MappedFile m_a_rm = mmap_remap(PatchFileId::ADDR_POINTS);
        MappedFile m_i_rm = mmap_remap(PatchFileId::INTERP_WAYS);
        RemapRef w_rm = {(const uint32_t*)m_w_rm.data, m_w_rm.size / 4};
        RemapRef a_rm = {(const uint32_t*)m_a_rm.data, m_a_rm.size / 4};
        RemapRef i_rm = {(const uint32_t*)m_i_rm.data, m_i_rm.size / 4};

        // mmap old files (zero RSS until pages are accessed, then only working set)
        MappedFile m_geo = mmap_file(cur_dir + "/geo_cells.bin");
        MappedFile m_se = mmap_file(cur_dir + "/street_entries.bin");
        MappedFile m_ae = mmap_file(cur_dir + "/addr_entries.bin");
        MappedFile m_ie = mmap_file(cur_dir + "/interp_entries.bin");
        madvise(const_cast<char*>(m_se.data), m_se.size, MADV_SEQUENTIAL);
        madvise(const_cast<char*>(m_ae.data), m_ae.size, MADV_SEQUENTIAL);
        madvise(const_cast<char*>(m_ie.data), m_ie.size, MADV_SEQUENTIAL);
        size_t n_old = m_geo.size / 20;

        // Build sorted correction vectors + removed set (better cache locality than hash maps)
        std::unordered_set<uint64_t> rm_set(geo_removed.begin(), geo_removed.end());
        struct CorrEntry { uint64_t cid; const std::vector<uint32_t>* ids; };
        std::vector<CorrEntry> cs, ca, ci_map;
        for (auto& [fid, list] : entry_corrections) {
            auto* v = (fid == (uint32_t)PatchFileId::STREET_ENTRIES) ? &cs :
                      (fid == (uint32_t)PatchFileId::ADDR_ENTRIES) ? &ca :
                      (fid == (uint32_t)PatchFileId::INTERP_ENTRIES) ? &ci_map : nullptr;
            if (v) for (auto& c : list) v->push_back({c.cell_id, &c.ids});
        }
        auto corr_cmp = [](const CorrEntry& a, const CorrEntry& b) { return a.cid < b.cid; };
        std::sort(cs.begin(), cs.end(), corr_cmp);
        std::sort(ca.begin(), ca.end(), corr_cmp);
        std::sort(ci_map.begin(), ci_map.end(), corr_cmp);
        // Binary search helper
        auto corr_find = [](const std::vector<CorrEntry>& v, uint64_t cid) -> const std::vector<uint32_t>* {
            auto it = std::lower_bound(v.begin(), v.end(), CorrEntry{cid, nullptr},
                [](const CorrEntry& a, const CorrEntry& b) { return a.cid < b.cid; });
            return (it != v.end() && it->cid == cid) ? it->ids : nullptr;
        };
        // Added cells sorted
        std::sort(geo_added.begin(), geo_added.end());
        log_phase("  Setup", t_entry);

        // Open 4 output files
        FILE* f_geo = fopen((out_dir + "/geo_cells.bin").c_str(), "wb");
        FILE* f_se = fopen((out_dir + "/street_entries.bin").c_str(), "wb");
        FILE* f_ae = fopen((out_dir + "/addr_entries.bin").c_str(), "wb");
        FILE* f_ie = fopen((out_dir + "/interp_entries.bin").c_str(), "wb");
        constexpr uint32_t NO = 0xFFFFFFFF;

        // Reusable buffer (one per entry type to avoid aliasing issues)
        std::vector<uint32_t> buf;
        buf.reserve(4096);

        // Helper: parse IDs from mmap'd entry file
        auto parse = [&buf](const MappedFile& f, uint32_t off) {
            buf.clear();
            if (off == NO || off + 2 > f.size) return;
            uint16_t c; memcpy(&c, f.data + off, 2);
            if (off + 2 + (size_t)c * 4 > f.size) return;
            buf.resize(c);
            memcpy(buf.data(), f.data + off + 2, c * 4);
        };
        auto remap = [](std::vector<uint32_t>& ids, const RemapRef& rm) {
            constexpr uint32_t NO2 = 0xFFFFFFFF;
            for (auto& id : ids) if (id < rm.size() && rm[id] != NO2) id = rm[id];
            std::sort(ids.begin(), ids.end());
        };
        // Write entry and return offset, or NO if empty
        auto emit = [&NO](FILE* f, const uint32_t* ids, size_t n) -> uint32_t {
            if (n == 0) return NO;
            uint32_t off = (uint32_t)ftell(f);
            uint16_t c = (uint16_t)n;
            fwrite(&c, 2, 1, f); fwrite(ids, 4, n, f);
            return off;
        };

        // Merge-walk: old cells + added cells in sorted order
        // Both are sorted by cell_id. Merge them, skip removed.
        size_t old_i = 0, add_i = 0;
        size_t cells_written = 0;

        auto process_cell = [&](uint64_t cid, int32_t oi) {
            // For each entry type: check correction → remap old → write
            auto do_entry = [&](const MappedFile& old_e, size_t geo_off, const RemapRef& rm,
                                const std::vector<CorrEntry>& corr,
                                FILE* outf, uint8_t flag_bit) -> uint32_t {
                // Check flag
                bool has = false;
                if (oi >= 0 && (size_t)oi * 20 + geo_off + 4 <= m_geo.size) {
                    uint32_t off; memcpy(&off, m_geo.data + oi * 20 + geo_off, 4);
                    has = (off != NO);
                } else if (oi >= 0) {
                    std::cerr << "OOB: oi=" << oi << " geo_off=" << geo_off << " m_geo.size=" << m_geo.size << std::endl;
                    return NO;
                }
                auto fc = flag_corrections.find(cid);
                if (fc != flag_corrections.end()) has = (fc->second & flag_bit) != 0;
                auto* corr_ids = corr_find(corr, cid);
                if (corr_ids) has = true;
                if (!has) return NO;

                if (corr_ids) return emit(outf, corr_ids->data(), corr_ids->size());

                if (oi < 0) return NO;
                if ((size_t)oi * 20 + geo_off + 4 > m_geo.size) return NO;
                uint32_t off; memcpy(&off, m_geo.data + oi * 20 + geo_off, 4);
                parse(old_e, off);
                if (buf.empty()) return NO;
                remap(buf, rm);
                return emit(outf, buf.data(), buf.size());
            };

            uint32_t so = do_entry(m_se, 8, w_rm, cs, f_se, 1);
            uint32_t ao = do_entry(m_ae, 12, a_rm, ca, f_ae, 2);
            uint32_t io = do_entry(m_ie, 16, i_rm, ci_map, f_ie, 4);
            fwrite(&cid, 8, 1, f_geo); fwrite(&so, 4, 1, f_geo); fwrite(&ao, 4, 1, f_geo); fwrite(&io, 4, 1, f_geo);
            cells_written++;
        };

        while (old_i < n_old || add_i < geo_added.size()) {
            uint64_t old_cid = UINT64_MAX, add_cid = UINT64_MAX;
            if (old_i < n_old) memcpy(&old_cid, m_geo.data + old_i * 20, 8);
            if (add_i < geo_added.size()) add_cid = geo_added[add_i];

            if (old_cid <= add_cid) {
                if (!rm_set.count(old_cid))
                    process_cell(old_cid, (int32_t)old_i);
                old_i++;
                if (old_cid == add_cid) add_i++; // skip duplicate add
            } else {
                process_cell(add_cid, -1);
                add_i++;
            }

            if (cells_written % 10000000 == 0 && cells_written > 0) {
                std::cerr << "    " << cells_written << " cells, se=" << ftell(f_se)/1024/1024
                          << "M ae=" << ftell(f_ae)/1024/1024 << "M rss=" << get_rss_mb() << "M" << std::endl;
            }
        }

        fclose(f_geo); fclose(f_se); fclose(f_ae); fclose(f_ie);
        unmap_file(m_geo); unmap_file(m_se); unmap_file(m_ae); unmap_file(m_ie);
        unmap_file(m_w_rm); unmap_file(m_a_rm); unmap_file(m_i_rm);
        std::cerr << "  Geo: " << cells_written << " cells written" << std::endl;
        malloc_trim(0);
        log_phase("  Geo entries complete", t_entry);

        // Admin: small, use existing rebuild + corrections
        {
            uint32_t no_data = 0xFFFFFFFF;
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            MappedFile m_ad_rm = mmap_remap(PatchFileId::ADMIN_POLYGONS);
            if (m_ad_rm.data) {
                const uint32_t* ad_vec = (const uint32_t*)m_ad_rm.data;
                size_t ad_count = m_ad_rm.size / 4;
                for (uint32_t i = 0; i < ad_count; i++)
                    if (ad_vec[i] != 0) ad_rm[i] = ad_vec[i] - 1; // decode +1 encoding
                unmap_file(m_ad_rm);
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_adme = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_adme, ad_rm, admin_added, admin_removed);

            auto ecit = entry_corrections.find((uint32_t)PatchFileId::ADMIN_ENTRIES);
            if (ecit != entry_corrections.end()) {
                std::unordered_map<uint64_t, const std::vector<uint32_t>*> ac_corr;
                for (auto& c : ecit->second) ac_corr[c.cell_id] = &c.ids;
                size_t n = admin.admin_cells_data.size() / 12;
                FILE* fac = fopen((out_dir + "/admin_cells.bin").c_str(), "wb");
                FILE* fae = fopen((out_dir + "/admin_entries.bin").c_str(), "wb");
                uint32_t ae_wpos = 0;
                for (size_t i = 0; i < n; i++) {
                    uint64_t cid; memcpy(&cid, admin.admin_cells_data.data()+i*12, 8);
                    fwrite(&cid, 8, 1, fac);
                    auto cit = ac_corr.find(cid);
                    if (cit != ac_corr.end()) {
                        uint32_t off = cit->second->empty() ? no_data : ae_wpos;
                        fwrite(&off, 4, 1, fac);
                        if (!cit->second->empty()) {
                            uint16_t c = cit->second->size();
                            fwrite(&c, 2, 1, fae); fwrite(cit->second->data(), 4, c, fae);
                            ae_wpos += 2 + c*4;
                        }
                    } else {
                        uint32_t old_off; memcpy(&old_off, admin.admin_cells_data.data()+i*12+8, 4);
                        if (old_off != no_data && old_off+2 <= admin.admin_entries_data.size()) {
                            uint32_t new_off = ae_wpos; fwrite(&new_off, 4, 1, fac);
                            uint16_t c; memcpy(&c, admin.admin_entries_data.data()+old_off, 2);
                            fwrite(admin.admin_entries_data.data()+old_off, 1, 2+c*4, fae);
                            ae_wpos += 2+c*4;
                        } else {
                            fwrite(&no_data, 4, 1, fac);
                        }
                    }
                }
                fclose(fac); fclose(fae);
                std::cerr << "  Admin: " << n << " cells, " << ac_corr.size() << " corrections" << std::endl;
            } else {
                write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
                write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            }
            log_phase("  Admin", t_entry);
        }
    }

    // Cleanup
    { std::string cmd = "rm -rf '" + tmpdir + "'"; system(cmd.c_str()); }
    log_phase("Total", t_start);
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

-        // Replay merge sequence — stream output, apply remap/fixups per-record inline

+        // Read entire merge sequence into local buffer (fast sequential decompression)

-        size_t seq_start_pos = ps.position();

+        std::vector<char> seq_buf(seq_size);

+        ps.read(seq_buf.data(), seq_size);


Now update the merge replay loop to read from `seq_buf` instead of `ps`:


        size_t old_rec = 0, new_rec = 0, old_bytes = 0, written = 0;
        while (seq_bytes_read < seq_size) {
            uint8_t op = ps.read_u8(); seq_bytes_read++;
            uint32_t count = ps.read_u32(); seq_bytes_read += 4;

        size_t old_rec = 0, new_rec = 0, old_bytes = 0, written = 0;
        while (seq_pos < seq_size) {
            uint8_t op = (uint8_t)seq_buf[seq_pos++];
            uint32_t count; memcpy(&count, seq_buf.data() + seq_pos, 4); seq_pos += 4;