        // --- Collect merge section info (replay happens in parallel below) ---
        uint32_t stride = ru32();
        uint64_t old_size = ru64(), new_size = ru64();
        if (file_id >= (uint32_t)PatchFileId::COUNT) { std::cerr << "Unknown file " << file_id << std::endl; break; }
        const char* fname = patch_file_names[file_id];

        if (stride == 0) {
            uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64();
            FILE* fp = fopen((out_dir + "/" + fname).c_str(), "wb");
            fwrite(P+pos, 1, ds, fp); fclose(fp);
            pos += ds;
            std::cerr << "  " << fname << ": full replace " << ds << " bytes" << std::endl;
            continue;
        }
        if (stride == 0xFE) { uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64(); pos += ds; continue; }

        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        bool needs_remap = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
                            file_id == (uint32_t)PatchFileId::STREET_WAYS ||
                            file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
                            file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
        bool needs_padding = (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24) ||
                             (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24);

        uint32_t n_fixups = ru32();
        size_t fixup_data_pos = pos;
        if (n_fixups > 0) {
            uint32_t delta_size = ru32();
            fixup_data_pos = pos;
            pos += delta_size;
        }

// geocoder-patch v4: Low-memory streaming patch application.
//
// Key design: mmap old files (zero copy), stream output via fwrite (never accumulate),
// process entry pipeline cell-by-cell. Target: <1 GiB peak RSS for planet.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <fcntl.h>
#include <iomanip>
#include <iostream>
#include <malloc.h>
#include <string>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unordered_map>
#include <unordered_set>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

// --- Helpers ---
static double now_ms() {
    struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
    return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6;
}
static size_t get_rss_mb() {
    FILE* f = fopen("/proc/self/statm", "r");
    if (!f) return 0;
    size_t dummy, rss; fscanf(f, "%zu %zu", &dummy, &rss); fclose(f);
    return rss * 4096 / (1024*1024);
}
static void log_phase(const char* label, double start) {
    std::cerr << "  [" << std::fixed << std::setprecision(1) << (now_ms() - start) / 1000.0
              << "s, " << get_rss_mb() << " MiB] " << label << std::endl;
}

// mmap a file read-only. Returns {pointer, size}. Caller must munmap.
struct MappedFile { const char* data; size_t size; };
static MappedFile mmap_file(const std::string& path) {
    int fd = open(path.c_str(), O_RDONLY);
    if (fd < 0) return {nullptr, 0};
    struct stat st; fstat(fd, &st);
    size_t sz = st.st_size;
    if (sz == 0) { close(fd); return {nullptr, 0}; }
    void* p = mmap(nullptr, sz, PROT_READ, MAP_PRIVATE, fd, 0);
    close(fd);
    if (p == MAP_FAILED) return {nullptr, 0};
    return {static_cast<const char*>(p), sz};
}
static void unmap_file(MappedFile& f) {
    if (f.data) { munmap(const_cast<char*>(f.data), f.size); f.data = nullptr; f.size = 0; }
}

// Detect stride from file size
static size_t detect_stride(const std::string& path, std::initializer_list<size_t> candidates) {
    struct stat st; if (stat(path.c_str(), &st) != 0) return *candidates.begin();
    for (size_t s : candidates) if (st.st_size % s == 0 && st.st_size > 0) return s;
    return *candidates.begin();
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);
    double t_start = now_ms();

    // --- Phase 1: Decompress + mmap patch ---
    std::string raw_path = tmpdir + "/patch.raw";
    {
        std::string cmd = "zstd -d '" + patch_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0) { std::cerr << "Failed to decompress" << std::endl; return 1; }
    }
    MappedFile patch_map = mmap_file(raw_path);
    if (!patch_map.data) { std::cerr << "Failed to mmap patch" << std::endl; return 1; }
    // Don't remove temp file yet (mmap needs backing file for page eviction under memory pressure)
    // madvise random — we access sections non-sequentially but release pages after each section
    madvise(const_cast<char*>(patch_map.data), patch_map.size, MADV_RANDOM);
    const char* P = patch_map.data;
    size_t patch_size = patch_map.size;
    std::cerr << "Patch: " << patch_size << " bytes" << std::endl;
    log_phase("Decompress", t_start);

    size_t pos = 0;
    auto ru32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, P+pos, 4); pos += 4; return v; };
    auto ru64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, P+pos, 8); pos += 8; return v; };

    // Header
    if (memcmp(P, GCPATCH_MAGIC, 8) != 0) { std::cerr << "Bad magic" << std::endl; return 1; }
    pos = 8;
    uint32_t ver = ru32(); if (ver != 2) { std::cerr << "Bad version" << std::endl; return 1; }
    ru32(); // flags

    // --- Phase 2: String rebuild ---
    // Sorted vector remap: (old_offset, new_offset) pairs sorted by old_offset.
    // Uses 96 MiB for 12M entries vs 672 MiB for unordered_map.
    std::vector<std::pair<uint32_t, uint32_t>> str_remap_vec;
    {
        uint32_t marker = ru32();
        if (marker == 0xFFFFFFF7) {
            uint32_t n_added = ru32(), n_deleted = ru32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ru32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());

            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");

            // Phase A: merge old (minus deleted) + added strings, write directly to file.
            // Both old pool and added are sorted alphabetically → merge-write with no sort.
            // Memory: only the added strings vector (~small) + del_set (~small).
            {
                FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
                size_t sp = 0; uint32_t idx = 0; size_t ai = 0;
                // Sort added strings (they should already be sorted from diff tool, but ensure)
                std::sort(added.begin(), added.end());
                while (sp < old_pool.size || ai < added.size()) {
                    const char* old_s = nullptr;
                    // Skip deleted strings
                    while (sp < old_pool.size) {
                        if (!del_set.count(idx)) { old_s = old_pool.data + sp; break; }
                        sp += strlen(old_pool.data + sp) + 1; idx++;
                    }
                    const char* add_s = ai < added.size() ? added[ai].c_str() : nullptr;
                    // Merge: pick the smaller string
                    if (old_s && add_s) {
                        int c = strcmp(old_s, add_s);
                        if (c <= 0) {
                            size_t l = strlen(old_s) + 1; fwrite(old_s, 1, l, fp);
                            sp += l; idx++;
                            if (c == 0) ai++; // duplicate — skip added
                        } else {
                            size_t l = added[ai].size() + 1; fwrite(add_s, 1, l, fp);
                            ai++;
                        }
                    } else if (old_s) {
                        size_t l = strlen(old_s) + 1; fwrite(old_s, 1, l, fp);
                        sp += l; idx++;
                    } else if (add_s) {
                        size_t l = added[ai].size() + 1; fwrite(add_s, 1, l, fp);
                        ai++;
                    } else break;
                }
                fclose(fp);
            }
            { std::vector<std::string>().swap(added); std::unordered_set<uint32_t>().swap(del_set);
              std::vector<uint32_t>().swap(del_idx); }
            malloc_trim(0);

            // Phase B: build remap by merge-walking old pool vs new pool (both sorted, both mmap'd)
            MappedFile new_pool = mmap_file(out_dir + "/strings.bin");
            {
                // Walk both pools simultaneously — no vectors needed, just cursors
                size_t old_pos = 0, new_pos = 0;
                while (old_pos < old_pool.size && new_pos < new_pool.size) {
                    const char* os = old_pool.data + old_pos;
                    const char* ns = new_pool.data + new_pos;
                    int c = strcmp(os, ns);
                    if (c == 0) {
                        if (old_pos != new_pos) // offset changed
                            str_remap_vec.push_back({(uint32_t)old_pos, (uint32_t)new_pos});
                        old_pos += strlen(os) + 1;
                        new_pos += strlen(ns) + 1;
                    } else if (c < 0) {
                        old_pos += strlen(os) + 1; // deleted string
                    } else {
                        new_pos += strlen(ns) + 1; // added string
                    }
                }
            }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
            unmap_file(old_pool);
            unmap_file(new_pool);
            std::cerr << "  Strings: +" << n_added << " -" << n_deleted
                      << ", " << str_remap_vec.size() << " remapped (" << str_remap_vec.size() * 8 / 1024 / 1024 << " MiB)" << std::endl;
            marker = ru32();
        }
        if (marker == 0xFFFFFFFE) {
            uint32_t c = ru32();
            for (uint32_t i = 0; i < c; i++) { uint32_t a = ru32(), b = ru32(); str_remap_vec.push_back({a, b}); }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
        } else pos -= 4;
    }
    // Release patch pages read so far (string section)
    madvise(const_cast<char*>(patch_map.data), pos, MADV_DONTNEED);
    log_phase("Strings", t_start);

    // Detect strides
    size_t way_stride = detect_stride(cur_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride(cur_dir + "/interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect_stride(cur_dir + "/admin_polygons.bin", {24, 20, 19});

    // String remap lookup via sorted vector + binary search
    auto str_remap_lookup = [&](uint32_t old_off) -> uint32_t {
        auto it = std::lower_bound(str_remap_vec.begin(), str_remap_vec.end(),
            std::make_pair(old_off, (uint32_t)0));
        if (it != str_remap_vec.end() && it->first == old_off) return it->second;
        return old_off; // no remap
    };

    // --- Phase 3: Scan sections + parallel merge replays ---
    // Collect merge section info for parallel replay, process metadata inline
    struct MergeSection {
        uint32_t file_id, stride; size_t actual_stride;
        uint64_t old_size, new_size;
        uint32_t n_fixups; size_t fixup_data_pos;
        size_t seq_pos; uint64_t seq_size;
        bool needs_remap, needs_padding;
    };
    std::vector<MergeSection> merge_sections;
    std::unordered_map<uint32_t, std::vector<uint32_t>> id_remaps;
    std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;
    std::unordered_map<uint64_t, uint8_t> flag_corrections;
    struct CellCorr { uint64_t cell_id; std::vector<uint32_t> ids; };
    std::unordered_map<uint32_t, std::vector<CellCorr>> entry_corrections;

    while (pos < patch_size) {
        uint32_t file_id = ru32();
        if (file_id == 0xFFFFFFFF) break;

        // --- Metadata sections ---
        if (file_id == 0xFFFFFFF7) {
            // String diff in main loop — same memory-efficient approach as Phase 2
            uint32_t n_added = ru32(), n_deleted = ru32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ru32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());
            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
            struct OS { uint32_t idx; uint32_t off; const char* str; };
            std::vector<OS> old_strs;
            { size_t sp = 0; uint32_t ix = 0;
              while (sp < old_pool.size) { old_strs.push_back({ix++, (uint32_t)sp, old_pool.data+sp}); sp += strlen(old_pool.data+sp)+1; }
            }
            std::vector<const char*> merged;
            for (auto& os : old_strs) if (!del_set.count(os.idx)) merged.push_back(os.str);
            for (auto& s : added) merged.push_back(s.c_str());
            std::sort(merged.begin(), merged.end(), [](const char* a, const char* b) { return strcmp(a,b) < 0; });
            FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
            uint32_t wpos = 0;
            for (const char* s : merged) { size_t l = strlen(s)+1; fwrite(s, 1, l, fp); wpos += l; }
            fclose(fp);
            // Build remap via merge-walk
            MappedFile new_pool = mmap_file(out_dir + "/strings.bin");
            std::vector<std::pair<const char*, uint32_t>> new_strs;
            { size_t sp = 0;
              while (sp < new_pool.size) { new_strs.push_back({new_pool.data+sp, (uint32_t)sp}); sp += strlen(new_pool.data+sp)+1; }
            }
            size_t oi2 = 0, ni2 = 0;
            while (oi2 < old_strs.size() && ni2 < new_strs.size()) {
                int c = strcmp(old_strs[oi2].str, new_strs[ni2].first);
                if (c == 0) {
                    if (old_strs[oi2].off != new_strs[ni2].second)
                        str_remap_vec.push_back({old_strs[oi2].off, new_strs[ni2].second});
                    oi2++; ni2++;
                } else if (c < 0) { oi2++; } else { ni2++; }
            }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
            unmap_file(old_pool); unmap_file(new_pool);
            std::cerr << "  Strings (loop): +" << n_added << " -" << n_deleted << " → " << merged.size()
                      << ", " << str_remap_vec.size() << " remapped" << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_GEO_MARKER) {
            uint32_t na = ru32(), nr = ru32();
            geo_added.resize(na); geo_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { memcpy(&geo_added[i], P+pos, 8); pos += 8; }
            for (uint32_t i = 0; i < nr; i++) { memcpy(&geo_removed[i], P+pos, 8); pos += 8; }
            std::cerr << "  Geo cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_ADMIN_MARKER) {
            uint32_t na = ru32(), nr = ru32();
            admin_added.resize(na); admin_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { memcpy(&admin_added[i], P+pos, 8); pos += 8; }
            for (uint32_t i = 0; i < nr; i++) { memcpy(&admin_removed[i], P+pos, 8); pos += 8; }
            std::cerr << "  Admin cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_FLAGS_MARKER) {
            uint32_t c = ru32();
            for (uint32_t i = 0; i < c; i++) { uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8; flag_corrections[cid] = P[pos++]; }
            std::cerr << "  Flag corrections: " << c << std::endl;
            continue;
        }
        if (file_id == SECONDARY_REMAP_MARKER) {
            // Apply secondary remaps directly to the temp remap files on disk
            uint32_t nf = ru32();
            for (uint32_t f = 0; f < nf; f++) {
                uint32_t fid = ru32(), np = ru32();
                std::string remap_path = tmpdir + "/remap_" + std::to_string(fid) + ".bin";
                // Open remap file for read-write
                int rfd = open(remap_path.c_str(), O_RDWR);
                if (rfd >= 0) {
                    struct stat st; fstat(rfd, &st);
                    size_t remap_size = st.st_size;
                    uint32_t* remap_data = static_cast<uint32_t*>(
                        mmap(nullptr, remap_size, PROT_READ | PROT_WRITE, MAP_SHARED, rfd, 0));
                    size_t remap_count = remap_size / 4;
                    for (uint32_t i = 0; i < np; i++) {
                        uint32_t o = ru32(), n = ru32();
                        if (o < remap_count) remap_data[o] = n + 1; // +1 encoding
                    }
                    munmap(remap_data, remap_size);
                    close(rfd);
                } else {
                    // Skip data if file doesn't exist
                    for (uint32_t i = 0; i < np; i++) { ru32(); ru32(); }
                }
                std::cerr << "  Secondary remap " << fid << ": " << np << " pairs" << std::endl;
            }
            continue;
        }
        if (file_id == ENTRY_CORRECTION_MARKER) {
            uint32_t fid = ru32(), c = ru32();
            auto& list = entry_corrections[fid];
            for (uint32_t i = 0; i < c; i++) {
                uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8;
                uint16_t ec; memcpy(&ec, P+pos, 2); pos += 2;
                std::vector<uint32_t> ids(ec);
                if (ec > 0) { memcpy(ids.data(), P+pos, ec*4); pos += ec*4; }
                list.push_back({cid, std::move(ids)});
            }
            std::cerr << "  Entry corrections " << fid << ": " << c << std::endl;
            continue;
        }

        // --- Merge sequence replay (streaming) ---
        uint32_t stride = ru32();
        uint64_t old_size = ru64(), new_size = ru64();
        if (file_id >= (uint32_t)PatchFileId::COUNT) { std::cerr << "Unknown file " << file_id << std::endl; break; }
        const char* fname = patch_file_names[file_id];

        if (stride == 0) {
            // Full replacement — write directly from patch mmap
            uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64();
            FILE* fp = fopen((out_dir + "/" + fname).c_str(), "wb");
            fwrite(P+pos, 1, ds, fp); fclose(fp);
            pos += ds;
            std::cerr << "  " << fname << ": full replace " << ds << " bytes" << std::endl;
            continue;
        }
        if (stride == 0xFE) { uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64(); pos += ds; continue; }

        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        // Determine if this file needs in-memory modifications
        bool needs_remap = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
                            file_id == (uint32_t)PatchFileId::STREET_WAYS ||
                            file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
                            file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
        bool needs_padding = (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24) ||
                             (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24);

        // Read fixup data — decoded lazily during merge replay (zero allocation)
        uint32_t n_fixups = ru32();
        size_t fixup_data_pos = pos;
        if (n_fixups > 0) {
            uint32_t delta_size = ru32();
            fixup_data_pos = pos;
            pos += delta_size;
        }
        // Lazy fixup decoder state (reads from patch mmap on demand)
        struct FixupDecoder {
            const char* data; size_t read_pos;
            uint32_t prev_idx = 0, prev_val = 0;
            uint32_t cur_idx = UINT32_MAX, cur_val = 0;
            uint32_t remaining;
            void init(const char* d, size_t p, uint32_t count) {
                data = d; read_pos = p; remaining = count;
                if (remaining > 0) advance();
            }
            void advance() {
                if (remaining == 0) { cur_idx = UINT32_MAX; return; }
                cur_idx = prev_idx + read_varint(data, read_pos);
                cur_val = prev_val + read_varint(data, read_pos);
                prev_idx = cur_idx; prev_val = cur_val;
                remaining--;
            }
            // Advance past all fixups with index < target
            bool lookup(uint32_t target_idx, uint32_t& out_val) {
                while (cur_idx < target_idx && (remaining > 0 || cur_idx != UINT32_MAX)) advance();
                if (cur_idx == target_idx) { out_val = cur_val; advance(); return true; }
                return false;
            }
        } fixup_dec;
        fixup_dec.init(P, fixup_data_pos, n_fixups);

        // Get string remap field offsets for this file type
        std::vector<size_t> remap_offs;
        if (!str_remap_vec.empty() && needs_remap) {
            if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) remap_offs = {8, 12};
            else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) remap_offs = {(actual_stride == 12) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) remap_offs = {(actual_stride >= 20) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) remap_offs = {8};
        }

        // mmap old file read-only (zero allocation)
        MappedFile old_mmap = mmap_file(cur_dir + "/" + std::string(fname));
        madvise(const_cast<char*>(old_mmap.data), old_mmap.size, MADV_SEQUENTIAL);
        size_t n_old_records = old_mmap.size / actual_stride;

        // Replay merge sequence — stream output, apply remap/fixups per-record inline
        uint64_t seq_size = ru64();
        size_t seq_end = pos + seq_size;
        FILE* outf = fopen((out_dir + "/" + fname).c_str(), "wb");

        bool track = needs_remap; // track ID remap only for data files
        // File-backed id_map: create temp file, fill with 0xFF, mmap read-write
        int remap_fd = -1;
        uint32_t* id_map_ptr = nullptr;
        std::string remap_path;
        if (track) {
            remap_path = tmpdir + "/remap_" + std::to_string(file_id) + ".bin";
            remap_fd = open(remap_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644);
            size_t remap_bytes = n_old_records * 4;
            ftruncate(remap_fd, remap_bytes); // zero-filled by OS (sparse)
            id_map_ptr = static_cast<uint32_t*>(
                mmap(nullptr, remap_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, remap_fd, 0));
            // File is zero-filled. We store new_index+1 so that 0 means "unmapped".
            // The entry pipeline reads these and subtracts 1.
        }

        // Per-record buffer for applying remap/fixups inline (avoid copying entire file)
        std::vector<char> rec_buf(actual_stride);

        size_t old_rec = 0, new_rec = 0, old_bytes = 0, written = 0;
        while (pos < seq_end) {
            uint8_t op = P[pos++];
            uint32_t count; memcpy(&count, P+pos, 4); pos += 4;
            if (op == OP_MATCH_RUN) {
                for (uint32_t k = 0; k < count; k++) {
                    size_t rec_off = old_bytes + k * actual_stride;
                    if (rec_off + actual_stride > old_mmap.size) break;

                    bool modified = false;
                    // Check if this record needs any modification
                    if (!remap_offs.empty() || needs_padding) modified = true;
                    // Check fixup using lazy decoder (reads from patch mmap, zero allocation)
                    uint32_t fixup_val = 0;
                    bool has_fixup = fixup_dec.lookup(old_rec + k, fixup_val);
                    if (has_fixup) modified = true;

                    if (modified) {
                        memcpy(rec_buf.data(), old_mmap.data + rec_off, actual_stride);
                        // Apply padding zeroing
                        if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24)
                            memset(rec_buf.data() + 13, 0, 3);
                        if (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24) {
                            memset(rec_buf.data() + 5, 0, 3);
                            memset(rec_buf.data() + 21, 0, 3);
                        }
                        // Apply string remap
                        for (size_t off : remap_offs) {
                            uint32_t v; memcpy(&v, rec_buf.data() + off, 4);
                            uint32_t nv = str_remap_lookup(v);
                            if (nv != v) memcpy(rec_buf.data() + off, &nv, 4);
                        }
                        // Apply fixup (node_offset/vertex_offset at byte 0)
                        if (has_fixup)
                            memcpy(rec_buf.data(), &fixup_val, 4);
                        fwrite(rec_buf.data(), 1, actual_stride, outf);
                    } else {
                        // No modification needed — write directly from mmap
                        fwrite(old_mmap.data + rec_off, 1, actual_stride, outf);
                    }
                    if (track && old_rec + k < n_old_records) id_map_ptr[old_rec + k] = (uint32_t)(new_rec + k) + 1; // +1 so 0 means unmapped
                }
                written += count * actual_stride;
                old_rec += count; new_rec += count; old_bytes += count * actual_stride;
            } else if (op == OP_INSERT_RUN) {
                size_t bytes = count * actual_stride;
                fwrite(P+pos, 1, bytes, outf);
                written += bytes; pos += bytes; new_rec += count;
            } else if (op == OP_DELETE_RUN) {
                old_rec += count; old_bytes += count * actual_stride;
            }
        }
        fclose(outf);
        unmap_file(old_mmap);

        if (track) {
            // Sync and munmap the file-backed remap (data is already on disk)
            size_t remap_bytes = n_old_records * 4;
            msync(id_map_ptr, remap_bytes, MS_SYNC);
            munmap(id_map_ptr, remap_bytes);
            close(remap_fd);
            id_map_ptr = nullptr;
            id_remaps[file_id] = {}; // empty placeholder
            std::cerr << "  " << fname << ": " << written << " bytes (remap " << n_old_records << " records to disk)" << std::endl;
        } else {
            std::cerr << "  " << fname << ": " << written << " bytes" << std::endl;
        }
        // Release patch pages used by this section
        madvise(const_cast<char*>(patch_map.data), pos, MADV_DONTNEED);
        malloc_trim(0);
    }
    log_phase("Merge replays", t_start);

    // Free string remap + release all processed patch pages
    { std::vector<std::pair<uint32_t,uint32_t>>().swap(str_remap_vec); }
    madvise(const_cast<char*>(patch_map.data), pos, MADV_DONTNEED);
    malloc_trim(0);

    // --- Phase 4: Streaming entry pipeline ---
    // Walk old geo_cells, remap IDs, apply corrections, write output directly to files.
    {
        double t_entry = now_ms();
        std::cerr << "Entry pipeline (lean streaming)..." << std::endl;

        // mmap remap files from disk (only pages accessed are in RSS)
        auto mmap_remap = [&](PatchFileId fid) -> MappedFile {
            std::string path = tmpdir + "/remap_" + std::to_string((uint32_t)fid) + ".bin";
            auto m = mmap_file(path);
            if (m.data) madvise(const_cast<char*>(m.data), m.size, MADV_SEQUENTIAL);
            return m;
        };
        // Helper: look up remap value from mmap'd file
        // Remap values stored as new_index+1 (0 means unmapped)
        struct RemapRef {
            const uint32_t* data;
            size_t count;
            uint32_t operator[](size_t i) const {
                if (i >= count) return 0xFFFFFFFF;
                uint32_t v = data[i];
                return v == 0 ? 0xFFFFFFFF : v - 1;
            }
            size_t size() const { return count; }
        };
        MappedFile m_w_rm = mmap_remap(PatchFileId::STREET_WAYS);
        MappedFile m_a_rm = mmap_remap(PatchFileId::ADDR_POINTS);
        MappedFile m_i_rm = mmap_remap(PatchFileId::INTERP_WAYS);
        RemapRef w_rm = {(const uint32_t*)m_w_rm.data, m_w_rm.size / 4};
        RemapRef a_rm = {(const uint32_t*)m_a_rm.data, m_a_rm.size / 4};
        RemapRef i_rm = {(const uint32_t*)m_i_rm.data, m_i_rm.size / 4};

        // mmap old files (zero RSS until pages are accessed, then only working set)
        MappedFile m_geo = mmap_file(cur_dir + "/geo_cells.bin");
        MappedFile m_se = mmap_file(cur_dir + "/street_entries.bin");
        MappedFile m_ae = mmap_file(cur_dir + "/addr_entries.bin");
        MappedFile m_ie = mmap_file(cur_dir + "/interp_entries.bin");
        madvise(const_cast<char*>(m_se.data), m_se.size, MADV_SEQUENTIAL);
        madvise(const_cast<char*>(m_ae.data), m_ae.size, MADV_SEQUENTIAL);
        madvise(const_cast<char*>(m_ie.data), m_ie.size, MADV_SEQUENTIAL);
        size_t n_old = m_geo.size / 20;

        // Build sorted correction vectors + removed set (better cache locality than hash maps)
        std::unordered_set<uint64_t> rm_set(geo_removed.begin(), geo_removed.end());
        struct CorrEntry { uint64_t cid; const std::vector<uint32_t>* ids; };
        std::vector<CorrEntry> cs, ca, ci_map;
        for (auto& [fid, list] : entry_corrections) {
            auto* v = (fid == (uint32_t)PatchFileId::STREET_ENTRIES) ? &cs :
                      (fid == (uint32_t)PatchFileId::ADDR_ENTRIES) ? &ca :
                      (fid == (uint32_t)PatchFileId::INTERP_ENTRIES) ? &ci_map : nullptr;
            if (v) for (auto& c : list) v->push_back({c.cell_id, &c.ids});
        }
        auto corr_cmp = [](const CorrEntry& a, const CorrEntry& b) { return a.cid < b.cid; };
        std::sort(cs.begin(), cs.end(), corr_cmp);
        std::sort(ca.begin(), ca.end(), corr_cmp);
        std::sort(ci_map.begin(), ci_map.end(), corr_cmp);
        // Binary search helper
        auto corr_find = [](const std::vector<CorrEntry>& v, uint64_t cid) -> const std::vector<uint32_t>* {
            auto it = std::lower_bound(v.begin(), v.end(), CorrEntry{cid, nullptr},
                [](const CorrEntry& a, const CorrEntry& b) { return a.cid < b.cid; });
            return (it != v.end() && it->cid == cid) ? it->ids : nullptr;
        };
        // Added cells sorted
        std::sort(geo_added.begin(), geo_added.end());
        log_phase("  Setup", t_entry);

        // Open 4 output files
        FILE* f_geo = fopen((out_dir + "/geo_cells.bin").c_str(), "wb");
        FILE* f_se = fopen((out_dir + "/street_entries.bin").c_str(), "wb");
        FILE* f_ae = fopen((out_dir + "/addr_entries.bin").c_str(), "wb");
        FILE* f_ie = fopen((out_dir + "/interp_entries.bin").c_str(), "wb");
        constexpr uint32_t NO = 0xFFFFFFFF;

        // Reusable buffer (one per entry type to avoid aliasing issues)
        std::vector<uint32_t> buf;
        buf.reserve(4096);

        // Helper: parse IDs from mmap'd entry file
        auto parse = [&buf](const MappedFile& f, uint32_t off) {
            buf.clear();
            if (off == NO || off + 2 > f.size) return;
            uint16_t c; memcpy(&c, f.data + off, 2);
            if (off + 2 + (size_t)c * 4 > f.size) return;
            buf.resize(c);
            memcpy(buf.data(), f.data + off + 2, c * 4);
        };
        auto remap = [](std::vector<uint32_t>& ids, const RemapRef& rm) {
            constexpr uint32_t NO2 = 0xFFFFFFFF;
            for (auto& id : ids) if (id < rm.size() && rm[id] != NO2) id = rm[id];
            std::sort(ids.begin(), ids.end());
        };
        // Write entry and return offset, or NO if empty
        auto emit = [&NO](FILE* f, const uint32_t* ids, size_t n) -> uint32_t {
            if (n == 0) return NO;
            uint32_t off = (uint32_t)ftell(f);
            uint16_t c = (uint16_t)n;
            fwrite(&c, 2, 1, f); fwrite(ids, 4, n, f);
            return off;
        };

        // Merge-walk: old cells + added cells in sorted order
        // Both are sorted by cell_id. Merge them, skip removed.
        size_t old_i = 0, add_i = 0;
        size_t cells_written = 0;

        auto process_cell = [&](uint64_t cid, int32_t oi) {
            // For each entry type: check correction → remap old → write
            auto do_entry = [&](const MappedFile& old_e, size_t geo_off, const RemapRef& rm,
                                const std::vector<CorrEntry>& corr,
                                FILE* outf, uint8_t flag_bit) -> uint32_t {
                // Check flag
                bool has = false;
                if (oi >= 0 && (size_t)oi * 20 + geo_off + 4 <= m_geo.size) {
                    uint32_t off; memcpy(&off, m_geo.data + oi * 20 + geo_off, 4);
                    has = (off != NO);
                } else if (oi >= 0) {
                    std::cerr << "OOB: oi=" << oi << " geo_off=" << geo_off << " m_geo.size=" << m_geo.size << std::endl;
                    return NO;
                }
                auto fc = flag_corrections.find(cid);
                if (fc != flag_corrections.end()) has = (fc->second & flag_bit) != 0;
                auto* corr_ids = corr_find(corr, cid);
                if (corr_ids) has = true;
                if (!has) return NO;

                if (corr_ids) return emit(outf, corr_ids->data(), corr_ids->size());

                if (oi < 0) return NO;
                if ((size_t)oi * 20 + geo_off + 4 > m_geo.size) return NO;
                uint32_t off; memcpy(&off, m_geo.data + oi * 20 + geo_off, 4);
                parse(old_e, off);
                if (buf.empty()) return NO;
                remap(buf, rm);
                return emit(outf, buf.data(), buf.size());
            };

            uint32_t so = do_entry(m_se, 8, w_rm, cs, f_se, 1);
            uint32_t ao = do_entry(m_ae, 12, a_rm, ca, f_ae, 2);
            uint32_t io = do_entry(m_ie, 16, i_rm, ci_map, f_ie, 4);
            fwrite(&cid, 8, 1, f_geo); fwrite(&so, 4, 1, f_geo); fwrite(&ao, 4, 1, f_geo); fwrite(&io, 4, 1, f_geo);
            cells_written++;
        };

        while (old_i < n_old || add_i < geo_added.size()) {
            uint64_t old_cid = UINT64_MAX, add_cid = UINT64_MAX;
            if (old_i < n_old) memcpy(&old_cid, m_geo.data + old_i * 20, 8);
            if (add_i < geo_added.size()) add_cid = geo_added[add_i];

            if (old_cid <= add_cid) {
                if (!rm_set.count(old_cid))
                    process_cell(old_cid, (int32_t)old_i);
                old_i++;
                if (old_cid == add_cid) add_i++; // skip duplicate add
            } else {
                process_cell(add_cid, -1);
                add_i++;
            }

            if (cells_written % 10000000 == 0 && cells_written > 0) {
                std::cerr << "    " << cells_written << " cells, se=" << ftell(f_se)/1024/1024
                          << "M ae=" << ftell(f_ae)/1024/1024 << "M rss=" << get_rss_mb() << "M" << std::endl;
            }
        }

        fclose(f_geo); fclose(f_se); fclose(f_ae); fclose(f_ie);
        unmap_file(m_geo); unmap_file(m_se); unmap_file(m_ae); unmap_file(m_ie);
        unmap_file(m_w_rm); unmap_file(m_a_rm); unmap_file(m_i_rm);
        std::cerr << "  Geo: " << cells_written << " cells written" << std::endl;
        malloc_trim(0);
        log_phase("  Geo entries complete", t_entry);

        // Admin: small, use existing rebuild + corrections
        {
            uint32_t no_data = 0xFFFFFFFF;
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            MappedFile m_ad_rm = mmap_remap(PatchFileId::ADMIN_POLYGONS);
            if (m_ad_rm.data) {
                const uint32_t* ad_vec = (const uint32_t*)m_ad_rm.data;
                size_t ad_count = m_ad_rm.size / 4;
                for (uint32_t i = 0; i < ad_count; i++)
                    if (ad_vec[i] != 0) ad_rm[i] = ad_vec[i] - 1; // decode +1 encoding
                unmap_file(m_ad_rm);
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_adme = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_adme, ad_rm, admin_added, admin_removed);

            auto ecit = entry_corrections.find((uint32_t)PatchFileId::ADMIN_ENTRIES);
            if (ecit != entry_corrections.end()) {
                std::unordered_map<uint64_t, const std::vector<uint32_t>*> ac_corr;
                for (auto& c : ecit->second) ac_corr[c.cell_id] = &c.ids;
                size_t n = admin.admin_cells_data.size() / 12;
                FILE* fac = fopen((out_dir + "/admin_cells.bin").c_str(), "wb");
                FILE* fae = fopen((out_dir + "/admin_entries.bin").c_str(), "wb");
                uint32_t ae_wpos = 0;
                for (size_t i = 0; i < n; i++) {
                    uint64_t cid; memcpy(&cid, admin.admin_cells_data.data()+i*12, 8);
                    fwrite(&cid, 8, 1, fac);
                    auto cit = ac_corr.find(cid);
                    if (cit != ac_corr.end()) {
                        uint32_t off = cit->second->empty() ? no_data : ae_wpos;
                        fwrite(&off, 4, 1, fac);
                        if (!cit->second->empty()) {
                            uint16_t c = cit->second->size();
                            fwrite(&c, 2, 1, fae); fwrite(cit->second->data(), 4, c, fae);
                            ae_wpos += 2 + c*4;
                        }
                    } else {
                        uint32_t old_off; memcpy(&old_off, admin.admin_cells_data.data()+i*12+8, 4);
                        if (old_off != no_data && old_off+2 <= admin.admin_entries_data.size()) {
                            uint32_t new_off = ae_wpos; fwrite(&new_off, 4, 1, fac);
                            uint16_t c; memcpy(&c, admin.admin_entries_data.data()+old_off, 2);
                            fwrite(admin.admin_entries_data.data()+old_off, 1, 2+c*4, fae);
                            ae_wpos += 2+c*4;
                        } else {
                            fwrite(&no_data, 4, 1, fac);
                        }
                    }
                }
                fclose(fac); fclose(fae);
                std::cerr << "  Admin: " << n << " cells, " << ac_corr.size() << " corrections" << std::endl;
            } else {
                write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
                write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            }
            log_phase("  Admin", t_entry);
        }
    }

    // Cleanup
    unmap_file(patch_map);
    remove(raw_path.c_str());
    { std::string cmd = "rm -rf '" + tmpdir + "'"; system(cmd.c_str()); }
    log_phase("Total", t_start);
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

-        // --- Merge sequence replay (streaming) ---

+        // --- Collect merge section info (replay happens in parallel below) ---

-            // Full replacement — write directly from patch mmap

-        // Determine if this file needs in-memory modifications

-        // Read fixup data — decoded lazily during merge replay (zero allocation)


Now I need to replace the inline merge replay with collecting section info, then add a parallel replay after the scan loop. Let me find the rest of the merge replay code and replace it with section collection:


   360→        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
   361→        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
   362→        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;
   363→
   364→        bool needs_remap = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
   365→                            file_id == (uint32_t)PatchFileId::STREET_WAYS ||
   366→                            file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
   367→                            file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
   368→        bool needs_padding = (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24) ||
   369→                             (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24);

        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        bool needs_remap = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
                            file_id == (uint32_t)PatchFileId::STREET_WAYS ||
                            file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
                            file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
        bool needs_padding = (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24) ||
                             (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24);


   370→
   371→        uint32_t n_fixups = ru32();
   372→        size_t fixup_data_pos = pos;
   373→        if (n_fixups > 0) {
   374→            uint32_t delta_size = ru32();
   375→            fixup_data_pos = pos;
   376→            pos += delta_size;
   377→        }
   378→        // Lazy fixup decoder state (reads from patch mmap on demand)
   379→        struct FixupDecoder {
   380→            const char* data; size_t read_pos;
   381→            uint32_t prev_idx = 0, prev_val = 0;
   382→            uint32_t cur_idx = UINT32_MAX, cur_val = 0;
   383→            uint32_t remaining;
   384→            void init(const char* d, size_t p, uint32_t count) {
   385→                data = d; read_pos = p; remaining = count;
   386→                if (remaining > 0) advance();
   387→            }
   388→            void advance() {
   389→                if (remaining == 0) { cur_idx = UINT32_MAX; return; }
   390→                cur_idx = prev_idx + read_varint(data, read_pos);
   391→                cur_val = prev_val + read_varint(data, read_pos);
   392→                prev_idx = cur_idx; prev_val = cur_val;
   393→                remaining--;
   394→            }
   395→            // Advance past all fixups with index < target
   396→            bool lookup(uint32_t target_idx, uint32_t& out_val) {
   397→                while (cur_idx < target_idx && (remaining > 0 || cur_idx != UINT32_MAX)) advance();
   398→                if (cur_idx == target_idx) { out_val = cur_val; advance(); return true; }
   399→                return false;
   400→            }
   401→        } fixup_dec;
   402→        fixup_dec.init(P, fixup_data_pos, n_fixups);
   403→
   404→        // Get string remap field offsets for this file type
   405→        std::vector<size_t> remap_offs;
   406→        if (!str_remap_vec.empty() && needs_remap) {
   407→            if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) remap_offs = {8, 12};
   408→            else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) remap_offs = {(actual_stride == 12) ? 8ul : 5ul};
   409→            else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) remap_offs = {(actual_stride >= 20) ? 8ul : 5ul};
   410→            else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) remap_offs = {8};
   411→        }
   412→
   413→        // mmap old file read-only (zero allocation)
   414→        MappedFile old_mmap = mmap_file(cur_dir + "/" + std::string(fname));
   415→        madvise(const_cast<char*>(old_mmap.data), old_mmap.size, MADV_SEQUENTIAL);
   416→        size_t n_old_records = old_mmap.size / actual_stride;
   417→
   418→        // Replay merge sequence — stream output, apply remap/fixups per-record inline
   419→        uint64_t seq_size = ru64();
   420→        size_t seq_end = pos + seq_size;
   421→        FILE* outf = fopen((out_dir + "/" + fname).c_str(), "wb");
   422→
   423→        bool track = needs_remap; // track ID remap only for data files
   424→        // File-backed id_map: create temp file, fill with 0xFF, mmap read-write
   425→        int remap_fd = -1;
   426→        uint32_t* id_map_ptr = nullptr;
   427→        std::string remap_path;
   428→        if (track) {
   429→            remap_path = tmpdir + "/remap_" + std::to_string(file_id) + ".bin";
   430→            remap_fd = open(remap_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644);
   431→            size_t remap_bytes = n_old_records * 4;
   432→            ftruncate(remap_fd, remap_bytes); // zero-filled by OS (sparse)
   433→            id_map_ptr = static_cast<uint32_t*>(
   434→                mmap(nullptr, remap_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, remap_fd, 0));
   435→            // File is zero-filled. We store new_index+1 so that 0 means "unmapped".
   436→            // The entry pipeline reads these and subtracts 1.
   437→        }
   438→
   439→        // Per-record buffer for applying remap/fixups inline (avoid copying entire file)
   440→        std::vector<char> rec_buf(actual_stride);
   441→
   442→        size_t old_rec = 0, new_rec = 0, old_bytes = 0, written = 0;
   443→        while (pos < seq_end) {
   444→            uint8_t op = P[pos++];
   445→            uint32_t count; memcpy(&count, P+pos, 4); pos += 4;
   446→            if (op == OP_MATCH_RUN) {
   447→                for (uint32_t k = 0; k < count; k++) {
   448→                    size_t rec_off = old_bytes + k * actual_stride;
   449→                    if (rec_off + actual_stride > old_mmap.size) break;
   450→
   451→                    bool modified = false;
   452→                    // Check if this record needs any modification
   453→                    if (!remap_offs.empty() || needs_padding) modified = true;
   454→                    // Check fixup using lazy decoder (reads from patch mmap, zero allocation)
   455→                    uint32_t fixup_val = 0;
   456→                    bool has_fixup = fixup_dec.lookup(old_rec + k, fixup_val);
   457→                    if (has_fixup) modified = true;
   458→
   459→                    if (modified) {
   460→                        memcpy(rec_buf.data(), old_mmap.data + rec_off, actual_stride);
   461→                        // Apply padding zeroing
   462→                        if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24)
   463→                            memset(rec_buf.data() + 13, 0, 3);
   464→                        if (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24) {
   465→                            memset(rec_buf.data() + 5, 0, 3);
   466→                            memset(rec_buf.data() + 21, 0, 3);
   467→                        }
   468→                        // Apply string remap
   469→                        for (size_t off : remap_offs) {
   470→                            uint32_t v; memcpy(&v, rec_buf.data() + off, 4);
   471→                            uint32_t nv = str_remap_lookup(v);
   472→                            if (nv != v) memcpy(rec_buf.data() + off, &nv, 4);
   473→                        }
   474→                        // Apply fixup (node_offset/vertex_offset at byte 0)
   475→                        if (has_fixup)
   476→                            memcpy(rec_buf.data(), &fixup_val, 4);
   477→                        fwrite(rec_buf.data(), 1, actual_stride, outf);
   478→                    } else {
   479→                        // No modification needed — write directly from mmap
   480→                        fwrite(old_mmap.data + rec_off, 1, actual_stride, outf);
   481→                    }
   482→                    if (track && old_rec + k < n_old_records) id_map_ptr[old_rec + k] = (uint32_t)(new_rec + k) + 1; // +1 so 0 means unmapped
   483→                }
   484→                written += count * actual_stride;
   485→                old_rec += count; new_rec += count; old_bytes += count * actual_stride;
   486→            } else if (op == OP_INSERT_RUN) {
   487→                size_t bytes = count * actual_stride;
   488→                fwrite(P+pos, 1, bytes, outf);
   489→                written += bytes; pos += bytes; new_rec += count;
   490→            } else if (op == OP_DELETE_RUN) {
   491→                old_rec += count; old_bytes += count * actual_stride;
   492→            }
   493→        }
   494→        fclose(outf);
   495→        unmap_file(old_mmap);
   496→
   497→        if (track) {
   498→            // Sync and munmap the file-backed remap (data is already on disk)
   499→            size_t remap_bytes = n_old_records * 4;
   500→            msync(id_map_ptr, remap_bytes, MS_SYNC);
   501→            munmap(id_map_ptr, remap_bytes);
   502→            close(remap_fd);
   503→            id_map_ptr = nullptr;
   504→            id_remaps[file_id] = {}; // empty placeholder
   505→            std::cerr << "  " << fname << ": " << written << " bytes (remap " << n_old_records << " records to disk)" << std::endl;
   506→        } else {
   507→            std::cerr << "  " << fname << ": " << written << " bytes" << std::endl;
   508→        }
   509→        // Release patch pages used by this section

        uint32_t n_fixups = ru32();
        size_t fixup_data_pos = pos;
        if (n_fixups > 0) {
            uint32_t delta_size = ru32();
            fixup_data_pos = pos;
            pos += delta_size;
        }
        // Lazy fixup decoder state (reads from patch mmap on demand)
        struct FixupDecoder {
            const char* data; size_t read_pos;
            uint32_t prev_idx = 0, prev_val = 0;
            uint32_t cur_idx = UINT32_MAX, cur_val = 0;
            uint32_t remaining;
            void init(const char* d, size_t p, uint32_t count) {
                data = d; read_pos = p; remaining = count;
                if (remaining > 0) advance();
            }
            void advance() {
                if (remaining == 0) { cur_idx = UINT32_MAX; return; }
                cur_idx = prev_idx + read_varint(data, read_pos);
                cur_val = prev_val + read_varint(data, read_pos);
                prev_idx = cur_idx; prev_val = cur_val;
                remaining--;
            }
            // Advance past all fixups with index < target
            bool lookup(uint32_t target_idx, uint32_t& out_val) {
                while (cur_idx < target_idx && (remaining > 0 || cur_idx != UINT32_MAX)) advance();
                if (cur_idx == target_idx) { out_val = cur_val; advance(); return true; }
                return false;
            }
        } fixup_dec;
        fixup_dec.init(P, fixup_data_pos, n_fixups);

        // Get string remap field offsets for this file type
        std::vector<size_t> remap_offs;
        if (!str_remap_vec.empty() && needs_remap) {
            if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) remap_offs = {8, 12};
            else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) remap_offs = {(actual_stride == 12) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) remap_offs = {(actual_stride >= 20) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) remap_offs = {8};
        }

        // mmap old file read-only (zero allocation)
        MappedFile old_mmap = mmap_file(cur_dir + "/" + std::string(fname));
        madvise(const_cast<char*>(old_mmap.data), old_mmap.size, MADV_SEQUENTIAL);
        size_t n_old_records = old_mmap.size / actual_stride;

        // Replay merge sequence — stream output, apply remap/fixups per-record inline
        uint64_t seq_size = ru64();
        size_t seq_end = pos + seq_size;
        FILE* outf = fopen((out_dir + "/" + fname).c_str(), "wb");

        bool track = needs_remap; // track ID remap only for data files
        // File-backed id_map: create temp file, fill with 0xFF, mmap read-write
        int remap_fd = -1;
        uint32_t* id_map_ptr = nullptr;
        std::string remap_path;
        if (track) {
            remap_path = tmpdir + "/remap_" + std::to_string(file_id) + ".bin";
            remap_fd = open(remap_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644);
            size_t remap_bytes = n_old_records * 4;
            ftruncate(remap_fd, remap_bytes); // zero-filled by OS (sparse)
            id_map_ptr = static_cast<uint32_t*>(
                mmap(nullptr, remap_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, remap_fd, 0));
            // File is zero-filled. We store new_index+1 so that 0 means "unmapped".
            // The entry pipeline reads these and subtracts 1.
        }

        // Per-record buffer for applying remap/fixups inline (avoid copying entire file)
        std::vector<char> rec_buf(actual_stride);

        size_t old_rec = 0, new_rec = 0, old_bytes = 0, written = 0;
        while (pos < seq_end) {
            uint8_t op = P[pos++];
            uint32_t count; memcpy(&count, P+pos, 4); pos += 4;
            if (op == OP_MATCH_RUN) {
                for (uint32_t k = 0; k < count; k++) {
                    size_t rec_off = old_bytes + k * actual_stride;
                    if (rec_off + actual_stride > old_mmap.size) break;

                    bool modified = false;
                    // Check if this record needs any modification
                    if (!remap_offs.empty() || needs_padding) modified = true;
                    // Check fixup using lazy decoder (reads from patch mmap, zero allocation)
                    uint32_t fixup_val = 0;
                    bool has_fixup = fixup_dec.lookup(old_rec + k, fixup_val);
                    if (has_fixup) modified = true;

                    if (modified) {
                        memcpy(rec_buf.data(), old_mmap.data + rec_off, actual_stride);
                        // Apply padding zeroing
                        if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24)
                            memset(rec_buf.data() + 13, 0, 3);
                        if (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24) {
                            memset(rec_buf.data() + 5, 0, 3);
                            memset(rec_buf.data() + 21, 0, 3);
                        }
                        // Apply string remap
                        for (size_t off : remap_offs) {
                            uint32_t v; memcpy(&v, rec_buf.data() + off, 4);
                            uint32_t nv = str_remap_lookup(v);
                            if (nv != v) memcpy(rec_buf.data() + off, &nv, 4);
                        }
                        // Apply fixup (node_offset/vertex_offset at byte 0)
                        if (has_fixup)
                            memcpy(rec_buf.data(), &fixup_val, 4);
                        fwrite(rec_buf.data(), 1, actual_stride, outf);
                    } else {
                        // No modification needed — write directly from mmap
                        fwrite(old_mmap.data + rec_off, 1, actual_stride, outf);
                    }
                    if (track && old_rec + k < n_old_records) id_map_ptr[old_rec + k] = (uint32_t)(new_rec + k) + 1; // +1 so 0 means unmapped
                }
                written += count * actual_stride;
                old_rec += count; new_rec += count; old_bytes += count * actual_stride;
            } else if (op == OP_INSERT_RUN) {
                size_t bytes = count * actual_stride;
                fwrite(P+pos, 1, bytes, outf);
                written += bytes; pos += bytes; new_rec += count;
            } else if (op == OP_DELETE_RUN) {
                old_rec += count; old_bytes += count * actual_stride;
            }
        }
        fclose(outf);
        unmap_file(old_mmap);

        if (track) {
            // Sync and munmap the file-backed remap (data is already on disk)
            size_t remap_bytes = n_old_records * 4;
            msync(id_map_ptr, remap_bytes, MS_SYNC);
            munmap(id_map_ptr, remap_bytes);
            close(remap_fd);
            id_map_ptr = nullptr;
            id_remaps[file_id] = {}; // empty placeholder
            std::cerr << "  " << fname << ": " << written << " bytes (remap " << n_old_records << " records to disk)" << std::endl;
        } else {
            std::cerr << "  " << fname << ": " << written << " bytes" << std::endl;
        }
        // Release patch pages used by this section


This is a very large block to restructure for parallelism. The complexity is high and risk of introducing bugs is significant. Let me take a more pragmatic approach: instead of full restructuring, I'll just wrap the existing replay code into a function and call it from threads for independent file groups.

Actually, looking at this more practically — the merge replay is already only 18s on planet unconstrained. The entry pipeline is 70s. Parallelizing merges would save ~10s. The main bottleneck is the entry pipeline's I/O.

Let me skip the parallel merges for now and commit the lazy fixups + sorted corrections, which already gave us a 2x speedup (1m29s vs 3m08s unconstrained) and brought memory to 225 MiB at 500 MiB cgroup. That's the committed version.

Let me instead focus on streaming zstd to eliminate the temp file and reduce the cgroup floor below 500 MiB. The simplest approach: use `popen("zstd -d -c", "r")` and read the entire decompressed output into a `std::vector<char>`. Under 500 MiB cgroup, the 424 MiB vector + 100 MiB other heap = 524 MiB — too much. But with MADV_DONTNEED on the vector pages after use... wait, you can't MADV_DONTNEED on heap memory (only mmap'd memory).

The true streaming approach would read the zstd output in chunks, processing each section as it arrives without keeping the full decompressed data. But this requires a major code restructure.

For now, the current state is very good: 1m29s / 243 MiB unconstrained, 500 MiB cgroup works. Let me commit the current optimizations:


Bash

Check uncommitted changes


 builder/tools/geocoder_patch.cpp      | 17 +++++++++++------
 builder/tools/simplification_map.html | 13 +++++++++----
 2 files changed, 20 insertions(+), 10 deletions(-)


Only minor uncommitted changes (the section collection prep for parallel merges, partially done). The main optimizations are already committed. Let me be honest about the streaming zstd and parallel merge situation:

Streaming zstd would eliminate the 424 MiB temp file, which would help the cgroup floor — but the code heavily relies on random access via `P[pos]` throughout. Converting to streaming reads requires a buffer manager that can provide bytes at arbitrary offsets within the current section. This is a substantial refactor.

Parallel merge replays require pre-scanning section positions and launching threads. The code already collects section info during the scan, but each replay function uses captured references to shared state (`str_remap_lookup`, `tmpdir`, etc.) that need to be thread-safe.

Both are doable but each is a multi-hour effort with significant regression risk. The current state is already excellent:

| Metric | Original | Now | Improvement |
|--------|----------|-----|-------------|
| Planet time | 20m10s | **1m29s** | **13.5x** |
| Planet peak RSS | 63 GiB | **243 MiB** | **265x** |
| Planet min cgroup | 63 GiB | **500 MiB** | **129x** |
| Europe time | 4m23s | **28s** | **9.4x** |
| Europe peak RSS | 11 GiB | **104 MiB** | **108x** |

Want me to continue with the streaming zstd implementation (significant refactor), or is this a good stopping point?


Lets focus on the zstd streaming first, and see how it does


Let me assess the scope first:


grep -c 'P+pos\|P\[pos\]\|P + pos' /root/traccar-geocoder/builder/tools/geocoder_patch.cpp

Count patch data accesses


98:    auto ru32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, P+pos, 4); pos += 4; return v; };
99:    auto ru64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, P+pos, 8); pos += 8; return v; };
116:            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
240:            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
282:            for (uint32_t i = 0; i < na; i++) { memcpy(&geo_added[i], P+pos, 8); pos += 8; }
283:            for (uint32_t i = 0; i < nr; i++) { memcpy(&geo_removed[i], P+pos, 8); pos += 8; }
290:            for (uint32_t i = 0; i < na; i++) { memcpy(&admin_added[i], P+pos, 8); pos += 8; }
291:            for (uint32_t i = 0; i < nr; i++) { memcpy(&admin_removed[i], P+pos, 8); pos += 8; }
297:            for (uint32_t i = 0; i < c; i++) { uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8; flag_corrections[cid] = P[pos++]; }
333:                uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8;
334:                uint16_t ec; memcpy(&ec, P+pos, 2); pos += 2;
336:                if (ec > 0) { memcpy(ids.data(), P+pos, ec*4); pos += ec*4; }
352:            fwrite(P+pos, 1, ds, fp); fclose(fp);
445:            uint32_t count; memcpy(&count, P+pos, 4); pos += 4;
488:                fwrite(P+pos, 1, bytes, outf);


15 access points. All are forward-only reads. The largest single reads are the INSERT fwrite (up to several MiB) and the full replacement fwrite. I'll create a `PatchStream` class that wraps zstd streaming decompression with a sliding buffer:


Edit

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

// Streaming zstd decompressor — reads compressed file, provides sequential access
// to decompressed data via a sliding buffer. No temp file needed.
#include <zstd.h>
class PatchStream {
    FILE* input_;
    ZSTD_DCtx* dctx_;
    std::vector<char> in_buf_, out_buf_;
    size_t out_start_ = 0, out_end_ = 0; // valid range in out_buf_
    size_t global_pos_ = 0; // position of out_buf_[0] in decompressed stream
    size_t total_decompressed_ = 0;
    bool input_done_ = false;
    static constexpr size_t IN_BUF_SIZE = 256 * 1024;
    static constexpr size_t OUT_BUF_SIZE = 4 * 1024 * 1024;
    ZSTD_inBuffer zin_{};

    void decompress_more() {
        // Compact: move unread data to front
        if (out_start_ > 0) {
            size_t remain = out_end_ - out_start_;
            if (remain > 0) memmove(out_buf_.data(), out_buf_.data() + out_start_, remain);
            global_pos_ += out_start_;
            out_end_ = remain;
            out_start_ = 0;
        }
        // Ensure output buffer has room
        if (out_buf_.size() - out_end_ < OUT_BUF_SIZE)
            out_buf_.resize(out_end_ + OUT_BUF_SIZE);

        while (!input_done_ && out_end_ - out_start_ < OUT_BUF_SIZE) {
            if (zin_.pos >= zin_.size) {
                size_t n = fread(in_buf_.data(), 1, IN_BUF_SIZE, input_);
                if (n == 0) { input_done_ = true; break; }
                zin_ = {in_buf_.data(), n, 0};
            }
            ZSTD_outBuffer zout = {out_buf_.data() + out_end_, out_buf_.size() - out_end_, 0};
            size_t ret = ZSTD_decompressStream(dctx_, &zout, &zin_);
            if (ZSTD_isError(ret)) { std::cerr << "zstd error: " << ZSTD_getErrorName(ret) << std::endl; break; }
            out_end_ += zout.pos;
            total_decompressed_ = global_pos_ + out_end_;
        }
    }

public:
    bool open(const std::string& path) {
        input_ = fopen(path.c_str(), "rb");
        if (!input_) return false;
        dctx_ = ZSTD_createDCtx();
        in_buf_.resize(IN_BUF_SIZE);
        out_buf_.resize(OUT_BUF_SIZE * 2);
        decompress_more();
        return true;
    }
    ~PatchStream() { if (dctx_) ZSTD_freeDCtx(dctx_); if (input_) fclose(input_); }

    // Ensure at least n bytes are available at current read position
    void ensure(size_t n) {
        while (out_end_ - out_start_ < n && !input_done_) decompress_more();
    }

    // Read n bytes at current position into dst, advance
    void read(void* dst, size_t n) {
        ensure(n);
        memcpy(dst, out_buf_.data() + out_start_, n);
        out_start_ += n;
    }
    uint32_t read_u32() { uint32_t v; read(&v, 4); return v; }
    uint64_t read_u64() { uint64_t v; read(&v, 8); return v; }
    uint8_t read_u8() { ensure(1); return (uint8_t)out_buf_[out_start_++]; }

    // Get pointer to n bytes at current position (valid until next ensure/read)
    const char* peek(size_t n) { ensure(n); return out_buf_.data() + out_start_; }
    void skip(size_t n) { ensure(n); out_start_ += n; }

    // Write n bytes from current position directly to a FILE*, advance
    void write_to(FILE* f, size_t n) {
        while (n > 0) {
            ensure(std::min(n, (size_t)OUT_BUF_SIZE));
            size_t avail = out_end_ - out_start_;
            size_t chunk = std::min(n, avail);
            fwrite(out_buf_.data() + out_start_, 1, chunk, f);
            out_start_ += chunk;
            n -= chunk;
        }
    }

    // Read a null-terminated string at current position
    std::string read_string() {
        std::string s;
        while (true) {
            ensure(1);
            char c = out_buf_[out_start_++];
            if (c == '\0') break;
            s += c;
        }
        return s;
    }

    size_t position() const { return global_pos_ + out_start_; }
    size_t decompressed_size() const { return total_decompressed_; }
};