            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");

            // Phase A: build merged pool, write to file, free all temporaries
            // Walk old pool directly (no old_ptrs vector) to minimize memory
            {
                std::vector<const char*> merged;
                // Count old strings to reserve
                size_t n_old = 0;
                { size_t sp = 0; while (sp < old_pool.size) { sp += strlen(old_pool.data + sp) + 1; n_old++; } }
                merged.reserve(n_old + added.size());
                { size_t sp = 0; uint32_t idx = 0;
                  while (sp < old_pool.size) {
                      if (!del_set.count(idx)) merged.push_back(old_pool.data + sp);
                      sp += strlen(old_pool.data + sp) + 1; idx++;
                  }
                }
                for (auto& s : added) merged.push_back(s.c_str());
                std::sort(merged.begin(), merged.end(), [](const char* a, const char* b) { return strcmp(a, b) < 0; });
                FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
                for (const char* s : merged) { size_t l = strlen(s) + 1; fwrite(s, 1, l, fp); }
                fclose(fp);
            }
            // Free temporaries
            { std::vector<std::string>().swap(added); std::unordered_set<uint32_t>().swap(del_set);
              std::vector<uint32_t>().swap(del_idx); }
            malloc_trim(0);

// geocoder-patch v4: Low-memory streaming patch application.
//
// Key design: mmap old files (zero copy), stream output via fwrite (never accumulate),
// process entry pipeline cell-by-cell. Target: <1 GiB peak RSS for planet.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <fcntl.h>
#include <iomanip>
#include <iostream>
#include <malloc.h>
#include <string>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unordered_map>
#include <unordered_set>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

// --- Helpers ---
static double now_ms() {
    struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
    return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6;
}
static size_t get_rss_mb() {
    FILE* f = fopen("/proc/self/statm", "r");
    if (!f) return 0;
    size_t dummy, rss; fscanf(f, "%zu %zu", &dummy, &rss); fclose(f);
    return rss * 4096 / (1024*1024);
}
static void log_phase(const char* label, double start) {
    std::cerr << "  [" << std::fixed << std::setprecision(1) << (now_ms() - start) / 1000.0
              << "s, " << get_rss_mb() << " MiB] " << label << std::endl;
}

// mmap a file read-only. Returns {pointer, size}. Caller must munmap.
struct MappedFile { const char* data; size_t size; };
static MappedFile mmap_file(const std::string& path) {
    int fd = open(path.c_str(), O_RDONLY);
    if (fd < 0) return {nullptr, 0};
    struct stat st; fstat(fd, &st);
    size_t sz = st.st_size;
    if (sz == 0) { close(fd); return {nullptr, 0}; }
    void* p = mmap(nullptr, sz, PROT_READ, MAP_PRIVATE, fd, 0);
    close(fd);
    if (p == MAP_FAILED) return {nullptr, 0};
    return {static_cast<const char*>(p), sz};
}
static void unmap_file(MappedFile& f) {
    if (f.data) { munmap(const_cast<char*>(f.data), f.size); f.data = nullptr; f.size = 0; }
}

// Detect stride from file size
static size_t detect_stride(const std::string& path, std::initializer_list<size_t> candidates) {
    struct stat st; if (stat(path.c_str(), &st) != 0) return *candidates.begin();
    for (size_t s : candidates) if (st.st_size % s == 0 && st.st_size > 0) return s;
    return *candidates.begin();
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);
    double t_start = now_ms();

    // --- Phase 1: Decompress + mmap patch ---
    std::string raw_path = tmpdir + "/patch.raw";
    {
        std::string cmd = "zstd -d '" + patch_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0) { std::cerr << "Failed to decompress" << std::endl; return 1; }
    }
    MappedFile patch_map = mmap_file(raw_path);
    if (!patch_map.data) { std::cerr << "Failed to mmap patch" << std::endl; return 1; }
    remove(raw_path.c_str());
    const char* P = patch_map.data;
    size_t patch_size = patch_map.size;
    std::cerr << "Patch: " << patch_size << " bytes" << std::endl;
    log_phase("Decompress", t_start);

    size_t pos = 0;
    auto ru32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, P+pos, 4); pos += 4; return v; };
    auto ru64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, P+pos, 8); pos += 8; return v; };

    // Header
    if (memcmp(P, GCPATCH_MAGIC, 8) != 0) { std::cerr << "Bad magic" << std::endl; return 1; }
    pos = 8;
    uint32_t ver = ru32(); if (ver != 2) { std::cerr << "Bad version" << std::endl; return 1; }
    ru32(); // flags

    // --- Phase 2: String rebuild ---
    // Sorted vector remap: (old_offset, new_offset) pairs sorted by old_offset.
    // Uses 96 MiB for 12M entries vs 672 MiB for unordered_map.
    std::vector<std::pair<uint32_t, uint32_t>> str_remap_vec;
    {
        uint32_t marker = ru32();
        if (marker == 0xFFFFFFF7) {
            uint32_t n_added = ru32(), n_deleted = ru32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ru32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());

            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");

            // Phase A: build merged pool, write to file, free all temporaries
            {
                // Parse old strings just for the merge (lightweight: only need idx + str ptr)
                std::vector<const char*> old_ptrs; // indexed by string index
                { size_t sp = 0;
                  while (sp < old_pool.size) { old_ptrs.push_back(old_pool.data + sp); sp += strlen(old_pool.data + sp) + 1; }
                }
                std::vector<const char*> merged;
                merged.reserve(old_ptrs.size() + added.size());
                for (uint32_t i = 0; i < old_ptrs.size(); i++)
                    if (!del_set.count(i)) merged.push_back(old_ptrs[i]);
                for (auto& s : added) merged.push_back(s.c_str());
                std::sort(merged.begin(), merged.end(), [](const char* a, const char* b) { return strcmp(a, b) < 0; });
                FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
                for (const char* s : merged) { size_t l = strlen(s) + 1; fwrite(s, 1, l, fp); }
                fclose(fp);
            } // merged, old_ptrs, added, del_set, del_idx all freed here
            malloc_trim(0);

            // Phase B: build remap by merge-walking old pool vs new pool (both sorted, both mmap'd)
            MappedFile new_pool = mmap_file(out_dir + "/strings.bin");
            {
                // Walk both pools simultaneously — no vectors needed, just cursors
                size_t old_pos = 0, new_pos = 0;
                while (old_pos < old_pool.size && new_pos < new_pool.size) {
                    const char* os = old_pool.data + old_pos;
                    const char* ns = new_pool.data + new_pos;
                    int c = strcmp(os, ns);
                    if (c == 0) {
                        if (old_pos != new_pos) // offset changed
                            str_remap_vec.push_back({(uint32_t)old_pos, (uint32_t)new_pos});
                        old_pos += strlen(os) + 1;
                        new_pos += strlen(ns) + 1;
                    } else if (c < 0) {
                        old_pos += strlen(os) + 1; // deleted string
                    } else {
                        new_pos += strlen(ns) + 1; // added string
                    }
                }
            }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
            unmap_file(old_pool);
            unmap_file(new_pool);
            std::cerr << "  Strings: +" << n_added << " -" << n_deleted
                      << ", " << str_remap_vec.size() << " remapped (" << str_remap_vec.size() * 8 / 1024 / 1024 << " MiB)" << std::endl;
            marker = ru32();
        }
        if (marker == 0xFFFFFFFE) {
            uint32_t c = ru32();
            for (uint32_t i = 0; i < c; i++) { uint32_t a = ru32(), b = ru32(); str_remap_vec.push_back({a, b}); }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
        } else pos -= 4;
    }
    log_phase("Strings", t_start);

    // Detect strides
    size_t way_stride = detect_stride(cur_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride(cur_dir + "/interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect_stride(cur_dir + "/admin_polygons.bin", {24, 20, 19});

    // String remap lookup via sorted vector + binary search
    auto str_remap_lookup = [&](uint32_t old_off) -> uint32_t {
        auto it = std::lower_bound(str_remap_vec.begin(), str_remap_vec.end(),
            std::make_pair(old_off, (uint32_t)0));
        if (it != str_remap_vec.end() && it->first == old_off) return it->second;
        return old_off; // no remap
    };

    // --- Phase 3: Merge replays (streaming output) ---
    // ID remaps: file_id → vector<uint32_t> where remap[old_idx] = new_idx
    std::unordered_map<uint32_t, std::vector<uint32_t>> id_remaps;
    std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;
    std::unordered_map<uint64_t, uint8_t> flag_corrections;
    struct CellCorr { uint64_t cell_id; std::vector<uint32_t> ids; };
    std::unordered_map<uint32_t, std::vector<CellCorr>> entry_corrections;

    while (pos < patch_size) {
        uint32_t file_id = ru32();
        if (file_id == 0xFFFFFFFF) break;

        // --- Metadata sections ---
        if (file_id == 0xFFFFFFF7) {
            // String diff in main loop — same memory-efficient approach as Phase 2
            uint32_t n_added = ru32(), n_deleted = ru32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ru32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());
            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
            struct OS { uint32_t idx; uint32_t off; const char* str; };
            std::vector<OS> old_strs;
            { size_t sp = 0; uint32_t ix = 0;
              while (sp < old_pool.size) { old_strs.push_back({ix++, (uint32_t)sp, old_pool.data+sp}); sp += strlen(old_pool.data+sp)+1; }
            }
            std::vector<const char*> merged;
            for (auto& os : old_strs) if (!del_set.count(os.idx)) merged.push_back(os.str);
            for (auto& s : added) merged.push_back(s.c_str());
            std::sort(merged.begin(), merged.end(), [](const char* a, const char* b) { return strcmp(a,b) < 0; });
            FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
            uint32_t wpos = 0;
            for (const char* s : merged) { size_t l = strlen(s)+1; fwrite(s, 1, l, fp); wpos += l; }
            fclose(fp);
            // Build remap via merge-walk
            MappedFile new_pool = mmap_file(out_dir + "/strings.bin");
            std::vector<std::pair<const char*, uint32_t>> new_strs;
            { size_t sp = 0;
              while (sp < new_pool.size) { new_strs.push_back({new_pool.data+sp, (uint32_t)sp}); sp += strlen(new_pool.data+sp)+1; }
            }
            size_t oi2 = 0, ni2 = 0;
            while (oi2 < old_strs.size() && ni2 < new_strs.size()) {
                int c = strcmp(old_strs[oi2].str, new_strs[ni2].first);
                if (c == 0) {
                    if (old_strs[oi2].off != new_strs[ni2].second)
                        str_remap_vec.push_back({old_strs[oi2].off, new_strs[ni2].second});
                    oi2++; ni2++;
                } else if (c < 0) { oi2++; } else { ni2++; }
            }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
            unmap_file(old_pool); unmap_file(new_pool);
            std::cerr << "  Strings (loop): +" << n_added << " -" << n_deleted << " → " << merged.size()
                      << ", " << str_remap_vec.size() << " remapped" << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_GEO_MARKER) {
            uint32_t na = ru32(), nr = ru32();
            geo_added.resize(na); geo_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { memcpy(&geo_added[i], P+pos, 8); pos += 8; }
            for (uint32_t i = 0; i < nr; i++) { memcpy(&geo_removed[i], P+pos, 8); pos += 8; }
            std::cerr << "  Geo cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_ADMIN_MARKER) {
            uint32_t na = ru32(), nr = ru32();
            admin_added.resize(na); admin_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { memcpy(&admin_added[i], P+pos, 8); pos += 8; }
            for (uint32_t i = 0; i < nr; i++) { memcpy(&admin_removed[i], P+pos, 8); pos += 8; }
            std::cerr << "  Admin cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_FLAGS_MARKER) {
            uint32_t c = ru32();
            for (uint32_t i = 0; i < c; i++) { uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8; flag_corrections[cid] = P[pos++]; }
            std::cerr << "  Flag corrections: " << c << std::endl;
            continue;
        }
        if (file_id == SECONDARY_REMAP_MARKER) {
            // Apply secondary remaps directly to the temp remap files on disk
            uint32_t nf = ru32();
            for (uint32_t f = 0; f < nf; f++) {
                uint32_t fid = ru32(), np = ru32();
                std::string remap_path = tmpdir + "/remap_" + std::to_string(fid) + ".bin";
                // Open remap file for read-write
                int rfd = open(remap_path.c_str(), O_RDWR);
                if (rfd >= 0) {
                    struct stat st; fstat(rfd, &st);
                    size_t remap_size = st.st_size;
                    uint32_t* remap_data = static_cast<uint32_t*>(
                        mmap(nullptr, remap_size, PROT_READ | PROT_WRITE, MAP_SHARED, rfd, 0));
                    size_t remap_count = remap_size / 4;
                    for (uint32_t i = 0; i < np; i++) {
                        uint32_t o = ru32(), n = ru32();
                        if (o < remap_count) remap_data[o] = n + 1; // +1 encoding
                    }
                    munmap(remap_data, remap_size);
                    close(rfd);
                } else {
                    // Skip data if file doesn't exist
                    for (uint32_t i = 0; i < np; i++) { ru32(); ru32(); }
                }
                std::cerr << "  Secondary remap " << fid << ": " << np << " pairs" << std::endl;
            }
            continue;
        }
        if (file_id == ENTRY_CORRECTION_MARKER) {
            uint32_t fid = ru32(), c = ru32();
            auto& list = entry_corrections[fid];
            for (uint32_t i = 0; i < c; i++) {
                uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8;
                uint16_t ec; memcpy(&ec, P+pos, 2); pos += 2;
                std::vector<uint32_t> ids(ec);
                if (ec > 0) { memcpy(ids.data(), P+pos, ec*4); pos += ec*4; }
                list.push_back({cid, std::move(ids)});
            }
            std::cerr << "  Entry corrections " << fid << ": " << c << std::endl;
            continue;
        }

        // --- Merge sequence replay (streaming) ---
        uint32_t stride = ru32();
        uint64_t old_size = ru64(), new_size = ru64();
        if (file_id >= (uint32_t)PatchFileId::COUNT) { std::cerr << "Unknown file " << file_id << std::endl; break; }
        const char* fname = patch_file_names[file_id];

        if (stride == 0) {
            // Full replacement — write directly from patch mmap
            uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64();
            FILE* fp = fopen((out_dir + "/" + fname).c_str(), "wb");
            fwrite(P+pos, 1, ds, fp); fclose(fp);
            pos += ds;
            std::cerr << "  " << fname << ": full replace " << ds << " bytes" << std::endl;
            continue;
        }
        if (stride == 0xFE) { uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64(); pos += ds; continue; }

        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        // Determine if this file needs in-memory modifications
        bool needs_remap = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
                            file_id == (uint32_t)PatchFileId::STREET_WAYS ||
                            file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
                            file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
        bool needs_padding = (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24) ||
                             (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24);

        // Read fixup data position (applied inline during merge replay)
        uint32_t n_fixups = ru32();
        size_t fixup_data_pos = pos; // position of delta-encoded fixup data in patch
        if (n_fixups > 0) {
            uint32_t delta_size = ru32();
            fixup_data_pos = pos;
            pos += delta_size; // skip past fixup data
        }
        // Build sorted fixup array (decode delta-encoded, ~2 bytes/entry instead of ~40 bytes/entry in hash map)
        std::vector<std::pair<uint32_t,uint32_t>> fixups_sorted;
        if (n_fixups > 0) {
            fixups_sorted.reserve(n_fixups);
            uint32_t pi = 0, pv = 0; size_t fp = fixup_data_pos;
            for (uint32_t i = 0; i < n_fixups; i++) {
                uint32_t idx = pi + read_varint(P, fp);
                uint32_t val = pv + read_varint(P, fp);
                fixups_sorted.push_back({idx, val});
                pi = idx; pv = val;
            }
        }

        // Get string remap field offsets for this file type
        std::vector<size_t> remap_offs;
        if (!str_remap_vec.empty() && needs_remap) {
            if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) remap_offs = {8, 12};
            else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) remap_offs = {(actual_stride == 12) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) remap_offs = {(actual_stride >= 20) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) remap_offs = {8};
        }

        // mmap old file read-only (zero allocation)
        MappedFile old_mmap = mmap_file(cur_dir + "/" + std::string(fname));
        madvise(const_cast<char*>(old_mmap.data), old_mmap.size, MADV_SEQUENTIAL);
        size_t n_old_records = old_mmap.size / actual_stride;

        // Replay merge sequence — stream output, apply remap/fixups per-record inline
        uint64_t seq_size = ru64();
        size_t seq_end = pos + seq_size;
        FILE* outf = fopen((out_dir + "/" + fname).c_str(), "wb");

        bool track = needs_remap; // track ID remap only for data files
        // File-backed id_map: create temp file, fill with 0xFF, mmap read-write
        int remap_fd = -1;
        uint32_t* id_map_ptr = nullptr;
        std::string remap_path;
        if (track) {
            remap_path = tmpdir + "/remap_" + std::to_string(file_id) + ".bin";
            remap_fd = open(remap_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644);
            size_t remap_bytes = n_old_records * 4;
            ftruncate(remap_fd, remap_bytes); // zero-filled by OS (sparse)
            id_map_ptr = static_cast<uint32_t*>(
                mmap(nullptr, remap_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, remap_fd, 0));
            // File is zero-filled. We store new_index+1 so that 0 means "unmapped".
            // The entry pipeline reads these and subtracts 1.
        }

        // Per-record buffer for applying remap/fixups inline (avoid copying entire file)
        std::vector<char> rec_buf(actual_stride);

        size_t old_rec = 0, new_rec = 0, old_bytes = 0, written = 0;
        size_t fixup_cursor = 0; // cursor into fixups_sorted for sequential access
        while (pos < seq_end) {
            uint8_t op = P[pos++];
            uint32_t count; memcpy(&count, P+pos, 4); pos += 4;
            if (op == OP_MATCH_RUN) {
                for (uint32_t k = 0; k < count; k++) {
                    size_t rec_off = old_bytes + k * actual_stride;
                    if (rec_off + actual_stride > old_mmap.size) break;

                    bool modified = false;
                    // Check if this record needs any modification
                    if (!remap_offs.empty() || needs_padding) modified = true;
                    // Check fixup using sequential cursor (fixups are sorted by record index)
                    uint32_t fixup_val = 0;
                    bool has_fixup = false;
                    while (fixup_cursor < fixups_sorted.size() && fixups_sorted[fixup_cursor].first < old_rec + k)
                        fixup_cursor++;
                    if (fixup_cursor < fixups_sorted.size() && fixups_sorted[fixup_cursor].first == old_rec + k) {
                        has_fixup = true; fixup_val = fixups_sorted[fixup_cursor].second;
                        modified = true; fixup_cursor++;
                    }

                    if (modified) {
                        memcpy(rec_buf.data(), old_mmap.data + rec_off, actual_stride);
                        // Apply padding zeroing
                        if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24)
                            memset(rec_buf.data() + 13, 0, 3);
                        if (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24) {
                            memset(rec_buf.data() + 5, 0, 3);
                            memset(rec_buf.data() + 21, 0, 3);
                        }
                        // Apply string remap
                        for (size_t off : remap_offs) {
                            uint32_t v; memcpy(&v, rec_buf.data() + off, 4);
                            uint32_t nv = str_remap_lookup(v);
                            if (nv != v) memcpy(rec_buf.data() + off, &nv, 4);
                        }
                        // Apply fixup (node_offset/vertex_offset at byte 0)
                        if (has_fixup)
                            memcpy(rec_buf.data(), &fixup_val, 4);
                        fwrite(rec_buf.data(), 1, actual_stride, outf);
                    } else {
                        // No modification needed — write directly from mmap
                        fwrite(old_mmap.data + rec_off, 1, actual_stride, outf);
                    }
                    if (track && old_rec + k < n_old_records) id_map_ptr[old_rec + k] = (uint32_t)(new_rec + k) + 1; // +1 so 0 means unmapped
                }
                written += count * actual_stride;
                old_rec += count; new_rec += count; old_bytes += count * actual_stride;
            } else if (op == OP_INSERT_RUN) {
                size_t bytes = count * actual_stride;
                fwrite(P+pos, 1, bytes, outf);
                written += bytes; pos += bytes; new_rec += count;
            } else if (op == OP_DELETE_RUN) {
                old_rec += count; old_bytes += count * actual_stride;
            }
        }
        fclose(outf);
        unmap_file(old_mmap);
        fixups_sorted.clear(); fixups_sorted.shrink_to_fit();

        if (track) {
            // Sync and munmap the file-backed remap (data is already on disk)
            size_t remap_bytes = n_old_records * 4;
            msync(id_map_ptr, remap_bytes, MS_SYNC);
            munmap(id_map_ptr, remap_bytes);
            close(remap_fd);
            id_map_ptr = nullptr;
            id_remaps[file_id] = {}; // empty placeholder
            std::cerr << "  " << fname << ": " << written << " bytes (remap " << n_old_records << " records to disk)" << std::endl;
        } else {
            std::cerr << "  " << fname << ": " << written << " bytes" << std::endl;
        }
        malloc_trim(0);
    }
    log_phase("Merge replays", t_start);

    // Free string remap (no longer needed)
    { std::vector<std::pair<uint32_t,uint32_t>>().swap(str_remap_vec); }
    malloc_trim(0);

    // --- Phase 4: Streaming entry pipeline ---
    // Walk old geo_cells, remap IDs, apply corrections, write output directly to files.
    {
        double t_entry = now_ms();
        std::cerr << "Entry pipeline (lean streaming)..." << std::endl;

        // mmap remap files from disk (only pages accessed are in RSS)
        auto mmap_remap = [&](PatchFileId fid) -> MappedFile {
            std::string path = tmpdir + "/remap_" + std::to_string((uint32_t)fid) + ".bin";
            auto m = mmap_file(path);
            if (m.data) madvise(const_cast<char*>(m.data), m.size, MADV_SEQUENTIAL);
            return m;
        };
        // Helper: look up remap value from mmap'd file
        // Remap values stored as new_index+1 (0 means unmapped)
        struct RemapRef {
            const uint32_t* data;
            size_t count;
            uint32_t operator[](size_t i) const {
                if (i >= count) return 0xFFFFFFFF;
                uint32_t v = data[i];
                return v == 0 ? 0xFFFFFFFF : v - 1;
            }
            size_t size() const { return count; }
        };
        MappedFile m_w_rm = mmap_remap(PatchFileId::STREET_WAYS);
        MappedFile m_a_rm = mmap_remap(PatchFileId::ADDR_POINTS);
        MappedFile m_i_rm = mmap_remap(PatchFileId::INTERP_WAYS);
        RemapRef w_rm = {(const uint32_t*)m_w_rm.data, m_w_rm.size / 4};
        RemapRef a_rm = {(const uint32_t*)m_a_rm.data, m_a_rm.size / 4};
        RemapRef i_rm = {(const uint32_t*)m_i_rm.data, m_i_rm.size / 4};

        // mmap old files (zero RSS until pages are accessed, then only working set)
        MappedFile m_geo = mmap_file(cur_dir + "/geo_cells.bin");
        MappedFile m_se = mmap_file(cur_dir + "/street_entries.bin");
        MappedFile m_ae = mmap_file(cur_dir + "/addr_entries.bin");
        MappedFile m_ie = mmap_file(cur_dir + "/interp_entries.bin");
        madvise(const_cast<char*>(m_se.data), m_se.size, MADV_SEQUENTIAL);
        madvise(const_cast<char*>(m_ae.data), m_ae.size, MADV_SEQUENTIAL);
        madvise(const_cast<char*>(m_ie.data), m_ie.size, MADV_SEQUENTIAL);
        size_t n_old = m_geo.size / 20;

        // Build correction maps + removed set (small memory)
        std::unordered_set<uint64_t> rm_set(geo_removed.begin(), geo_removed.end());
        std::unordered_map<uint64_t, const std::vector<uint32_t>*> cs, ca, ci_map;
        for (auto& [fid, list] : entry_corrections) {
            auto* m = (fid == (uint32_t)PatchFileId::STREET_ENTRIES) ? &cs :
                      (fid == (uint32_t)PatchFileId::ADDR_ENTRIES) ? &ca :
                      (fid == (uint32_t)PatchFileId::INTERP_ENTRIES) ? &ci_map : nullptr;
            if (m) for (auto& c : list) (*m)[c.cell_id] = &c.ids;
        }
        // Added cells sorted
        std::sort(geo_added.begin(), geo_added.end());
        log_phase("  Setup", t_entry);

        // Open 4 output files
        FILE* f_geo = fopen((out_dir + "/geo_cells.bin").c_str(), "wb");
        FILE* f_se = fopen((out_dir + "/street_entries.bin").c_str(), "wb");
        FILE* f_ae = fopen((out_dir + "/addr_entries.bin").c_str(), "wb");
        FILE* f_ie = fopen((out_dir + "/interp_entries.bin").c_str(), "wb");
        constexpr uint32_t NO = 0xFFFFFFFF;

        // Reusable buffer (one per entry type to avoid aliasing issues)
        std::vector<uint32_t> buf;
        buf.reserve(4096);

        // Helper: parse IDs from mmap'd entry file
        auto parse = [&buf](const MappedFile& f, uint32_t off) {
            buf.clear();
            if (off == NO || off + 2 > f.size) return;
            uint16_t c; memcpy(&c, f.data + off, 2);
            if (off + 2 + (size_t)c * 4 > f.size) return;
            buf.resize(c);
            memcpy(buf.data(), f.data + off + 2, c * 4);
        };
        auto remap = [](std::vector<uint32_t>& ids, const RemapRef& rm) {
            constexpr uint32_t NO2 = 0xFFFFFFFF;
            for (auto& id : ids) if (id < rm.size() && rm[id] != NO2) id = rm[id];
            std::sort(ids.begin(), ids.end());
        };
        // Write entry and return offset, or NO if empty
        auto emit = [&NO](FILE* f, const uint32_t* ids, size_t n) -> uint32_t {
            if (n == 0) return NO;
            uint32_t off = (uint32_t)ftell(f);
            uint16_t c = (uint16_t)n;
            fwrite(&c, 2, 1, f); fwrite(ids, 4, n, f);
            return off;
        };

        // Merge-walk: old cells + added cells in sorted order
        // Both are sorted by cell_id. Merge them, skip removed.
        size_t old_i = 0, add_i = 0;
        size_t cells_written = 0;

        auto process_cell = [&](uint64_t cid, int32_t oi) {
            // For each entry type: check correction → remap old → write
            auto do_entry = [&](const MappedFile& old_e, size_t geo_off, const RemapRef& rm,
                                const std::unordered_map<uint64_t, const std::vector<uint32_t>*>& corr,
                                FILE* outf, uint8_t flag_bit) -> uint32_t {
                // Check flag
                bool has = false;
                if (oi >= 0 && (size_t)oi * 20 + geo_off + 4 <= m_geo.size) {
                    uint32_t off; memcpy(&off, m_geo.data + oi * 20 + geo_off, 4);
                    has = (off != NO);
                } else if (oi >= 0) {
                    std::cerr << "OOB: oi=" << oi << " geo_off=" << geo_off << " m_geo.size=" << m_geo.size << std::endl;
                    return NO;
                }
                auto fc = flag_corrections.find(cid);
                if (fc != flag_corrections.end()) has = (fc->second & flag_bit) != 0;
                if (corr.count(cid)) has = true;
                if (!has) return NO;

                auto cit = corr.find(cid);
                if (cit != corr.end()) return emit(outf, cit->second->data(), cit->second->size());

                if (oi < 0) return NO;
                if ((size_t)oi * 20 + geo_off + 4 > m_geo.size) return NO;
                uint32_t off; memcpy(&off, m_geo.data + oi * 20 + geo_off, 4);
                parse(old_e, off);
                if (buf.empty()) return NO;
                remap(buf, rm);
                return emit(outf, buf.data(), buf.size());
            };

            uint32_t so = do_entry(m_se, 8, w_rm, cs, f_se, 1);
            uint32_t ao = do_entry(m_ae, 12, a_rm, ca, f_ae, 2);
            uint32_t io = do_entry(m_ie, 16, i_rm, ci_map, f_ie, 4);
            fwrite(&cid, 8, 1, f_geo); fwrite(&so, 4, 1, f_geo); fwrite(&ao, 4, 1, f_geo); fwrite(&io, 4, 1, f_geo);
            cells_written++;
        };

        while (old_i < n_old || add_i < geo_added.size()) {
            uint64_t old_cid = UINT64_MAX, add_cid = UINT64_MAX;
            if (old_i < n_old) memcpy(&old_cid, m_geo.data + old_i * 20, 8);
            if (add_i < geo_added.size()) add_cid = geo_added[add_i];

            if (old_cid <= add_cid) {
                if (!rm_set.count(old_cid))
                    process_cell(old_cid, (int32_t)old_i);
                old_i++;
                if (old_cid == add_cid) add_i++; // skip duplicate add
            } else {
                process_cell(add_cid, -1);
                add_i++;
            }

            if (cells_written % 10000000 == 0 && cells_written > 0) {
                std::cerr << "    " << cells_written << " cells, se=" << ftell(f_se)/1024/1024
                          << "M ae=" << ftell(f_ae)/1024/1024 << "M rss=" << get_rss_mb() << "M" << std::endl;
            }
        }

        fclose(f_geo); fclose(f_se); fclose(f_ae); fclose(f_ie);
        unmap_file(m_geo); unmap_file(m_se); unmap_file(m_ae); unmap_file(m_ie);
        unmap_file(m_w_rm); unmap_file(m_a_rm); unmap_file(m_i_rm);
        std::cerr << "  Geo: " << cells_written << " cells written" << std::endl;
        malloc_trim(0);
        log_phase("  Geo entries complete", t_entry);

        // Admin: small, use existing rebuild + corrections
        {
            uint32_t no_data = 0xFFFFFFFF;
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            MappedFile m_ad_rm = mmap_remap(PatchFileId::ADMIN_POLYGONS);
            if (m_ad_rm.data) {
                const uint32_t* ad_vec = (const uint32_t*)m_ad_rm.data;
                size_t ad_count = m_ad_rm.size / 4;
                for (uint32_t i = 0; i < ad_count; i++)
                    if (ad_vec[i] != 0) ad_rm[i] = ad_vec[i] - 1; // decode +1 encoding
                unmap_file(m_ad_rm);
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_adme = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_adme, ad_rm, admin_added, admin_removed);

            auto ecit = entry_corrections.find((uint32_t)PatchFileId::ADMIN_ENTRIES);
            if (ecit != entry_corrections.end()) {
                std::unordered_map<uint64_t, const std::vector<uint32_t>*> ac_corr;
                for (auto& c : ecit->second) ac_corr[c.cell_id] = &c.ids;
                size_t n = admin.admin_cells_data.size() / 12;
                FILE* fac = fopen((out_dir + "/admin_cells.bin").c_str(), "wb");
                FILE* fae = fopen((out_dir + "/admin_entries.bin").c_str(), "wb");
                uint32_t ae_wpos = 0;
                for (size_t i = 0; i < n; i++) {
                    uint64_t cid; memcpy(&cid, admin.admin_cells_data.data()+i*12, 8);
                    fwrite(&cid, 8, 1, fac);
                    auto cit = ac_corr.find(cid);
                    if (cit != ac_corr.end()) {
                        uint32_t off = cit->second->empty() ? no_data : ae_wpos;
                        fwrite(&off, 4, 1, fac);
                        if (!cit->second->empty()) {
                            uint16_t c = cit->second->size();
                            fwrite(&c, 2, 1, fae); fwrite(cit->second->data(), 4, c, fae);
                            ae_wpos += 2 + c*4;
                        }
                    } else {
                        uint32_t old_off; memcpy(&old_off, admin.admin_cells_data.data()+i*12+8, 4);
                        if (old_off != no_data && old_off+2 <= admin.admin_entries_data.size()) {
                            uint32_t new_off = ae_wpos; fwrite(&new_off, 4, 1, fac);
                            uint16_t c; memcpy(&c, admin.admin_entries_data.data()+old_off, 2);
                            fwrite(admin.admin_entries_data.data()+old_off, 1, 2+c*4, fae);
                            ae_wpos += 2+c*4;
                        } else {
                            fwrite(&no_data, 4, 1, fac);
                        }
                    }
                }
                fclose(fac); fclose(fae);
                std::cerr << "  Admin: " << n << " cells, " << ac_corr.size() << " corrections" << std::endl;
            } else {
                write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
                write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            }
            log_phase("  Admin", t_entry);
        }
    }

    // Cleanup
    unmap_file(patch_map);
    { std::string cmd = "rm -rf '" + tmpdir + "'"; system(cmd.c_str()); }
    log_phase("Total", t_start);
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

+            // Walk old pool directly (no old_ptrs vector) to minimize memory

-                // Parse old strings just for the merge (lightweight: only need idx + str ptr)

-                std::vector<const char*> old_ptrs; // indexed by string index

-                  while (sp < old_pool.size) { old_ptrs.push_back(old_pool.data + sp); sp += strlen(old_pool.data + sp) + 1; }

-                merged.reserve(old_ptrs.size() + added.size());

-                for (uint32_t i = 0; i < old_ptrs.size(); i++)

-                    if (!del_set.count(i)) merged.push_back(old_ptrs[i]);

+                // Count old strings to reserve

+                { size_t sp = 0; while (sp < old_pool.size) { sp += strlen(old_pool.data + sp) + 1; n_old++; } }

+                merged.reserve(n_old + added.size());

+                { size_t sp = 0; uint32_t idx = 0;

+                  while (sp < old_pool.size) {

+                      if (!del_set.count(idx)) merged.push_back(old_pool.data + sp);

+                      sp += strlen(old_pool.data + sp) + 1; idx++;

-            } // merged, old_ptrs, added, del_set, del_idx all freed here

+            { std::vector<std::string>().swap(added); std::unordered_set<uint32_t>().swap(del_set);

+              std::vector<uint32_t>().swap(del_idx); }


Bash

make -j$(nproc) geocoder-patch 2>&1 | tail -3 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "rm -f /home/michtest/geocoder-patch" 2>&1 && sshpass -p 'elon-is-insane' scp -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:/home/michtest/ 2>&1 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "rm -rf /home/michtest/patched-planet-stream/ && systemd-run --user --scope -p MemoryMax=500M -- ./geocoder-patch planet-B/ validated-patches/planet-mar16-to-mar23-v2.gcpatch -o patched-planet-stream/ 2>&1 | grep -E '\\[.*MiB\\]' && echo '---' && for f in planet-C/*; do b=\$(basename \$f); cmp -s patched-planet-stream/\$b planet-C/\$b || echo \"MISMATCH: \$b\"; done && echo All_checked" 2>&1

Planet 500 MiB cgroup test


make: *** No rule to make target 'geocoder-patch'.  Stop.
  [0.3s, 3 MiB] Decompress
  [0.3s, 3 MiB] Strings
---
MISMATCH: addr_entries.bin
MISMATCH: admin_cells.bin
MISMATCH: admin_entries.bin
MISMATCH: admin_polygons.bin
MISMATCH: admin_vertices.bin
MISMATCH: geo_cells.bin
MISMATCH: interp_entries.bin
MISMATCH: interp_nodes.bin
MISMATCH: interp_ways.bin
MISMATCH: manifest.json
MISMATCH: street_entries.bin
MISMATCH: street_nodes.bin
All_checked


Make failed from wrong dir again. Let me fix:


cd /root/traccar-geocoder/builder/build && make -j$(nproc) geocoder-patch 2>&1 | tail -3 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "rm -f /home/michtest/geocoder-patch" 2>&1 && sshpass -p 'elon-is-insane' scp -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:/home/michtest/ 2>&1 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "rm -rf /home/michtest/patched-planet-stream/ && systemd-run --user --scope -p MemoryMax=500M -- ./geocoder-patch planet-B/ validated-patches/planet-mar16-to-mar23-v2.gcpatch -o patched-planet-stream/ 2>&1 | grep -E '\\[.*MiB\\]' && echo '---' && for f in planet-C/*; do b=\$(basename \$f); cmp -s patched-planet-stream/\$b planet-C/\$b || echo \"MISMATCH: \$b\"; done && echo All_checked" 2>&1

Planet 500 MiB - take 2


[ 50%] Building CXX object CMakeFiles/geocoder-patch.dir/tools/geocoder_patch.cpp.o
[100%] Linking CXX executable geocoder-patch
[100%] Built target geocoder-patch
  [0.3s, 3 MiB] Decompress
  [0.3s, 3 MiB] Strings
---
MISMATCH: addr_entries.bin
MISMATCH: admin_cells.bin
MISMATCH: admin_entries.bin
MISMATCH: admin_polygons.bin
MISMATCH: admin_vertices.bin
MISMATCH: geo_cells.bin
MISMATCH: interp_entries.bin
MISMATCH: interp_nodes.bin
MISMATCH: interp_ways.bin
MISMATCH: manifest.json
MISMATCH: street_entries.bin
MISMATCH: street_nodes.bin
All_checked


Still OOM at 500 MiB. The `merged` vector is 12M × 8 = 96 MiB, plus the `del_set` hash set (~80 MiB for 4.5K entries — actually this is tiny), plus the string pool mmap pages being accessed during the walk + sort. The sort accesses all 12M pointers which faults in the string pool pages (~215 MiB).

Total: merged (96) + old pool pages (215) + sort workspace + patch mmap pages (~50 accessed) + kernel overhead = ~400 MiB. Plus cgroup overhead. 500 MiB should theoretically work.

The issue might be the `added` strings vector. Let me check: `added` has 10K strings, which is tiny. The `del_set` has 4.5K entries — also tiny. The real issue is the sort touching all 12M string pointers, which faults in most of the old_pool (215 MiB).

Let me try 550 MiB:


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "rm -rf /home/michtest/patched-planet-stream/ && systemd-run --user --scope -p MemoryMax=550M -- ./geocoder-patch planet-B/ validated-patches/planet-mar16-to-mar23-v2.gcpatch -o patched-planet-stream/ 2>&1 | grep -E '\\[.*MiB\\]' && echo '---' && for f in planet-C/*; do b=\$(basename \$f); cmp -s patched-planet-stream/\$b planet-C/\$b || echo \"MISMATCH: \$b\"; done && echo All_checked" 2>&1

Planet 550 MiB test


  [0.3s, 3 MiB] Decompress
  [0.3s, 3 MiB] Strings
---
MISMATCH: addr_entries.bin
MISMATCH: admin_cells.bin
MISMATCH: admin_entries.bin
MISMATCH: admin_polygons.bin
MISMATCH: admin_vertices.bin
MISMATCH: geo_cells.bin
MISMATCH: interp_entries.bin
MISMATCH: interp_nodes.bin
MISMATCH: interp_ways.bin
MISMATCH: manifest.json
MISMATCH: street_entries.bin
MISMATCH: street_nodes.bin
All_checked


Still fails at 550 MiB. The string rebuild Phase A needs >550 MiB. The `merged` vector at 96 MiB + old pool mmap (215 MiB pages accessed during sort) + new pool write buffer + kernel overhead ≈ 500 MiB. But the cgroup counts mmap'd pages as well.

The core issue: sorting 12M string pointers requires accessing (and paging in) most of the 215 MiB old string pool. That's unavoidable with the current approach.

Alternative: don't sort — use the fact that old pool is already sorted. Walk old pool in order (already sorted), skip deleted strings, merge-insert added strings (also sorted from the diff tool). This is a merge of two sorted sequences — O(n+m), no sort, sequential access pattern. The `added` strings are already sorted (they come from the diff tool's string diff which walks pools alphabetically).


Edit

            // Phase A: build merged pool, write to file, free all temporaries
            // Walk old pool directly (no old_ptrs vector) to minimize memory
            {
                std::vector<const char*> merged;
                // Count old strings to reserve
                size_t n_old = 0;
                { size_t sp = 0; while (sp < old_pool.size) { sp += strlen(old_pool.data + sp) + 1; n_old++; } }
                merged.reserve(n_old + added.size());
                { size_t sp = 0; uint32_t idx = 0;
                  while (sp < old_pool.size) {
                      if (!del_set.count(idx)) merged.push_back(old_pool.data + sp);
                      sp += strlen(old_pool.data + sp) + 1; idx++;
                  }
                }
                for (auto& s : added) merged.push_back(s.c_str());
                std::sort(merged.begin(), merged.end(), [](const char* a, const char* b) { return strcmp(a, b) < 0; });
                FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
                for (const char* s : merged) { size_t l = strlen(s) + 1; fwrite(s, 1, l, fp); }
                fclose(fp);
            }
            // Free temporaries
            { std::vector<std::string>().swap(added); std::unordered_set<uint32_t>().swap(del_set);
              std::vector<uint32_t>().swap(del_idx); }
            malloc_trim(0);

            // Phase A: merge old (minus deleted) + added strings, write directly to file.
            // Both old pool and added are sorted alphabetically → merge-write with no sort.
            // Memory: only the added strings vector (~small) + del_set (~small).
            {
                FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
                size_t sp = 0; uint32_t idx = 0; size_t ai = 0;
                // Sort added strings (they should already be sorted from diff tool, but ensure)
                std::sort(added.begin(), added.end());
                while (sp < old_pool.size || ai < added.size()) {
                    const char* old_s = nullptr;
                    // Skip deleted strings
                    while (sp < old_pool.size) {
                        if (!del_set.count(idx)) { old_s = old_pool.data + sp; break; }
                        sp += strlen(old_pool.data + sp) + 1; idx++;
                    }
                    const char* add_s = ai < added.size() ? added[ai].c_str() : nullptr;
                    // Merge: pick the smaller string
                    if (old_s && add_s) {
                        int c = strcmp(old_s, add_s);
                        if (c <= 0) {
                            size_t l = strlen(old_s) + 1; fwrite(old_s, 1, l, fp);
                            sp += l; idx++;
                            if (c == 0) ai++; // duplicate — skip added
                        } else {
                            size_t l = added[ai].size() + 1; fwrite(add_s, 1, l, fp);
                            ai++;
                        }
                    } else if (old_s) {
                        size_t l = strlen(old_s) + 1; fwrite(old_s, 1, l, fp);
                        sp += l; idx++;
                    } else if (add_s) {
                        size_t l = added[ai].size() + 1; fwrite(add_s, 1, l, fp);
                        ai++;
                    } else break;
                }
                fclose(fp);
            }
            { std::vector<std::string>().swap(added); std::unordered_set<uint32_t>().swap(del_set);
              std::vector<uint32_t>().swap(del_idx); }
            malloc_trim(0);


The file /root/traccar-geocoder/builder/tools/geocoder_patch.cpp has been updated successfully.

            // Phase A: merge old (minus deleted) + added strings, write directly to file.
            // Both old pool and added are sorted alphabetically → merge-write with no sort.
            // Memory: only the added strings vector (~small) + del_set (~small).
            {
                FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
                size_t sp = 0; uint32_t idx = 0; size_t ai = 0;
                // Sort added strings (they should already be sorted from diff tool, but ensure)
                std::sort(added.begin(), added.end());
                while (sp < old_pool.size || ai < added.size()) {
                    const char* old_s = nullptr;
                    // Skip deleted strings
                    while (sp < old_pool.size) {
                        if (!del_set.count(idx)) { old_s = old_pool.data + sp; break; }
                        sp += strlen(old_pool.data + sp) + 1; idx++;
                    }
                    const char* add_s = ai < added.size() ? added[ai].c_str() : nullptr;
                    // Merge: pick the smaller string
                    if (old_s && add_s) {
                        int c = strcmp(old_s, add_s);
                        if (c <= 0) {
                            size_t l = strlen(old_s) + 1; fwrite(old_s, 1, l, fp);
                            sp += l; idx++;
                            if (c == 0) ai++; // duplicate — skip added
                        } else {
                            size_t l = added[ai].size() + 1; fwrite(add_s, 1, l, fp);
                            ai++;
                        }
                    } else if (old_s) {
                        size_t l = strlen(old_s) + 1; fwrite(old_s, 1, l, fp);
                        sp += l; idx++;
                    } else if (add_s) {
                        size_t l = added[ai].size() + 1; fwrite(add_s, 1, l, fp);
                        ai++;
                    } else break;
                }
                fclose(fp);
            }
            { std::vector<std::string>().swap(added); std::unordered_set<uint32_t>().swap(del_set);
              std::vector<uint32_t>().swap(del_idx); }
            malloc_trim(0);

// geocoder-patch v4: Low-memory streaming patch application.
//
// Key design: mmap old files (zero copy), stream output via fwrite (never accumulate),
// process entry pipeline cell-by-cell. Target: <1 GiB peak RSS for planet.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <fcntl.h>
#include <iomanip>
#include <iostream>
#include <malloc.h>
#include <string>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unordered_map>
#include <unordered_set>
#include <unistd.h>
#include <vector>

#include "patch_format.h"

enum MergeOp : uint8_t { OP_MATCH_RUN = 0, OP_INSERT_RUN = 1, OP_DELETE_RUN = 2 };

// --- Helpers ---
static double now_ms() {
    struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
    return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6;
}
static size_t get_rss_mb() {
    FILE* f = fopen("/proc/self/statm", "r");
    if (!f) return 0;
    size_t dummy, rss; fscanf(f, "%zu %zu", &dummy, &rss); fclose(f);
    return rss * 4096 / (1024*1024);
}
static void log_phase(const char* label, double start) {
    std::cerr << "  [" << std::fixed << std::setprecision(1) << (now_ms() - start) / 1000.0
              << "s, " << get_rss_mb() << " MiB] " << label << std::endl;
}

// mmap a file read-only. Returns {pointer, size}. Caller must munmap.
struct MappedFile { const char* data; size_t size; };
static MappedFile mmap_file(const std::string& path) {
    int fd = open(path.c_str(), O_RDONLY);
    if (fd < 0) return {nullptr, 0};
    struct stat st; fstat(fd, &st);
    size_t sz = st.st_size;
    if (sz == 0) { close(fd); return {nullptr, 0}; }
    void* p = mmap(nullptr, sz, PROT_READ, MAP_PRIVATE, fd, 0);
    close(fd);
    if (p == MAP_FAILED) return {nullptr, 0};
    return {static_cast<const char*>(p), sz};
}
static void unmap_file(MappedFile& f) {
    if (f.data) { munmap(const_cast<char*>(f.data), f.size); f.data = nullptr; f.size = 0; }
}

// Detect stride from file size
static size_t detect_stride(const std::string& path, std::initializer_list<size_t> candidates) {
    struct stat st; if (stat(path.c_str(), &st) != 0) return *candidates.begin();
    for (size_t s : candidates) if (st.st_size % s == 0 && st.st_size > 0) return s;
    return *candidates.begin();
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }
    std::string cur_dir = argv[1], patch_path = argv[2], out_dir = argv[4];
    ensure_dir(out_dir);
    std::string tmpdir = "/tmp/geocoder-patch-" + std::to_string(getpid());
    ensure_dir(tmpdir);
    double t_start = now_ms();

    // --- Phase 1: Decompress + mmap patch ---
    std::string raw_path = tmpdir + "/patch.raw";
    {
        std::string cmd = "zstd -d '" + patch_path + "' -o '" + raw_path + "' -f --quiet 2>/dev/null";
        if (system(cmd.c_str()) != 0) { std::cerr << "Failed to decompress" << std::endl; return 1; }
    }
    MappedFile patch_map = mmap_file(raw_path);
    if (!patch_map.data) { std::cerr << "Failed to mmap patch" << std::endl; return 1; }
    remove(raw_path.c_str());
    const char* P = patch_map.data;
    size_t patch_size = patch_map.size;
    std::cerr << "Patch: " << patch_size << " bytes" << std::endl;
    log_phase("Decompress", t_start);

    size_t pos = 0;
    auto ru32 = [&]() -> uint32_t { uint32_t v; memcpy(&v, P+pos, 4); pos += 4; return v; };
    auto ru64 = [&]() -> uint64_t { uint64_t v; memcpy(&v, P+pos, 8); pos += 8; return v; };

    // Header
    if (memcmp(P, GCPATCH_MAGIC, 8) != 0) { std::cerr << "Bad magic" << std::endl; return 1; }
    pos = 8;
    uint32_t ver = ru32(); if (ver != 2) { std::cerr << "Bad version" << std::endl; return 1; }
    ru32(); // flags

    // --- Phase 2: String rebuild ---
    // Sorted vector remap: (old_offset, new_offset) pairs sorted by old_offset.
    // Uses 96 MiB for 12M entries vs 672 MiB for unordered_map.
    std::vector<std::pair<uint32_t, uint32_t>> str_remap_vec;
    {
        uint32_t marker = ru32();
        if (marker == 0xFFFFFFF7) {
            uint32_t n_added = ru32(), n_deleted = ru32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ru32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());

            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");

            // Phase A: build merged pool, write to file, free all temporaries
            // Walk old pool directly (no old_ptrs vector) to minimize memory
            {
                std::vector<const char*> merged;
                // Count old strings to reserve
                size_t n_old = 0;
                { size_t sp = 0; while (sp < old_pool.size) { sp += strlen(old_pool.data + sp) + 1; n_old++; } }
                merged.reserve(n_old + added.size());
                { size_t sp = 0; uint32_t idx = 0;
                  while (sp < old_pool.size) {
                      if (!del_set.count(idx)) merged.push_back(old_pool.data + sp);
                      sp += strlen(old_pool.data + sp) + 1; idx++;
                  }
                }
                for (auto& s : added) merged.push_back(s.c_str());
                std::sort(merged.begin(), merged.end(), [](const char* a, const char* b) { return strcmp(a, b) < 0; });
                FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
                for (const char* s : merged) { size_t l = strlen(s) + 1; fwrite(s, 1, l, fp); }
                fclose(fp);
            }
            // Free temporaries
            { std::vector<std::string>().swap(added); std::unordered_set<uint32_t>().swap(del_set);
              std::vector<uint32_t>().swap(del_idx); }
            malloc_trim(0);

            // Phase B: build remap by merge-walking old pool vs new pool (both sorted, both mmap'd)
            MappedFile new_pool = mmap_file(out_dir + "/strings.bin");
            {
                // Walk both pools simultaneously — no vectors needed, just cursors
                size_t old_pos = 0, new_pos = 0;
                while (old_pos < old_pool.size && new_pos < new_pool.size) {
                    const char* os = old_pool.data + old_pos;
                    const char* ns = new_pool.data + new_pos;
                    int c = strcmp(os, ns);
                    if (c == 0) {
                        if (old_pos != new_pos) // offset changed
                            str_remap_vec.push_back({(uint32_t)old_pos, (uint32_t)new_pos});
                        old_pos += strlen(os) + 1;
                        new_pos += strlen(ns) + 1;
                    } else if (c < 0) {
                        old_pos += strlen(os) + 1; // deleted string
                    } else {
                        new_pos += strlen(ns) + 1; // added string
                    }
                }
            }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
            unmap_file(old_pool);
            unmap_file(new_pool);
            std::cerr << "  Strings: +" << n_added << " -" << n_deleted
                      << ", " << str_remap_vec.size() << " remapped (" << str_remap_vec.size() * 8 / 1024 / 1024 << " MiB)" << std::endl;
            marker = ru32();
        }
        if (marker == 0xFFFFFFFE) {
            uint32_t c = ru32();
            for (uint32_t i = 0; i < c; i++) { uint32_t a = ru32(), b = ru32(); str_remap_vec.push_back({a, b}); }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
        } else pos -= 4;
    }
    log_phase("Strings", t_start);

    // Detect strides
    size_t way_stride = detect_stride(cur_dir + "/street_ways.bin", {12, 9});
    size_t interp_stride = detect_stride(cur_dir + "/interp_ways.bin", {24, 20, 18});
    size_t admin_stride = detect_stride(cur_dir + "/admin_polygons.bin", {24, 20, 19});

    // String remap lookup via sorted vector + binary search
    auto str_remap_lookup = [&](uint32_t old_off) -> uint32_t {
        auto it = std::lower_bound(str_remap_vec.begin(), str_remap_vec.end(),
            std::make_pair(old_off, (uint32_t)0));
        if (it != str_remap_vec.end() && it->first == old_off) return it->second;
        return old_off; // no remap
    };

    // --- Phase 3: Merge replays (streaming output) ---
    // ID remaps: file_id → vector<uint32_t> where remap[old_idx] = new_idx
    std::unordered_map<uint32_t, std::vector<uint32_t>> id_remaps;
    std::vector<uint64_t> geo_added, geo_removed, admin_added, admin_removed;
    std::unordered_map<uint64_t, uint8_t> flag_corrections;
    struct CellCorr { uint64_t cell_id; std::vector<uint32_t> ids; };
    std::unordered_map<uint32_t, std::vector<CellCorr>> entry_corrections;

    while (pos < patch_size) {
        uint32_t file_id = ru32();
        if (file_id == 0xFFFFFFFF) break;

        // --- Metadata sections ---
        if (file_id == 0xFFFFFFF7) {
            // String diff in main loop — same memory-efficient approach as Phase 2
            uint32_t n_added = ru32(), n_deleted = ru32();
            std::vector<std::string> added;
            for (uint32_t i = 0; i < n_added; i++) { const char* s = P+pos; added.push_back(s); pos += strlen(s)+1; }
            std::vector<uint32_t> del_idx(n_deleted);
            for (uint32_t i = 0; i < n_deleted; i++) del_idx[i] = ru32();
            std::unordered_set<uint32_t> del_set(del_idx.begin(), del_idx.end());
            MappedFile old_pool = mmap_file(cur_dir + "/strings.bin");
            struct OS { uint32_t idx; uint32_t off; const char* str; };
            std::vector<OS> old_strs;
            { size_t sp = 0; uint32_t ix = 0;
              while (sp < old_pool.size) { old_strs.push_back({ix++, (uint32_t)sp, old_pool.data+sp}); sp += strlen(old_pool.data+sp)+1; }
            }
            std::vector<const char*> merged;
            for (auto& os : old_strs) if (!del_set.count(os.idx)) merged.push_back(os.str);
            for (auto& s : added) merged.push_back(s.c_str());
            std::sort(merged.begin(), merged.end(), [](const char* a, const char* b) { return strcmp(a,b) < 0; });
            FILE* fp = fopen((out_dir + "/strings.bin").c_str(), "wb");
            uint32_t wpos = 0;
            for (const char* s : merged) { size_t l = strlen(s)+1; fwrite(s, 1, l, fp); wpos += l; }
            fclose(fp);
            // Build remap via merge-walk
            MappedFile new_pool = mmap_file(out_dir + "/strings.bin");
            std::vector<std::pair<const char*, uint32_t>> new_strs;
            { size_t sp = 0;
              while (sp < new_pool.size) { new_strs.push_back({new_pool.data+sp, (uint32_t)sp}); sp += strlen(new_pool.data+sp)+1; }
            }
            size_t oi2 = 0, ni2 = 0;
            while (oi2 < old_strs.size() && ni2 < new_strs.size()) {
                int c = strcmp(old_strs[oi2].str, new_strs[ni2].first);
                if (c == 0) {
                    if (old_strs[oi2].off != new_strs[ni2].second)
                        str_remap_vec.push_back({old_strs[oi2].off, new_strs[ni2].second});
                    oi2++; ni2++;
                } else if (c < 0) { oi2++; } else { ni2++; }
            }
            std::sort(str_remap_vec.begin(), str_remap_vec.end());
            unmap_file(old_pool); unmap_file(new_pool);
            std::cerr << "  Strings (loop): +" << n_added << " -" << n_deleted << " → " << merged.size()
                      << ", " << str_remap_vec.size() << " remapped" << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_GEO_MARKER) {
            uint32_t na = ru32(), nr = ru32();
            geo_added.resize(na); geo_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { memcpy(&geo_added[i], P+pos, 8); pos += 8; }
            for (uint32_t i = 0; i < nr; i++) { memcpy(&geo_removed[i], P+pos, 8); pos += 8; }
            std::cerr << "  Geo cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_CHANGES_ADMIN_MARKER) {
            uint32_t na = ru32(), nr = ru32();
            admin_added.resize(na); admin_removed.resize(nr);
            for (uint32_t i = 0; i < na; i++) { memcpy(&admin_added[i], P+pos, 8); pos += 8; }
            for (uint32_t i = 0; i < nr; i++) { memcpy(&admin_removed[i], P+pos, 8); pos += 8; }
            std::cerr << "  Admin cells: +" << na << " -" << nr << std::endl;
            continue;
        }
        if (file_id == CELL_FLAGS_MARKER) {
            uint32_t c = ru32();
            for (uint32_t i = 0; i < c; i++) { uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8; flag_corrections[cid] = P[pos++]; }
            std::cerr << "  Flag corrections: " << c << std::endl;
            continue;
        }
        if (file_id == SECONDARY_REMAP_MARKER) {
            // Apply secondary remaps directly to the temp remap files on disk
            uint32_t nf = ru32();
            for (uint32_t f = 0; f < nf; f++) {
                uint32_t fid = ru32(), np = ru32();
                std::string remap_path = tmpdir + "/remap_" + std::to_string(fid) + ".bin";
                // Open remap file for read-write
                int rfd = open(remap_path.c_str(), O_RDWR);
                if (rfd >= 0) {
                    struct stat st; fstat(rfd, &st);
                    size_t remap_size = st.st_size;
                    uint32_t* remap_data = static_cast<uint32_t*>(
                        mmap(nullptr, remap_size, PROT_READ | PROT_WRITE, MAP_SHARED, rfd, 0));
                    size_t remap_count = remap_size / 4;
                    for (uint32_t i = 0; i < np; i++) {
                        uint32_t o = ru32(), n = ru32();
                        if (o < remap_count) remap_data[o] = n + 1; // +1 encoding
                    }
                    munmap(remap_data, remap_size);
                    close(rfd);
                } else {
                    // Skip data if file doesn't exist
                    for (uint32_t i = 0; i < np; i++) { ru32(); ru32(); }
                }
                std::cerr << "  Secondary remap " << fid << ": " << np << " pairs" << std::endl;
            }
            continue;
        }
        if (file_id == ENTRY_CORRECTION_MARKER) {
            uint32_t fid = ru32(), c = ru32();
            auto& list = entry_corrections[fid];
            for (uint32_t i = 0; i < c; i++) {
                uint64_t cid; memcpy(&cid, P+pos, 8); pos += 8;
                uint16_t ec; memcpy(&ec, P+pos, 2); pos += 2;
                std::vector<uint32_t> ids(ec);
                if (ec > 0) { memcpy(ids.data(), P+pos, ec*4); pos += ec*4; }
                list.push_back({cid, std::move(ids)});
            }
            std::cerr << "  Entry corrections " << fid << ": " << c << std::endl;
            continue;
        }

        // --- Merge sequence replay (streaming) ---
        uint32_t stride = ru32();
        uint64_t old_size = ru64(), new_size = ru64();
        if (file_id >= (uint32_t)PatchFileId::COUNT) { std::cerr << "Unknown file " << file_id << std::endl; break; }
        const char* fname = patch_file_names[file_id];

        if (stride == 0) {
            // Full replacement — write directly from patch mmap
            uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64();
            FILE* fp = fopen((out_dir + "/" + fname).c_str(), "wb");
            fwrite(P+pos, 1, ds, fp); fclose(fp);
            pos += ds;
            std::cerr << "  " << fname << ": full replace " << ds << " bytes" << std::endl;
            continue;
        }
        if (stride == 0xFE) { uint32_t nf = ru32(); (void)nf; uint64_t ds = ru64(); pos += ds; continue; }

        size_t actual_stride = stride;
        if (file_id == (uint32_t)PatchFileId::STREET_WAYS) actual_stride = way_stride;
        else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) actual_stride = interp_stride;
        else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) actual_stride = admin_stride;

        // Determine if this file needs in-memory modifications
        bool needs_remap = (file_id == (uint32_t)PatchFileId::ADDR_POINTS ||
                            file_id == (uint32_t)PatchFileId::STREET_WAYS ||
                            file_id == (uint32_t)PatchFileId::INTERP_WAYS ||
                            file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS);
        bool needs_padding = (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24) ||
                             (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24);

        // Read fixup data position (applied inline during merge replay)
        uint32_t n_fixups = ru32();
        size_t fixup_data_pos = pos; // position of delta-encoded fixup data in patch
        if (n_fixups > 0) {
            uint32_t delta_size = ru32();
            fixup_data_pos = pos;
            pos += delta_size; // skip past fixup data
        }
        // Build sorted fixup array (decode delta-encoded, ~2 bytes/entry instead of ~40 bytes/entry in hash map)
        std::vector<std::pair<uint32_t,uint32_t>> fixups_sorted;
        if (n_fixups > 0) {
            fixups_sorted.reserve(n_fixups);
            uint32_t pi = 0, pv = 0; size_t fp = fixup_data_pos;
            for (uint32_t i = 0; i < n_fixups; i++) {
                uint32_t idx = pi + read_varint(P, fp);
                uint32_t val = pv + read_varint(P, fp);
                fixups_sorted.push_back({idx, val});
                pi = idx; pv = val;
            }
        }

        // Get string remap field offsets for this file type
        std::vector<size_t> remap_offs;
        if (!str_remap_vec.empty() && needs_remap) {
            if (file_id == (uint32_t)PatchFileId::ADDR_POINTS) remap_offs = {8, 12};
            else if (file_id == (uint32_t)PatchFileId::STREET_WAYS) remap_offs = {(actual_stride == 12) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::INTERP_WAYS) remap_offs = {(actual_stride >= 20) ? 8ul : 5ul};
            else if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS) remap_offs = {8};
        }

        // mmap old file read-only (zero allocation)
        MappedFile old_mmap = mmap_file(cur_dir + "/" + std::string(fname));
        madvise(const_cast<char*>(old_mmap.data), old_mmap.size, MADV_SEQUENTIAL);
        size_t n_old_records = old_mmap.size / actual_stride;

        // Replay merge sequence — stream output, apply remap/fixups per-record inline
        uint64_t seq_size = ru64();
        size_t seq_end = pos + seq_size;
        FILE* outf = fopen((out_dir + "/" + fname).c_str(), "wb");

        bool track = needs_remap; // track ID remap only for data files
        // File-backed id_map: create temp file, fill with 0xFF, mmap read-write
        int remap_fd = -1;
        uint32_t* id_map_ptr = nullptr;
        std::string remap_path;
        if (track) {
            remap_path = tmpdir + "/remap_" + std::to_string(file_id) + ".bin";
            remap_fd = open(remap_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644);
            size_t remap_bytes = n_old_records * 4;
            ftruncate(remap_fd, remap_bytes); // zero-filled by OS (sparse)
            id_map_ptr = static_cast<uint32_t*>(
                mmap(nullptr, remap_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, remap_fd, 0));
            // File is zero-filled. We store new_index+1 so that 0 means "unmapped".
            // The entry pipeline reads these and subtracts 1.
        }

        // Per-record buffer for applying remap/fixups inline (avoid copying entire file)
        std::vector<char> rec_buf(actual_stride);

        size_t old_rec = 0, new_rec = 0, old_bytes = 0, written = 0;
        size_t fixup_cursor = 0; // cursor into fixups_sorted for sequential access
        while (pos < seq_end) {
            uint8_t op = P[pos++];
            uint32_t count; memcpy(&count, P+pos, 4); pos += 4;
            if (op == OP_MATCH_RUN) {
                for (uint32_t k = 0; k < count; k++) {
                    size_t rec_off = old_bytes + k * actual_stride;
                    if (rec_off + actual_stride > old_mmap.size) break;

                    bool modified = false;
                    // Check if this record needs any modification
                    if (!remap_offs.empty() || needs_padding) modified = true;
                    // Check fixup using sequential cursor (fixups are sorted by record index)
                    uint32_t fixup_val = 0;
                    bool has_fixup = false;
                    while (fixup_cursor < fixups_sorted.size() && fixups_sorted[fixup_cursor].first < old_rec + k)
                        fixup_cursor++;
                    if (fixup_cursor < fixups_sorted.size() && fixups_sorted[fixup_cursor].first == old_rec + k) {
                        has_fixup = true; fixup_val = fixups_sorted[fixup_cursor].second;
                        modified = true; fixup_cursor++;
                    }

                    if (modified) {
                        memcpy(rec_buf.data(), old_mmap.data + rec_off, actual_stride);
                        // Apply padding zeroing
                        if (file_id == (uint32_t)PatchFileId::ADMIN_POLYGONS && actual_stride == 24)
                            memset(rec_buf.data() + 13, 0, 3);
                        if (file_id == (uint32_t)PatchFileId::INTERP_WAYS && actual_stride == 24) {
                            memset(rec_buf.data() + 5, 0, 3);
                            memset(rec_buf.data() + 21, 0, 3);
                        }
                        // Apply string remap
                        for (size_t off : remap_offs) {
                            uint32_t v; memcpy(&v, rec_buf.data() + off, 4);
                            uint32_t nv = str_remap_lookup(v);
                            if (nv != v) memcpy(rec_buf.data() + off, &nv, 4);
                        }
                        // Apply fixup (node_offset/vertex_offset at byte 0)
                        if (has_fixup)
                            memcpy(rec_buf.data(), &fixup_val, 4);
                        fwrite(rec_buf.data(), 1, actual_stride, outf);
                    } else {
                        // No modification needed — write directly from mmap
                        fwrite(old_mmap.data + rec_off, 1, actual_stride, outf);
                    }
                    if (track && old_rec + k < n_old_records) id_map_ptr[old_rec + k] = (uint32_t)(new_rec + k) + 1; // +1 so 0 means unmapped
                }
                written += count * actual_stride;
                old_rec += count; new_rec += count; old_bytes += count * actual_stride;
            } else if (op == OP_INSERT_RUN) {
                size_t bytes = count * actual_stride;
                fwrite(P+pos, 1, bytes, outf);
                written += bytes; pos += bytes; new_rec += count;
            } else if (op == OP_DELETE_RUN) {
                old_rec += count; old_bytes += count * actual_stride;
            }
        }
        fclose(outf);
        unmap_file(old_mmap);
        fixups_sorted.clear(); fixups_sorted.shrink_to_fit();

        if (track) {
            // Sync and munmap the file-backed remap (data is already on disk)
            size_t remap_bytes = n_old_records * 4;
            msync(id_map_ptr, remap_bytes, MS_SYNC);
            munmap(id_map_ptr, remap_bytes);
            close(remap_fd);
            id_map_ptr = nullptr;
            id_remaps[file_id] = {}; // empty placeholder
            std::cerr << "  " << fname << ": " << written << " bytes (remap " << n_old_records << " records to disk)" << std::endl;
        } else {
            std::cerr << "  " << fname << ": " << written << " bytes" << std::endl;
        }
        malloc_trim(0);
    }
    log_phase("Merge replays", t_start);

    // Free string remap (no longer needed)
    { std::vector<std::pair<uint32_t,uint32_t>>().swap(str_remap_vec); }
    malloc_trim(0);

    // --- Phase 4: Streaming entry pipeline ---
    // Walk old geo_cells, remap IDs, apply corrections, write output directly to files.
    {
        double t_entry = now_ms();
        std::cerr << "Entry pipeline (lean streaming)..." << std::endl;

        // mmap remap files from disk (only pages accessed are in RSS)
        auto mmap_remap = [&](PatchFileId fid) -> MappedFile {
            std::string path = tmpdir + "/remap_" + std::to_string((uint32_t)fid) + ".bin";
            auto m = mmap_file(path);
            if (m.data) madvise(const_cast<char*>(m.data), m.size, MADV_SEQUENTIAL);
            return m;
        };
        // Helper: look up remap value from mmap'd file
        // Remap values stored as new_index+1 (0 means unmapped)
        struct RemapRef {
            const uint32_t* data;
            size_t count;
            uint32_t operator[](size_t i) const {
                if (i >= count) return 0xFFFFFFFF;
                uint32_t v = data[i];
                return v == 0 ? 0xFFFFFFFF : v - 1;
            }
            size_t size() const { return count; }
        };
        MappedFile m_w_rm = mmap_remap(PatchFileId::STREET_WAYS);
        MappedFile m_a_rm = mmap_remap(PatchFileId::ADDR_POINTS);
        MappedFile m_i_rm = mmap_remap(PatchFileId::INTERP_WAYS);
        RemapRef w_rm = {(const uint32_t*)m_w_rm.data, m_w_rm.size / 4};
        RemapRef a_rm = {(const uint32_t*)m_a_rm.data, m_a_rm.size / 4};
        RemapRef i_rm = {(const uint32_t*)m_i_rm.data, m_i_rm.size / 4};

        // mmap old files (zero RSS until pages are accessed, then only working set)
        MappedFile m_geo = mmap_file(cur_dir + "/geo_cells.bin");
        MappedFile m_se = mmap_file(cur_dir + "/street_entries.bin");
        MappedFile m_ae = mmap_file(cur_dir + "/addr_entries.bin");
        MappedFile m_ie = mmap_file(cur_dir + "/interp_entries.bin");
        madvise(const_cast<char*>(m_se.data), m_se.size, MADV_SEQUENTIAL);
        madvise(const_cast<char*>(m_ae.data), m_ae.size, MADV_SEQUENTIAL);
        madvise(const_cast<char*>(m_ie.data), m_ie.size, MADV_SEQUENTIAL);
        size_t n_old = m_geo.size / 20;

        // Build correction maps + removed set (small memory)
        std::unordered_set<uint64_t> rm_set(geo_removed.begin(), geo_removed.end());
        std::unordered_map<uint64_t, const std::vector<uint32_t>*> cs, ca, ci_map;
        for (auto& [fid, list] : entry_corrections) {
            auto* m = (fid == (uint32_t)PatchFileId::STREET_ENTRIES) ? &cs :
                      (fid == (uint32_t)PatchFileId::ADDR_ENTRIES) ? &ca :
                      (fid == (uint32_t)PatchFileId::INTERP_ENTRIES) ? &ci_map : nullptr;
            if (m) for (auto& c : list) (*m)[c.cell_id] = &c.ids;
        }
        // Added cells sorted
        std::sort(geo_added.begin(), geo_added.end());
        log_phase("  Setup", t_entry);

        // Open 4 output files
        FILE* f_geo = fopen((out_dir + "/geo_cells.bin").c_str(), "wb");
        FILE* f_se = fopen((out_dir + "/street_entries.bin").c_str(), "wb");
        FILE* f_ae = fopen((out_dir + "/addr_entries.bin").c_str(), "wb");
        FILE* f_ie = fopen((out_dir + "/interp_entries.bin").c_str(), "wb");
        constexpr uint32_t NO = 0xFFFFFFFF;

        // Reusable buffer (one per entry type to avoid aliasing issues)
        std::vector<uint32_t> buf;
        buf.reserve(4096);

        // Helper: parse IDs from mmap'd entry file
        auto parse = [&buf](const MappedFile& f, uint32_t off) {
            buf.clear();
            if (off == NO || off + 2 > f.size) return;
            uint16_t c; memcpy(&c, f.data + off, 2);
            if (off + 2 + (size_t)c * 4 > f.size) return;
            buf.resize(c);
            memcpy(buf.data(), f.data + off + 2, c * 4);
        };
        auto remap = [](std::vector<uint32_t>& ids, const RemapRef& rm) {
            constexpr uint32_t NO2 = 0xFFFFFFFF;
            for (auto& id : ids) if (id < rm.size() && rm[id] != NO2) id = rm[id];
            std::sort(ids.begin(), ids.end());
        };
        // Write entry and return offset, or NO if empty
        auto emit = [&NO](FILE* f, const uint32_t* ids, size_t n) -> uint32_t {
            if (n == 0) return NO;
            uint32_t off = (uint32_t)ftell(f);
            uint16_t c = (uint16_t)n;
            fwrite(&c, 2, 1, f); fwrite(ids, 4, n, f);
            return off;
        };

        // Merge-walk: old cells + added cells in sorted order
        // Both are sorted by cell_id. Merge them, skip removed.
        size_t old_i = 0, add_i = 0;
        size_t cells_written = 0;

        auto process_cell = [&](uint64_t cid, int32_t oi) {
            // For each entry type: check correction → remap old → write
            auto do_entry = [&](const MappedFile& old_e, size_t geo_off, const RemapRef& rm,
                                const std::unordered_map<uint64_t, const std::vector<uint32_t>*>& corr,
                                FILE* outf, uint8_t flag_bit) -> uint32_t {
                // Check flag
                bool has = false;
                if (oi >= 0 && (size_t)oi * 20 + geo_off + 4 <= m_geo.size) {
                    uint32_t off; memcpy(&off, m_geo.data + oi * 20 + geo_off, 4);
                    has = (off != NO);
                } else if (oi >= 0) {
                    std::cerr << "OOB: oi=" << oi << " geo_off=" << geo_off << " m_geo.size=" << m_geo.size << std::endl;
                    return NO;
                }
                auto fc = flag_corrections.find(cid);
                if (fc != flag_corrections.end()) has = (fc->second & flag_bit) != 0;
                if (corr.count(cid)) has = true;
                if (!has) return NO;

                auto cit = corr.find(cid);
                if (cit != corr.end()) return emit(outf, cit->second->data(), cit->second->size());

                if (oi < 0) return NO;
                if ((size_t)oi * 20 + geo_off + 4 > m_geo.size) return NO;
                uint32_t off; memcpy(&off, m_geo.data + oi * 20 + geo_off, 4);
                parse(old_e, off);
                if (buf.empty()) return NO;
                remap(buf, rm);
                return emit(outf, buf.data(), buf.size());
            };

            uint32_t so = do_entry(m_se, 8, w_rm, cs, f_se, 1);
            uint32_t ao = do_entry(m_ae, 12, a_rm, ca, f_ae, 2);
            uint32_t io = do_entry(m_ie, 16, i_rm, ci_map, f_ie, 4);
            fwrite(&cid, 8, 1, f_geo); fwrite(&so, 4, 1, f_geo); fwrite(&ao, 4, 1, f_geo); fwrite(&io, 4, 1, f_geo);
            cells_written++;
        };

        while (old_i < n_old || add_i < geo_added.size()) {
            uint64_t old_cid = UINT64_MAX, add_cid = UINT64_MAX;
            if (old_i < n_old) memcpy(&old_cid, m_geo.data + old_i * 20, 8);
            if (add_i < geo_added.size()) add_cid = geo_added[add_i];

            if (old_cid <= add_cid) {
                if (!rm_set.count(old_cid))
                    process_cell(old_cid, (int32_t)old_i);
                old_i++;
                if (old_cid == add_cid) add_i++; // skip duplicate add
            } else {
                process_cell(add_cid, -1);
                add_i++;
            }

            if (cells_written % 10000000 == 0 && cells_written > 0) {
                std::cerr << "    " << cells_written << " cells, se=" << ftell(f_se)/1024/1024
                          << "M ae=" << ftell(f_ae)/1024/1024 << "M rss=" << get_rss_mb() << "M" << std::endl;
            }
        }

        fclose(f_geo); fclose(f_se); fclose(f_ae); fclose(f_ie);
        unmap_file(m_geo); unmap_file(m_se); unmap_file(m_ae); unmap_file(m_ie);
        unmap_file(m_w_rm); unmap_file(m_a_rm); unmap_file(m_i_rm);
        std::cerr << "  Geo: " << cells_written << " cells written" << std::endl;
        malloc_trim(0);
        log_phase("  Geo entries complete", t_entry);

        // Admin: small, use existing rebuild + corrections
        {
            uint32_t no_data = 0xFFFFFFFF;
            std::unordered_map<uint32_t,uint32_t> ad_rm;
            MappedFile m_ad_rm = mmap_remap(PatchFileId::ADMIN_POLYGONS);
            if (m_ad_rm.data) {
                const uint32_t* ad_vec = (const uint32_t*)m_ad_rm.data;
                size_t ad_count = m_ad_rm.size / 4;
                for (uint32_t i = 0; i < ad_count; i++)
                    if (ad_vec[i] != 0) ad_rm[i] = ad_vec[i] - 1; // decode +1 encoding
                unmap_file(m_ad_rm);
            }
            auto old_ac = read_file(cur_dir + "/admin_cells.bin");
            auto old_adme = read_file(cur_dir + "/admin_entries.bin");
            auto admin = rebuild_admin_from_remap(old_ac, old_adme, ad_rm, admin_added, admin_removed);

            auto ecit = entry_corrections.find((uint32_t)PatchFileId::ADMIN_ENTRIES);
            if (ecit != entry_corrections.end()) {
                std::unordered_map<uint64_t, const std::vector<uint32_t>*> ac_corr;
                for (auto& c : ecit->second) ac_corr[c.cell_id] = &c.ids;
                size_t n = admin.admin_cells_data.size() / 12;
                FILE* fac = fopen((out_dir + "/admin_cells.bin").c_str(), "wb");
                FILE* fae = fopen((out_dir + "/admin_entries.bin").c_str(), "wb");
                uint32_t ae_wpos = 0;
                for (size_t i = 0; i < n; i++) {
                    uint64_t cid; memcpy(&cid, admin.admin_cells_data.data()+i*12, 8);
                    fwrite(&cid, 8, 1, fac);
                    auto cit = ac_corr.find(cid);
                    if (cit != ac_corr.end()) {
                        uint32_t off = cit->second->empty() ? no_data : ae_wpos;
                        fwrite(&off, 4, 1, fac);
                        if (!cit->second->empty()) {
                            uint16_t c = cit->second->size();
                            fwrite(&c, 2, 1, fae); fwrite(cit->second->data(), 4, c, fae);
                            ae_wpos += 2 + c*4;
                        }
                    } else {
                        uint32_t old_off; memcpy(&old_off, admin.admin_cells_data.data()+i*12+8, 4);
                        if (old_off != no_data && old_off+2 <= admin.admin_entries_data.size()) {
                            uint32_t new_off = ae_wpos; fwrite(&new_off, 4, 1, fac);
                            uint16_t c; memcpy(&c, admin.admin_entries_data.data()+old_off, 2);
                            fwrite(admin.admin_entries_data.data()+old_off, 1, 2+c*4, fae);
                            ae_wpos += 2+c*4;
                        } else {
                            fwrite(&no_data, 4, 1, fac);
                        }
                    }
                }
                fclose(fac); fclose(fae);
                std::cerr << "  Admin: " << n << " cells, " << ac_corr.size() << " corrections" << std::endl;
            } else {
                write_file(out_dir + "/admin_cells.bin", admin.admin_cells_data);
                write_file(out_dir + "/admin_entries.bin", admin.admin_entries_data);
            }
            log_phase("  Admin", t_entry);
        }
    }

    // Cleanup
    unmap_file(patch_map);
    { std::string cmd = "rm -rf '" + tmpdir + "'"; system(cmd.c_str()); }
    log_phase("Total", t_start);
    std::cerr << "Patch applied. Output in " << out_dir << std::endl;
    return 0;
}

-            // Phase A: build merged pool, write to file, free all temporaries

-            // Walk old pool directly (no old_ptrs vector) to minimize memory

+            // Phase A: merge old (minus deleted) + added strings, write directly to file.

+            // Both old pool and added are sorted alphabetically → merge-write with no sort.

+            // Memory: only the added strings vector (~small) + del_set (~small).

-                std::vector<const char*> merged;

-                // Count old strings to reserve

-                { size_t sp = 0; while (sp < old_pool.size) { sp += strlen(old_pool.data + sp) + 1; n_old++; } }

-                merged.reserve(n_old + added.size());

-                { size_t sp = 0; uint32_t idx = 0;

-                  while (sp < old_pool.size) {

-                      if (!del_set.count(idx)) merged.push_back(old_pool.data + sp);

-                      sp += strlen(old_pool.data + sp) + 1; idx++;

-                for (auto& s : added) merged.push_back(s.c_str());

-                std::sort(merged.begin(), merged.end(), [](const char* a, const char* b) { return strcmp(a, b) < 0; });

-                for (const char* s : merged) { size_t l = strlen(s) + 1; fwrite(s, 1, l, fp); }

+                size_t sp = 0; uint32_t idx = 0; size_t ai = 0;

+                // Sort added strings (they should already be sorted from diff tool, but ensure)

+                std::sort(added.begin(), added.end());

+                while (sp < old_pool.size || ai < added.size()) {

+                    const char* old_s = nullptr;

+                    // Skip deleted strings

+                    while (sp < old_pool.size) {

+                        if (!del_set.count(idx)) { old_s = old_pool.data + sp; break; }

+                        sp += strlen(old_pool.data + sp) + 1; idx++;

+                    const char* add_s = ai < added.size() ? added[ai].c_str() : nullptr;

+                    // Merge: pick the smaller string

+                    if (old_s && add_s) {

+                        int c = strcmp(old_s, add_s);

+                            size_t l = strlen(old_s) + 1; fwrite(old_s, 1, l, fp);

+                            if (c == 0) ai++; // duplicate — skip added

+                            size_t l = added[ai].size() + 1; fwrite(add_s, 1, l, fp);

+                    } else if (old_s) {

+                    } else if (add_s) {


This eliminates the `merged` vector entirely — strings are written directly to the output file during the merge-walk. The only memory used is the `added` strings (~100 KB for 10K strings) + `del_set` (tiny) + sequential mmap page access.


Bash

make -j$(nproc) geocoder-patch 2>&1 | tail -3 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "rm -f /home/michtest/geocoder-patch" 2>&1 && sshpass -p 'elon-is-insane' scp -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:/home/michtest/ 2>&1 && sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "rm -rf /home/michtest/patched-planet-stream/ && systemd-run --user --scope -p MemoryMax=500M -- ./geocoder-patch planet-B/ validated-patches/planet-mar16-to-mar23-v2.gcpatch -o patched-planet-stream/ 2>&1 | grep -E '\\[.*MiB\\]' && echo '---' && for f in planet-C/*; do b=\$(basename \$f); cmp -s patched-planet-stream/\$b planet-C/\$b || echo \"MISMATCH: \$b\"; done && echo All_checked" 2>&1

Planet 500 MiB - merge-write approach


Still OOM at 500 MiB during Strings. The merge-write approach should use almost no heap. Let me check what's consuming memory — the patch mmap itself is 424 MiB! The decompressed patch file is mmap'd at the start and stays mapped throughout. At 424 MiB, that alone exceeds 500 MiB with any additional heap.


sshpass -p 'elon-is-insane' ssh -o ProxyCommand="tailscale nc %h %p" -o StrictHostKeyChecking=no michtest@100.113.88.121 "ls -lh /home/michtest/validated-patches/planet-mar16-to-mar23-v2.gcpatch && zstd -d -c /home/michtest/validated-patches/planet-mar16-to-mar23-v2.gcpatch 2>/dev/null | wc -c" 2>&1

Check patch decompressed size


-rw-rw-r-- 1 michtest michtest 74M Mar 29 07:55 /home/michtest/validated-patches/planet-mar16-to-mar23-v2.gcpatch
423813763


424 MiB decompressed patch. The mmap of this file alone takes 424 MiB of virtual address space, and when accessed, those pages count toward the cgroup memory limit. During the string rebuild, we access the string diff section of the patch (~100 KB of data), but the cgroup counts ALL mapped pages as potentially-used.

The fix: decompress using streaming zstd and don't mmap the full patch. Instead, read sections sequentially from the decompressed stream. But this requires restructuring the entire parsing approach.

A simpler fix: after reading the string section, `madvise(MADV_DONTNEED)` on the pages we've already processed to let the kernel reclaim them:


Edit

    MappedFile patch_map = mmap_file(raw_path);
    if (!patch_map.data) { std::cerr << "Failed to mmap patch" << std::endl; return 1; }
    remove(raw_path.c_str());

    MappedFile patch_map = mmap_file(raw_path);
    if (!patch_map.data) { std::cerr << "Failed to mmap patch" << std::endl; return 1; }
    // Keep temp file for mmap backing (remove after munmap at end)
    // madvise sequential for initial parse