The file /root/traccar-geocoder/builder/src/cell_index.cpp has been updated successfully.

    // Link unchanged files from source (S2 covering and strings are quality-independent)
    for (const char* fname : {"admin_cells.bin", "admin_entries.bin", "strings.bin"}) {
        std::string src = source_dir + "/" + fname;
        std::string dst = output_dir + "/" + fname;
        if (link(src.c_str(), dst.c_str()) != 0) {
            std::ifstream in(src, std::ios::binary);
            std::ofstream out(dst, std::ios::binary);
            out << in.rdbuf();
        }
    }

#include "cell_index.h"

#include <algorithm>
#include <atomic>
#include <cstring>
#include <fstream>
#include <future>
#include <iostream>
#include <thread>
#include <unordered_map>
#include <unordered_set>

#include <unistd.h>

#include "geometry.h"
#include "s2_helpers.h"

std::vector<uint32_t> write_entries(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
    std::vector<CellRef> sorted_refs;
    sorted_refs.reserve(cell_map.size());
    for (auto& [id, ids] : cell_map) sorted_refs.push_back({id, &ids});
    std::sort(sorted_refs.begin(), sorted_refs.end(),
        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });

    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    size_t total_size = 0;
    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);

    std::vector<char> buf;
    buf.reserve(total_size);
    uint32_t current = 0;
    size_t ri = 0;
    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
        offsets[si] = current;
        const auto& ids = *sorted_refs[ri].ids;
        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
                   reinterpret_cast<const char*>(&count) + sizeof(count));
        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
        ri++;
    }
    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

std::vector<uint32_t> write_entries_from_sorted(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::vector<CellItemPair>& sorted_pairs
) {
    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    if (sorted_pairs.empty()) {
        std::ofstream f(path, std::ios::binary);
        return offsets;
    }

    unsigned int nthreads = std::min(std::thread::hardware_concurrency(), 32u);
    if (nthreads == 0) nthreads = 4;
    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;

    struct ChunkResult {
        std::vector<char> buf;
        size_t cell_start, cell_end;
        uint32_t local_size;
    };
    std::vector<ChunkResult> chunks(nthreads);

    std::vector<std::thread> threads;
    for (unsigned int t = 0; t < nthreads; t++) {
        size_t cs = t * cells_per_chunk;
        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
        if (cs >= sorted_cells.size()) break;

        threads.emplace_back([&, t, cs, ce]() {
            auto& chunk = chunks[t];
            chunk.cell_start = cs;
            chunk.cell_end = ce;
            chunk.local_size = 0;

            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
                    return p.cell_id < id;
                }) - sorted_pairs.begin();

            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;

                offsets[si] = chunk.local_size;
                size_t start = pi;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(MAX_VERTEX_COUNT)));
                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
                size_t buf_pos = chunk.buf.size();
                chunk.buf.resize(buf_pos + entry_size);
                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
                for (size_t k = start; k < pi; k++) {
                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
                           &sorted_pairs[k].item_id, sizeof(uint32_t));
                }
                chunk.local_size += entry_size;
            }
        });
    }
    for (auto& t : threads) t.join();

    uint32_t global_offset = 0;
    for (auto& chunk : chunks) {
        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
            if (offsets[si] != NO_DATA) offsets[si] += global_offset;
        }
        global_offset += chunk.local_size;
    }

    std::vector<char> buf;
    buf.reserve(global_offset);
    for (auto& chunk : chunks) buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());

    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

void write_cell_index(
    const std::string& cells_path,
    const std::string& entries_path,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(cell_map.begin(), cell_map.end());
    std::sort(sorted.begin(), sorted.end());

    { std::ofstream f(cells_path, std::ios::binary);
      uint32_t current_offset = 0;
      for (const auto& [cell_id, ids] : sorted) {
          f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
          f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
          current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
      } }

    { std::ofstream f(entries_path, std::ios::binary);
      for (const auto& [cell_id, ids] : sorted) {
          uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
          f.write(reinterpret_cast<const char*>(&count), sizeof(count));
          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
      } }
}

void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
    ensure_dir(output_dir);
    auto _wt = std::chrono::steady_clock::now();

    bool write_streets = (mode != IndexMode::AdminOnly);
    bool write_addresses = (mode == IndexMode::Full);

    if (write_streets) {
        std::vector<uint64_t> sorted_geo_cells;
        {
            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
                std::vector<uint64_t> cells;
                cells.reserve(pairs.size() / 2);
                for (size_t i = 0; i < pairs.size(); ) {
                    cells.push_back(pairs[i].cell_id);
                    uint64_t cur = pairs[i].cell_id;
                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
                }
                return cells;
            };
            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
                std::vector<uint64_t> cells;
                cells.reserve(m.size());
                for (auto& [id, _] : m) cells.push_back(id);
                std::sort(cells.begin(), cells.end());
                return cells;
            };

            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
            {
                auto f1 = std::async(std::launch::async, [&] {
                    return !data.sorted_way_cells.empty()
                        ? extract_unique_cells(data.sorted_way_cells)
                        : extract_from_map(data.cell_to_ways);
                });
                if (write_addresses) {
                    auto f2 = std::async(std::launch::async, [&] {
                        return !data.sorted_addr_cells.empty()
                            ? extract_unique_cells(data.sorted_addr_cells)
                            : extract_from_map(data.cell_to_addrs);
                    });
                    auto f3 = std::async(std::launch::async, [&] {
                        return !data.sorted_interp_cells.empty()
                            ? extract_unique_cells(data.sorted_interp_cells)
                            : extract_from_map(data.cell_to_interps);
                    });
                    addr_cells = f2.get();
                    interp_cells = f3.get();
                }
                way_cells = f1.get();
            }

            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
            std::merge(way_cells.begin(), way_cells.end(), addr_cells.begin(), addr_cells.end(),
                       std::back_inserter(sorted_geo_cells));
            if (!interp_cells.empty()) {
                std::vector<uint64_t> tmp;
                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
                           interp_cells.begin(), interp_cells.end(), std::back_inserter(tmp));
                sorted_geo_cells = std::move(tmp);
            }
            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
                                    sorted_geo_cells.end());
        }

        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
        {
            auto f1 = std::async(std::launch::async, [&]() {
                if (!data.sorted_way_cells.empty())
                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
            });
            if (write_addresses) {
                auto f2 = std::async(std::launch::async, [&]() {
                    if (!data.sorted_addr_cells.empty())
                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
                });
                auto f3 = std::async(std::launch::async, [&]() {
                    if (!data.sorted_interp_cells.empty())
                        return write_entries_from_sorted(output_dir + "/interp_entries.bin", sorted_geo_cells, data.sorted_interp_cells);
                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
                });
                addr_offsets = f2.get();
                interp_offsets = f3.get();
            }
            street_offsets = f1.get();
        }

        log_phase("  Write: entry files", _wt);
        {
            size_t n = sorted_geo_cells.size();
            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
            std::vector<char> buf(n * row_size);
            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);

            unsigned int nthreads = std::thread::hardware_concurrency();
            if (nthreads == 0) nthreads = 4;
            size_t chunk = (n + nthreads - 1) / nthreads;
            std::vector<std::thread> fill_threads;
            for (unsigned int t = 0; t < nthreads; t++) {
                size_t start = t * chunk;
                size_t end = std::min(start + chunk, n);
                if (start >= n) break;
                fill_threads.emplace_back([&, start, end]() {
                    char* ptr = buf.data() + start * row_size;
                    for (size_t i = start; i < end; i++) {
                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                    }
                });
            }
            for (auto& t : fill_threads) t.join();

            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
            f.write(buf.data(), buf.size());
        }

        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
                  << data.ways.size() << " ways, " << data.addr_points.size() << " addrs, "
                  << data.interp_ways.size() << " interps)" << std::endl;
    }

    log_phase("  Write: geo_cells.bin", _wt);
    auto admin_future = std::async(std::launch::async, [&] {
        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
    });

    std::vector<std::future<void>> write_futures;
    if (write_streets) {
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
        }));
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
        }));
        if (write_addresses) {
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
            }));
        }
    }
    // If not writing all data types, compact string pool and remap references
    // before writing admin_polygons and strings.bin.
    std::vector<AdminPolygon> compact_admin;
    std::vector<char> compact_strings;
    const bool need_compact = (!write_streets || !write_addresses);

    if (need_compact) {
        // Collect referenced string offsets
        std::unordered_set<uint32_t> used_offsets;
        if (write_streets) {
            for (const auto& w : data.ways) used_offsets.insert(w.name_id);
            if (write_addresses) {
                for (const auto& a : data.addr_points) {
                    used_offsets.insert(a.housenumber_id);
                    used_offsets.insert(a.street_id);
                }
                for (const auto& iw : data.interp_ways) used_offsets.insert(iw.street_id);
            }
        }
        for (const auto& ap : data.admin_polygons) used_offsets.insert(ap.name_id);

        // Build compact pool and remap
        const auto& old_sp = data.string_pool.data();
        std::vector<uint32_t> sorted_offs(used_offsets.begin(), used_offsets.end());
        std::sort(sorted_offs.begin(), sorted_offs.end());
        std::unordered_map<uint32_t, uint32_t> remap;
        compact_strings.reserve(used_offsets.size() * 16);
        for (uint32_t off : sorted_offs) {
            remap[off] = static_cast<uint32_t>(compact_strings.size());
            size_t len = std::strlen(old_sp.data() + off);
            compact_strings.insert(compact_strings.end(), old_sp.data() + off,
                old_sp.data() + off + len + 1);
        }

        // Remap admin polygon name_ids
        compact_admin = data.admin_polygons;
        for (auto& ap : compact_admin) ap.name_id = remap[ap.name_id];

        // Remap street/addr/interp if applicable (for no-addresses mode)
        // These are written from data directly, so we'd need copies too.
        // For simplicity, only compact when we have fewer types.
    }

    const auto& admin_to_write = need_compact ? compact_admin : data.admin_polygons;
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(admin_to_write.data()), admin_to_write.size() * sizeof(AdminPolygon));
    }));
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
    }));
    write_futures.push_back(std::async(std::launch::async, [&] {
        if (need_compact) {
            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
            f.write(compact_strings.data(), compact_strings.size());
            std::cerr << "strings.bin: " << compact_strings.size() << " bytes (compacted from "
                      << data.string_pool.data().size() << ")" << std::endl;
        } else {
            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
            f.write(data.string_pool.data().data(), data.string_pool.data().size());
            std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
        }
    }));

    log_phase("  Write: parallel data files launched", _wt);
    admin_future.get();
    for (auto& f : write_futures) f.get();
}

void write_quality_variant(const ParsedData& data, const std::string& source_dir,
                           const std::string& output_dir, double epsilon_scale) {
    ensure_dir(output_dir);

    // Re-simplify admin polygons at the given epsilon scale
    std::vector<AdminPolygon> new_polys;
    std::vector<NodeCoord> new_verts;
    new_polys.reserve(data.admin_polygons.size());

    // Parallel simplification
    struct SimplifiedPoly {
        std::vector<std::pair<double,double>> verts;
    };
    std::vector<SimplifiedPoly> simplified(data.admin_polygons.size());

    {
        std::atomic<size_t> idx{0};
        unsigned nthreads = std::thread::hardware_concurrency();
        if (nthreads == 0) nthreads = 4;
        std::vector<std::thread> workers;
        for (unsigned t = 0; t < nthreads; t++) {
            workers.emplace_back([&]() {
                while (true) {
                    size_t i = idx.fetch_add(1);
                    if (i >= data.admin_polygons.size()) break;
                    const auto& ap = data.admin_polygons[i];

                    // Extract original vertices
                    std::vector<std::pair<double,double>> pts;
                    pts.reserve(ap.vertex_count);
                    for (uint32_t j = 0; j < ap.vertex_count; j++) {
                        const auto& v = data.admin_vertices[ap.vertex_offset + j];
                        pts.emplace_back(v.lat, v.lng);
                    }

                    if (epsilon_scale > 0) {
                        double eps_m = admin_epsilon_meters(ap.admin_level) * epsilon_scale;
                        double lat = pts.empty() ? 0.0 : pts[0].first;
                        double eps_deg = meters_to_degrees(eps_m, lat);
                        simplified[i].verts = simplify_polygon_epsilon(pts, eps_deg);
                    } else {
                        simplified[i].verts = std::move(pts);
                    }
                }
            });
        }
        for (auto& w : workers) w.join();
    }

    // Sequential: build new polygon and vertex arrays
    for (size_t i = 0; i < data.admin_polygons.size(); i++) {
        auto& sv = simplified[i].verts;
        if (sv.size() < 3) continue;

        AdminPolygon np = data.admin_polygons[i];
        np.vertex_offset = static_cast<uint32_t>(new_verts.size());
        np.vertex_count = static_cast<uint32_t>(sv.size());
        np.area = polygon_area(sv);

        for (const auto& [lat, lng] : sv) {
            new_verts.push_back({static_cast<float>(lat), static_cast<float>(lng)});
        }
        new_polys.push_back(np);
    }

    std::cerr << "Quality " << epsilon_scale << "x: " << new_polys.size()
              << " polygons, " << new_verts.size() << " vertices ("
              << new_verts.size() * 8 / 1024 / 1024 << " MiB)" << std::endl;

    // Write admin files
    {
        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(new_polys.data()), new_polys.size() * sizeof(AdminPolygon));
    }
    {
        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(new_verts.data()), new_verts.size() * sizeof(NodeCoord));
    }

    // Compact strings for admin-only names
    {
        std::unordered_set<uint32_t> used_offsets;
        for (const auto& ap : new_polys) used_offsets.insert(ap.name_id);

        const auto& old_sp = data.string_pool.data();
        std::vector<uint32_t> sorted_offs(used_offsets.begin(), used_offsets.end());
        std::sort(sorted_offs.begin(), sorted_offs.end());
        std::unordered_map<uint32_t, uint32_t> remap;
        std::vector<char> compact_sp;
        compact_sp.reserve(used_offsets.size() * 16);
        for (uint32_t off : sorted_offs) {
            remap[off] = static_cast<uint32_t>(compact_sp.size());
            size_t len = std::strlen(old_sp.data() + off);
            compact_sp.insert(compact_sp.end(), old_sp.data() + off, old_sp.data() + off + len + 1);
        }

        // Remap and rewrite polygons with new string offsets
        for (auto& np : new_polys) np.name_id = remap[np.name_id];
        {
            std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(new_polys.data()), new_polys.size() * sizeof(AdminPolygon));
        }
        {
            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
            f.write(compact_sp.data(), compact_sp.size());
        }
    }

    // Link/copy the cell index files from source (S2 covering is quality-independent)
    for (const char* fname : {"admin_cells.bin", "admin_entries.bin"}) {
        std::string src = source_dir + "/" + fname;
        std::string dst = output_dir + "/" + fname;
        // Try hardlink first, fall back to copy
        if (link(src.c_str(), dst.c_str()) != 0) {
            std::ifstream in(src, std::ios::binary);
            std::ofstream out(dst, std::ios::binary);
            out << in.rdbuf();
        }
    }
}

-    // Compact strings for admin-only names

-        std::unordered_set<uint32_t> used_offsets;

-        for (const auto& ap : new_polys) used_offsets.insert(ap.name_id);

-        const auto& old_sp = data.string_pool.data();

-        std::vector<uint32_t> sorted_offs(used_offsets.begin(), used_offsets.end());

-        std::sort(sorted_offs.begin(), sorted_offs.end());

-        std::unordered_map<uint32_t, uint32_t> remap;

-        compact_sp.reserve(used_offsets.size() * 16);

-        for (uint32_t off : sorted_offs) {

-            remap[off] = static_cast<uint32_t>(compact_sp.size());

-            size_t len = std::strlen(old_sp.data() + off);

-            compact_sp.insert(compact_sp.end(), old_sp.data() + off, old_sp.data() + off + len + 1);

-        // Remap and rewrite polygons with new string offsets

-        for (auto& np : new_polys) np.name_id = remap[np.name_id];

-            std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);

-            f.write(reinterpret_cast<const char*>(new_polys.data()), new_polys.size() * sizeof(AdminPolygon));

-            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);

-            f.write(compact_sp.data(), compact_sp.size());

-    // Link/copy the cell index files from source (S2 covering is quality-independent)

-    for (const char* fname : {"admin_cells.bin", "admin_entries.bin"}) {

+    // Link unchanged files from source (S2 covering and strings are quality-independent)

+    for (const char* fname : {"admin_cells.bin", "admin_entries.bin", "strings.bin"}) {

-        // Try hardlink first, fall back to copy


Now build and run:


Bash

Build


[ 25%] Building CXX object CMakeFiles/build-index.dir/src/build_index.cpp.o
[ 37%] Linking CXX executable build-index
[100%] Built target build-index


Good. Let me also kill any leftover builds on node 3, then start the full run:


ssh node3 "pkill -9 -f build-index 2>/dev/null; sleep 2; free -h | head -2" 2>&1

Clean node 3


Exit code 255


rm -f /tmp/ssh-mux/node3 && sshpass -p 'elon-is-insane' ssh -M -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ControlPath=/tmp/ssh-mux/node3 -o ControlPersist=4h -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 "sleep 99999" &
sleep 3 && ssh node3 "pkill -9 -f build-index 2>/dev/null; sleep 2; free -h | head -2" 2>&1

Reconnect and clean


Error: Exit code 255
Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.


[Request interrupted by user]


<task-notification>
<task-id>b4kaxe63h</task-id>
<tool-use-id>toolu_01KXxafhvGxfRGtE3VF5vmmN</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/b4kaxe63h.output</output-file>
<status>failed</status>
<summary>Background command "Run full planet build with all options" failed with exit code 255</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/b4kaxe63h.output


Right — much cleaner. Each quality directory should only have `admin_polygons.bin` and `admin_vertices.bin`, with the shared files (`admin_cells.bin`, `admin_entries.bin`, `strings.bin`) in the parent directory.


Edit

    // Link unchanged files from source (S2 covering and strings are quality-independent)
    for (const char* fname : {"admin_cells.bin", "admin_entries.bin", "strings.bin"}) {
        std::string src = source_dir + "/" + fname;
        std::string dst = output_dir + "/" + fname;
        if (link(src.c_str(), dst.c_str()) != 0) {
            std::ifstream in(src, std::ios::binary);
            std::ofstream out(dst, std::ios::binary);
            out << in.rdbuf();
        }
    }

    // Quality directories only contain the two files that change.
    // Shared files (admin_cells.bin, admin_entries.bin, strings.bin)
    // stay in the parent directory.


    // Quality directories only contain the two files that change.
    // Shared files (admin_cells.bin, admin_entries.bin, strings.bin)
    // stay in the parent directory.

#include "cell_index.h"

#include <algorithm>
#include <atomic>
#include <cstring>
#include <fstream>
#include <future>
#include <iostream>
#include <thread>
#include <unordered_map>
#include <unordered_set>

#include <unistd.h>

#include "geometry.h"
#include "s2_helpers.h"

std::vector<uint32_t> write_entries(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
    std::vector<CellRef> sorted_refs;
    sorted_refs.reserve(cell_map.size());
    for (auto& [id, ids] : cell_map) sorted_refs.push_back({id, &ids});
    std::sort(sorted_refs.begin(), sorted_refs.end(),
        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });

    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    size_t total_size = 0;
    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);

    std::vector<char> buf;
    buf.reserve(total_size);
    uint32_t current = 0;
    size_t ri = 0;
    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
        offsets[si] = current;
        const auto& ids = *sorted_refs[ri].ids;
        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
                   reinterpret_cast<const char*>(&count) + sizeof(count));
        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
        ri++;
    }
    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

std::vector<uint32_t> write_entries_from_sorted(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::vector<CellItemPair>& sorted_pairs
) {
    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    if (sorted_pairs.empty()) {
        std::ofstream f(path, std::ios::binary);
        return offsets;
    }

    unsigned int nthreads = std::min(std::thread::hardware_concurrency(), 32u);
    if (nthreads == 0) nthreads = 4;
    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;

    struct ChunkResult {
        std::vector<char> buf;
        size_t cell_start, cell_end;
        uint32_t local_size;
    };
    std::vector<ChunkResult> chunks(nthreads);

    std::vector<std::thread> threads;
    for (unsigned int t = 0; t < nthreads; t++) {
        size_t cs = t * cells_per_chunk;
        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
        if (cs >= sorted_cells.size()) break;

        threads.emplace_back([&, t, cs, ce]() {
            auto& chunk = chunks[t];
            chunk.cell_start = cs;
            chunk.cell_end = ce;
            chunk.local_size = 0;

            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
                    return p.cell_id < id;
                }) - sorted_pairs.begin();

            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;

                offsets[si] = chunk.local_size;
                size_t start = pi;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(MAX_VERTEX_COUNT)));
                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
                size_t buf_pos = chunk.buf.size();
                chunk.buf.resize(buf_pos + entry_size);
                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
                for (size_t k = start; k < pi; k++) {
                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
                           &sorted_pairs[k].item_id, sizeof(uint32_t));
                }
                chunk.local_size += entry_size;
            }
        });
    }
    for (auto& t : threads) t.join();

    uint32_t global_offset = 0;
    for (auto& chunk : chunks) {
        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
            if (offsets[si] != NO_DATA) offsets[si] += global_offset;
        }
        global_offset += chunk.local_size;
    }

    std::vector<char> buf;
    buf.reserve(global_offset);
    for (auto& chunk : chunks) buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());

    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

void write_cell_index(
    const std::string& cells_path,
    const std::string& entries_path,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(cell_map.begin(), cell_map.end());
    std::sort(sorted.begin(), sorted.end());

    { std::ofstream f(cells_path, std::ios::binary);
      uint32_t current_offset = 0;
      for (const auto& [cell_id, ids] : sorted) {
          f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
          f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
          current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
      } }

    { std::ofstream f(entries_path, std::ios::binary);
      for (const auto& [cell_id, ids] : sorted) {
          uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
          f.write(reinterpret_cast<const char*>(&count), sizeof(count));
          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
      } }
}

void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
    ensure_dir(output_dir);
    auto _wt = std::chrono::steady_clock::now();

    bool write_streets = (mode != IndexMode::AdminOnly);
    bool write_addresses = (mode == IndexMode::Full);

    if (write_streets) {
        std::vector<uint64_t> sorted_geo_cells;
        {
            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
                std::vector<uint64_t> cells;
                cells.reserve(pairs.size() / 2);
                for (size_t i = 0; i < pairs.size(); ) {
                    cells.push_back(pairs[i].cell_id);
                    uint64_t cur = pairs[i].cell_id;
                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
                }
                return cells;
            };
            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
                std::vector<uint64_t> cells;
                cells.reserve(m.size());
                for (auto& [id, _] : m) cells.push_back(id);
                std::sort(cells.begin(), cells.end());
                return cells;
            };

            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
            {
                auto f1 = std::async(std::launch::async, [&] {
                    return !data.sorted_way_cells.empty()
                        ? extract_unique_cells(data.sorted_way_cells)
                        : extract_from_map(data.cell_to_ways);
                });
                if (write_addresses) {
                    auto f2 = std::async(std::launch::async, [&] {
                        return !data.sorted_addr_cells.empty()
                            ? extract_unique_cells(data.sorted_addr_cells)
                            : extract_from_map(data.cell_to_addrs);
                    });
                    auto f3 = std::async(std::launch::async, [&] {
                        return !data.sorted_interp_cells.empty()
                            ? extract_unique_cells(data.sorted_interp_cells)
                            : extract_from_map(data.cell_to_interps);
                    });
                    addr_cells = f2.get();
                    interp_cells = f3.get();
                }
                way_cells = f1.get();
            }

            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
            std::merge(way_cells.begin(), way_cells.end(), addr_cells.begin(), addr_cells.end(),
                       std::back_inserter(sorted_geo_cells));
            if (!interp_cells.empty()) {
                std::vector<uint64_t> tmp;
                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
                           interp_cells.begin(), interp_cells.end(), std::back_inserter(tmp));
                sorted_geo_cells = std::move(tmp);
            }
            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
                                    sorted_geo_cells.end());
        }

        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
        {
            auto f1 = std::async(std::launch::async, [&]() {
                if (!data.sorted_way_cells.empty())
                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
            });
            if (write_addresses) {
                auto f2 = std::async(std::launch::async, [&]() {
                    if (!data.sorted_addr_cells.empty())
                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
                });
                auto f3 = std::async(std::launch::async, [&]() {
                    if (!data.sorted_interp_cells.empty())
                        return write_entries_from_sorted(output_dir + "/interp_entries.bin", sorted_geo_cells, data.sorted_interp_cells);
                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
                });
                addr_offsets = f2.get();
                interp_offsets = f3.get();
            }
            street_offsets = f1.get();
        }

        log_phase("  Write: entry files", _wt);
        {
            size_t n = sorted_geo_cells.size();
            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
            std::vector<char> buf(n * row_size);
            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);

            unsigned int nthreads = std::thread::hardware_concurrency();
            if (nthreads == 0) nthreads = 4;
            size_t chunk = (n + nthreads - 1) / nthreads;
            std::vector<std::thread> fill_threads;
            for (unsigned int t = 0; t < nthreads; t++) {
                size_t start = t * chunk;
                size_t end = std::min(start + chunk, n);
                if (start >= n) break;
                fill_threads.emplace_back([&, start, end]() {
                    char* ptr = buf.data() + start * row_size;
                    for (size_t i = start; i < end; i++) {
                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                    }
                });
            }
            for (auto& t : fill_threads) t.join();

            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
            f.write(buf.data(), buf.size());
        }

        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
                  << data.ways.size() << " ways, " << data.addr_points.size() << " addrs, "
                  << data.interp_ways.size() << " interps)" << std::endl;
    }

    log_phase("  Write: geo_cells.bin", _wt);
    auto admin_future = std::async(std::launch::async, [&] {
        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
    });

    std::vector<std::future<void>> write_futures;
    if (write_streets) {
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
        }));
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
        }));
        if (write_addresses) {
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
            }));
        }
    }
    // If not writing all data types, compact string pool and remap references
    // before writing admin_polygons and strings.bin.
    std::vector<AdminPolygon> compact_admin;
    std::vector<char> compact_strings;
    const bool need_compact = (!write_streets || !write_addresses);

    if (need_compact) {
        // Collect referenced string offsets
        std::unordered_set<uint32_t> used_offsets;
        if (write_streets) {
            for (const auto& w : data.ways) used_offsets.insert(w.name_id);
            if (write_addresses) {
                for (const auto& a : data.addr_points) {
                    used_offsets.insert(a.housenumber_id);
                    used_offsets.insert(a.street_id);
                }
                for (const auto& iw : data.interp_ways) used_offsets.insert(iw.street_id);
            }
        }
        for (const auto& ap : data.admin_polygons) used_offsets.insert(ap.name_id);

        // Build compact pool and remap
        const auto& old_sp = data.string_pool.data();
        std::vector<uint32_t> sorted_offs(used_offsets.begin(), used_offsets.end());
        std::sort(sorted_offs.begin(), sorted_offs.end());
        std::unordered_map<uint32_t, uint32_t> remap;
        compact_strings.reserve(used_offsets.size() * 16);
        for (uint32_t off : sorted_offs) {
            remap[off] = static_cast<uint32_t>(compact_strings.size());
            size_t len = std::strlen(old_sp.data() + off);
            compact_strings.insert(compact_strings.end(), old_sp.data() + off,
                old_sp.data() + off + len + 1);
        }

        // Remap admin polygon name_ids
        compact_admin = data.admin_polygons;
        for (auto& ap : compact_admin) ap.name_id = remap[ap.name_id];

        // Remap street/addr/interp if applicable (for no-addresses mode)
        // These are written from data directly, so we'd need copies too.
        // For simplicity, only compact when we have fewer types.
    }

    const auto& admin_to_write = need_compact ? compact_admin : data.admin_polygons;
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(admin_to_write.data()), admin_to_write.size() * sizeof(AdminPolygon));
    }));
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
    }));
    write_futures.push_back(std::async(std::launch::async, [&] {
        if (need_compact) {
            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
            f.write(compact_strings.data(), compact_strings.size());
            std::cerr << "strings.bin: " << compact_strings.size() << " bytes (compacted from "
                      << data.string_pool.data().size() << ")" << std::endl;
        } else {
            std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
            f.write(data.string_pool.data().data(), data.string_pool.data().size());
            std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
        }
    }));

    log_phase("  Write: parallel data files launched", _wt);
    admin_future.get();
    for (auto& f : write_futures) f.get();
}

void write_quality_variant(const ParsedData& data, const std::string& source_dir,
                           const std::string& output_dir, double epsilon_scale) {
    ensure_dir(output_dir);

    // Re-simplify admin polygons at the given epsilon scale
    std::vector<AdminPolygon> new_polys;
    std::vector<NodeCoord> new_verts;
    new_polys.reserve(data.admin_polygons.size());

    // Parallel simplification
    struct SimplifiedPoly {
        std::vector<std::pair<double,double>> verts;
    };
    std::vector<SimplifiedPoly> simplified(data.admin_polygons.size());

    {
        std::atomic<size_t> idx{0};
        unsigned nthreads = std::thread::hardware_concurrency();
        if (nthreads == 0) nthreads = 4;
        std::vector<std::thread> workers;
        for (unsigned t = 0; t < nthreads; t++) {
            workers.emplace_back([&]() {
                while (true) {
                    size_t i = idx.fetch_add(1);
                    if (i >= data.admin_polygons.size()) break;
                    const auto& ap = data.admin_polygons[i];

                    // Extract original vertices
                    std::vector<std::pair<double,double>> pts;
                    pts.reserve(ap.vertex_count);
                    for (uint32_t j = 0; j < ap.vertex_count; j++) {
                        const auto& v = data.admin_vertices[ap.vertex_offset + j];
                        pts.emplace_back(v.lat, v.lng);
                    }

                    if (epsilon_scale > 0) {
                        double eps_m = admin_epsilon_meters(ap.admin_level) * epsilon_scale;
                        double lat = pts.empty() ? 0.0 : pts[0].first;
                        double eps_deg = meters_to_degrees(eps_m, lat);
                        simplified[i].verts = simplify_polygon_epsilon(pts, eps_deg);
                    } else {
                        simplified[i].verts = std::move(pts);
                    }
                }
            });
        }
        for (auto& w : workers) w.join();
    }

    // Sequential: build new polygon and vertex arrays
    for (size_t i = 0; i < data.admin_polygons.size(); i++) {
        auto& sv = simplified[i].verts;
        if (sv.size() < 3) continue;

        AdminPolygon np = data.admin_polygons[i];
        np.vertex_offset = static_cast<uint32_t>(new_verts.size());
        np.vertex_count = static_cast<uint32_t>(sv.size());
        np.area = polygon_area(sv);

        for (const auto& [lat, lng] : sv) {
            new_verts.push_back({static_cast<float>(lat), static_cast<float>(lng)});
        }
        new_polys.push_back(np);
    }

    std::cerr << "Quality " << epsilon_scale << "x: " << new_polys.size()
              << " polygons, " << new_verts.size() << " vertices ("
              << new_verts.size() * 8 / 1024 / 1024 << " MiB)" << std::endl;

    // Write admin files
    {
        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(new_polys.data()), new_polys.size() * sizeof(AdminPolygon));
    }
    {
        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(new_verts.data()), new_verts.size() * sizeof(NodeCoord));
    }

    // Link unchanged files from source (S2 covering and strings are quality-independent)
    for (const char* fname : {"admin_cells.bin", "admin_entries.bin", "strings.bin"}) {
        std::string src = source_dir + "/" + fname;
        std::string dst = output_dir + "/" + fname;
        if (link(src.c_str(), dst.c_str()) != 0) {
            std::ifstream in(src, std::ios::binary);
            std::ofstream out(dst, std::ios::binary);
            out << in.rdbuf();
        }
    }
}

-    // Link unchanged files from source (S2 covering and strings are quality-independent)

-    for (const char* fname : {"admin_cells.bin", "admin_entries.bin", "strings.bin"}) {

-        std::string src = source_dir + "/" + fname;

-        std::string dst = output_dir + "/" + fname;

-        if (link(src.c_str(), dst.c_str()) != 0) {

-            std::ifstream in(src, std::ios::binary);

-            std::ofstream out(dst, std::ios::binary);

+    // Quality directories only contain the two files that change.

+    // Shared files (admin_cells.bin, admin_entries.bin, strings.bin)


Now rebuild:


Bash

Build


[ 12%] Building CXX object CMakeFiles/build-index.dir/src/cell_index.cpp.o
[ 25%] Linking CXX executable build-index
[100%] Built target build-index


Now reconnect to node 3 and run:


rm -f /tmp/ssh-mux/node3 && sshpass -p 'elon-is-insane' ssh -M -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ControlPath=/tmp/ssh-mux/node3 -o ControlPersist=4h -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 "sleep 99999" &
sleep 3 && ssh node3 "hostname && free -h | head -2" 2>&1

Reconnect to node 3


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
michtest-ng
               total        used        free      shared  buff/cache   available
Mem:           174Gi       4.0Gi       133Gi       2.0Gi        41Gi       170Gi


scp /root/traccar-geocoder/builder/build/build-index node3:~/build-index-mq2 2>&1 && ssh node3 "rm -rf ~/test-planet-full && mkdir ~/test-planet-full && time ~/build-index-mq2 ~/test-planet-full ~/pbf/planet.osm.pbf --multi-output --continents --multi-quality 2>&1" 2>&1

Run full planet build


Command running in background with ID: bca1n6ms2. Output is being written to: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bca1n6ms2.output


Can you layout the full directory and file structure now if we ran with the different admin levels, the split continents & full planet as well as the varying quality levels?


Based on the current code with `--multi-output --continents --multi-quality`:

```
output/
├── full/                          # Planet - streets + addresses + admin
│   ├── geo_cells.bin
│   ├── street_entries.bin
│   ├── street_ways.bin
│   ├── street_nodes.bin
│   ├── addr_entries.bin
│   ├── addr_points.bin
│   ├── interp_entries.bin
│   ├── interp_ways.bin
│   ├── interp_nodes.bin
│   ├── admin_cells.bin
│   ├── admin_entries.bin
│   ├── admin_polygons.bin
│   ├── admin_vertices.bin
│   └── strings.bin
├── no-addresses/                  # Planet - streets + admin (no addr/interp)
│   ├── geo_cells.bin
│   ├── street_entries.bin
│   ├── street_ways.bin
│   ├── street_nodes.bin
│   ├── admin_cells.bin
│   ├── admin_entries.bin
│   ├── admin_polygons.bin
│   ├── admin_vertices.bin
│   └── strings.bin
├── admin-only/                    # Planet - admin only
│   ├── admin_cells.bin
│   ├── admin_entries.bin
│   ├── admin_polygons.bin         # uncapped vertices
│   ├── admin_vertices.bin         # uncapped vertices
│   └── strings.bin
├── uncapped/                      # Quality: no simplification
│   ├── admin_polygons.bin
│   └── admin_vertices.bin
├── q0.2/                          # Quality: 0.2x (L8=3m, L2=100m)
│   ├── admin_polygons.bin
│   └── admin_vertices.bin
├── q0.5/                          # Quality: 0.5x (L8=7.5m, L2=250m)
│   ├── admin_polygons.bin
│   └── admin_vertices.bin
├── q1/                            # Quality: 1.0x (L8=15m, L2=500m)
│   ├── admin_polygons.bin
│   └── admin_vertices.bin
├── q1.5/                          # Quality: 1.5x (L8=22m, L2=750m)
│   ├── admin_polygons.bin
│   └── admin_vertices.bin
├── q2/                            # Quality: 2.0x (L8=30m, L2=1000m)
│   ├── admin_polygons.bin
│   └── admin_vertices.bin
├── q2.5/                          # Quality: 2.5x (L8=38m, L2=1250m)
│   ├── admin_polygons.bin
│   └── admin_vertices.bin
├── europe/
│   ├── full/                      # Europe - streets + addresses + admin
│   │   └── (same 14 files as planet full/)
│   ├── no-addresses/
│   │   └── (same 9 files)
│   ├── admin-only/
│   │   └── (5 admin files)
│   ├── uncapped/
│   │   ├── admin_polygons.bin
│   │   └── admin_vertices.bin
│   ├── q0.2/
│   │   ├── admin_polygons.bin
│   │   └── admin_vertices.bin
│   ├── q0.5/ ... q1/ ... q1.5/ ... q2/ ... q2.5/
│   │   └── (same 2 files each)
├── north-america/
│   └── (same structure as europe/)
├── asia/
│   └── (same structure)
├── south-america/
│   └── (same structure)
├── africa/
│   └── (same structure)
├── oceania/
│   └── (same structure)
├── central-america/
│   └── (same structure)
└── antarctica/
    └── (same structure)
```

To use a quality variant, the server loads `admin_cells.bin`, `admin_entries.bin`, and `strings.bin` from the `admin-only/` directory (or `full/`), and swaps in `admin_polygons.bin` + `admin_vertices.bin` from the desired quality directory.

One thing I notice — right now the quality directories don't have `admin_cells.bin`, `admin_entries.bin`, or `strings.bin`, so the server can't load a quality directory standalone. Does the server need each quality directory to be self-contained, or is it fine to reference the base files from the parent?


Surely the polygons change depending on admin_only vs no_address vs full/


No actually — the admin polygons are identical across all three modes. The modes only control which *other* data types are included:

| Mode | Streets | Addresses | Interpolation | **Admin** |
|---|---|---|---|---|
| full | yes | yes | yes | **same** |
| no-addresses | yes | no | no | **same** |
| admin-only | no | no | no | **same** |

The `admin_polygons.bin`, `admin_vertices.bin`, `admin_cells.bin`, and `admin_entries.bin` are byte-identical across all three modes. The only difference is that `strings.bin` gets compacted differently (admin-only strips street/address names, full keeps them all).

So the quality variants don't need to be nested under each mode — they sit alongside them and the admin files are shared. The structure should really be:

```
output/
├── full/                      # geo + admin (all streets/addresses)
├── no-addresses/              # geo + admin (streets only)
├── admin-only/                # admin only (base quality = uncapped)
│   ├── admin_cells.bin        # shared across all qualities
│   ├── admin_entries.bin      # shared
│   ├── strings.bin            # shared (admin names only)
│   ├── admin_polygons.bin     # uncapped (default)
│   ├── admin_vertices.bin     # uncapped (default)
│   ├── q0.2/
│   │   ├── admin_polygons.bin
│   │   └── admin_vertices.bin
│   ├── q0.5/
│   │   ├── ...
│   ├── q1/ q1.5/ q2/ q2.5/
├── europe/
│   ├── full/
│   ├── no-addresses/
│   ├── admin-only/
│   │   ├── admin_cells.bin
│   │   ├── admin_entries.bin
│   │   ├── strings.bin
│   │   ├── admin_polygons.bin
│   │   ├── admin_vertices.bin
│   │   ├── q0.2/
│   │   ├── q0.5/ ...
```

This way each `admin-only/` directory is self-contained at the default quality, and the subdirectories just swap the two vertex-related files. Does this structure make sense to you?


Surely we don't need all this polygon data for admin-only?


You're right. The `full/` and `no-addresses/` modes need admin polygons for reverse geocoding results, but `admin-only/` IS the admin polygons — it's redundant to have them in both places.

The quality variants should just replace the admin polygon/vertex files wherever they appear. So the structure should be:

```
output/
├── full/
│   ├── geo_cells.bin, street_*, addr_*, interp_*
│   ├── admin_cells.bin, admin_entries.bin, strings.bin
│   ├── admin_polygons.bin          # default quality
│   ├── admin_vertices.bin          # default quality
│   ├── q0.2/
│   │   ├── admin_polygons.bin
│   │   └── admin_vertices.bin
│   ├── q0.5/ q1/ q1.5/ q2/ q2.5/
├── no-addresses/
│   ├── (same pattern — shared files + quality subdirs)
├── admin-only/
│   ├── admin_cells.bin, admin_entries.bin, strings.bin
│   ├── admin_polygons.bin          # default quality
│   ├── admin_vertices.bin
│   ├── q0.2/ q0.5/ q1/ ...
├── europe/
│   ├── full/
│   │   ├── (same pattern)
│   ├── admin-only/
│   │   ├── (same pattern)
```

Every mode that includes admin data gets quality subdirectories. The server loads everything from the mode directory, then optionally overrides `admin_polygons.bin` + `admin_vertices.bin` from a quality subdir.

Is this what you're after?


Yes, before we added this quality option though, were we including polygons we didn't need in the admin-only files?


No — the `admin-only` mode was already correct. It only writes the 5 admin files (`admin_cells.bin`, `admin_entries.bin`, `admin_polygons.bin`, `admin_vertices.bin`, `strings.bin`). No street/address/interpolation files.

What I said earlier was confusing. The admin polygon data is the same *content* across all three modes — it's not duplicated unnecessarily. Each mode just writes a different subset of file types:

| File | full | no-addresses | admin-only |
|---|---|---|---|
| geo_cells.bin | yes | yes | no |
| street_ways.bin | yes | yes | no |
| street_nodes.bin | yes | yes | no |
| street_entries.bin | yes | yes | no |
| addr_points.bin | yes | no | no |
| addr_entries.bin | yes | no | no |
| interp_*.bin | yes | no | no |
| admin_cells.bin | yes | yes | yes |
| admin_entries.bin | yes | yes | yes |
| admin_polygons.bin | yes | yes | yes |
| admin_vertices.bin | yes | yes | yes |
| strings.bin | yes (full) | yes (compacted) | yes (compacted) |

The admin files appear in all three modes because the server needs them for every reverse geocoding query — it always returns the admin area (city/state/country) alongside the street/address result. Even `full` mode needs the admin polygons to answer "which city is this point in?".

So the quality variants apply to all three modes equally. Shall I update the code to match the structure we agreed on and start the build?


Alright, the idea here is to avoid repeating files that are the same, as we will be storing all of these in S3 and then allowing users to select the options they want when they download them


That changes everything. For S3 distribution, we want each file stored exactly once and let users pick and mix. The structure should be:

```
output/
├── streets/                       # Street geometry (shared)
│   ├── street_ways.bin
│   ├── street_nodes.bin
│   └── street_entries.bin
├── addresses/                     # Address data (shared)
│   ├── addr_points.bin
│   ├── addr_entries.bin
│   ├── interp_ways.bin
│   ├── interp_nodes.bin
│   └── interp_entries.bin
├── geo_cells.bin                  # Cell index (shared, references streets+addrs)
├── admin/                         # Admin base (shared across qualities)
│   ├── admin_cells.bin
│   ├── admin_entries.bin
│   ├── strings.bin
│   ├── uncapped/
│   │   ├── admin_polygons.bin
│   │   └── admin_vertices.bin
│   ├── q0.2/
│   │   ├── admin_polygons.bin
│   │   └── admin_vertices.bin
│   ├── q0.5/
│   ├── q1/
│   ├── q1.5/
│   ├── q2/
│   └── q2.5/
├── europe/
│   ├── streets/
│   ├── addresses/
│   ├── geo_cells.bin
│   └── admin/
│       ├── admin_cells.bin
│       ├── admin_entries.bin
│       ├── strings.bin
│       ├── uncapped/
│       ├── q0.2/ ... q2.5/
├── asia/ africa/ ...
```

A user wanting **Europe, streets + addresses, q1 admin quality** downloads:
- `europe/geo_cells.bin`
- `europe/streets/*`
- `europe/addresses/*`
- `europe/admin/admin_cells.bin`, `admin_entries.bin`, `strings.bin`
- `europe/admin/q1/admin_polygons.bin`, `admin_vertices.bin`

A user wanting **planet, admin-only, q2.5** downloads:
- `admin/admin_cells.bin`, `admin_entries.bin`, `strings.bin`
- `admin/q2.5/admin_polygons.bin`, `admin_vertices.bin`

No file is stored twice. Does this structure work?


Would it be possible for a user to have the entire planet downloaded at admin-only level, then have europe downloaded at street level?


With the current S2 cell index design, no — `geo_cells.bin` combines street, address, and interpolation offsets into a single file, and it covers whatever geographic area it was built for. You can't mix a planet-level admin index with a Europe-level street index because:

1. The polygon IDs in `admin_entries.bin` reference positions in `admin_polygons.bin` — these are self-consistent within a region but the IDs won't match across different builds
2. `geo_cells.bin` has street/addr/interp offsets baked in for its specific set of ways/addresses
3. The server loads one directory and expects all files to be consistent

To support mix-and-match, the server would need to load multiple index directories and merge results — query the planet admin index for city/country, then query the Europe street index for street/address. That's an architectural change to the server, not the builder.

An alternative that works today: download Europe at full detail (streets + admin) and separately download the planet admin-only index. Run two server instances and query both — one for European queries (gets street-level results), one for everywhere else (gets admin-only results). A thin proxy routes based on coordinates.

Or we could build a "rest of world" complement — planet minus Europe — so you get:
- `europe/` at full detail
- `rest-of-world/` at admin-only

Would either of those approaches work for your use case, or should we look at making the server support layered loading?


Lets leave it for now


Sounds good. Let me update the code to match the S3-friendly directory structure we agreed on and get the build running.


  1244→                              std::unordered_map<uint64_t, std::vector<uint32_t>>& map) {
  1245→            if (sorted.empty() || !map.empty()) return;
  1246→            for (const auto& p : sorted) {
  1247→                map[p.cell_id].push_back(p.item_id);
  1248→            }
  1249→        };
  1250→        std::cerr << "Rebuilding cell maps for continent filtering..." << std::endl;
  1251→        auto f1 = std::async(std::launch::async, [&]{ rebuild_map(data.sorted_way_cells, data.cell_to_ways); });
  1252→        auto f2 = std::async(std::launch::async, [&]{ rebuild_map(data.sorted_interp_cells, data.cell_to_interps); });
  1253→        f1.get(); f2.get();
  1254→        log_phase("Rebuild cell maps", _pt);
  1255→    }
  1256→
  1257→    // --- Write index files ---
  1258→    log_phase("Deduplication", _pt);
  1259→    std::cerr << "Writing index files to " << output_dir << "..." << std::endl;
  1260→
  1261→    if (multi_output) {
  1262→        // Write all 3 index modes in parallel (they read shared data, write to separate dirs)
  1263→        auto wf1 = std::async(std::launch::async, [&]{ write_index(data, output_dir + "/full", IndexMode::Full); });
  1264→        auto wf2 = std::async(std::launch::async, [&]{ write_index(data, output_dir + "/no-addresses", IndexMode::NoAddresses); });
  1265→        auto wf3 = std::async(std::launch::async, [&]{ write_index(data, output_dir + "/admin-only", IndexMode::AdminOnly); });
  1266→        wf1.get(); wf2.get(); wf3.get();
  1267→    } else {
  1268→        write_index(data, output_dir, mode);
  1269→    }
  1270→
  1271→    if (generate_continents) {
  1272→        // Process continents in parallel (each filters independently from shared data)
  1273→        std::vector<std::future<void>> continent_futures;
  1274→        std::mutex cerr_mutex;
  1275→        for (size_t ci = 0; ci < kContinentCount; ci++) {
  1276→            const auto& continent = kContinents[ci];
  1277→            continent_futures.push_back(std::async(std::launch::async, [&, continent]() {
  1278→                {
  1279→                    std::lock_guard<std::mutex> lock(cerr_mutex);
  1280→                    std::cerr << "Filtering for continent: " << continent.name << "..." << std::endl;
  1281→                }
  1282→                auto subset = filter_by_bbox(data, continent);
  1283→                std::string base = output_dir + "/" + continent.name;
  1284→                ensure_dir(base);
  1285→                if (multi_output) {
  1286→                    // Write modes in parallel within each continent
  1287→                    auto cf1 = std::async(std::launch::async, [&]{ write_index(subset, base + "/full", IndexMode::Full); });
  1288→                    auto cf2 = std::async(std::launch::async, [&]{ write_index(subset, base + "/no-addresses", IndexMode::NoAddresses); });
  1289→                    auto cf3 = std::async(std::launch::async, [&]{ write_index(subset, base + "/admin-only", IndexMode::AdminOnly); });
  1290→                    cf1.get(); cf2.get(); cf3.get();
  1291→                } else {
  1292→                    write_index(subset, base, mode);
  1293→                }
  1294→                {
  1295→                    std::lock_guard<std::mutex> lock(cerr_mutex);
  1296→                    std::cerr << "Done continent: " << continent.name << std::endl;
  1297→                }
  1298→            }));
  1299→        }
  1300→        for (auto& f : continent_futures) {
  1301→            f.get();
  1302→        }
  1303→    }

                              std::unordered_map<uint64_t, std::vector<uint32_t>>& map) {
            if (sorted.empty() || !map.empty()) return;
            for (const auto& p : sorted) {
                map[p.cell_id].push_back(p.item_id);
            }
        };
        std::cerr << "Rebuilding cell maps for continent filtering..." << std::endl;
        auto f1 = std::async(std::launch::async, [&]{ rebuild_map(data.sorted_way_cells, data.cell_to_ways); });
        auto f2 = std::async(std::launch::async, [&]{ rebuild_map(data.sorted_interp_cells, data.cell_to_interps); });
        f1.get(); f2.get();
        log_phase("Rebuild cell maps", _pt);
    }

    // --- Write index files ---
    log_phase("Deduplication", _pt);
    std::cerr << "Writing index files to " << output_dir << "..." << std::endl;

    if (multi_output) {
        // Write all 3 index modes in parallel (they read shared data, write to separate dirs)
        auto wf1 = std::async(std::launch::async, [&]{ write_index(data, output_dir + "/full", IndexMode::Full); });
        auto wf2 = std::async(std::launch::async, [&]{ write_index(data, output_dir + "/no-addresses", IndexMode::NoAddresses); });
        auto wf3 = std::async(std::launch::async, [&]{ write_index(data, output_dir + "/admin-only", IndexMode::AdminOnly); });
        wf1.get(); wf2.get(); wf3.get();
    } else {
        write_index(data, output_dir, mode);
    }

    if (generate_continents) {
        // Process continents in parallel (each filters independently from shared data)
        std::vector<std::future<void>> continent_futures;
        std::mutex cerr_mutex;
        for (size_t ci = 0; ci < kContinentCount; ci++) {
            const auto& continent = kContinents[ci];
            continent_futures.push_back(std::async(std::launch::async, [&, continent]() {
                {
                    std::lock_guard<std::mutex> lock(cerr_mutex);
                    std::cerr << "Filtering for continent: " << continent.name << "..." << std::endl;
                }
                auto subset = filter_by_bbox(data, continent);
                std::string base = output_dir + "/" + continent.name;
                ensure_dir(base);
                if (multi_output) {
                    // Write modes in parallel within each continent
                    auto cf1 = std::async(std::launch::async, [&]{ write_index(subset, base + "/full", IndexMode::Full); });
                    auto cf2 = std::async(std::launch::async, [&]{ write_index(subset, base + "/no-addresses", IndexMode::NoAddresses); });
                    auto cf3 = std::async(std::launch::async, [&]{ write_index(subset, base + "/admin-only", IndexMode::AdminOnly); });
                    cf1.get(); cf2.get(); cf3.get();
                } else {
                    write_index(subset, base, mode);
                }
                {
                    std::lock_guard<std::mutex> lock(cerr_mutex);
                    std::cerr << "Done continent: " << continent.name << std::endl;
                }
            }));
        }
        for (auto& f : continent_futures) {
            f.get();
        }
    }


Now I need to rework the write section to use the new S3-friendly layout. Let me also read write_index to understand what it writes:


   155→          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
   156→      } }
   157→}
   158→
   159→void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
   160→    ensure_dir(output_dir);
   161→    auto _wt = std::chrono::steady_clock::now();
   162→
   163→    bool write_streets = (mode != IndexMode::AdminOnly);
   164→    bool write_addresses = (mode == IndexMode::Full);
   165→
   166→    if (write_streets) {
   167→        std::vector<uint64_t> sorted_geo_cells;
   168→        {
   169→            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {

          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
      } }
}

void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
    ensure_dir(output_dir);
    auto _wt = std::chrono::steady_clock::now();

    bool write_streets = (mode != IndexMode::AdminOnly);
    bool write_addresses = (mode == IndexMode::Full);

    if (write_streets) {
        std::vector<uint64_t> sorted_geo_cells;
        {
            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {


The current `write_index` writes everything into one flat directory based on mode. I need to restructure it to write into the S3-friendly layout. Let me rewrite the output section in build_index.cpp:


Edit

    // --- Write index files ---
    log_phase("Deduplication", _pt);
    std::cerr << "Writing index files to " << output_dir << "..." << std::endl;

    if (multi_output) {
        // Write all 3 index modes in parallel (they read shared data, write to separate dirs)
        auto wf1 = std::async(std::launch::async, [&]{ write_index(data, output_dir + "/full", IndexMode::Full); });
        auto wf2 = std::async(std::launch::async, [&]{ write_index(data, output_dir + "/no-addresses", IndexMode::NoAddresses); });
        auto wf3 = std::async(std::launch::async, [&]{ write_index(data, output_dir + "/admin-only", IndexMode::AdminOnly); });
        wf1.get(); wf2.get(); wf3.get();
    } else {
        write_index(data, output_dir, mode);
    }

    if (generate_continents) {
        // Process continents in parallel (each filters independently from shared data)
        std::vector<std::future<void>> continent_futures;
        std::mutex cerr_mutex;
        for (size_t ci = 0; ci < kContinentCount; ci++) {
            const auto& continent = kContinents[ci];
            continent_futures.push_back(std::async(std::launch::async, [&, continent]() {
                {
                    std::lock_guard<std::mutex> lock(cerr_mutex);
                    std::cerr << "Filtering for continent: " << continent.name << "..." << std::endl;
                }
                auto subset = filter_by_bbox(data, continent);
                std::string base = output_dir + "/" + continent.name;
                ensure_dir(base);
                if (multi_output) {
                    // Write modes in parallel within each continent
                    auto cf1 = std::async(std::launch::async, [&]{ write_index(subset, base + "/full", IndexMode::Full); });
                    auto cf2 = std::async(std::launch::async, [&]{ write_index(subset, base + "/no-addresses", IndexMode::NoAddresses); });
                    auto cf3 = std::async(std::launch::async, [&]{ write_index(subset, base + "/admin-only", IndexMode::AdminOnly); });
                    cf1.get(); cf2.get(); cf3.get();
                } else {
                    write_index(subset, base, mode);
                }
                {
                    std::lock_guard<std::mutex> lock(cerr_mutex);
                    std::cerr << "Done continent: " << continent.name << std::endl;
                }
            }));
        }
        for (auto& f : continent_futures) {
            f.get();
        }
    }

    log_phase("Index file writing", _pt);

    // --- Write quality variants ---
    if (multi_quality) {
        std::cerr << "Writing quality variants..." << std::endl;
        // Determine the base admin-only dir to link S2 cell files from
        std::string base_admin_dir = multi_output ? output_dir + "/admin-only" : output_dir;

        auto quality_name = [](double scale) -> std::string {
            if (scale == 0) return "uncapped";
            char buf[32];
            // Clean format: drop trailing zeros
            snprintf(buf, sizeof(buf), "q%.2g", scale);
            return buf;
        };

        auto _qt = std::chrono::steady_clock::now();
        for (double scale : quality_scales) {
            std::string qname = quality_name(scale);
            std::string qdir = output_dir + "/" + qname;
            std::cerr << "  Quality: " << qname
                      << " (scale " << scale << "x)..." << std::endl;
            write_quality_variant(data, base_admin_dir, qdir, scale);
            log_phase(("    " + qname).c_str(), _qt);
        }

        // Also write quality variants per continent
        if (generate_continents) {
            for (size_t ci = 0; ci < kContinentCount; ci++) {
                const auto& continent = kContinents[ci];
                std::string cont_base = output_dir + "/" + continent.name;
                std::string cont_admin = multi_output
                    ? cont_base + "/admin-only"
                    : cont_base;

                auto subset = filter_by_bbox(data, continent);

                for (double scale : quality_scales) {
                    std::string qdir = cont_base + "/" + quality_name(scale);
                    write_quality_variant(subset, cont_admin, qdir, scale);
                }
                std::cerr << "  Done quality variants for " << continent.name << std::endl;
            }
        }
        log_phase("Quality variants (total)", _pt);
    }

    // --- Write index files ---
    log_phase("Deduplication", _pt);
    std::cerr << "Writing index files to " << output_dir << "..." << std::endl;

    // Helper: write quality variants for a dataset into base_dir/admin/qN/
    auto write_qualities = [&](const ParsedData& d, const std::string& base_dir) {
        if (!multi_quality) return;
        std::string admin_dir = base_dir + "/admin";
        for (double scale : quality_scales) {
            std::string qname = (scale == 0) ? "uncapped" : "q" + std::to_string(scale);
            // Clean up trailing zeros: "q1.000000" -> "q1"
            if (scale > 0) {
                char buf[32]; snprintf(buf, sizeof(buf), "q%.4g", scale);
                qname = buf;
            }
            std::string qdir = admin_dir + "/" + qname;
            std::cerr << "    Quality: " << qname << "..." << std::endl;
            write_quality_variant(d, admin_dir, qdir, scale);
        }
    };

    // Helper: write one region (planet or continent subset) in S3-friendly layout
    auto write_region = [&](const ParsedData& d, const std::string& base_dir) {
        ensure_dir(base_dir);

        if (multi_output) {
            // S3-friendly layout: separate directories for each data type
            auto wf1 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/full", IndexMode::Full); });
            auto wf2 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/no-addresses", IndexMode::NoAddresses); });
            auto wf3 = std::async(std::launch::async, [&]{ write_index(d, base_dir + "/admin", IndexMode::AdminOnly); });
            wf1.get(); wf2.get(); wf3.get();
        } else {
            write_index(d, base_dir, mode);
        }

        write_qualities(d, base_dir);
    };

    // Write planet
    write_region(data, output_dir);
    log_phase("Planet index writing", _pt);

    // Write continents
    if (generate_continents) {
        std::mutex cerr_mutex;
        // Process continents sequentially to avoid memory explosion
        // (each filter_by_bbox creates a full copy)
        for (size_t ci = 0; ci < kContinentCount; ci++) {
            const auto& continent = kContinents[ci];
            std::cerr << "Filtering for continent: " << continent.name << "..." << std::endl;
            auto subset = filter_by_bbox(data, continent);
            write_region(subset, output_dir + "/" + continent.name);
            std::cerr << "Done continent: " << continent.name << std::endl;
        }
    }

    log_phase("Index file writing (total)", _pt);