#include "cell_index.h"

#include <algorithm>
#include <cstring>
#include <fstream>
#include <future>
#include <iostream>
#include <thread>

std::vector<uint32_t> write_entries(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
    std::vector<CellRef> sorted_refs;
    sorted_refs.reserve(cell_map.size());
    for (auto& [id, ids] : cell_map) sorted_refs.push_back({id, &ids});
    std::sort(sorted_refs.begin(), sorted_refs.end(),
        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });

    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    size_t total_size = 0;
    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);

    std::vector<char> buf;
    buf.reserve(total_size);
    uint32_t current = 0;
    size_t ri = 0;
    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
        offsets[si] = current;
        const auto& ids = *sorted_refs[ri].ids;
        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
                   reinterpret_cast<const char*>(&count) + sizeof(count));
        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
        ri++;
    }
    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

std::vector<uint32_t> write_entries_from_sorted(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::vector<CellItemPair>& sorted_pairs
) {
    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    if (sorted_pairs.empty()) {
        std::ofstream f(path, std::ios::binary);
        return offsets;
    }

    unsigned int nthreads = std::min(std::thread::hardware_concurrency(), 32u);
    if (nthreads == 0) nthreads = 4;
    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;

    struct ChunkResult {
        std::vector<char> buf;
        size_t cell_start, cell_end;
        uint32_t local_size;
    };
    std::vector<ChunkResult> chunks(nthreads);

    std::vector<std::thread> threads;
    for (unsigned int t = 0; t < nthreads; t++) {
        size_t cs = t * cells_per_chunk;
        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
        if (cs >= sorted_cells.size()) break;

        threads.emplace_back([&, t, cs, ce]() {
            auto& chunk = chunks[t];
            chunk.cell_start = cs;
            chunk.cell_end = ce;
            chunk.local_size = 0;

            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
                    return p.cell_id < id;
                }) - sorted_pairs.begin();

            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;

                offsets[si] = chunk.local_size;
                size_t start = pi;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(MAX_VERTEX_COUNT)));
                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
                size_t buf_pos = chunk.buf.size();
                chunk.buf.resize(buf_pos + entry_size);
                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
                for (size_t k = start; k < pi; k++) {
                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
                           &sorted_pairs[k].item_id, sizeof(uint32_t));
                }
                chunk.local_size += entry_size;
            }
        });
    }
    for (auto& t : threads) t.join();

    uint32_t global_offset = 0;
    for (auto& chunk : chunks) {
        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
            if (offsets[si] != NO_DATA) offsets[si] += global_offset;
        }
        global_offset += chunk.local_size;
    }

    std::vector<char> buf;
    buf.reserve(global_offset);
    for (auto& chunk : chunks) buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());

    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

void write_cell_index(
    const std::string& cells_path,
    const std::string& entries_path,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(cell_map.begin(), cell_map.end());
    std::sort(sorted.begin(), sorted.end());

    { std::ofstream f(cells_path, std::ios::binary);
      uint32_t current_offset = 0;
      for (const auto& [cell_id, ids] : sorted) {
          f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
          f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
          current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
      } }

    { std::ofstream f(entries_path, std::ios::binary);
      for (const auto& [cell_id, ids] : sorted) {
          uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
          f.write(reinterpret_cast<const char*>(&count), sizeof(count));
          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
      } }
}

void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
    ensure_dir(output_dir);
    auto _wt = std::chrono::steady_clock::now();

    bool write_streets = (mode != IndexMode::AdminOnly);
    bool write_addresses = (mode == IndexMode::Full);

    if (write_streets) {
        std::vector<uint64_t> sorted_geo_cells;
        {
            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
                std::vector<uint64_t> cells;
                cells.reserve(pairs.size() / 2);
                for (size_t i = 0; i < pairs.size(); ) {
                    cells.push_back(pairs[i].cell_id);
                    uint64_t cur = pairs[i].cell_id;
                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
                }
                return cells;
            };
            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
                std::vector<uint64_t> cells;
                cells.reserve(m.size());
                for (auto& [id, _] : m) cells.push_back(id);
                std::sort(cells.begin(), cells.end());
                return cells;
            };

            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
            {
                auto f1 = std::async(std::launch::async, [&] {
                    return !data.sorted_way_cells.empty()
                        ? extract_unique_cells(data.sorted_way_cells)
                        : extract_from_map(data.cell_to_ways);
                });
                if (write_addresses) {
                    auto f2 = std::async(std::launch::async, [&] {
                        return !data.sorted_addr_cells.empty()
                            ? extract_unique_cells(data.sorted_addr_cells)
                            : extract_from_map(data.cell_to_addrs);
                    });
                    auto f3 = std::async(std::launch::async, [&] { return extract_from_map(data.cell_to_interps); });
                    addr_cells = f2.get();
                    interp_cells = f3.get();
                }
                way_cells = f1.get();
            }

            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
            std::merge(way_cells.begin(), way_cells.end(), addr_cells.begin(), addr_cells.end(),
                       std::back_inserter(sorted_geo_cells));
            if (!interp_cells.empty()) {
                std::vector<uint64_t> tmp;
                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
                           interp_cells.begin(), interp_cells.end(), std::back_inserter(tmp));
                sorted_geo_cells = std::move(tmp);
            }
            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
                                    sorted_geo_cells.end());
        }

        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
        {
            auto f1 = std::async(std::launch::async, [&]() {
                if (!data.sorted_way_cells.empty())
                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
            });
            if (write_addresses) {
                auto f2 = std::async(std::launch::async, [&]() {
                    if (!data.sorted_addr_cells.empty())
                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
                });
                auto f3 = std::async(std::launch::async, [&]() {
                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
                });
                addr_offsets = f2.get();
                interp_offsets = f3.get();
            }
            street_offsets = f1.get();
        }

        log_phase("  Write: entry files", _wt);
        {
            size_t n = sorted_geo_cells.size();
            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
            std::vector<char> buf(n * row_size);
            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);

            unsigned int nthreads = std::thread::hardware_concurrency();
            if (nthreads == 0) nthreads = 4;
            size_t chunk = (n + nthreads - 1) / nthreads;
            std::vector<std::thread> fill_threads;
            for (unsigned int t = 0; t < nthreads; t++) {
                size_t start = t * chunk;
                size_t end = std::min(start + chunk, n);
                if (start >= n) break;
                fill_threads.emplace_back([&, start, end]() {
                    char* ptr = buf.data() + start * row_size;
                    for (size_t i = start; i < end; i++) {
                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                    }
                });
            }
            for (auto& t : fill_threads) t.join();

            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
            f.write(buf.data(), buf.size());
        }

        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
                  << data.ways.size() << " ways, " << data.addr_points.size() << " addrs, "
                  << data.interp_ways.size() << " interps)" << std::endl;
    }

    log_phase("  Write: geo_cells.bin", _wt);
    auto admin_future = std::async(std::launch::async, [&] {
        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
    });

    std::vector<std::future<void>> write_futures;
    if (write_streets) {
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
        }));
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
        }));
        if (write_addresses) {
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
            }));
        }
    }
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(data.admin_polygons.data()), data.admin_polygons.size() * sizeof(AdminPolygon));
    }));
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
    }));
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
        f.write(data.string_pool.data().data(), data.string_pool.data().size());
        std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
    }));

    log_phase("  Write: parallel data files launched", _wt);
    admin_future.get();
    for (auto& f : write_futures) f.get();
}


File created successfully at: /root/traccar-geocoder/builder/src/cell_index.cpp

#include "cell_index.h"

#include <algorithm>
#include <cstring>
#include <fstream>
#include <future>
#include <iostream>
#include <thread>

std::vector<uint32_t> write_entries(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
    std::vector<CellRef> sorted_refs;
    sorted_refs.reserve(cell_map.size());
    for (auto& [id, ids] : cell_map) sorted_refs.push_back({id, &ids});
    std::sort(sorted_refs.begin(), sorted_refs.end(),
        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });

    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    size_t total_size = 0;
    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);

    std::vector<char> buf;
    buf.reserve(total_size);
    uint32_t current = 0;
    size_t ri = 0;
    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
        offsets[si] = current;
        const auto& ids = *sorted_refs[ri].ids;
        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
                   reinterpret_cast<const char*>(&count) + sizeof(count));
        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
        ri++;
    }
    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

std::vector<uint32_t> write_entries_from_sorted(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::vector<CellItemPair>& sorted_pairs
) {
    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    if (sorted_pairs.empty()) {
        std::ofstream f(path, std::ios::binary);
        return offsets;
    }

    unsigned int nthreads = std::min(std::thread::hardware_concurrency(), 32u);
    if (nthreads == 0) nthreads = 4;
    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;

    struct ChunkResult {
        std::vector<char> buf;
        size_t cell_start, cell_end;
        uint32_t local_size;
    };
    std::vector<ChunkResult> chunks(nthreads);

    std::vector<std::thread> threads;
    for (unsigned int t = 0; t < nthreads; t++) {
        size_t cs = t * cells_per_chunk;
        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
        if (cs >= sorted_cells.size()) break;

        threads.emplace_back([&, t, cs, ce]() {
            auto& chunk = chunks[t];
            chunk.cell_start = cs;
            chunk.cell_end = ce;
            chunk.local_size = 0;

            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
                    return p.cell_id < id;
                }) - sorted_pairs.begin();

            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;

                offsets[si] = chunk.local_size;
                size_t start = pi;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(MAX_VERTEX_COUNT)));
                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
                size_t buf_pos = chunk.buf.size();
                chunk.buf.resize(buf_pos + entry_size);
                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
                for (size_t k = start; k < pi; k++) {
                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
                           &sorted_pairs[k].item_id, sizeof(uint32_t));
                }
                chunk.local_size += entry_size;
            }
        });
    }
    for (auto& t : threads) t.join();

    uint32_t global_offset = 0;
    for (auto& chunk : chunks) {
        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
            if (offsets[si] != NO_DATA) offsets[si] += global_offset;
        }
        global_offset += chunk.local_size;
    }

    std::vector<char> buf;
    buf.reserve(global_offset);
    for (auto& chunk : chunks) buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());

    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

void write_cell_index(
    const std::string& cells_path,
    const std::string& entries_path,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(cell_map.begin(), cell_map.end());
    std::sort(sorted.begin(), sorted.end());

    { std::ofstream f(cells_path, std::ios::binary);
      uint32_t current_offset = 0;
      for (const auto& [cell_id, ids] : sorted) {
          f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
          f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
          current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
      } }

    { std::ofstream f(entries_path, std::ios::binary);
      for (const auto& [cell_id, ids] : sorted) {
          uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
          f.write(reinterpret_cast<const char*>(&count), sizeof(count));
          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
      } }
}

void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
    ensure_dir(output_dir);
    auto _wt = std::chrono::steady_clock::now();

    bool write_streets = (mode != IndexMode::AdminOnly);
    bool write_addresses = (mode == IndexMode::Full);

    if (write_streets) {
        std::vector<uint64_t> sorted_geo_cells;
        {
            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
                std::vector<uint64_t> cells;
                cells.reserve(pairs.size() / 2);
                for (size_t i = 0; i < pairs.size(); ) {
                    cells.push_back(pairs[i].cell_id);
                    uint64_t cur = pairs[i].cell_id;
                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
                }
                return cells;
            };
            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
                std::vector<uint64_t> cells;
                cells.reserve(m.size());
                for (auto& [id, _] : m) cells.push_back(id);
                std::sort(cells.begin(), cells.end());
                return cells;
            };

            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
            {
                auto f1 = std::async(std::launch::async, [&] {
                    return !data.sorted_way_cells.empty()
                        ? extract_unique_cells(data.sorted_way_cells)
                        : extract_from_map(data.cell_to_ways);
                });
                if (write_addresses) {
                    auto f2 = std::async(std::launch::async, [&] {
                        return !data.sorted_addr_cells.empty()
                            ? extract_unique_cells(data.sorted_addr_cells)
                            : extract_from_map(data.cell_to_addrs);
                    });
                    auto f3 = std::async(std::launch::async, [&] { return extract_from_map(data.cell_to_interps); });
                    addr_cells = f2.get();
                    interp_cells = f3.get();
                }
                way_cells = f1.get();
            }

            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
            std::merge(way_cells.begin(), way_cells.end(), addr_cells.begin(), addr_cells.end(),
                       std::back_inserter(sorted_geo_cells));
            if (!interp_cells.empty()) {
                std::vector<uint64_t> tmp;
                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
                           interp_cells.begin(), interp_cells.end(), std::back_inserter(tmp));
                sorted_geo_cells = std::move(tmp);
            }
            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
                                    sorted_geo_cells.end());
        }

        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
        {
            auto f1 = std::async(std::launch::async, [&]() {
                if (!data.sorted_way_cells.empty())
                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
            });
            if (write_addresses) {
                auto f2 = std::async(std::launch::async, [&]() {
                    if (!data.sorted_addr_cells.empty())
                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
                });
                auto f3 = std::async(std::launch::async, [&]() {
                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
                });
                addr_offsets = f2.get();
                interp_offsets = f3.get();
            }
            street_offsets = f1.get();
        }

        log_phase("  Write: entry files", _wt);
        {
            size_t n = sorted_geo_cells.size();
            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
            std::vector<char> buf(n * row_size);
            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);

            unsigned int nthreads = std::thread::hardware_concurrency();
            if (nthreads == 0) nthreads = 4;
            size_t chunk = (n + nthreads - 1) / nthreads;
            std::vector<std::thread> fill_threads;
            for (unsigned int t = 0; t < nthreads; t++) {
                size_t start = t * chunk;
                size_t end = std::min(start + chunk, n);
                if (start >= n) break;
                fill_threads.emplace_back([&, start, end]() {
                    char* ptr = buf.data() + start * row_size;
                    for (size_t i = start; i < end; i++) {
                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                    }
                });
            }
            for (auto& t : fill_threads) t.join();

            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
            f.write(buf.data(), buf.size());
        }

        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
                  << data.ways.size() << " ways, " << data.addr_points.size() << " addrs, "
                  << data.interp_ways.size() << " interps)" << std::endl;
    }

    log_phase("  Write: geo_cells.bin", _wt);
    auto admin_future = std::async(std::launch::async, [&] {
        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
    });

    std::vector<std::future<void>> write_futures;
    if (write_streets) {
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
        }));
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
        }));
        if (write_addresses) {
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
            }));
        }
    }
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(data.admin_polygons.data()), data.admin_polygons.size() * sizeof(AdminPolygon));
    }));
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
    }));
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
        f.write(data.string_pool.data().data(), data.string_pool.data().size());
        std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
    }));

    log_phase("  Write: parallel data files launched", _wt);
    admin_future.get();
    for (auto& f : write_futures) f.get();
}


Updated task #9 status


Updated task #10 status


Now the critical step — trim build_index.cpp to just main() and update includes. I need to rewrite the file to only include main() with the new module headers:


  1628→// --- Main ---
  1629→
  1630→int main(int argc, char* argv[]) {
  1631→    if (argc < 2) {
  1632→        std::cerr << "Usage: build-index <output-dir> <input.osm.pbf> [options]" << std::endl;

// --- Main ---

int main(int argc, char* argv[]) {
    if (argc < 2) {
        std::cerr << "Usage: build-index <output-dir> <input.osm.pbf> [options]" << std::endl;


Now I need to read the main function (lines 1630-2755) and create a new build_index.cpp with just the includes and main:


Error: File content (15181 tokens) exceeds maximum allowed tokens (10000). Use offset and limit parameters to read specific portions of the file, or search for specific content instead of reading the whole file.