  1269→// Write entries file from a hash map (used for addr and interp cell maps).
  1270→static std::vector<uint32_t> write_entries(
  1271→    const std::string& path,
  1272→    const std::vector<uint64_t>& sorted_cells,
  1273→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
  1274→) {
  1275→    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
  1276→    std::vector<CellRef> sorted_refs;
  1277→    sorted_refs.reserve(cell_map.size());
  1278→    for (auto& [id, ids] : cell_map) {
  1279→        sorted_refs.push_back({id, &ids});
  1280→    }
  1281→    std::sort(sorted_refs.begin(), sorted_refs.end(),
  1282→        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });
  1283→
  1284→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
  1285→    size_t total_size = 0;
  1286→    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);
  1287→
  1288→    std::vector<char> buf;
  1289→    buf.reserve(total_size);
  1290→    uint32_t current = 0;
  1291→    size_t ri = 0;
  1292→    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
  1293→        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
  1294→        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
  1295→        offsets[si] = current;
  1296→        const auto& ids = *sorted_refs[ri].ids;
  1297→        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(65535)));
  1298→        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
  1299→                   reinterpret_cast<const char*>(&count) + sizeof(count));
  1300→        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
  1301→                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
  1302→        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
  1303→        ri++;
  1304→    }
  1305→    std::ofstream f(path, std::ios::binary);
  1306→    f.write(buf.data(), buf.size());
  1307→    return offsets;
  1308→}
  1309→
  1310→// Write entries file from sorted pairs — parallel chunked merge-join.
  1311→// Splits sorted_cells into chunks, each thread builds its chunk independently,
  1312→// then prefix-sum computes global offsets and buffers are concatenated.
  1313→static std::vector<uint32_t> write_entries_from_sorted(
  1314→    const std::string& path,
  1315→    const std::vector<uint64_t>& sorted_cells,
  1316→    const std::vector<CellItemPair>& sorted_pairs
  1317→) {
  1318→    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
  1319→    if (sorted_pairs.empty()) {
  1320→        std::ofstream f(path, std::ios::binary);
  1321→        return offsets;
  1322→    }
  1323→
  1324→    // Find where each chunk of sorted_cells starts in sorted_pairs
  1325→    // using binary search (both are sorted by cell_id)
  1326→    unsigned int nthreads = std::min(std::thread::hardware_concurrency(), 32u);
  1327→    if (nthreads == 0) nthreads = 4;
  1328→    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;
  1329→
  1330→    struct ChunkResult {
  1331→        std::vector<char> buf;
  1332→        size_t cell_start, cell_end; // range in sorted_cells
  1333→        uint32_t local_size; // total bytes in this chunk's buffer
  1334→    };
  1335→    std::vector<ChunkResult> chunks(nthreads);
  1336→
  1337→    // Parallel: each thread processes its range of sorted_cells
  1338→    std::vector<std::thread> threads;
  1339→    for (unsigned int t = 0; t < nthreads; t++) {
  1340→        size_t cs = t * cells_per_chunk;
  1341→        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
  1342→        if (cs >= sorted_cells.size()) break;
  1343→
  1344→        threads.emplace_back([&, t, cs, ce]() {
  1345→            auto& chunk = chunks[t];
  1346→            chunk.cell_start = cs;
  1347→            chunk.cell_end = ce;
  1348→            chunk.local_size = 0;
  1349→
  1350→            // Binary search for where this chunk's first cell appears in sorted_pairs
  1351→            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
  1352→                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
  1353→                    return p.cell_id < id;
  1354→                }) - sorted_pairs.begin();
  1355→
  1356→            // Merge-join for this chunk
  1357→            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
  1358→                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
  1359→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
  1360→                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;
  1361→
  1362→                offsets[si] = chunk.local_size; // local offset, adjusted later
  1363→                size_t start = pi;
  1364→                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
  1365→                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(65535)));
  1366→                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
  1367→                size_t buf_pos = chunk.buf.size();
  1368→                chunk.buf.resize(buf_pos + entry_size);
  1369→                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
  1370→                for (size_t k = start; k < pi; k++) {
  1371→                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
  1372→                           &sorted_pairs[k].item_id, sizeof(uint32_t));
  1373→                }
  1374→                chunk.local_size += entry_size;
  1375→            }
  1376→        });
  1377→    }
  1378→    for (auto& t : threads) t.join();
  1379→
  1380→    // Prefix-sum: compute global offsets for each chunk
  1381→    uint32_t global_offset = 0;
  1382→    for (auto& chunk : chunks) {
  1383→        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
  1384→            if (offsets[si] != NO_DATA) {
  1385→                offsets[si] += global_offset;
  1386→            }
  1387→        }
  1388→        global_offset += chunk.local_size;
  1389→    }
  1390→
  1391→    // Concatenate buffers and write
  1392→    std::vector<char> buf;
  1393→    buf.reserve(global_offset);
  1394→    for (auto& chunk : chunks) {
  1395→        buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());
  1396→    }
  1397→
  1398→    std::ofstream f(path, std::ios::binary);
  1399→    f.write(buf.data(), buf.size());
  1400→    return offsets;
  1401→}
  1402→
  1403→static void write_cell_index(
  1404→    const std::string& cells_path,
  1405→    const std::string& entries_path,
  1406→    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
  1407→) {
  1408→    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(
  1409→        cell_map.begin(), cell_map.end());
  1410→    std::sort(sorted.begin(), sorted.end());
  1411→
  1412→    {
  1413→        std::ofstream f(cells_path, std::ios::binary);
  1414→        uint32_t current_offset = 0;
  1415→        for (const auto& [cell_id, ids] : sorted) {
  1416→            f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
  1417→            f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
  1418→            current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
  1419→        }
  1420→    }
  1421→
  1422→    {
  1423→        std::ofstream f(entries_path, std::ios::binary);
  1424→        for (const auto& [cell_id, ids] : sorted) {
  1425→            uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(65535)));
  1426→            f.write(reinterpret_cast<const char*>(&count), sizeof(count));
  1427→            f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
  1428→        }
  1429→    }
  1430→}
  1431→
  1432→// --- Write all index files ---
  1433→
  1434→static void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
  1435→    ensure_dir(output_dir);
  1436→    auto _wt = std::chrono::steady_clock::now();
  1437→
  1438→    bool write_streets = (mode != IndexMode::AdminOnly);
  1439→    bool write_addresses = (mode == IndexMode::Full);
  1440→
  1441→    if (write_streets) {
  1442→        // Build sorted_geo_cells from sorted pairs where available (avoids hash map iteration).
  1443→        // Extract unique cell IDs from each sorted pair vector in parallel, then merge.
  1444→        std::vector<uint64_t> sorted_geo_cells;
  1445→        {
  1446→            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
  1447→                std::vector<uint64_t> cells;
  1448→                cells.reserve(pairs.size() / 2);
  1449→                for (size_t i = 0; i < pairs.size(); ) {
  1450→                    cells.push_back(pairs[i].cell_id);
  1451→                    uint64_t cur = pairs[i].cell_id;
  1452→                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
  1453→                }
  1454→                return cells;
  1455→            };
  1456→            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
  1457→                std::vector<uint64_t> cells;
  1458→                cells.reserve(m.size());
  1459→                for (auto& [id, _] : m) cells.push_back(id);
  1460→                std::sort(cells.begin(), cells.end());
  1461→                return cells;
  1462→            };
  1463→
  1464→            // Extract unique cell IDs in parallel
  1465→            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
  1466→            {
  1467→                auto f1 = std::async(std::launch::async, [&] {
  1468→                    return !data.sorted_way_cells.empty()
  1469→                        ? extract_unique_cells(data.sorted_way_cells)
  1470→                        : extract_from_map(data.cell_to_ways);
  1471→                });
  1472→                if (write_addresses) {
  1473→                    auto f2 = std::async(std::launch::async, [&] {
  1474→                        return !data.sorted_addr_cells.empty()
  1475→                            ? extract_unique_cells(data.sorted_addr_cells)
  1476→                            : extract_from_map(data.cell_to_addrs);
  1477→                    });
  1478→                    auto f3 = std::async(std::launch::async, [&] {
  1479→                        return extract_from_map(data.cell_to_interps);
  1480→                    });
  1481→                    addr_cells = f2.get();
  1482→                    interp_cells = f3.get();
  1483→                }
  1484→                way_cells = f1.get();
  1485→            }
  1486→
  1487→            // Merge sorted unique cell ID vectors
  1488→            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
  1489→            std::merge(way_cells.begin(), way_cells.end(),
  1490→                       addr_cells.begin(), addr_cells.end(),
  1491→                       std::back_inserter(sorted_geo_cells));
  1492→            if (!interp_cells.empty()) {
  1493→                std::vector<uint64_t> tmp;
  1494→                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
  1495→                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
  1496→                           interp_cells.begin(), interp_cells.end(),
  1497→                           std::back_inserter(tmp));
  1498→                sorted_geo_cells = std::move(tmp);
  1499→            }
  1500→            // Deduplicate (merge may have duplicates from overlapping cells)
  1501→            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
  1502→                                    sorted_geo_cells.end());
  1503→        }
  1504→
  1505→        // Write entry files in parallel (returns positional offset vectors, not hash maps)
  1506→        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
  1507→        {
  1508→            auto f1 = std::async(std::launch::async, [&]() {
  1509→                // Use direct sorted-pair writing for ways (skips hash map extraction + sort)
  1510→                if (!data.sorted_way_cells.empty()) {
  1511→                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
  1512→                }
  1513→                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
  1514→            });
  1515→            if (write_addresses) {
  1516→                auto f2 = std::async(std::launch::async, [&]() {
  1517→                    if (!data.sorted_addr_cells.empty()) {
  1518→                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
  1519→                    }
  1520→                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
  1521→                });
  1522→                auto f3 = std::async(std::launch::async, [&]() {
  1523→                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
  1524→                });
  1525→                addr_offsets = f2.get();
  1526→                interp_offsets = f3.get();
  1527→            }
  1528→            street_offsets = f1.get();
  1529→        }
  1530→
  1531→        log_phase("  Write: entry files", _wt);
  1532→        // Build geo_cells.bin — parallel buffer fill + single write
  1533→        {
  1534→            size_t n = sorted_geo_cells.size();
  1535→            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
  1536→            std::vector<char> buf(n * row_size);
  1537→
  1538→            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
  1539→            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);
  1540→
  1541→            // Parallel buffer fill — split into chunks across threads
  1542→            unsigned int nthreads = std::thread::hardware_concurrency();
  1543→            if (nthreads == 0) nthreads = 4;
  1544→            size_t chunk = (n + nthreads - 1) / nthreads;
  1545→            std::vector<std::thread> fill_threads;
  1546→            for (unsigned int t = 0; t < nthreads; t++) {
  1547→                size_t start = t * chunk;
  1548→                size_t end = std::min(start + chunk, n);
  1549→                if (start >= n) break;
  1550→                fill_threads.emplace_back([&, start, end]() {
  1551→                    char* ptr = buf.data() + start * row_size;
  1552→                    for (size_t i = start; i < end; i++) {
  1553→                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
  1554→                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
  1555→                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
  1556→                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
  1557→                    }
  1558→                });
  1559→            }
  1560→            for (auto& t : fill_threads) t.join();
  1561→
  1562→            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
  1563→            f.write(buf.data(), buf.size());
  1564→        }
  1565→
  1566→        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
  1567→                  << data.ways.size() << " ways, "
  1568→                  << data.addr_points.size() << " addrs, "
  1569→                  << data.interp_ways.size() << " interps)" << std::endl;
  1570→    }
  1571→
  1572→    log_phase("  Write: geo_cells.bin", _wt);
  1573→    // Write admin cell index, raw data files, and admin polygons in parallel
  1574→    auto admin_future = std::async(std::launch::async, [&] {
  1575→        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
  1576→        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
  1577→    });
  1578→
  1579→    std::vector<std::future<void>> write_futures;
  1580→
  1581→    if (write_streets) {
  1582→        write_futures.push_back(std::async(std::launch::async, [&] {
  1583→            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
  1584→            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
  1585→        }));
  1586→        write_futures.push_back(std::async(std::launch::async, [&] {
  1587→            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
  1588→            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
  1589→        }));
  1590→        if (write_addresses) {
  1591→            write_futures.push_back(std::async(std::launch::async, [&] {
  1592→                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
  1593→                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
  1594→            }));
  1595→            write_futures.push_back(std::async(std::launch::async, [&] {
  1596→                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
  1597→                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
  1598→            }));
  1599→            write_futures.push_back(std::async(std::launch::async, [&] {
  1600→                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
  1601→                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
  1602→            }));
  1603→        }
  1604→    }
  1605→
  1606→    write_futures.push_back(std::async(std::launch::async, [&] {
  1607→        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
  1608→        f.write(reinterpret_cast<const char*>(data.admin_polygons.data()), data.admin_polygons.size() * sizeof(AdminPolygon));
  1609→    }));
  1610→
  1611→    write_futures.push_back(std::async(std::launch::async, [&] {
  1612→        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
  1613→        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
  1614→    }));
  1615→
  1616→    write_futures.push_back(std::async(std::launch::async, [&] {
  1617→        std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
  1618→        f.write(data.string_pool.data().data(), data.string_pool.data().size());
  1619→        std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
  1620→    }));
  1621→
  1622→    // Wait for all parallel writes
  1623→    log_phase("  Write: parallel data files launched", _wt);
  1624→    admin_future.get();
  1625→    for (auto& f : write_futures) f.get();
  1626→}
  1627→
  1628→// --- Main ---

// Write entries file from a hash map (used for addr and interp cell maps).
static std::vector<uint32_t> write_entries(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
    std::vector<CellRef> sorted_refs;
    sorted_refs.reserve(cell_map.size());
    for (auto& [id, ids] : cell_map) {
        sorted_refs.push_back({id, &ids});
    }
    std::sort(sorted_refs.begin(), sorted_refs.end(),
        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });

    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    size_t total_size = 0;
    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);

    std::vector<char> buf;
    buf.reserve(total_size);
    uint32_t current = 0;
    size_t ri = 0;
    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
        offsets[si] = current;
        const auto& ids = *sorted_refs[ri].ids;
        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(65535)));
        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
                   reinterpret_cast<const char*>(&count) + sizeof(count));
        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
        ri++;
    }
    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

// Write entries file from sorted pairs — parallel chunked merge-join.
// Splits sorted_cells into chunks, each thread builds its chunk independently,
// then prefix-sum computes global offsets and buffers are concatenated.
static std::vector<uint32_t> write_entries_from_sorted(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::vector<CellItemPair>& sorted_pairs
) {
    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    if (sorted_pairs.empty()) {
        std::ofstream f(path, std::ios::binary);
        return offsets;
    }

    // Find where each chunk of sorted_cells starts in sorted_pairs
    // using binary search (both are sorted by cell_id)
    unsigned int nthreads = std::min(std::thread::hardware_concurrency(), 32u);
    if (nthreads == 0) nthreads = 4;
    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;

    struct ChunkResult {
        std::vector<char> buf;
        size_t cell_start, cell_end; // range in sorted_cells
        uint32_t local_size; // total bytes in this chunk's buffer
    };
    std::vector<ChunkResult> chunks(nthreads);

    // Parallel: each thread processes its range of sorted_cells
    std::vector<std::thread> threads;
    for (unsigned int t = 0; t < nthreads; t++) {
        size_t cs = t * cells_per_chunk;
        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
        if (cs >= sorted_cells.size()) break;

        threads.emplace_back([&, t, cs, ce]() {
            auto& chunk = chunks[t];
            chunk.cell_start = cs;
            chunk.cell_end = ce;
            chunk.local_size = 0;

            // Binary search for where this chunk's first cell appears in sorted_pairs
            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
                    return p.cell_id < id;
                }) - sorted_pairs.begin();

            // Merge-join for this chunk
            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;

                offsets[si] = chunk.local_size; // local offset, adjusted later
                size_t start = pi;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(65535)));
                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
                size_t buf_pos = chunk.buf.size();
                chunk.buf.resize(buf_pos + entry_size);
                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
                for (size_t k = start; k < pi; k++) {
                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
                           &sorted_pairs[k].item_id, sizeof(uint32_t));
                }
                chunk.local_size += entry_size;
            }
        });
    }
    for (auto& t : threads) t.join();

    // Prefix-sum: compute global offsets for each chunk
    uint32_t global_offset = 0;
    for (auto& chunk : chunks) {
        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
            if (offsets[si] != NO_DATA) {
                offsets[si] += global_offset;
            }
        }
        global_offset += chunk.local_size;
    }

    // Concatenate buffers and write
    std::vector<char> buf;
    buf.reserve(global_offset);
    for (auto& chunk : chunks) {
        buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());
    }

    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

static void write_cell_index(
    const std::string& cells_path,
    const std::string& entries_path,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(
        cell_map.begin(), cell_map.end());
    std::sort(sorted.begin(), sorted.end());

    {
        std::ofstream f(cells_path, std::ios::binary);
        uint32_t current_offset = 0;
        for (const auto& [cell_id, ids] : sorted) {
            f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
            f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
            current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
        }
    }

    {
        std::ofstream f(entries_path, std::ios::binary);
        for (const auto& [cell_id, ids] : sorted) {
            uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(65535)));
            f.write(reinterpret_cast<const char*>(&count), sizeof(count));
            f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
        }
    }
}

// --- Write all index files ---

static void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
    ensure_dir(output_dir);
    auto _wt = std::chrono::steady_clock::now();

    bool write_streets = (mode != IndexMode::AdminOnly);
    bool write_addresses = (mode == IndexMode::Full);

    if (write_streets) {
        // Build sorted_geo_cells from sorted pairs where available (avoids hash map iteration).
        // Extract unique cell IDs from each sorted pair vector in parallel, then merge.
        std::vector<uint64_t> sorted_geo_cells;
        {
            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
                std::vector<uint64_t> cells;
                cells.reserve(pairs.size() / 2);
                for (size_t i = 0; i < pairs.size(); ) {
                    cells.push_back(pairs[i].cell_id);
                    uint64_t cur = pairs[i].cell_id;
                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
                }
                return cells;
            };
            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
                std::vector<uint64_t> cells;
                cells.reserve(m.size());
                for (auto& [id, _] : m) cells.push_back(id);
                std::sort(cells.begin(), cells.end());
                return cells;
            };

            // Extract unique cell IDs in parallel
            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
            {
                auto f1 = std::async(std::launch::async, [&] {
                    return !data.sorted_way_cells.empty()
                        ? extract_unique_cells(data.sorted_way_cells)
                        : extract_from_map(data.cell_to_ways);
                });
                if (write_addresses) {
                    auto f2 = std::async(std::launch::async, [&] {
                        return !data.sorted_addr_cells.empty()
                            ? extract_unique_cells(data.sorted_addr_cells)
                            : extract_from_map(data.cell_to_addrs);
                    });
                    auto f3 = std::async(std::launch::async, [&] {
                        return extract_from_map(data.cell_to_interps);
                    });
                    addr_cells = f2.get();
                    interp_cells = f3.get();
                }
                way_cells = f1.get();
            }

            // Merge sorted unique cell ID vectors
            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
            std::merge(way_cells.begin(), way_cells.end(),
                       addr_cells.begin(), addr_cells.end(),
                       std::back_inserter(sorted_geo_cells));
            if (!interp_cells.empty()) {
                std::vector<uint64_t> tmp;
                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
                           interp_cells.begin(), interp_cells.end(),
                           std::back_inserter(tmp));
                sorted_geo_cells = std::move(tmp);
            }
            // Deduplicate (merge may have duplicates from overlapping cells)
            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
                                    sorted_geo_cells.end());
        }

        // Write entry files in parallel (returns positional offset vectors, not hash maps)
        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
        {
            auto f1 = std::async(std::launch::async, [&]() {
                // Use direct sorted-pair writing for ways (skips hash map extraction + sort)
                if (!data.sorted_way_cells.empty()) {
                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
                }
                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
            });
            if (write_addresses) {
                auto f2 = std::async(std::launch::async, [&]() {
                    if (!data.sorted_addr_cells.empty()) {
                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
                    }
                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
                });
                auto f3 = std::async(std::launch::async, [&]() {
                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
                });
                addr_offsets = f2.get();
                interp_offsets = f3.get();
            }
            street_offsets = f1.get();
        }

        log_phase("  Write: entry files", _wt);
        // Build geo_cells.bin — parallel buffer fill + single write
        {
            size_t n = sorted_geo_cells.size();
            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
            std::vector<char> buf(n * row_size);

            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);

            // Parallel buffer fill — split into chunks across threads
            unsigned int nthreads = std::thread::hardware_concurrency();
            if (nthreads == 0) nthreads = 4;
            size_t chunk = (n + nthreads - 1) / nthreads;
            std::vector<std::thread> fill_threads;
            for (unsigned int t = 0; t < nthreads; t++) {
                size_t start = t * chunk;
                size_t end = std::min(start + chunk, n);
                if (start >= n) break;
                fill_threads.emplace_back([&, start, end]() {
                    char* ptr = buf.data() + start * row_size;
                    for (size_t i = start; i < end; i++) {
                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                    }
                });
            }
            for (auto& t : fill_threads) t.join();

            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
            f.write(buf.data(), buf.size());
        }

        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
                  << data.ways.size() << " ways, "
                  << data.addr_points.size() << " addrs, "
                  << data.interp_ways.size() << " interps)" << std::endl;
    }

    log_phase("  Write: geo_cells.bin", _wt);
    // Write admin cell index, raw data files, and admin polygons in parallel
    auto admin_future = std::async(std::launch::async, [&] {
        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
    });

    std::vector<std::future<void>> write_futures;

    if (write_streets) {
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
        }));
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
        }));
        if (write_addresses) {
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
            }));
        }
    }

    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(data.admin_polygons.data()), data.admin_polygons.size() * sizeof(AdminPolygon));
    }));

    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
    }));

    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
        f.write(data.string_pool.data().data(), data.string_pool.data().size());
        std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
    }));

    // Wait for all parallel writes
    log_phase("  Write: parallel data files launched", _wt);
    admin_future.get();
    for (auto& f : write_futures) f.get();
}

// --- Main ---


Write

#include "cell_index.h"

#include <algorithm>
#include <cstring>
#include <fstream>
#include <future>
#include <iostream>
#include <thread>

std::vector<uint32_t> write_entries(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
    std::vector<CellRef> sorted_refs;
    sorted_refs.reserve(cell_map.size());
    for (auto& [id, ids] : cell_map) sorted_refs.push_back({id, &ids});
    std::sort(sorted_refs.begin(), sorted_refs.end(),
        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });

    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    size_t total_size = 0;
    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);

    std::vector<char> buf;
    buf.reserve(total_size);
    uint32_t current = 0;
    size_t ri = 0;
    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
        offsets[si] = current;
        const auto& ids = *sorted_refs[ri].ids;
        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
                   reinterpret_cast<const char*>(&count) + sizeof(count));
        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
        ri++;
    }
    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

std::vector<uint32_t> write_entries_from_sorted(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::vector<CellItemPair>& sorted_pairs
) {
    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    if (sorted_pairs.empty()) {
        std::ofstream f(path, std::ios::binary);
        return offsets;
    }

    unsigned int nthreads = std::min(std::thread::hardware_concurrency(), 32u);
    if (nthreads == 0) nthreads = 4;
    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;

    struct ChunkResult {
        std::vector<char> buf;
        size_t cell_start, cell_end;
        uint32_t local_size;
    };
    std::vector<ChunkResult> chunks(nthreads);

    std::vector<std::thread> threads;
    for (unsigned int t = 0; t < nthreads; t++) {
        size_t cs = t * cells_per_chunk;
        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
        if (cs >= sorted_cells.size()) break;

        threads.emplace_back([&, t, cs, ce]() {
            auto& chunk = chunks[t];
            chunk.cell_start = cs;
            chunk.cell_end = ce;
            chunk.local_size = 0;

            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
                    return p.cell_id < id;
                }) - sorted_pairs.begin();

            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;

                offsets[si] = chunk.local_size;
                size_t start = pi;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(MAX_VERTEX_COUNT)));
                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
                size_t buf_pos = chunk.buf.size();
                chunk.buf.resize(buf_pos + entry_size);
                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
                for (size_t k = start; k < pi; k++) {
                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
                           &sorted_pairs[k].item_id, sizeof(uint32_t));
                }
                chunk.local_size += entry_size;
            }
        });
    }
    for (auto& t : threads) t.join();

    uint32_t global_offset = 0;
    for (auto& chunk : chunks) {
        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
            if (offsets[si] != NO_DATA) offsets[si] += global_offset;
        }
        global_offset += chunk.local_size;
    }

    std::vector<char> buf;
    buf.reserve(global_offset);
    for (auto& chunk : chunks) buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());

    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

void write_cell_index(
    const std::string& cells_path,
    const std::string& entries_path,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(cell_map.begin(), cell_map.end());
    std::sort(sorted.begin(), sorted.end());

    { std::ofstream f(cells_path, std::ios::binary);
      uint32_t current_offset = 0;
      for (const auto& [cell_id, ids] : sorted) {
          f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
          f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
          current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
      } }

    { std::ofstream f(entries_path, std::ios::binary);
      for (const auto& [cell_id, ids] : sorted) {
          uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
          f.write(reinterpret_cast<const char*>(&count), sizeof(count));
          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
      } }
}

void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
    ensure_dir(output_dir);
    auto _wt = std::chrono::steady_clock::now();

    bool write_streets = (mode != IndexMode::AdminOnly);
    bool write_addresses = (mode == IndexMode::Full);

    if (write_streets) {
        std::vector<uint64_t> sorted_geo_cells;
        {
            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
                std::vector<uint64_t> cells;
                cells.reserve(pairs.size() / 2);
                for (size_t i = 0; i < pairs.size(); ) {
                    cells.push_back(pairs[i].cell_id);
                    uint64_t cur = pairs[i].cell_id;
                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
                }
                return cells;
            };
            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
                std::vector<uint64_t> cells;
                cells.reserve(m.size());
                for (auto& [id, _] : m) cells.push_back(id);
                std::sort(cells.begin(), cells.end());
                return cells;
            };

            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
            {
                auto f1 = std::async(std::launch::async, [&] {
                    return !data.sorted_way_cells.empty()
                        ? extract_unique_cells(data.sorted_way_cells)
                        : extract_from_map(data.cell_to_ways);
                });
                if (write_addresses) {
                    auto f2 = std::async(std::launch::async, [&] {
                        return !data.sorted_addr_cells.empty()
                            ? extract_unique_cells(data.sorted_addr_cells)
                            : extract_from_map(data.cell_to_addrs);
                    });
                    auto f3 = std::async(std::launch::async, [&] { return extract_from_map(data.cell_to_interps); });
                    addr_cells = f2.get();
                    interp_cells = f3.get();
                }
                way_cells = f1.get();
            }

            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
            std::merge(way_cells.begin(), way_cells.end(), addr_cells.begin(), addr_cells.end(),
                       std::back_inserter(sorted_geo_cells));
            if (!interp_cells.empty()) {
                std::vector<uint64_t> tmp;
                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
                           interp_cells.begin(), interp_cells.end(), std::back_inserter(tmp));
                sorted_geo_cells = std::move(tmp);
            }
            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
                                    sorted_geo_cells.end());
        }

        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
        {
            auto f1 = std::async(std::launch::async, [&]() {
                if (!data.sorted_way_cells.empty())
                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
            });
            if (write_addresses) {
                auto f2 = std::async(std::launch::async, [&]() {
                    if (!data.sorted_addr_cells.empty())
                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
                });
                auto f3 = std::async(std::launch::async, [&]() {
                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
                });
                addr_offsets = f2.get();
                interp_offsets = f3.get();
            }
            street_offsets = f1.get();
        }

        log_phase("  Write: entry files", _wt);
        {
            size_t n = sorted_geo_cells.size();
            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
            std::vector<char> buf(n * row_size);
            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);

            unsigned int nthreads = std::thread::hardware_concurrency();
            if (nthreads == 0) nthreads = 4;
            size_t chunk = (n + nthreads - 1) / nthreads;
            std::vector<std::thread> fill_threads;
            for (unsigned int t = 0; t < nthreads; t++) {
                size_t start = t * chunk;
                size_t end = std::min(start + chunk, n);
                if (start >= n) break;
                fill_threads.emplace_back([&, start, end]() {
                    char* ptr = buf.data() + start * row_size;
                    for (size_t i = start; i < end; i++) {
                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                    }
                });
            }
            for (auto& t : fill_threads) t.join();

            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
            f.write(buf.data(), buf.size());
        }

        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
                  << data.ways.size() << " ways, " << data.addr_points.size() << " addrs, "
                  << data.interp_ways.size() << " interps)" << std::endl;
    }

    log_phase("  Write: geo_cells.bin", _wt);
    auto admin_future = std::async(std::launch::async, [&] {
        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
    });

    std::vector<std::future<void>> write_futures;
    if (write_streets) {
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
        }));
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
        }));
        if (write_addresses) {
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
            }));
        }
    }
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(data.admin_polygons.data()), data.admin_polygons.size() * sizeof(AdminPolygon));
    }));
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
    }));
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
        f.write(data.string_pool.data().data(), data.string_pool.data().size());
        std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
    }));

    log_phase("  Write: parallel data files launched", _wt);
    admin_future.get();
    for (auto& f : write_futures) f.get();
}


File created successfully at: /root/traccar-geocoder/builder/src/cell_index.cpp

#include "cell_index.h"

#include <algorithm>
#include <cstring>
#include <fstream>
#include <future>
#include <iostream>
#include <thread>

std::vector<uint32_t> write_entries(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    struct CellRef { uint64_t cell_id; const std::vector<uint32_t>* ids; };
    std::vector<CellRef> sorted_refs;
    sorted_refs.reserve(cell_map.size());
    for (auto& [id, ids] : cell_map) sorted_refs.push_back({id, &ids});
    std::sort(sorted_refs.begin(), sorted_refs.end(),
        [](const CellRef& a, const CellRef& b) { return a.cell_id < b.cell_id; });

    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    size_t total_size = 0;
    for (auto& r : sorted_refs) total_size += sizeof(uint16_t) + r.ids->size() * sizeof(uint32_t);

    std::vector<char> buf;
    buf.reserve(total_size);
    uint32_t current = 0;
    size_t ri = 0;
    for (uint32_t si = 0; si < sorted_cells.size() && ri < sorted_refs.size(); si++) {
        if (sorted_cells[si] < sorted_refs[ri].cell_id) continue;
        if (sorted_cells[si] > sorted_refs[ri].cell_id) { si--; ri++; continue; }
        offsets[si] = current;
        const auto& ids = *sorted_refs[ri].ids;
        uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
        buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
                   reinterpret_cast<const char*>(&count) + sizeof(count));
        buf.insert(buf.end(), reinterpret_cast<const char*>(ids.data()),
                   reinterpret_cast<const char*>(ids.data()) + ids.size() * sizeof(uint32_t));
        current += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
        ri++;
    }
    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

std::vector<uint32_t> write_entries_from_sorted(
    const std::string& path,
    const std::vector<uint64_t>& sorted_cells,
    const std::vector<CellItemPair>& sorted_pairs
) {
    std::vector<uint32_t> offsets(sorted_cells.size(), NO_DATA);
    if (sorted_pairs.empty()) {
        std::ofstream f(path, std::ios::binary);
        return offsets;
    }

    unsigned int nthreads = std::min(std::thread::hardware_concurrency(), 32u);
    if (nthreads == 0) nthreads = 4;
    size_t cells_per_chunk = (sorted_cells.size() + nthreads - 1) / nthreads;

    struct ChunkResult {
        std::vector<char> buf;
        size_t cell_start, cell_end;
        uint32_t local_size;
    };
    std::vector<ChunkResult> chunks(nthreads);

    std::vector<std::thread> threads;
    for (unsigned int t = 0; t < nthreads; t++) {
        size_t cs = t * cells_per_chunk;
        size_t ce = std::min(cs + cells_per_chunk, sorted_cells.size());
        if (cs >= sorted_cells.size()) break;

        threads.emplace_back([&, t, cs, ce]() {
            auto& chunk = chunks[t];
            chunk.cell_start = cs;
            chunk.cell_end = ce;
            chunk.local_size = 0;

            size_t pi = std::lower_bound(sorted_pairs.begin(), sorted_pairs.end(),
                sorted_cells[cs], [](const CellItemPair& p, uint64_t id) {
                    return p.cell_id < id;
                }) - sorted_pairs.begin();

            for (size_t si = cs; si < ce && pi < sorted_pairs.size(); si++) {
                if (sorted_cells[si] < sorted_pairs[pi].cell_id) continue;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id < sorted_cells[si]) pi++;
                if (pi >= sorted_pairs.size() || sorted_pairs[pi].cell_id != sorted_cells[si]) continue;

                offsets[si] = chunk.local_size;
                size_t start = pi;
                while (pi < sorted_pairs.size() && sorted_pairs[pi].cell_id == sorted_cells[si]) pi++;
                uint16_t count = static_cast<uint16_t>(std::min(pi - start, size_t(MAX_VERTEX_COUNT)));
                size_t entry_size = sizeof(uint16_t) + (pi - start) * sizeof(uint32_t);
                size_t buf_pos = chunk.buf.size();
                chunk.buf.resize(buf_pos + entry_size);
                memcpy(chunk.buf.data() + buf_pos, &count, sizeof(count));
                for (size_t k = start; k < pi; k++) {
                    memcpy(chunk.buf.data() + buf_pos + sizeof(uint16_t) + (k - start) * sizeof(uint32_t),
                           &sorted_pairs[k].item_id, sizeof(uint32_t));
                }
                chunk.local_size += entry_size;
            }
        });
    }
    for (auto& t : threads) t.join();

    uint32_t global_offset = 0;
    for (auto& chunk : chunks) {
        for (size_t si = chunk.cell_start; si < chunk.cell_end; si++) {
            if (offsets[si] != NO_DATA) offsets[si] += global_offset;
        }
        global_offset += chunk.local_size;
    }

    std::vector<char> buf;
    buf.reserve(global_offset);
    for (auto& chunk : chunks) buf.insert(buf.end(), chunk.buf.begin(), chunk.buf.end());

    std::ofstream f(path, std::ios::binary);
    f.write(buf.data(), buf.size());
    return offsets;
}

void write_cell_index(
    const std::string& cells_path,
    const std::string& entries_path,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map
) {
    std::vector<std::pair<uint64_t, std::vector<uint32_t>>> sorted(cell_map.begin(), cell_map.end());
    std::sort(sorted.begin(), sorted.end());

    { std::ofstream f(cells_path, std::ios::binary);
      uint32_t current_offset = 0;
      for (const auto& [cell_id, ids] : sorted) {
          f.write(reinterpret_cast<const char*>(&cell_id), sizeof(cell_id));
          f.write(reinterpret_cast<const char*>(&current_offset), sizeof(current_offset));
          current_offset += sizeof(uint16_t) + ids.size() * sizeof(uint32_t);
      } }

    { std::ofstream f(entries_path, std::ios::binary);
      for (const auto& [cell_id, ids] : sorted) {
          uint16_t count = static_cast<uint16_t>(std::min(ids.size(), size_t(MAX_VERTEX_COUNT)));
          f.write(reinterpret_cast<const char*>(&count), sizeof(count));
          f.write(reinterpret_cast<const char*>(ids.data()), ids.size() * sizeof(uint32_t));
      } }
}

void write_index(const ParsedData& data, const std::string& output_dir, IndexMode mode) {
    ensure_dir(output_dir);
    auto _wt = std::chrono::steady_clock::now();

    bool write_streets = (mode != IndexMode::AdminOnly);
    bool write_addresses = (mode == IndexMode::Full);

    if (write_streets) {
        std::vector<uint64_t> sorted_geo_cells;
        {
            auto extract_unique_cells = [](const std::vector<CellItemPair>& pairs) {
                std::vector<uint64_t> cells;
                cells.reserve(pairs.size() / 2);
                for (size_t i = 0; i < pairs.size(); ) {
                    cells.push_back(pairs[i].cell_id);
                    uint64_t cur = pairs[i].cell_id;
                    while (i < pairs.size() && pairs[i].cell_id == cur) i++;
                }
                return cells;
            };
            auto extract_from_map = [](const std::unordered_map<uint64_t, std::vector<uint32_t>>& m) {
                std::vector<uint64_t> cells;
                cells.reserve(m.size());
                for (auto& [id, _] : m) cells.push_back(id);
                std::sort(cells.begin(), cells.end());
                return cells;
            };

            std::vector<uint64_t> way_cells, addr_cells, interp_cells;
            {
                auto f1 = std::async(std::launch::async, [&] {
                    return !data.sorted_way_cells.empty()
                        ? extract_unique_cells(data.sorted_way_cells)
                        : extract_from_map(data.cell_to_ways);
                });
                if (write_addresses) {
                    auto f2 = std::async(std::launch::async, [&] {
                        return !data.sorted_addr_cells.empty()
                            ? extract_unique_cells(data.sorted_addr_cells)
                            : extract_from_map(data.cell_to_addrs);
                    });
                    auto f3 = std::async(std::launch::async, [&] { return extract_from_map(data.cell_to_interps); });
                    addr_cells = f2.get();
                    interp_cells = f3.get();
                }
                way_cells = f1.get();
            }

            sorted_geo_cells.reserve(way_cells.size() + addr_cells.size() + interp_cells.size());
            std::merge(way_cells.begin(), way_cells.end(), addr_cells.begin(), addr_cells.end(),
                       std::back_inserter(sorted_geo_cells));
            if (!interp_cells.empty()) {
                std::vector<uint64_t> tmp;
                tmp.reserve(sorted_geo_cells.size() + interp_cells.size());
                std::merge(sorted_geo_cells.begin(), sorted_geo_cells.end(),
                           interp_cells.begin(), interp_cells.end(), std::back_inserter(tmp));
                sorted_geo_cells = std::move(tmp);
            }
            sorted_geo_cells.erase(std::unique(sorted_geo_cells.begin(), sorted_geo_cells.end()),
                                    sorted_geo_cells.end());
        }

        std::vector<uint32_t> street_offsets, addr_offsets, interp_offsets;
        {
            auto f1 = std::async(std::launch::async, [&]() {
                if (!data.sorted_way_cells.empty())
                    return write_entries_from_sorted(output_dir + "/street_entries.bin", sorted_geo_cells, data.sorted_way_cells);
                return write_entries(output_dir + "/street_entries.bin", sorted_geo_cells, data.cell_to_ways);
            });
            if (write_addresses) {
                auto f2 = std::async(std::launch::async, [&]() {
                    if (!data.sorted_addr_cells.empty())
                        return write_entries_from_sorted(output_dir + "/addr_entries.bin", sorted_geo_cells, data.sorted_addr_cells);
                    return write_entries(output_dir + "/addr_entries.bin", sorted_geo_cells, data.cell_to_addrs);
                });
                auto f3 = std::async(std::launch::async, [&]() {
                    return write_entries(output_dir + "/interp_entries.bin", sorted_geo_cells, data.cell_to_interps);
                });
                addr_offsets = f2.get();
                interp_offsets = f3.get();
            }
            street_offsets = f1.get();
        }

        log_phase("  Write: entry files", _wt);
        {
            size_t n = sorted_geo_cells.size();
            size_t row_size = sizeof(uint64_t) + 3 * sizeof(uint32_t);
            std::vector<char> buf(n * row_size);
            if (addr_offsets.empty()) addr_offsets.resize(n, NO_DATA);
            if (interp_offsets.empty()) interp_offsets.resize(n, NO_DATA);

            unsigned int nthreads = std::thread::hardware_concurrency();
            if (nthreads == 0) nthreads = 4;
            size_t chunk = (n + nthreads - 1) / nthreads;
            std::vector<std::thread> fill_threads;
            for (unsigned int t = 0; t < nthreads; t++) {
                size_t start = t * chunk;
                size_t end = std::min(start + chunk, n);
                if (start >= n) break;
                fill_threads.emplace_back([&, start, end]() {
                    char* ptr = buf.data() + start * row_size;
                    for (size_t i = start; i < end; i++) {
                        memcpy(ptr, &sorted_geo_cells[i], sizeof(uint64_t)); ptr += sizeof(uint64_t);
                        memcpy(ptr, &street_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &addr_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                        memcpy(ptr, &interp_offsets[i], sizeof(uint32_t)); ptr += sizeof(uint32_t);
                    }
                });
            }
            for (auto& t : fill_threads) t.join();

            std::ofstream f(output_dir + "/geo_cells.bin", std::ios::binary);
            f.write(buf.data(), buf.size());
        }

        std::cerr << "geo index: " << sorted_geo_cells.size() << " cells ("
                  << data.ways.size() << " ways, " << data.addr_points.size() << " addrs, "
                  << data.interp_ways.size() << " interps)" << std::endl;
    }

    log_phase("  Write: geo_cells.bin", _wt);
    auto admin_future = std::async(std::launch::async, [&] {
        write_cell_index(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin", data.cell_to_admin);
        std::cerr << "admin index: " << data.cell_to_admin.size() << " cells, " << data.admin_polygons.size() << " polygons" << std::endl;
    });

    std::vector<std::future<void>> write_futures;
    if (write_streets) {
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_ways.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.ways.data()), data.ways.size() * sizeof(WayHeader));
        }));
        write_futures.push_back(std::async(std::launch::async, [&] {
            std::ofstream f(output_dir + "/street_nodes.bin", std::ios::binary);
            f.write(reinterpret_cast<const char*>(data.street_nodes.data()), data.street_nodes.size() * sizeof(NodeCoord));
        }));
        if (write_addresses) {
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/addr_points.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.addr_points.data()), data.addr_points.size() * sizeof(AddrPoint));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_ways.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_ways.data()), data.interp_ways.size() * sizeof(InterpWay));
            }));
            write_futures.push_back(std::async(std::launch::async, [&] {
                std::ofstream f(output_dir + "/interp_nodes.bin", std::ios::binary);
                f.write(reinterpret_cast<const char*>(data.interp_nodes.data()), data.interp_nodes.size() * sizeof(NodeCoord));
            }));
        }
    }
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_polygons.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(data.admin_polygons.data()), data.admin_polygons.size() * sizeof(AdminPolygon));
    }));
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/admin_vertices.bin", std::ios::binary);
        f.write(reinterpret_cast<const char*>(data.admin_vertices.data()), data.admin_vertices.size() * sizeof(NodeCoord));
    }));
    write_futures.push_back(std::async(std::launch::async, [&] {
        std::ofstream f(output_dir + "/strings.bin", std::ios::binary);
        f.write(data.string_pool.data().data(), data.string_pool.data().size());
        std::cerr << "strings.bin: " << data.string_pool.data().size() << " bytes" << std::endl;
    }));

    log_phase("  Write: parallel data files launched", _wt);
    admin_future.get();
    for (auto& f : write_futures) f.get();
}


Updated task #9 status


Updated task #10 status


Now the critical step — trim build_index.cpp to just main() and update includes. I need to rewrite the file to only include main() with the new module headers:


  1628→// --- Main ---
  1629→
  1630→int main(int argc, char* argv[]) {
  1631→    if (argc < 2) {
  1632→        std::cerr << "Usage: build-index <output-dir> <input.osm.pbf> [options]" << std::endl;

// --- Main ---

int main(int argc, char* argv[]) {
    if (argc < 2) {
        std::cerr << "Usage: build-index <output-dir> <input.osm.pbf> [options]" << std::endl;


Now I need to read the main function (lines 1630-2755) and create a new build_index.cpp with just the includes and main:


Error: File content (15181 tokens) exceeds maximum allowed tokens (10000). Use offset and limit parameters to read specific portions of the file, or search for specific content instead of reading the whole file.