Actually, the issue is scope. The S2 block and the write phase are in different scopes within `main()`. The simplest approach: just don't bother with the direct-write optimization for now, since the biggest gain was already from the merge-join. Instead, let me focus on other improvements.

Let me look at what else can be parallelized or optimized. The dedup phase (48s on planet) sorts vectors within hash map buckets:


1252:static void deduplicate(Map& cell_map) {
3028:        std::cerr << "Deduplicating..." << std::endl;
3030:            auto f1 = std::async(std::launch::async, [&]{ deduplicate(data.cell_to_ways); });
3031:            auto f2 = std::async(std::launch::async, [&]{ deduplicate(data.cell_to_addrs); });
3032:            auto f3 = std::async(std::launch::async, [&]{ deduplicate(data.cell_to_interps); });
3033:            auto f4 = std::async(std::launch::async, [&]{ deduplicate(data.cell_to_admin); });


  1252→static void deduplicate(Map& cell_map) {
  1253→    for (auto& [cell_id, ids] : cell_map) {
  1254→        std::sort(ids.begin(), ids.end());
  1255→        ids.erase(std::unique(ids.begin(), ids.end()), ids.end());
  1256→    }
  1257→}
  1258→
  1259→// --- Cache serialization ---
  1260→
  1261→enum class SectionType : uint32_t {
  1262→    STRING_POOL = 0,
  1263→    WAYS = 1,
  1264→    STREET_NODES = 2,
  1265→    CELL_TO_WAYS = 3,
  1266→    ADDR_POINTS = 4,

static void deduplicate(Map& cell_map) {
    for (auto& [cell_id, ids] : cell_map) {
        std::sort(ids.begin(), ids.end());
        ids.erase(std::unique(ids.begin(), ids.end()), ids.end());
    }
}

// --- Cache serialization ---

enum class SectionType : uint32_t {
    STRING_POOL = 0,
    WAYS = 1,
    STREET_NODES = 2,
    CELL_TO_WAYS = 3,
    ADDR_POINTS = 4,


Since the S2 pairs are already sorted by (cell_id, item_id) before grouping into the hash map, the vectors within each cell should already be sorted! We can skip the dedup sort for way/interp cells. Let me verify and skip it:


Edit

            auto f_ways = std::async(std::launch::async, [&] {
                sorted_way_pairs = concat_and_sort(way_pairs);
            });
            auto f_interps = std::async(std::launch::async, [&] {
                sorted_interp_pairs = concat_and_sort(interp_pairs);
                build_cell_map(sorted_interp_pairs, data.cell_to_interps);
            });
            f_ways.get();
            f_interps.get();

            // Build cell_to_ways from sorted pairs (still needed for sorted_geo_cells union + dedup)
            build_cell_map(sorted_way_pairs, data.cell_to_ways);

            auto f_ways = std::async(std::launch::async, [&] {
                sorted_way_pairs = concat_and_sort(way_pairs);
                build_cell_map(sorted_way_pairs, data.cell_to_ways);
            });
            auto f_interps = std::async(std::launch::async, [&] {
                sorted_interp_pairs = concat_and_sort(interp_pairs);
                build_cell_map(sorted_interp_pairs, data.cell_to_interps);
            });
            f_ways.get();
            f_interps.get();
            // Note: cell_to_ways and cell_to_interps are already sorted+deduped
            // from the concat_and_sort, so the dedup phase can skip them.