  1643→        std::cerr << "  --mode <mode>          Index mode: full, no-addresses, admin-only (default: full)" << std::endl;
  1644→        std::cerr << "  --admin-only           Shorthand for --mode admin-only" << std::endl;
  1645→        std::cerr << "  --no-addresses         Shorthand for --mode no-addresses" << std::endl;
  1646→        return 1;
  1647→    }
  1648→
  1649→    std::string output_dir = argv[1];
  1650→    std::vector<std::string> input_files;
  1651→    std::string save_cache_path;
  1652→    std::string load_cache_path;
  1653→    bool multi_output = false;
  1654→    bool generate_continents = false;
  1655→    IndexMode mode = IndexMode::Full;
  1656→
  1657→    for (int i = 2; i < argc; i++) {
  1658→        std::string arg = argv[i];
  1659→        if (arg == "--street-level" && i + 1 < argc) {
  1660→            kStreetCellLevel = std::atoi(argv[++i]);
  1661→        } else if (arg == "--admin-level" && i + 1 < argc) {
  1662→            kAdminCellLevel = std::atoi(argv[++i]);
  1663→        } else if (arg == "--admin-only") {
  1664→            mode = IndexMode::AdminOnly;
  1665→        } else if (arg == "--no-addresses") {
  1666→            mode = IndexMode::NoAddresses;
  1667→        } else if (arg == "--max-admin-level" && i + 1 < argc) {
  1668→            kMaxAdminLevel = std::atoi(argv[++i]);
  1669→        } else if (arg == "--save-cache" && i + 1 < argc) {
  1670→            save_cache_path = argv[++i];
  1671→        } else if (arg == "--load-cache" && i + 1 < argc) {
  1672→            load_cache_path = argv[++i];
  1673→        } else if (arg == "--multi-output") {
  1674→            multi_output = true;
  1675→        } else if (arg == "--continents") {
  1676→            generate_continents = true;
  1677→        } else if (arg == "--mode" && i + 1 < argc) {
  1678→            std::string mode_str = argv[++i];
  1679→            if (mode_str == "full") {
  1680→                mode = IndexMode::Full;
  1681→            } else if (mode_str == "no-addresses") {
  1682→                mode = IndexMode::NoAddresses;
  1683→            } else if (mode_str == "admin-only") {
  1684→                mode = IndexMode::AdminOnly;
  1685→            } else {
  1686→                std::cerr << "Error: unknown mode '" << mode_str << "'" << std::endl;
  1687→                return 1;
  1688→            }
  1689→        } else {
  1690→            input_files.push_back(arg);
  1691→        }
  1692→    }
  1693→
  1694→    ParsedData data;
  1695→    auto _pt = std::chrono::steady_clock::now();
  1696→
  1697→    if (!load_cache_path.empty()) {
  1698→        // Load from cache
  1699→        if (!deserialize_cache(data, load_cache_path)) {
  1700→            std::cerr << "Error: failed to load cache" << std::endl;
  1701→            return 1;
  1702→        }
  1703→    } else {
  1704→        // Parse PBF files (always collect everything)
  1705→        if (input_files.empty()) {
  1706→            std::cerr << "Error: no input files specified and no --load-cache" << std::endl;
  1707→            return 1;
  1708→        }
  1709→
  1710→        // Create thread pool for concurrent admin polygon S2 covering
  1711→        unsigned int num_threads = std::max(1u, std::thread::hardware_concurrency() > 4 ? std::thread::hardware_concurrency() - 4 : 1u);
  1712→        std::cerr << "Using " << num_threads << " worker threads." << std::endl;
  1713→        AdminCoverPool admin_pool(num_threads);
  1714→        // BuildHandler no longer used — parallel processing handles everything
  1715→
  1716→        // Dense array node location index — lockless parallel writes
  1717→        // Planet OSM node IDs max ~12.5 billion. 8 bytes per Location = 100GB virtual.
  1718→        // MAP_NORESERVE means OS only allocates pages on write (~80GB for 10B nodes).
  1719→        static const size_t MAX_NODE_ID = MAX_NODE_ID_DEFAULT;
  1720→        struct DenseIndex {
  1721→            osmium::Location* data;
  1722→            size_t capacity;
  1723→
  1724→            DenseIndex() : capacity(MAX_NODE_ID) {
  1725→                void* ptr = mmap(nullptr, capacity * sizeof(osmium::Location),
  1726→                    PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE,
  1727→                    -1, 0);
  1728→                if (ptr == MAP_FAILED) {
  1729→                    std::cerr << "Error: failed to mmap dense node index ("
  1730→                              << (capacity * sizeof(osmium::Location) / (1024*1024*1024)) << "GB virtual)" << std::endl;
  1731→                    std::exit(1);
  1732→                }
  1733→                data = static_cast<osmium::Location*>(ptr);
  1734→                std::cerr << "Allocated dense node index: " << (capacity * sizeof(osmium::Location) / (1024*1024*1024))
  1735→                          << "GB virtual address space" << std::endl;
  1736→            }
  1737→
  1738→            ~DenseIndex() {
  1739→                munmap(data, capacity * sizeof(osmium::Location));
  1740→            }
  1741→
  1742→            // Lockless — each node ID maps to a unique array slot
  1743→            void set(osmium::unsigned_object_id_type id, osmium::Location loc) {
  1744→                if (id >= capacity) {
  1745→                    std::cerr << "FATAL: node ID " << id << " exceeds dense index capacity " << capacity << std::endl;
  1746→                    std::exit(1);
  1747→                }
  1748→                data[id] = loc;
  1749→            }
  1750→
  1751→            osmium::Location get(osmium::unsigned_object_id_type id) const {
  1752→                if (id >= capacity) return osmium::Location{};
  1753→                return data[id];
  1754→            }
  1755→
  1756→            void set_batch(const std::vector<std::pair<osmium::unsigned_object_id_type, osmium::Location>>& batch) {
  1757→                for (const auto& [id, loc] : batch) {
  1758→                    if (id >= capacity) {
  1759→                        std::cerr << "FATAL: node ID " << id << " exceeds dense index capacity " << capacity << std::endl;
  1760→                        std::exit(1);
  1761→                    }
  1762→                    data[id] = loc;
  1763→                }
  1764→            }
  1765→        } index;
  1766→
  1767→        for (const auto& input_file : input_files) {
  1768→            std::cerr << "Processing " << input_file << "..." << std::endl;
  1769→
  1770→            // --- Pass 1: collect relation members for parallel admin assembly ---
  1771→            std::cerr << "  Pass 1: scanning relations..." << std::endl;
  1772→            RelationCollector rel_collector(data.collected_relations);
  1773→
  1774→            {
  1775→                osmium::io::Reader reader1{input_file, osmium::osm_entity_bits::relation};
  1776→                osmium::apply(reader1, rel_collector);
  1777→                reader1.close();
  1778→            }
  1779→            std::cerr << "  Collected " << data.collected_relations.size()
  1780→                      << " admin/postal relations for parallel assembly." << std::endl;
  1781→
  1782→            // --- Combined Pass 2+3: nodes then ways in a single PBF read ---
  1783→            // PBF guarantees ordering: nodes → ways → relations.
  1784→            // We read everything in one pass, processing nodes first, then
  1785→            // transitioning to way processing when the first way block arrives.
  1786→            log_phase("Pass 1: relation scanning", _pt);
  1787→
  1788→            // Build admin_way_ids BEFORE the combined pass (needed during way processing)
  1789→            std::unordered_set<int64_t> admin_way_ids;
  1790→            for (const auto& rel : data.collected_relations) {
  1791→                for (const auto& [way_id, role] : rel.members) {
  1792→                    admin_way_ids.insert(way_id);
  1793→                }
  1794→            }
  1795→            std::cerr << "  Admin assembly needs " << admin_way_ids.size() << " way geometries." << std::endl;
  1796→
  1797→            std::cerr << "  Pass 2+3: processing nodes and ways in single read..." << std::endl;
  1798→
  1799→            {
  1800→                // Shared queue infrastructure for both phases
  1801→                std::mutex queue_mutex;
  1802→                std::condition_variable queue_cv;
  1803→                std::deque<osmium::memory::Buffer> block_queue;
  1804→                bool reader_done = false;
  1805→                const size_t MAX_QUEUE = MAX_BLOCK_QUEUE;
  1806→
  1807→                struct NodeThreadLocal {
  1808→                    std::vector<std::pair<double,double>> addr_coords;
  1809→                    std::vector<std::pair<std::string,std::string>> addr_strings;
  1810→                    uint64_t count = 0;
  1811→                };
  1812→                std::vector<NodeThreadLocal> ntld(num_threads);
  1813→                std::atomic<uint64_t> blocks_done{0};
  1814→
  1815→                // Worker threads
  1816→                std::vector<std::thread> node_workers;
  1817→                for (unsigned int t = 0; t < num_threads; t++) {
  1818→                    node_workers.emplace_back([&, t]() {
  1819→                        auto& local = ntld[t];
  1820→                        std::vector<std::pair<osmium::unsigned_object_id_type, osmium::Location>> loc_batch;
  1821→                        loc_batch.reserve(8192);
  1822→
  1823→                        while (true) {
  1824→                            osmium::memory::Buffer block;
  1825→                            {
  1826→                                std::unique_lock<std::mutex> lock(queue_mutex);
  1827→                                queue_cv.wait(lock, [&]{ return !block_queue.empty() || reader_done; });
  1828→                                if (block_queue.empty() && reader_done) break;
  1829→                                block = std::move(block_queue.front());
  1830→                                block_queue.pop_front();
  1831→                            }
  1832→                            queue_cv.notify_one(); // wake reader if it was waiting
  1833→
  1834→                            for (const auto& item : block) {
  1835→                                if (item.type() == osmium::item_type::node) {
  1836→                                    const auto& node = static_cast<const osmium::Node&>(item);
  1837→                                    if (!node.location().valid()) continue;
  1838→                                    loc_batch.push_back({node.positive_id(), node.location()});
  1839→
  1840→                                    const char* housenumber = node.tags()["addr:housenumber"];
  1841→                                    if (housenumber) {
  1842→                                        const char* street = node.tags()["addr:street"];


  1843→                                        if (street) {
  1844→                                            local.addr_coords.push_back({node.location().lat(), node.location().lon()});
  1845→                                            local.addr_strings.push_back({housenumber, street});
  1846→                                            local.count++;
  1847→                                        }
  1848→                                    }
  1849→                                }
  1850→                            }
  1851→
  1852→                            // Flush location batch (sharded index handles its own locking)
  1853→                            if (!loc_batch.empty()) {
  1854→                                index.set_batch(loc_batch);
  1855→                                loc_batch.clear();
  1856→                            }
  1857→
  1858→                            uint64_t n = blocks_done.fetch_add(1) + 1;
  1859→                            if (n % 1000 == 0) {
  1860→                                std::cerr << "  Processed " << n << " node blocks..." << std::endl;
  1861→                            }
  1862→                        }
  1863→
  1864→                        // Flush remaining
  1865→                        if (!loc_batch.empty()) {
  1866→                            index.set_batch(loc_batch);
  1867→                        }
  1868→                    });
  1869→                }
  1870→
  1871→                // Single reader for all entity types — processes nodes first,
  1872→                // then transitions to way processing when first way block arrives.
  1873→                osmium::io::Reader combined_reader{input_file};
  1874→                bool nodes_done = false;
  1875→                osmium::memory::Buffer first_way_buf; // stash the first way block
  1876→
  1877→                // Phase 1: Feed node blocks to node workers
  1878→                while (auto buf = combined_reader.read()) {
  1879→                    // Check if this buffer contains nodes or ways/relations
  1880→                    bool has_node = false;
  1881→                    for (const auto& item : buf) {
  1882→                        if (item.type() == osmium::item_type::node) { has_node = true; break; }
  1883→                    }
  1884→
  1885→                    if (has_node) {
  1886→                        std::unique_lock<std::mutex> lock(queue_mutex);
  1887→                        queue_cv.wait(lock, [&]{ return block_queue.size() < MAX_QUEUE; });
  1888→                        block_queue.push_back(std::move(buf));
  1889→                        queue_cv.notify_one();
  1890→                    } else {
  1891→                        // First non-node block — stash it and transition
  1892→                        first_way_buf = std::move(buf);
  1893→                        nodes_done = true;
  1894→                        break;
  1895→                    }
  1896→                }
  1897→
  1898→                // Signal node workers done
  1899→                {
  1900→                    std::lock_guard<std::mutex> lock(queue_mutex);
  1901→                    reader_done = true;
  1902→                }
  1903→                queue_cv.notify_all();
  1904→                for (auto& w : node_workers) w.join();
  1905→
  1906→                // Merge address points
  1907→                uint64_t total_addrs = 0;
  1908→                for (auto& local : ntld) {
  1909→                    for (size_t j = 0; j < local.addr_coords.size(); j++) {
  1910→                        uint64_t dummy = 0;
  1911→                        add_addr_point(data, local.addr_coords[j].first, local.addr_coords[j].second,
  1912→                                       local.addr_strings[j].first.c_str(),
  1913→                                       local.addr_strings[j].second.c_str(), dummy);
  1914→                    }
  1915→                    total_addrs += local.count;
  1916→                }
  1917→                std::cerr << "  Node processing complete: " << total_addrs
  1918→                          << " address points collected." << std::endl;
  1919→                log_phase("Pass 2: node processing", _pt);
  1920→                std::cerr << "  Processing ways with " << num_threads << " threads..." << std::endl;
  1921→
  1922→                // --- Phase 2: Way processing using same reader ---
  1923→
  1924→                // Thread-local data for parallel way processing
  1925→                struct ThreadLocalData {
  1926→                    std::vector<WayHeader> ways;
  1927→                    std::vector<NodeCoord> street_nodes;
  1928→                    std::vector<DeferredWay> deferred_ways;
  1929→                    std::vector<InterpWay> interp_ways;
  1930→                    std::vector<NodeCoord> interp_nodes;
  1931→                    std::vector<DeferredInterp> deferred_interps;
  1932→                    std::vector<AddrPoint> building_addrs;
  1933→                    std::vector<std::pair<double, double>> building_addr_coords; // lat,lng for S2 cell
  1934→                    std::vector<std::string> way_strings;      // way name strings
  1935→                    std::vector<std::pair<std::string,std::string>> addr_strings; // building addr {hn, street}
  1936→                    std::vector<std::string> interp_strings;   // interp street name strings
  1937→                    uint64_t way_count = 0;
  1938→                    uint64_t building_addr_count = 0;
  1939→                    uint64_t interp_count = 0;
  1940→                    // Way geometries for parallel admin assembly
  1941→                    struct WayGeomEntry {
  1942→                        int64_t way_id;
  1943→                        std::vector<std::pair<double,double>> coords;
  1944→                        int64_t first_node_id;
  1945→                        int64_t last_node_id;
  1946→                    };
  1947→                    std::vector<WayGeomEntry> way_geoms;
  1948→                    // Closed-way admin polygons (ways with boundary=administrative)
  1949→                    struct ClosedWayAdmin {
  1950→                        std::vector<std::pair<double,double>> vertices;
  1951→                        std::string name;
  1952→                        uint8_t admin_level;
  1953→                        std::string country_code;
  1954→                    };
  1955→                    std::vector<ClosedWayAdmin> closed_way_admins;
  1956→                };
  1957→
  1958→                // Read buffers and dispatch to thread pool
  1959→                // Streaming producer-consumer for way blocks
  1960→                std::mutex way_queue_mutex;
  1961→                std::condition_variable way_queue_cv;
  1962→                std::deque<osmium::memory::Buffer> way_block_queue;
  1963→                bool way_reader_done = false;
  1964→                const size_t WAY_MAX_QUEUE = MAX_BLOCK_QUEUE;
  1965→
  1966→                std::vector<ThreadLocalData> tld(num_threads);
  1967→                std::atomic<uint64_t> way_blocks_done{0};
  1968→
  1969→                std::vector<std::thread> workers;
  1970→                for (unsigned int t = 0; t < num_threads; t++) {
  1971→                    workers.emplace_back([&, t]() {
  1972→                        auto& local = tld[t];
  1973→                        while (true) {
  1974→                            osmium::memory::Buffer block;
  1975→                            {
  1976→                                std::unique_lock<std::mutex> lock(way_queue_mutex);
  1977→                                way_queue_cv.wait(lock, [&]{ return !way_block_queue.empty() || way_reader_done; });
  1978→                                if (way_block_queue.empty() && way_reader_done) break;
  1979→                                block = std::move(way_block_queue.front());
  1980→                                way_block_queue.pop_front();
  1981→                            }
  1982→                            way_queue_cv.notify_one();
  1983→
  1984→                            for (const auto& item : block) {
  1985→                                if (item.type() == osmium::item_type::way) {
  1986→                                    const auto& way = static_cast<const osmium::Way&>(item);
  1987→
  1988→                                    // Resolve node locations (read-only from shared index)
  1989→                                    // We need to check if nodes have valid locations
  1990→                                    const auto& wnodes = way.nodes();
  1991→
  1992→                                    // Address interpolation
  1993→                                    const char* interpolation = way.tags()["addr:interpolation"];
  1994→                                    if (interpolation) {
  1995→                                        if (wnodes.size() >= 2) {
  1996→                                            bool all_valid = true;
  1997→                                            for (const auto& nr : wnodes) {
  1998→                                                auto loc = index.get(nr.positive_ref());
  1999→                                                if (!loc.valid()) { all_valid = false; break; }
  2000→                                            }
  2001→                                            if (all_valid) {
  2002→                                                const char* street = way.tags()["addr:street"];
  2003→                                                if (street) {
  2004→                                                    uint32_t interp_id = static_cast<uint32_t>(local.interp_ways.size());
  2005→                                                    uint32_t node_offset = static_cast<uint32_t>(local.interp_nodes.size());
  2006→
  2007→                                                    for (const auto& nr : wnodes) {
  2008→                                                        auto loc = index.get(nr.positive_ref());
  2009→                                                        local.interp_nodes.push_back({
  2010→                                                            static_cast<float>(loc.lat()),
  2011→                                                            static_cast<float>(loc.lon())
  2012→                                                        });
  2013→                                                    }
  2014→
  2015→                                                    uint8_t interp_type = 0;
  2016→                                                    if (std::strcmp(interpolation, "even") == 0) interp_type = 1;
  2017→                                                    else if (std::strcmp(interpolation, "odd") == 0) interp_type = 2;
  2018→
  2019→                                                    InterpWay iw{};
  2020→                                                    iw.node_offset = node_offset;
  2021→                                                    iw.node_count = static_cast<uint8_t>(std::min(wnodes.size(), size_t(255)));
  2022→                                                    iw.street_id = 0; // placeholder, intern later
  2023→                                                    iw.start_number = 0;
  2024→                                                    iw.end_number = 0;
  2025→                                                    iw.interpolation = interp_type;
  2026→                                                    local.interp_ways.push_back(iw);
  2027→                                                    local.interp_strings.push_back(street);
  2028→                                                    local.deferred_interps.push_back({interp_id, node_offset, iw.node_count});
  2029→                                                    local.interp_count++;
  2030→                                                }
  2031→                                            }
  2032→                                        }
  2033→                                        continue;
  2034→                                    }
  2035→
  2036→                                    // Building addresses
  2037→                                    const char* housenumber = way.tags()["addr:housenumber"];
  2038→                                    if (housenumber) {
  2039→                                        const char* street = way.tags()["addr:street"];
  2040→                                        if (street && !wnodes.empty()) {
  2041→                                            double sum_lat = 0, sum_lng = 0;
  2042→                                            int valid = 0;
  2043→                                            for (const auto& nr : wnodes) {
  2044→                                                auto loc = index.get(nr.positive_ref());
  2045→                                                if (loc.valid()) {
  2046→                                                    sum_lat += loc.lat();
  2047→                                                    sum_lng += loc.lon();
  2048→                                                    valid++;
  2049→                                                }
  2050→                                            }
  2051→                                            if (valid > 0) {
  2052→                                                // Store coords for later S2 cell computation
  2053→                                                local.building_addr_coords.push_back({sum_lat / valid, sum_lng / valid});
  2054→                                                // Placeholder addr point — string IDs filled in during merge
  2055→                                                local.building_addrs.push_back({
  2056→                                                    static_cast<float>(sum_lat / valid),
  2057→                                                    static_cast<float>(sum_lng / valid),
  2058→                                                    0, 0 // placeholder string IDs
  2059→                                                });
  2060→                                                // Store strings for later interning
  2061→                                                local.addr_strings.push_back({housenumber, street});
  2062→                                                local.building_addr_count++;
  2063→                                            }
  2064→                                        }
  2065→                                    }
  2066→
  2067→                                    // Highway ways
  2068→                                    const char* highway = way.tags()["highway"];
  2069→                                    if (highway && is_included_highway(highway)) {
  2070→                                        const char* name = way.tags()["name"];
  2071→                                        if (name && wnodes.size() >= 2) {
  2072→                                            bool all_valid = true;
  2073→                                            for (const auto& nr : wnodes) {
  2074→                                                auto loc = index.get(nr.positive_ref());
  2075→                                                if (!loc.valid()) { all_valid = false; break; }
  2076→                                            }
  2077→                                            if (all_valid) {
  2078→                                                uint32_t way_id = static_cast<uint32_t>(local.ways.size());
  2079→                                                uint32_t node_offset = static_cast<uint32_t>(local.street_nodes.size());
  2080→
  2081→                                                for (const auto& nr : wnodes) {
  2082→                                                    auto loc = index.get(nr.positive_ref());
  2083→                                                    local.street_nodes.push_back({
  2084→                                                        static_cast<float>(loc.lat()),
  2085→                                                        static_cast<float>(loc.lon())
  2086→                                                    });
  2087→                                                }
  2088→
  2089→                                                WayHeader header{};
  2090→                                                header.node_offset = node_offset;
  2091→                                                header.node_count = static_cast<uint8_t>(std::min(wnodes.size(), size_t(255)));
  2092→                                                header.name_id = 0; // placeholder, intern later
  2093→                                                local.ways.push_back(header);
  2094→                                                local.way_strings.push_back(name);
  2095→                                                local.deferred_ways.push_back({way_id, node_offset, header.node_count});
  2096→                                                local.way_count++;
  2097→                                            }
  2098→                                        }
  2099→                                    }
  2100→
  2101→                                    // Store way geometry only for admin boundary member ways
  2102→                                    if (!wnodes.empty() && admin_way_ids.count(way.id())) {
  2103→                                        std::vector<std::pair<double,double>> geom;
  2104→                                        for (const auto& nr : wnodes) {
  2105→                                            auto loc = index.get(nr.positive_ref());
  2106→                                            if (loc.valid()) geom.push_back({loc.lat(), loc.lon()});
  2107→                                        }
  2108→                                        if (!geom.empty()) {
  2109→                                            int64_t first_nid = wnodes.front().positive_ref();
  2110→                                            int64_t last_nid = wnodes.back().positive_ref();
  2111→                                            local.way_geoms.push_back({way.id(), std::move(geom), first_nid, last_nid});
  2112→                                        }
  2113→                                    }
  2114→
  2115→                                    // Handle closed ways that are admin boundaries themselves
  2116→                                    // Osmium creates areas from closed ways independently of
  2117→                                    // multipolygon relations, even if the way is a relation member.
  2118→                                    {
  2119→                                        const char* boundary = way.tags()["boundary"];
  2120→                                        if (boundary) {
  2121→                                            bool is_admin = (std::strcmp(boundary, "administrative") == 0);
  2122→                                            bool is_postal = (std::strcmp(boundary, "postal_code") == 0);
  2123→                                            if ((is_admin || is_postal) && wnodes.size() >= 4 &&
  2124→                                                wnodes.front().ref() == wnodes.back().ref()) {
  2125→                                                // Closed way forming an admin polygon
  2126→                                                uint8_t al = 0;
  2127→                                                if (is_admin) {
  2128→                                                    const char* level_str = way.tags()["admin_level"];
  2129→                                                    if (level_str) al = static_cast<uint8_t>(std::atoi(level_str));
  2130→                                                } else {
  2131→                                                    al = 11;
  2132→                                                }
  2133→                                                int max_al = is_postal ? 11 : 10;
  2134→                                                if (al >= 2 && al <= max_al && (kMaxAdminLevel == 0 || al <= kMaxAdminLevel)) {
  2135→                                                    const char* aname = way.tags()["name"];
  2136→                                                    if (!aname && is_admin) continue; // admin needs name
  2137→                                                    std::string name_str;
  2138→                                                    if (is_postal) {
  2139→                                                        const char* pc = way.tags()["postal_code"];
  2140→                                                        if (!pc) pc = aname;
  2141→                                                        if (!pc) continue; // postal needs postal_code or name
  2142→                                                        name_str = pc;


  2143→                                                    } else {
  2144→                                                        name_str = aname;
  2145→                                                    }
  2146→                                                    {
  2147→                                                        std::vector<std::pair<double,double>> verts;
  2148→                                                        bool all_valid = true;
  2149→                                                        for (const auto& nr : wnodes) {
  2150→                                                            auto loc = index.get(nr.positive_ref());
  2151→                                                            if (loc.valid()) {
  2152→                                                                verts.push_back({loc.lat(), loc.lon()});
  2153→                                                            } else { all_valid = false; break; }
  2154→                                                        }
  2155→                                                        if (all_valid && verts.size() >= 3) {
  2156→                                                            std::string cc;
  2157→                                                            if (al == 2) {
  2158→                                                                const char* iso = way.tags()["ISO3166-1:alpha2"];
  2159→                                                                if (iso) cc = iso;
  2160→                                                            }
  2161→                                                            local.closed_way_admins.push_back({
  2162→                                                                std::move(verts),
  2163→                                                                std::move(name_str),
  2164→                                                                al, std::move(cc)
  2165→                                                            });
  2166→                                                        }
  2167→                                                    }
  2168→                                                }
  2169→                                            }
  2170→                                        }
  2171→                                    }
  2172→                                }
  2173→                            }
  2174→
  2175→                            uint64_t done = way_blocks_done.fetch_add(1) + 1;
  2176→                            if (done % 1000 == 0) {
  2177→                                std::cerr << "  Processed " << done << " way blocks..." << std::endl;
  2178→                            }
  2179→                        }
  2180→                    });
  2181→                }
  2182→
  2183→                // Way reader: use the combined_reader (continuing from where nodes left off)
  2184→                {
  2185→                    // First, push the stashed way buffer
  2186→                    if (first_way_buf) {
  2187→                        std::unique_lock<std::mutex> lock(way_queue_mutex);
  2188→                        way_queue_cv.wait(lock, [&]{ return way_block_queue.size() < WAY_MAX_QUEUE; });
  2189→                        way_block_queue.push_back(std::move(first_way_buf));
  2190→                        way_queue_cv.notify_one();
  2191→                    }
  2192→                    // Continue reading remaining blocks from the same reader
  2193→                    while (auto buf = combined_reader.read()) {
  2194→                        {
  2195→                            std::unique_lock<std::mutex> lock(way_queue_mutex);
  2196→                            way_queue_cv.wait(lock, [&]{ return way_block_queue.size() < WAY_MAX_QUEUE; });
  2197→                            way_block_queue.push_back(std::move(buf));
  2198→                        }
  2199→                        way_queue_cv.notify_one();
  2200→                    }
  2201→                    combined_reader.close();
  2202→                    {
  2203→                        std::lock_guard<std::mutex> lock(way_queue_mutex);
  2204→                        way_reader_done = true;
  2205→                    }
  2206→                    way_queue_cv.notify_all();
  2207→                }
  2208→
  2209→                for (auto& w : workers) w.join();
  2210→                std::cerr << "  Parallel way processing complete." << std::endl;
  2211→
  2212→                // Process areas/multipolygons — sequential fallback path
  2213→                // Merge thread-local way/interp data into main ParsedData
  2214→                std::cerr << "  Merging thread-local data..." << std::endl;
  2215→                uint64_t total_ways = 0, total_building_addrs = 0, total_interps = 0;
  2216→                for (auto& local : tld) {
  2217→                    uint32_t way_base = static_cast<uint32_t>(data.ways.size());
  2218→                    uint32_t node_base = static_cast<uint32_t>(data.street_nodes.size());
  2219→                    uint32_t interp_base = static_cast<uint32_t>(data.interp_ways.size());
  2220→                    uint32_t interp_node_base = static_cast<uint32_t>(data.interp_nodes.size());
  2221→
  2222→                    // Merge street ways
  2223→                    for (size_t i = 0; i < local.ways.size(); i++) {
  2224→                        auto h = local.ways[i];
  2225→                        h.node_offset += node_base;
  2226→                        h.name_id = data.string_pool.intern(local.way_strings[i]);
  2227→                        data.ways.push_back(h);
  2228→                    }
  2229→                    data.street_nodes.insert(data.street_nodes.end(),
  2230→                        local.street_nodes.begin(), local.street_nodes.end());
  2231→
  2232→                    // Remap deferred ways
  2233→                    for (auto dw : local.deferred_ways) {
  2234→                        dw.way_id += way_base;
  2235→                        dw.node_offset += node_base;
  2236→                        data.deferred_ways.push_back(dw);
  2237→                    }
  2238→
  2239→                    // Merge building addresses
  2240→                    for (size_t i = 0; i < local.building_addrs.size(); i++) {
  2241→                        uint64_t dummy = 0;
  2242→                        add_addr_point(data, local.building_addrs[i].lat, local.building_addrs[i].lng,
  2243→                                       local.addr_strings[i].first.c_str(),
  2244→                                       local.addr_strings[i].second.c_str(), dummy);
  2245→                    }
  2246→
  2247→                    // Merge interpolation ways
  2248→                    for (size_t i = 0; i < local.interp_ways.size(); i++) {
  2249→                        auto iw = local.interp_ways[i];
  2250→                        iw.node_offset += interp_node_base;
  2251→                        iw.street_id = data.string_pool.intern(local.interp_strings[i]);
  2252→                        data.interp_ways.push_back(iw);
  2253→                    }
  2254→                    data.interp_nodes.insert(data.interp_nodes.end(),
  2255→                        local.interp_nodes.begin(), local.interp_nodes.end());
  2256→
  2257→                    // Remap deferred interps
  2258→                    for (auto di : local.deferred_interps) {
  2259→                        di.interp_id += interp_base;
  2260→                        di.node_offset += interp_node_base;
  2261→                        data.deferred_interps.push_back(di);
  2262→                    }
  2263→
  2264→                    total_ways += local.way_count;
  2265→                    total_building_addrs += local.building_addr_count;
  2266→                    total_interps += local.interp_count;
  2267→
  2268→                    // Merge way geometries for admin assembly
  2269→                    {
  2270→                        for (auto& wg : local.way_geoms) {
  2271→                            ParsedData::WayGeometry g;
  2272→                            g.coords = std::move(wg.coords);
  2273→                            g.first_node_id = wg.first_node_id;
  2274→                            g.last_node_id = wg.last_node_id;
  2275→                            data.way_geometries[wg.way_id] = std::move(g);
  2276→                        }
  2277→                        local.way_geoms.clear();
  2278→                        local.way_geoms.shrink_to_fit();
  2279→                    }
  2280→                }
  2281→                std::cerr << "  Merged: " << total_ways << " ways, "
  2282→                          << total_building_addrs << " building addrs, "
  2283→                          << total_interps << " interps from parallel processing." << std::endl;
  2284→
  2285→                // --- Merge closed-way admin polygons ---
  2286→                {
  2287→                    uint64_t closed_way_admin_count = 0;
  2288→                    for (auto& local : tld) {
  2289→                        for (auto& cwa : local.closed_way_admins) {
  2290→                            const char* cc = cwa.country_code.empty() ? nullptr : cwa.country_code.c_str();
  2291→                            add_admin_polygon(data, cwa.vertices, cwa.name.c_str(),
  2292→                                              cwa.admin_level, cc, &admin_pool);
  2293→                            closed_way_admin_count++;
  2294→                        }
  2295→                        local.closed_way_admins.clear();
  2296→                    }
  2297→                    if (closed_way_admin_count > 0) {
  2298→                        std::cerr << "  Added " << closed_way_admin_count
  2299→                                  << " admin polygons from closed ways." << std::endl;
  2300→                    }
  2301→                }
  2302→
  2303→                // --- Parallel admin boundary assembly ---
  2304→                {
  2305→                    log_phase("Pass 2b: way processing", _pt);
  2306→                    std::cerr << "  Assembling admin polygons in parallel ("
  2307→                              << data.collected_relations.size() << " relations, "
  2308→                              << data.way_geometries.size() << " way geometries)..." << std::endl;
  2309→
  2310→                    // Thread-local admin results (to avoid locking add_admin_polygon)
  2311→                    struct AdminResult {
  2312→                        std::vector<std::pair<double,double>> vertices;
  2313→                        std::string name;
  2314→                        uint8_t admin_level;
  2315→                        std::string country_code;
  2316→                    };
  2317→
  2318→                    std::vector<std::vector<AdminResult>> thread_admin_results(num_threads);
  2319→                    std::atomic<size_t> rel_idx{0};
  2320→                    std::atomic<uint64_t> assembled_count{0};
  2321→
  2322→                    std::vector<std::thread> admin_workers;
  2323→                    for (unsigned int t = 0; t < num_threads; t++) {
  2324→                        admin_workers.emplace_back([&, t]() {
  2325→                            auto& local_results = thread_admin_results[t];
  2326→                            while (true) {
  2327→                                size_t i = rel_idx.fetch_add(1);
  2328→                                if (i >= data.collected_relations.size()) break;
  2329→
  2330→                                const auto& rel = data.collected_relations[i];
  2331→
  2332→                                // Skip level 2 border-line relations — these have names like
  2333→                                // "Deutschland - Österreich" (two countries separated by
  2334→                                // " - " or " — "). They are boundary lines, not country
  2335→                                // polygons, and produce self-intersecting geometry.
  2336→                                if (rel.admin_level == 2 && rel.country_code.empty() &&
  2337→                                    (rel.name.find(" - ") != std::string::npos ||
  2338→                                     rel.name.find(" \xe2\x80\x94 ") != std::string::npos)) { // " — " (em dash)
  2339→                                    assembled_count.fetch_add(1);
  2340→                                    continue;
  2341→                                }
  2342→
  2343→                                // Skip relations with missing member ways —
  2344→                                // these cross the extract boundary and would produce
  2345→                                // incorrect partial polygons
  2346→                                bool has_missing = false;
  2347→                                for (const auto& [way_id, role] : rel.members) {
  2348→                                    if (data.way_geometries.find(way_id) == data.way_geometries.end()) {
  2349→                                        has_missing = true;
  2350→                                        break;
  2351→                                    }
  2352→                                }
  2353→                                if (has_missing) {
  2354→                                    assembled_count.fetch_add(1);
  2355→                                    continue;
  2356→                                }
  2357→
  2358→                                auto rings = assemble_outer_rings(rel.members, data.way_geometries);
  2359→                                // If outer-only assembly failed, retry with all ways.
  2360→                                // Osmium ignores roles during assembly (check_roles=false) —
  2361→                                // inner ways sometimes form the boundary or bridge gaps.
  2362→                                // Filter retry results: discard rings with duplicate coords
  2363→                                // (figure-8 shapes from merging holes with outer boundary).
  2364→                                if (rings.empty()) {
  2365→                                    auto retry = assemble_outer_rings(rel.members, data.way_geometries, true);
  2366→                                    for (auto& ring : retry) {
  2367→                                        if (!ring_has_duplicate_coords(ring)) {
  2368→                                            rings.push_back(std::move(ring));
  2369→                                        }
  2370→                                    }
  2371→                                }
  2372→
  2373→                                // Diagnostic: log relations with 0 rings produced
  2374→                                if (rings.empty() && !rel.members.empty()) {
  2375→                                    int total_ways = 0, found = 0, missing = 0;
  2376→                                    for (const auto& [way_id, role] : rel.members) {
  2377→                                        total_ways++;
  2378→                                        auto it = data.way_geometries.find(way_id);
  2379→                                        if (it != data.way_geometries.end() && !it->second.coords.empty()) {
  2380→                                            found++;
  2381→                                        } else {
  2382→                                            missing++;
  2383→                                        }
  2384→                                    }
  2385→                                    if (missing == 0 && found > 0) {
  2386→                                        std::string detail = "  DEBUG '" + rel.name + "': ways:";
  2387→                                        for (const auto& [way_id, role] : rel.members) {
  2388→                                            auto it = data.way_geometries.find(way_id);
  2389→                                            if (it == data.way_geometries.end()) continue;
  2390→                                            detail += " [w" + std::to_string(way_id) +
  2391→                                                      " first=" + std::to_string(it->second.first_node_id) +
  2392→                                                      " last=" + std::to_string(it->second.last_node_id) +
  2393→                                                      " n=" + std::to_string(it->second.coords.size()) + "]";
  2394→                                        }
  2395→                                        std::cerr << detail << std::endl;
  2396→                                    }
  2397→                                    std::cerr << "  WARN: relation '" << rel.name
  2398→                                              << "' (level " << (int)rel.admin_level
  2399→                                              << ", " << rel.members.size() << " members): "
  2400→                                              << "0 rings from " << total_ways << " ways ("
  2401→                                              << found << " found, " << missing << " missing)" << std::endl;
  2402→                                }
  2403→
  2404→                                for (auto& ring : rings) {
  2405→                                    if (ring.size() >= 3) {
  2406→                                        AdminResult ar;
  2407→                                        ar.vertices = std::move(ring);
  2408→                                        ar.name = rel.name;
  2409→                                        ar.admin_level = rel.admin_level;
  2410→                                        ar.country_code = rel.country_code;
  2411→                                        local_results.push_back(std::move(ar));
  2412→                                    }
  2413→                                }
  2414→
  2415→                                uint64_t done = assembled_count.fetch_add(1) + 1;
  2416→                                if (done % 10000 == 0) {
  2417→                                    std::cerr << "  Assembled " << done / 1000
  2418→                                              << "K admin relations..." << std::endl;
  2419→                                }
  2420→                            }
  2421→                        });
  2422→                    }
  2423→                    for (auto& w : admin_workers) w.join();
  2424→
  2425→                    // Merge thread-local admin results into ParsedData (sequential, thread-safe)
  2426→                    uint64_t total_admin_rings = 0;
  2427→                    for (auto& local_results : thread_admin_results) {
  2428→                        for (auto& ar : local_results) {
  2429→                            const char* cc = ar.country_code.empty() ? nullptr : ar.country_code.c_str();
  2430→                            add_admin_polygon(data, ar.vertices, ar.name.c_str(),
  2431→                                              ar.admin_level, cc, &admin_pool);
  2432→                            total_admin_rings++;
  2433→                        }
  2434→                    }
  2435→                    std::cerr << "  Parallel admin assembly complete: "
  2436→                              << total_admin_rings << " polygon rings from "
  2437→                              << data.collected_relations.size() << " relations." << std::endl;
  2438→
  2439→                    // Free collected data
  2440→                    data.collected_relations.clear();
  2441→                    data.collected_relations.shrink_to_fit();
  2442→                    data.way_geometries.clear();


  2443→                    std::unordered_map<int64_t, ParsedData::WayGeometry>().swap(data.way_geometries);
  2444→                }
  2445→            }
  2446→        }
  2447→
  2448→        std::cerr << "Done reading:" << std::endl;
  2449→        std::cerr << "  " << data.ways.size() << " street ways" << std::endl;
  2450→        std::cerr << "  " << data.addr_points.size() << " address points" << std::endl;
  2451→        std::cerr << "  " << data.interp_ways.size() << " interpolation ways" << std::endl;
  2452→        std::cerr << "  " << data.admin_polygons.size() << " admin polygon rings" << std::endl;
  2453→
  2454→        // Drain admin polygon thread pool (may still be processing)
  2455→        std::cerr << "Waiting for admin polygon S2 covering to complete..." << std::endl;
  2456→        auto admin_results = admin_pool.drain();
  2457→        for (auto& [cell_id, ids] : admin_results) {
  2458→            auto& target = data.cell_to_admin[cell_id];
  2459→            target.insert(target.end(), ids.begin(), ids.end());
  2460→        }
  2461→        std::cerr << "Admin polygon S2 covering complete (" << data.cell_to_admin.size() << " cells)." << std::endl;
  2462→
  2463→        // --- Parallel S2 cell computation for ways and interpolation ---
  2464→        {
  2465→            log_phase("Admin assembly", _pt);
  2466→            std::cerr << "Computing S2 cells for ways with " << num_threads << " threads..." << std::endl;
  2467→
  2468→            // Flat-array approach: threads emit (cell_id, item_id) pairs into
  2469→            // thread-local vectors, then we sort globally and group by cell_id.
  2470→            // This avoids hash maps entirely — sort is cache-friendly O(n log n).
  2471→
  2472→            struct CellItem { uint64_t cell_id; uint32_t item_id; };
  2473→
  2474→            auto _s2t = std::chrono::steady_clock::now();
  2475→
  2476→            // Process streets: emit (cell_id, way_id) pairs
  2477→            std::cerr << "  Processing " << data.deferred_ways.size() << " street ways..." << std::endl;
  2478→            std::vector<std::vector<CellItem>> way_pairs(num_threads);
  2479→            {
  2480→                std::atomic<size_t> way_idx{0};
  2481→                std::vector<std::thread> threads;
  2482→                for (unsigned int t = 0; t < num_threads; t++) {
  2483→                    threads.emplace_back([&, t]() {
  2484→                        auto& local = way_pairs[t];
  2485→                        local.reserve(data.deferred_ways.size() / num_threads * 3);
  2486→                        std::vector<S2CellId> edge_cells;
  2487→                        std::vector<uint64_t> way_cells;
  2488→                        while (true) {
  2489→                            size_t i = way_idx.fetch_add(1);
  2490→                            if (i >= data.deferred_ways.size()) break;
  2491→                            const auto& dw = data.deferred_ways[i];
  2492→                            way_cells.clear();
  2493→                            for (uint8_t j = 0; j + 1 < dw.node_count; j++) {
  2494→                                const auto& n1 = data.street_nodes[dw.node_offset + j];
  2495→                                const auto& n2 = data.street_nodes[dw.node_offset + j + 1];
  2496→                                cover_edge(n1.lat, n1.lng, n2.lat, n2.lng, edge_cells);
  2497→                                for (const auto& c : edge_cells) way_cells.push_back(c.id());
  2498→                            }
  2499→                            std::sort(way_cells.begin(), way_cells.end());
  2500→                            way_cells.erase(std::unique(way_cells.begin(), way_cells.end()), way_cells.end());
  2501→                            for (uint64_t cell_id : way_cells) {
  2502→                                local.push_back({cell_id, dw.way_id});
  2503→                            }
  2504→                        }
  2505→                    });
  2506→                }
  2507→                for (auto& t : threads) t.join();
  2508→            }
  2509→            log_phase("  S2: street ways (parallel)", _s2t);
  2510→
  2511→            // Process interpolations: emit (cell_id, interp_id) pairs
  2512→            std::cerr << "  Processing " << data.deferred_interps.size() << " interpolation ways..." << std::endl;
  2513→            std::vector<std::vector<CellItem>> interp_pairs(num_threads);
  2514→            {
  2515→                std::atomic<size_t> interp_idx{0};
  2516→                std::vector<std::thread> threads;
  2517→                for (unsigned int t = 0; t < num_threads; t++) {
  2518→                    threads.emplace_back([&, t]() {
  2519→                        auto& local = interp_pairs[t];
  2520→                        std::vector<S2CellId> edge_cells;
  2521→                        std::vector<uint64_t> way_cells;
  2522→                        while (true) {
  2523→                            size_t i = interp_idx.fetch_add(1);
  2524→                            if (i >= data.deferred_interps.size()) break;
  2525→                            const auto& di = data.deferred_interps[i];
  2526→                            way_cells.clear();
  2527→                            for (uint8_t j = 0; j + 1 < di.node_count; j++) {
  2528→                                const auto& n1 = data.interp_nodes[di.node_offset + j];
  2529→                                const auto& n2 = data.interp_nodes[di.node_offset + j + 1];
  2530→                                cover_edge(n1.lat, n1.lng, n2.lat, n2.lng, edge_cells);
  2531→                                for (const auto& c : edge_cells) way_cells.push_back(c.id());
  2532→                            }
  2533→                            std::sort(way_cells.begin(), way_cells.end());
  2534→                            way_cells.erase(std::unique(way_cells.begin(), way_cells.end()), way_cells.end());
  2535→                            for (uint64_t cell_id : way_cells) {
  2536→                                local.push_back({cell_id, di.interp_id});
  2537→                            }
  2538→                        }
  2539→                    });
  2540→                }
  2541→                for (auto& t : threads) t.join();
  2542→            }
  2543→            log_phase("  S2: interp ways (parallel)", _s2t);
  2544→
  2545→            // Merge thread-local pairs into single vectors, sort, build cell maps
  2546→            std::cerr << "  Sorting and grouping cell pairs..." << std::endl;
  2547→            // Concatenate + sort helper for flat CellItem pairs
  2548→            // Parallel sort each thread's pairs, then k-way merge directly into
  2549→            // hash map + sorted output. No intermediate merged vector needed.
  2550→            auto parallel_sort_and_build = [](
  2551→                std::vector<std::vector<CellItem>>& thread_pairs,
  2552→                std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map,
  2553→                std::vector<CellItemPair>& sorted_out
  2554→            ) {
  2555→                // Step 1: Convert + sort each thread's data in parallel
  2556→                size_t total = 0;
  2557→                for (auto& v : thread_pairs) total += v.size();
  2558→
  2559→                std::vector<std::vector<CellItemPair>> chunks(thread_pairs.size());
  2560→                {
  2561→                    std::vector<std::thread> sort_threads;
  2562→                    for (size_t t = 0; t < thread_pairs.size(); t++) {
  2563→                        sort_threads.emplace_back([&, t]() {
  2564→                            auto& src = thread_pairs[t];
  2565→                            auto& dst = chunks[t];
  2566→                            dst.reserve(src.size());
  2567→                            for (auto& ci : src) dst.push_back({ci.cell_id, ci.item_id});
  2568→                            src.clear(); src.shrink_to_fit();
  2569→                            std::sort(dst.begin(), dst.end(), [](const CellItemPair& a, const CellItemPair& b) {
  2570→                                return a.cell_id < b.cell_id || (a.cell_id == b.cell_id && a.item_id < b.item_id);
  2571→                            });
  2572→                        });
  2573→                    }
  2574→                    for (auto& t : sort_threads) t.join();
  2575→                }
  2576→
  2577→                // Step 2: Concatenate sorted chunks and merge using parallel tree.
  2578→                // Each chunk is already sorted. Record run boundaries, concatenate,
  2579→                // then merge adjacent runs pairwise in parallel at each level.
  2580→                auto cmp = [](const CellItemPair& a, const CellItemPair& b) {
  2581→                    return a.cell_id < b.cell_id || (a.cell_id == b.cell_id && a.item_id < b.item_id);
  2582→                };
  2583→
  2584→                // Record run boundaries before concatenation
  2585→                std::vector<size_t> run_bounds = {0};
  2586→                sorted_out.clear();
  2587→                sorted_out.reserve(total);
  2588→                for (auto& chunk : chunks) {
  2589→                    sorted_out.insert(sorted_out.end(), chunk.begin(), chunk.end());
  2590→                    run_bounds.push_back(sorted_out.size());
  2591→                    chunk.clear(); chunk.shrink_to_fit();
  2592→                }
  2593→
  2594→                // Parallel tree merge: at each level, merge adjacent run pairs in parallel.
  2595→                // Level 0: merge runs (0,1), (2,3), ... → N/2 runs
  2596→                // Level 1: merge runs (01,23), (45,67), ... → N/4 runs
  2597→                // ... until one sorted run remains.
  2598→                while (run_bounds.size() > 2) {
  2599→                    std::vector<size_t> new_bounds = {0};
  2600→                    std::vector<std::thread> merge_threads;
  2601→                    for (size_t i = 0; i + 2 < run_bounds.size(); i += 2) {
  2602→                        size_t left = run_bounds[i];
  2603→                        size_t mid = run_bounds[i + 1];
  2604→                        size_t right = run_bounds[i + 2];
  2605→                        merge_threads.emplace_back([&, left, mid, right] {
  2606→                            std::inplace_merge(sorted_out.begin() + left,
  2607→                                               sorted_out.begin() + mid,
  2608→                                               sorted_out.begin() + right, cmp);
  2609→                        });
  2610→                        new_bounds.push_back(right);
  2611→                    }
  2612→                    // If odd number of runs, carry the last one forward
  2613→                    if (run_bounds.size() % 2 == 0) {
  2614→                        new_bounds.push_back(run_bounds.back());
  2615→                    }
  2616→                    for (auto& t : merge_threads) t.join();
  2617→                    run_bounds = std::move(new_bounds);
  2618→                }
  2619→
  2620→                // Skip building hash map — sorted_out is used directly for writing.
  2621→                // cell_map stays empty (only needed for cache/continent modes).
  2622→            };
  2623→
  2624→
  2625→            // Sort each thread's pairs in parallel, then k-way merge directly
  2626→            // into hash map + sorted output. No intermediate merged vector.
  2627→            auto f_ways = std::async(std::launch::async, [&] {
  2628→                parallel_sort_and_build(way_pairs, data.cell_to_ways, data.sorted_way_cells);
  2629→            });
  2630→            auto f_interps = std::async(std::launch::async, [&] {
  2631→                std::vector<CellItemPair> interp_sorted; // not needed for writing
  2632→                parallel_sort_and_build(interp_pairs, data.cell_to_interps, interp_sorted);
  2633→            });
  2634→            f_ways.get();
  2635→            f_interps.get();
  2636→            log_phase("  S2: sort + group into cell maps", _s2t);
  2637→
  2638→            // Free deferred work items
  2639→            data.deferred_ways.clear();
  2640→            data.deferred_ways.shrink_to_fit();
  2641→            data.deferred_interps.clear();
  2642→            data.deferred_interps.shrink_to_fit();
  2643→
  2644→            std::cerr << "S2 cell computation complete." << std::endl;
  2645→        }
  2646→
  2647→        // Resolve interpolation endpoints
  2648→        std::cerr << "Resolving interpolation endpoints..." << std::endl;
  2649→        resolve_interpolation_endpoints(data);
  2650→
  2651→        // Deduplicate + convert to sorted pairs for fast writing
  2652→        log_phase("S2 cell computation", _pt);
  2653→        std::cerr << "Deduplicating + sorting for write..." << std::endl;
  2654→        {
  2655→            // Convert addr hash map to sorted pairs.
  2656→            // Sort cell IDs only (30M unique), then build pairs in sorted order.
  2657→            // Much faster than sorting all 160M pairs.
  2658→            auto f2 = std::async(std::launch::async, [&] {
  2659→                // Extract and sort unique cell IDs
  2660→                std::vector<uint64_t> sorted_cells;
  2661→                sorted_cells.reserve(data.cell_to_addrs.size());
  2662→                size_t total_pairs = 0;
  2663→                for (auto& [cell_id, ids] : data.cell_to_addrs) {
  2664→                    sorted_cells.push_back(cell_id);
  2665→                    total_pairs += ids.size();
  2666→                }
  2667→                std::sort(sorted_cells.begin(), sorted_cells.end());
  2668→
  2669→                // Build sorted pairs by iterating sorted cell IDs
  2670→                std::vector<CellItemPair> pairs;
  2671→                pairs.reserve(total_pairs);
  2672→                for (uint64_t cell_id : sorted_cells) {
  2673→                    auto& ids = data.cell_to_addrs[cell_id];
  2674→                    std::sort(ids.begin(), ids.end()); // sort within cell
  2675→                    ids.erase(std::unique(ids.begin(), ids.end()), ids.end());
  2676→                    for (auto id : ids) pairs.push_back({cell_id, id});
  2677→                }
  2678→                data.sorted_addr_cells = std::move(pairs);
  2679→            });
  2680→            auto f4 = std::async(std::launch::async, [&]{ deduplicate(data.cell_to_admin); });
  2681→            f2.get(); f4.get();
  2682→        }
  2683→
  2684→        // Save cache if requested
  2685→        if (!save_cache_path.empty()) {
  2686→            serialize_cache(data, save_cache_path);
  2687→        }
  2688→    }
  2689→
  2690→    // --- Write index files ---
  2691→    log_phase("Deduplication", _pt);
  2692→    std::cerr << "Writing index files to " << output_dir << "..." << std::endl;
  2693→
  2694→    if (multi_output) {
  2695→        // Write all 3 index modes in parallel (they read shared data, write to separate dirs)
  2696→        auto wf1 = std::async(std::launch::async, [&]{ write_index(data, output_dir + "/full", IndexMode::Full); });
  2697→        auto wf2 = std::async(std::launch::async, [&]{ write_index(data, output_dir + "/no-addresses", IndexMode::NoAddresses); });
  2698→        auto wf3 = std::async(std::launch::async, [&]{ write_index(data, output_dir + "/admin-only", IndexMode::AdminOnly); });
  2699→        wf1.get(); wf2.get(); wf3.get();
  2700→    } else {
  2701→        write_index(data, output_dir, mode);
  2702→    }
  2703→
  2704→    if (generate_continents) {
  2705→        // Process continents in parallel (each filters independently from shared data)
  2706→        std::vector<std::future<void>> continent_futures;
  2707→        std::mutex cerr_mutex;
  2708→        for (const auto& continent : kContinents) {
  2709→            continent_futures.push_back(std::async(std::launch::async, [&, continent]() {
  2710→                {
  2711→                    std::lock_guard<std::mutex> lock(cerr_mutex);
  2712→                    std::cerr << "Filtering for continent: " << continent.name << "..." << std::endl;
  2713→                }
  2714→                auto subset = filter_by_bbox(data, continent);
  2715→                std::string base = output_dir + "/" + continent.name;
  2716→                ensure_dir(base);
  2717→                if (multi_output) {
  2718→                    // Write modes in parallel within each continent
  2719→                    auto cf1 = std::async(std::launch::async, [&]{ write_index(subset, base + "/full", IndexMode::Full); });
  2720→                    auto cf2 = std::async(std::launch::async, [&]{ write_index(subset, base + "/no-addresses", IndexMode::NoAddresses); });
  2721→                    auto cf3 = std::async(std::launch::async, [&]{ write_index(subset, base + "/admin-only", IndexMode::AdminOnly); });
  2722→                    cf1.get(); cf2.get(); cf3.get();
  2723→                } else {
  2724→                    write_index(subset, base, mode);
  2725→                }
  2726→                {
  2727→                    std::lock_guard<std::mutex> lock(cerr_mutex);
  2728→                    std::cerr << "Done continent: " << continent.name << std::endl;
  2729→                }
  2730→            }));
  2731→        }
  2732→        for (auto& f : continent_futures) {
  2733→            f.get();
  2734→        }
  2735→    }
  2736→
  2737→    log_phase("Index file writing", _pt);
  2738→    std::cerr << "Done." << std::endl;
  2739→    return 0;
  2740→}
  2741→


Now I have a complete picture of the file. Let me verify the existing header structure.


Bash

List source files


build_index.cpp  124.0K
geometry.h  7.2K
string_pool.h  694B
types.h  2.4K

4 files, 0 dirs (3.h 1.cpp)


I now have a complete understanding of the codebase. Here is the plan.

---

## Split Plan for build_index.cpp

### Design Principles

1. **Header-only for small, template, or inline code** that has no heavy dependencies.
2. **Separate .cpp files for code with heavy includes** (S2, osmium) to contain compile-time cost.
3. **ParsedData stays in a header** since it is referenced everywhere.
4. **Minimize cross-file coupling** -- each new file should have a clear, narrow interface.

### Proposed File Structure

#### New Headers (header-only)

**1. `src/parsed_data.h`** -- The ParsedData struct + utility helpers
- `ParsedData` struct (lines 164-194)
- `log_phase()` (lines 51-56)
- `ensure_dir()` (lines 42-44)
- `deduplicate()` template (lines 766-772)
- Includes: `types.h`, `string_pool.h`, plus STL containers
- Rationale: ParsedData is used by nearly every other module. Keeping it header-only with no S2/osmium includes means every consumer can include it cheaply. `log_phase`, `ensure_dir`, and `deduplicate` are tiny utilities that everything uses.

#### New .cpp Files (with corresponding headers)

**2. `src/s2_helpers.h` / `src/s2_helpers.cpp`** -- S2 cell computation
- The three `static` globals: `kStreetCellLevel`, `kAdminCellLevel`, `kMaxAdminLevel` (lines 198-200) -- move to non-static globals with `extern` declarations in the header
- `cover_edge()` both overloads (lines 205-249)
- `point_to_cell()` (line 251-253)
- `cover_polygon()` (lines 258-337)
- `AdminCoverPool` class (lines 65-158)
- `add_addr_point()` (lines 573-591) -- depends on `point_to_cell` and `ParsedData`
- `add_admin_polygon()` (lines 595-638) -- depends on `cover_polygon`, `simplify_polygon`, `ParsedData`
- Rationale: These are the S2-heavy functions. Grouping them isolates all S2 includes (`s2/s2cell_id.h`, `s2/s2region_coverer.h`, `s2/s2polygon.h`, etc.) into one translation unit. The `AdminCoverPool` calls `cover_polygon` directly, so it belongs here. The `add_*` helpers are thin wrappers over S2 functions that also touch ParsedData, and they are small enough to stay here rather than warrant their own file.

**3. `src/ring_assembly.h` / `src/ring_assembly.cpp`** -- Ring stitching
- `assemble_outer_rings()` + `BacktrackState` (lines 345-567)
- Rationale: This is a self-contained ~220-line algorithm. Its only dependencies are `ParsedData::WayGeometry`, `geometry.h` (`coord_key`, `ring_has_self_intersection`), and `types.h` constants (`BACKTRACK_*`). No S2 or osmium includes needed.

**4. `src/cache.h` / `src/cache.cpp`** -- Serialization
- `SectionType` enum (lines 776-789)
- Cache magic/version constants (lines 791-793)
- `serialize_vector()`, `deserialize_vector()` templates (lines 796-820)
- `serialize_cell_map()`, `deserialize_cell_map()` (lines 822-879)
- `serialize_cache()`, `deserialize_cache()` (lines 881-1015)
- Rationale: Pure binary I/O code, no S2 or osmium. Depends only on ParsedData and STL. About 240 lines.

**5. `src/continent_filter.h` / `src/continent_filter.cpp`** -- Bbox filtering
- `ContinentBBox` struct, `kContinents` array (lines 1019-1033)
- `cell_in_bbox()` (lines 1035-1042)
- `filter_by_bbox()` (lines 1044-1263)
- Rationale: About 250 lines, needs S2 only for `cell_in_bbox` (a single `S2CellId::ToLatLng()` call). Could be header-only for the struct/constants and .cpp for the filter logic. Depends on ParsedData.

**6. `src/cell_index.h` / `src/cell_index.cpp`** -- Index writing
- `write_entries()` (lines 1270-1308)
- `write_entries_from_sorted()` (lines 1313-1401)
- `write_cell_index()` (lines 1403-1430)
- `write_index()` (lines 1434-1626)
- Rationale: ~360 lines of I/O and parallel buffer construction. Depends on ParsedData and types.h constants. No S2 or osmium includes needed.

**7. `src/relation_collector.h`** -- Header-only osmium handler
- `RelationCollector` struct (lines 642-705)
- Rationale: Small (~65 lines), but requires `osmium/handler.hpp`. Header-only is fine since it is only included by `build_index.cpp` (the main file). Making it a separate .cpp just for one struct that is only used in one place adds unnecessary linking overhead.

**8. `src/interpolation.h` / `src/interpolation.cpp`** -- Endpoint resolution
- `resolve_interpolation_endpoints()` (lines 709-762)
- Rationale: Self-contained ~55 lines. Depends on ParsedData and `parse_house_number` from `geometry.h`. No S2 or osmium.

#### What Remains in build_index.cpp

- `#include` directives for the new headers
- `main()` function (lines 1630-2740) containing:
  - Argument parsing
  - DenseIndex (mmap-based node store) -- local to main, uses osmium types
  - The PBF processing pipeline (producer-consumer with node/way workers)
  - Thread-local data structs and merge logic
  - S2 cell computation orchestration (the `parallel_sort_and_build` lambda and its callers)
  - Cache save/load calls
  - Index writing calls
  - Continent generation calls

This keeps `main()` at roughly 1100 lines, which is still large but is a single sequential pipeline that is hard to decompose further without introducing artificial abstractions. The PBF pipeline is deeply intertwined (DenseIndex, thread-local structs, producer-consumer queues all reference each other).

### Dependency Graph

```
types.h  string_pool.h  geometry.h     (existing, no changes)
   \         |            /
    \        |           /
     parsed_data.h  (new: ParsedData, ensure_dir, log_phase, deduplicate)
    /    |    |    \      \
   /     |    |     \      \
s2_helpers  ring_assembly  cache  continent_filter  cell_index  interpolation
(.h/.cpp)   (.h/.cpp)    (.h/.cpp) (.h/.cpp)      (.h/.cpp)    (.h/.cpp)
   \         |    |        /          /              /
    \        |    |       /          /              /
     build_index.cpp  (main + PBF pipeline)
              |
     relation_collector.h (header-only, osmium)
```

### CMakeLists.txt Changes

Minimal change -- add the new .cpp files to the target:

```cmake
add_executable(build-index
    src/build_index.cpp
    src/s2_helpers.cpp
    src/ring_assembly.cpp
    src/cache.cpp
    src/continent_filter.cpp
    src/cell_index.cpp
    src/interpolation.cpp
)
```

### Key Decisions and Rationale

**Why not header-only for S2 functions?** The S2 includes are heavy (`s2/s2polygon.h`, `s2/s2region_coverer.h`, etc.). Keeping them in a .cpp file means only `s2_helpers.cpp` pays the parse cost. `build_index.cpp` includes `s2_helpers.h` which only exposes function signatures and the `AdminCoverPool` class declaration.

**Why keep `AdminCoverPool` with S2 helpers?** Its `worker_loop` calls `cover_polygon` directly. Separating them would require exposing `cover_polygon` as a non-static function anyway, and the pool is logically part of the S2 covering subsystem.

**Why not split main() further?** The PBF pipeline uses a local `DenseIndex` struct (mmap-based), thread-local data structs with many fields, and producer-consumer queues. These are all tightly coupled through captured references in lambda closures. Extracting them into functions would require passing 10+ parameters or creating a "PipelineContext" struct that just moves the complexity around.

**Why make `relation_collector.h` header-only?** It is only 65 lines and is only included in one file. A separate .cpp would add a compilation unit for no benefit.

**The `kStreetCellLevel` / `kAdminCellLevel` / `kMaxAdminLevel` globals** are currently `static` (file-scoped) and mutated by `main()` arg parsing. They need to become `extern` in `s2_helpers.h` and defined in `s2_helpers.cpp`. The arg-parsing code in `main()` will set them via the extern symbols.

### Implementation Sequence

1. Create `parsed_data.h` first (everything depends on it)
2. Create `s2_helpers.h/.cpp` (needed by ring_assembly via AdminCoverPool, and by build_index.cpp)
3. Create `ring_assembly.h/.cpp`
4. Create `relation_collector.h`
5. Create `cache.h/.cpp`
6. Create `interpolation.h/.cpp`
7. Create `continent_filter.h/.cpp`
8. Create `cell_index.h/.cpp`
9. Trim `build_index.cpp` to just main() + includes
10. Update `CMakeLists.txt`
11. Build and test

Steps 2-8 can be done in parallel since they all depend only on step 1.

### Critical Files for Implementation
- `/root/traccar-geocoder/builder/src/build_index.cpp` - The 2740-line file being split; source of all extracted code
- `/root/traccar-geocoder/builder/CMakeLists.txt` - Must add new .cpp source files to the build target
- `/root/traccar-geocoder/builder/src/types.h` - Existing header with struct definitions and constants; parsed_data.h will include it
- `/root/traccar-geocoder/builder/src/geometry.h` - Existing header with inline geometry functions; used by ring_assembly and s2_helpers
- `/root/traccar-geocoder/builder/src/string_pool.h` - Existing header; included transitively via parsed_data.h