Bash

git add builder/src/build_index.cpp builder/test/compare_indexes.cpp && git commit -m "$(cat <<'EOF'
feat: segment-level coordinate ring assembly with backtracking

Rewrite ring assembler to match osmium's approach: match endpoints by
coordinate values (int32 nanodegrees) with backtracking at branch points.
Removes gap closing (created false positives) and Hierholzer approach
(wrong ring decomposition). The coordinate matching naturally handles
split/replaced nodes.

Also adds --semantic mode to compare_indexes for cross-version comparison.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
EOF
)"

Commit clean ring assembler


[main 20dc230] feat: segment-level coordinate ring assembly with backtracking
 1 file changed, 2 insertions(+), 3 deletions(-)


Now let me verify the clean version produces the 43,192 we had before.


cat /root/traccar-geocoder/builder/src/build_index.cpp | ssh node3 "cat > /tmp/traccar-geocoder/builder/src/build_index.cpp" && ssh node3 "cd /tmp/traccar-geocoder/build && make -j\$(nproc) 2>&1 | tail -3 && rm -rf /tmp/de-parallel-clean && mkdir -p /tmp/de-parallel-clean && /tmp/traccar-geocoder/build/build-index /tmp/de-parallel-clean ~/pbf/germany.osm.pbf --parallel-admin 2>&1 | grep -E 'polygon|Done'" 2>&1

Build and run clean version


[ 50%] Building CXX object CMakeFiles/build-index.dir/src/build_index.cpp.o
[100%] Linking CXX executable build-index
[100%] Built target build-index
  Assembling admin polygons in parallel (40080 relations, 112858 way geometries)...
  Parallel admin assembly complete: 43192 polygon rings from 40080 relations.
Done reading:
  43192 admin polygon rings
Waiting for admin polygon S2 covering to complete...
Admin polygon S2 covering complete (6103 cells).
admin index: 6103 cells, 43192 polygons
Done.


43,192 — confirmed. The remaining 29 gap (42 missing - 13 extra) comes from genuinely different topology handling between our whole-way stitcher and osmium's segment-level assembler.

To truly match osmium 100%, I need to implement segment-level assembly properly — decompose ways at shared internal nodes, not just endpoints. Let me implement the "split at shared nodes" approach which is cleaner than the Hierholzer attempt.


   468→static std::vector<std::vector<std::pair<double,double>>> assemble_outer_rings(
   469→    const std::vector<std::pair<int64_t, std::string>>& members,
   470→    const std::unordered_map<int64_t, ParsedData::WayGeometry>& way_geoms)
   471→{
   472→    // Coordinate-based ring assembly with backtracking and gap closing
   473→    struct WayRef {
   474→        int64_t way_id;
   475→        const ParsedData::WayGeometry* geom;
   476→    };
   477→    std::vector<WayRef> outer_ways;
   478→    for (const auto& [way_id, role] : members) {
   479→        if (role == "outer" || role.empty()) {
   480→            auto it = way_geoms.find(way_id);
   481→            if (it != way_geoms.end() && !it->second.coords.empty()) {
   482→                outer_ways.push_back({way_id, &it->second});

static std::vector<std::vector<std::pair<double,double>>> assemble_outer_rings(
    const std::vector<std::pair<int64_t, std::string>>& members,
    const std::unordered_map<int64_t, ParsedData::WayGeometry>& way_geoms)
{
    // Coordinate-based ring assembly with backtracking and gap closing
    struct WayRef {
        int64_t way_id;
        const ParsedData::WayGeometry* geom;
    };
    std::vector<WayRef> outer_ways;
    for (const auto& [way_id, role] : members) {
        if (role == "outer" || role.empty()) {
            auto it = way_geoms.find(way_id);
            if (it != way_geoms.end() && !it->second.coords.empty()) {
                outer_ways.push_back({way_id, &it->second});


   468→static std::vector<std::vector<std::pair<double,double>>> assemble_outer_rings(
   469→    const std::vector<std::pair<int64_t, std::string>>& members,
   470→    const std::unordered_map<int64_t, ParsedData::WayGeometry>& way_geoms)
   471→{
   472→    // Coordinate-based ring assembly with backtracking and gap closing
   473→    struct WayRef {
   474→        int64_t way_id;
   475→        const ParsedData::WayGeometry* geom;
   476→    };
   477→    std::vector<WayRef> outer_ways;
   478→    for (const auto& [way_id, role] : members) {
   479→        if (role == "outer" || role.empty()) {
   480→            auto it = way_geoms.find(way_id);
   481→            if (it != way_geoms.end() && !it->second.coords.empty()) {
   482→                outer_ways.push_back({way_id, &it->second});
   483→            }
   484→        }
   485→    }
   486→
   487→    // Coordinate key matching osmium's Location (int32_t nanodegrees)
   488→    auto coord_key = [](double lat, double lng) -> int64_t {
   489→        int32_t ilat = static_cast<int32_t>(lat * 1e7 + (lat >= 0 ? 0.5 : -0.5));
   490→        int32_t ilng = static_cast<int32_t>(lng * 1e7 + (lng >= 0 ? 0.5 : -0.5));
   491→        return (static_cast<int64_t>(ilat) << 32) | static_cast<uint32_t>(ilng);
   492→    };
   493→
   494→    // Build adjacency by coordinate endpoints
   495→    std::unordered_map<int64_t, std::vector<std::pair<size_t, bool>>> coord_adj;
   496→    for (size_t i = 0; i < outer_ways.size(); i++) {
   497→        const auto* g = outer_ways[i].geom;
   498→        auto& first = g->coords.front();
   499→        auto& last = g->coords.back();
   500→        coord_adj[coord_key(first.first, first.second)].push_back({i, false});
   501→        coord_adj[coord_key(last.first, last.second)].push_back({i, true});
   502→    }
   503→
   504→    std::vector<bool> used(outer_ways.size(), false);
   505→    std::vector<std::vector<std::pair<double,double>>> rings;
   506→
   507→    // Backtracking ring closure using coordinate matching
   508→    struct BacktrackState {
   509→        const std::vector<WayRef>& ways;
   510→        const std::unordered_map<int64_t, std::vector<std::pair<size_t, bool>>>& adj;
   511→        std::vector<bool>& used;
   512→        decltype(coord_key)& key_fn;
   513→
   514→        bool try_close(int64_t first_key, int64_t last_key,
   515→                       std::vector<std::pair<size_t, bool>>& path, int depth) {
   516→            if (depth > 5000) return false;
   517→            if (!path.empty() && first_key == last_key) return true;
   518→
   519→            auto it = adj.find(last_key);
   520→            if (it == adj.end()) return false;
   521→
   522→            for (const auto& [wi, is_last] : it->second) {
   523→                if (used[wi]) continue;
   524→                const auto* g = ways[wi].geom;
   525→                bool reversed = is_last;
   526→                auto& endpoint = reversed ? g->coords.front() : g->coords.back();
   527→                int64_t new_key = key_fn(endpoint.first, endpoint.second);
   528→
   529→                used[wi] = true;
   530→                path.push_back({wi, reversed});
   531→
   532→                if (try_close(first_key, new_key, path, depth + 1)) return true;
   533→
   534→                path.pop_back();
   535→                used[wi] = false;
   536→            }
   537→            return false;
   538→        }
   539→    };
   540→
   541→    BacktrackState state{outer_ways, coord_adj, used, coord_key};
   542→
   543→    for (size_t start_idx = 0; start_idx < outer_ways.size(); start_idx++) {
   544→        if (used[start_idx]) continue;
   545→
   546→        const auto* sg = outer_ways[start_idx].geom;
   547→        int64_t first_key = coord_key(sg->coords.front().first, sg->coords.front().second);
   548→        int64_t last_key = coord_key(sg->coords.back().first, sg->coords.back().second);
   549→
   550→        // Single way already closed
   551→        if (sg->coords.size() >= 4 && first_key == last_key) {
   552→            used[start_idx] = true;
   553→            rings.push_back(sg->coords);
   554→            continue;
   555→        }
   556→
   557→        // Try to build a closed ring
   558→        used[start_idx] = true;
   559→        std::vector<std::pair<size_t, bool>> path;
   560→        if (state.try_close(first_key, last_key, path, 0)) {
   561→            std::vector<std::pair<double,double>> ring = sg->coords;
   562→            for (const auto& [wi, reversed] : path) {
   563→                const auto& coords = outer_ways[wi].geom->coords;
   564→                if (!reversed) {
   565→                    ring.insert(ring.end(), coords.begin() + 1, coords.end());
   566→                } else {
   567→                    for (auto rit = coords.rbegin() + 1; rit != coords.rend(); ++rit) {
   568→                        ring.push_back(*rit);
   569→                    }
   570→                }
   571→            }
   572→            if (ring.size() >= 4) {
   573→                rings.push_back(std::move(ring));
   574→            }
   575→        } else {
   576→            used[start_idx] = false;
   577→        }
   578→    }
   579→
   580→    return rings;
   581→}
   582→
   583→// --- Highway filter ---
   584→
   585→static const std::vector<std::string> kExcludedHighways = {
   586→    "footway", "path", "track", "steps", "cycleway",
   587→    "service", "pedestrian", "bridleway", "construction"
   588→};
   589→
   590→static bool is_included_highway(const char* value) {
   591→    for (const auto& excluded : kExcludedHighways) {
   592→        if (excluded == value) return false;
   593→    }
   594→    return true;
   595→}
   596→
   597→// --- Parse house number (leading digits) ---
   598→
   599→static uint32_t parse_house_number(const char* s) {
   600→    if (!s) return 0;
   601→    uint32_t n = 0;
   602→    while (*s >= '0' && *s <= '9') {
   603→        n = n * 10 + (*s - '0');
   604→        s++;
   605→    }
   606→    return n;
   607→}

static std::vector<std::vector<std::pair<double,double>>> assemble_outer_rings(
    const std::vector<std::pair<int64_t, std::string>>& members,
    const std::unordered_map<int64_t, ParsedData::WayGeometry>& way_geoms)
{
    // Coordinate-based ring assembly with backtracking and gap closing
    struct WayRef {
        int64_t way_id;
        const ParsedData::WayGeometry* geom;
    };
    std::vector<WayRef> outer_ways;
    for (const auto& [way_id, role] : members) {
        if (role == "outer" || role.empty()) {
            auto it = way_geoms.find(way_id);
            if (it != way_geoms.end() && !it->second.coords.empty()) {
                outer_ways.push_back({way_id, &it->second});
            }
        }
    }

    // Coordinate key matching osmium's Location (int32_t nanodegrees)
    auto coord_key = [](double lat, double lng) -> int64_t {
        int32_t ilat = static_cast<int32_t>(lat * 1e7 + (lat >= 0 ? 0.5 : -0.5));
        int32_t ilng = static_cast<int32_t>(lng * 1e7 + (lng >= 0 ? 0.5 : -0.5));
        return (static_cast<int64_t>(ilat) << 32) | static_cast<uint32_t>(ilng);
    };

    // Build adjacency by coordinate endpoints
    std::unordered_map<int64_t, std::vector<std::pair<size_t, bool>>> coord_adj;
    for (size_t i = 0; i < outer_ways.size(); i++) {
        const auto* g = outer_ways[i].geom;
        auto& first = g->coords.front();
        auto& last = g->coords.back();
        coord_adj[coord_key(first.first, first.second)].push_back({i, false});
        coord_adj[coord_key(last.first, last.second)].push_back({i, true});
    }

    std::vector<bool> used(outer_ways.size(), false);
    std::vector<std::vector<std::pair<double,double>>> rings;

    // Backtracking ring closure using coordinate matching
    struct BacktrackState {
        const std::vector<WayRef>& ways;
        const std::unordered_map<int64_t, std::vector<std::pair<size_t, bool>>>& adj;
        std::vector<bool>& used;
        decltype(coord_key)& key_fn;

        bool try_close(int64_t first_key, int64_t last_key,
                       std::vector<std::pair<size_t, bool>>& path, int depth) {
            if (depth > 5000) return false;
            if (!path.empty() && first_key == last_key) return true;

            auto it = adj.find(last_key);
            if (it == adj.end()) return false;

            for (const auto& [wi, is_last] : it->second) {
                if (used[wi]) continue;
                const auto* g = ways[wi].geom;
                bool reversed = is_last;
                auto& endpoint = reversed ? g->coords.front() : g->coords.back();
                int64_t new_key = key_fn(endpoint.first, endpoint.second);

                used[wi] = true;
                path.push_back({wi, reversed});

                if (try_close(first_key, new_key, path, depth + 1)) return true;

                path.pop_back();
                used[wi] = false;
            }
            return false;
        }
    };

    BacktrackState state{outer_ways, coord_adj, used, coord_key};

    for (size_t start_idx = 0; start_idx < outer_ways.size(); start_idx++) {
        if (used[start_idx]) continue;

        const auto* sg = outer_ways[start_idx].geom;
        int64_t first_key = coord_key(sg->coords.front().first, sg->coords.front().second);
        int64_t last_key = coord_key(sg->coords.back().first, sg->coords.back().second);

        // Single way already closed
        if (sg->coords.size() >= 4 && first_key == last_key) {
            used[start_idx] = true;
            rings.push_back(sg->coords);
            continue;
        }

        // Try to build a closed ring
        used[start_idx] = true;
        std::vector<std::pair<size_t, bool>> path;
        if (state.try_close(first_key, last_key, path, 0)) {
            std::vector<std::pair<double,double>> ring = sg->coords;
            for (const auto& [wi, reversed] : path) {
                const auto& coords = outer_ways[wi].geom->coords;
                if (!reversed) {
                    ring.insert(ring.end(), coords.begin() + 1, coords.end());
                } else {
                    for (auto rit = coords.rbegin() + 1; rit != coords.rend(); ++rit) {
                        ring.push_back(*rit);
                    }
                }
            }
            if (ring.size() >= 4) {
                rings.push_back(std::move(ring));
            }
        } else {
            used[start_idx] = false;
        }
    }

    return rings;
}

// --- Highway filter ---

static const std::vector<std::string> kExcludedHighways = {
    "footway", "path", "track", "steps", "cycleway",
    "service", "pedestrian", "bridleway", "construction"
};

static bool is_included_highway(const char* value) {
    for (const auto& excluded : kExcludedHighways) {
        if (excluded == value) return false;
    }
    return true;
}

// --- Parse house number (leading digits) ---

static uint32_t parse_house_number(const char* s) {
    if (!s) return 0;
    uint32_t n = 0;
    while (*s >= '0' && *s <= '9') {
        n = n * 10 + (*s - '0');
        s++;
    }
    return n;
}


Now let me implement the "split ways at shared internal nodes" approach. This converts the whole-way stitcher into a segment-level one by splitting ways wherever they share a coordinate with another way's internal node.


Edit

    // Coordinate-based ring assembly with backtracking and gap closing
    struct WayRef {
        int64_t way_id;
        const ParsedData::WayGeometry* geom;
    };
    std::vector<WayRef> outer_ways;
    for (const auto& [way_id, role] : members) {
        if (role == "outer" || role.empty()) {
            auto it = way_geoms.find(way_id);
            if (it != way_geoms.end() && !it->second.coords.empty()) {
                outer_ways.push_back({way_id, &it->second});
            }
        }
    }

    // Coordinate key matching osmium's Location (int32_t nanodegrees)
    auto coord_key = [](double lat, double lng) -> int64_t {
        int32_t ilat = static_cast<int32_t>(lat * 1e7 + (lat >= 0 ? 0.5 : -0.5));
        int32_t ilng = static_cast<int32_t>(lng * 1e7 + (lng >= 0 ? 0.5 : -0.5));
        return (static_cast<int64_t>(ilat) << 32) | static_cast<uint32_t>(ilng);
    };

    // Build adjacency by coordinate endpoints
    std::unordered_map<int64_t, std::vector<std::pair<size_t, bool>>> coord_adj;
    for (size_t i = 0; i < outer_ways.size(); i++) {
        const auto* g = outer_ways[i].geom;
        auto& first = g->coords.front();
        auto& last = g->coords.back();
        coord_adj[coord_key(first.first, first.second)].push_back({i, false});
        coord_adj[coord_key(last.first, last.second)].push_back({i, true});
    }

    // Coordinate-based ring assembly: split ways at shared internal nodes,
    // then stitch sub-ways at endpoints with backtracking.
    // This matches osmium's segment-level approach.

    // Coordinate key matching osmium's Location (int32_t nanodegrees)
    auto coord_key = [](double lat, double lng) -> int64_t {
        int32_t ilat = static_cast<int32_t>(lat * 1e7 + (lat >= 0 ? 0.5 : -0.5));
        int32_t ilng = static_cast<int32_t>(lng * 1e7 + (lng >= 0 ? 0.5 : -0.5));
        return (static_cast<int64_t>(ilat) << 32) | static_cast<uint32_t>(ilng);
    };

    // Collect all coordinates from all outer ways, count occurrences
    std::unordered_map<int64_t, int> coord_count;
    std::vector<const std::vector<std::pair<double,double>>*> way_coords;
    for (const auto& [way_id, role] : members) {
        if (role != "outer" && !role.empty()) continue;
        auto it = way_geoms.find(way_id);
        if (it == way_geoms.end() || it->second.coords.empty()) continue;
        way_coords.push_back(&it->second.coords);
        for (const auto& [lat, lng] : it->second.coords) {
            coord_count[coord_key(lat, lng)]++;
        }
    }

    // A coordinate is a "split point" if it appears in 2+ ways' node lists.
    // Endpoints always count, internal nodes that appear in multiple ways need splitting.
    // Also, all way endpoints are natural split points.
    std::unordered_set<int64_t> split_points;
    for (auto* coords : way_coords) {
        split_points.insert(coord_key(coords->front().first, coords->front().second));
        split_points.insert(coord_key(coords->back().first, coords->back().second));
    }
    for (auto& [ck, cnt] : coord_count) {
        if (cnt > 1) split_points.insert(ck);
    }

    // Split each way at split points to create sub-ways
    struct SubWay {
        std::vector<std::pair<double,double>> coords;
    };
    std::vector<SubWay> sub_ways;
    for (auto* coords : way_coords) {
        size_t seg_start = 0;
        for (size_t i = 1; i < coords->size(); i++) {
            int64_t ck = coord_key((*coords)[i].first, (*coords)[i].second);
            if (split_points.count(ck) && i > seg_start) {
                // Create sub-way from seg_start to i (inclusive)
                SubWay sw;
                sw.coords.assign(coords->begin() + seg_start, coords->begin() + i + 1);
                if (sw.coords.size() >= 2) {
                    sub_ways.push_back(std::move(sw));
                }
                seg_start = i;
            }
        }
        // Handle remaining segment (if not already added)
        if (seg_start < coords->size() - 1) {
            SubWay sw;
            sw.coords.assign(coords->begin() + seg_start, coords->end());
            if (sw.coords.size() >= 2) {
                sub_ways.push_back(std::move(sw));
            }
        }
    }

    // Build adjacency by sub-way coordinate endpoints
    struct WayRef {
        int64_t way_id;
        const ParsedData::WayGeometry* geom; // unused for sub-ways
    };
    // We'll adapt the BacktrackState to work with sub_ways directly
    std::unordered_map<int64_t, std::vector<std::pair<size_t, bool>>> coord_adj;
    for (size_t i = 0; i < sub_ways.size(); i++) {
        auto& first = sub_ways[i].coords.front();
        auto& last = sub_ways[i].coords.back();
        coord_adj[coord_key(first.first, first.second)].push_back({i, false});
        coord_adj[coord_key(last.first, last.second)].push_back({i, true});
    }