Bash

cat /root/traccar-geocoder/builder/src/build_index.cpp | ssh node3 "cat > /tmp/traccar-geocoder/builder/src/build_index.cpp" && ssh node3 "cd /tmp/traccar-geocoder/build && make -j\$(nproc) 2>&1 | tail -3 && rm -rf /tmp/de-par-final10 && mkdir -p /tmp/de-par-final10 && /tmp/traccar-geocoder/build/build-index /tmp/de-par-final10 ~/pbf/germany.osm.pbf --parallel-admin 2>&1 | grep -E 'polygon|Done|Dedup'" 2>&1

Build with name-aware dedup


[ 50%] Building CXX object CMakeFiles/build-index.dir/src/build_index.cpp.o
[100%] Linking CXX executable build-index
[100%] Built target build-index
  Added 37 admin polygons from closed ways.
  Assembling admin polygons in parallel (40080 relations, 112858 way geometries)...
  Deduplicated 3506 cross-relation duplicate rings.
  Parallel admin assembly complete: 39679 polygon rings from 40080 relations.
Done reading:
  39716 admin polygon rings
Waiting for admin polygon S2 covering to complete...
Admin polygon S2 covering complete (6103 cells).
Deduplicating...
admin index: 6103 cells, 39716 polygons
Done.


Still removing too many. Let me understand what these duplicates actually are.


ssh node3 "cat > /tmp/check_dupes.cpp << 'EOFCPP'
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <vector>
#include <unordered_map>
#include <algorithm>

struct AdminPolygon { uint32_t vertex_offset; uint16_t vertex_count; uint32_t name_id; uint8_t admin_level; float area; uint16_t country_code; };
struct NodeCoord { float lat; float lng; };

static std::vector<char> read_file(const std::string& path) {
    std::ifstream f(path, std::ios::binary | std::ios::ate); if (!f) return {};
    auto sz = f.tellg(); f.seekg(0);
    std::vector<char> d(sz); f.read(d.data(), sz); return d;
}

int main(int argc, char* argv[]) {
    // Check how many rings in the sequential build are duplicated across different name|level
    auto strings = read_file(std::string(argv[1]) + \"/strings.bin\");
    auto pdata = read_file(std::string(argv[1]) + \"/admin_polygons.bin\");
    auto vdata = read_file(std::string(argv[1]) + \"/admin_vertices.bin\");
    auto* polys = (AdminPolygon*)pdata.data();
    auto* verts = (NodeCoord*)vdata.data();
    size_t count = pdata.size() / sizeof(AdminPolygon);

    struct RingInfo { std::string name; uint8_t level; uint16_t vcount; };
    std::unordered_map<std::string, std::vector<RingInfo>> key_to_rings;

    for (size_t i = 0; i < count; i++) {
        const char* name = (polys[i].name_id < strings.size()) ? strings.data() + polys[i].name_id : \"?\";
        // Build key from sorted vertex coordinates
        std::vector<std::pair<float,float>> rverts;
        for (uint16_t j = 0; j < polys[i].vertex_count; j++) {
            auto& v = verts[polys[i].vertex_offset + j];
            rverts.push_back({v.lat, v.lng});
        }
        std::sort(rverts.begin(), rverts.end());
        std::string key;
        key.resize(rverts.size() * sizeof(std::pair<float,float>));
        memcpy(key.data(), rverts.data(), key.size());

        key_to_rings[key].push_back({name, polys[i].admin_level, polys[i].vertex_count});
    }

    int cross_name = 0;
    for (auto& [key, rings] : key_to_rings) {
        if (rings.size() <= 1) continue;
        bool diff_name = false;
        for (size_t k = 1; k < rings.size(); k++) {
            if (rings[k].name != rings[0].name || rings[k].level != rings[0].level) {
                diff_name = true; break;
            }
        }
        if (diff_name) {
            cross_name++;
            if (cross_name <= 5) {
                printf(\"Shared ring (verts=%d):\", rings[0].vcount);
                for (auto& r : rings) printf(\" [%s|%d]\", r.name.c_str(), r.level);
                printf(\"\\n\");
            }
        }
    }
    printf(\"Total cross-name duplicate ring groups: %d\\n\", cross_name);
    return 0;
}
EOFCPP
g++ -O2 -std=c++17 -o /tmp/check_dupes /tmp/check_dupes.cpp && /tmp/check_dupes /tmp/de-sequential-final" 2>&1

Check cross-name duplicate rings in sequential build


Shared ring (verts=499): [Samtgemeinde Harpstedt|7] [27243|11]
Shared ring (verts=9): [Prinzhöfte|8] [Klein Henstedt|10]
Shared ring (verts=499): [27777|11] [Ganderkesee|8]
Shared ring (verts=397): [27809|11] [Lemwerder|8]
Shared ring (verts=19): [Greiz|6] [Thüringen|4] [07973|11] [Greiz|8]
Total cross-name duplicate ring groups: 4549


**4549 cross-name duplicate ring groups in the sequential build!** This is expected — after Douglas-Peucker simplification, many rings at different admin levels end up with the same simplified vertices (e.g., a level 7 boundary and a level 11 postal code sharing the same simplified ring). These are NOT errors — they're legitimate identical rings from different admin areas.

Our dedup wrongly removes them. The cross-relation dedup approach is fundamentally flawed for this data. Let me remove it entirely and accept the 3 frequency differences as inherent to our per-relation assembly approach.


Edit

                    // Collect all admin results and deduplicate cross-relation rings
                    std::vector<AdminResult> all_admin_results;
                    for (auto& local_results : thread_admin_results) {
                        all_admin_results.insert(all_admin_results.end(),
                            std::make_move_iterator(local_results.begin()),
                            std::make_move_iterator(local_results.end()));
                    }

                    // Deduplicate: if two rings from different relations have identical
                    // vertex sets (from shared member ways), keep only the one belonging
                    // to the relation whose ring set produces the smallest total area.
                    // This matches osmium's behavior of preferring smallest-area assignments.
                    {
                        // Use exact raw bytes of sorted coordinates as key
                        // Only rings with identical coordinate sets will match
                        auto ring_key = [](const std::vector<std::pair<double,double>>& verts) -> std::string {
                            // Sort by raw coordinate values for order-independent comparison
                            std::vector<std::pair<double,double>> sorted = verts;
                            std::sort(sorted.begin(), sorted.end());
                            std::string key;
                            key.resize(sorted.size() * sizeof(std::pair<double,double>));
                            memcpy(key.data(), sorted.data(), key.size());
                            return key;
                        };

                        // Find duplicate rings
                        std::unordered_map<std::string, std::vector<size_t>> key_to_indices;
                        for (size_t i = 0; i < all_admin_results.size(); i++) {
                            key_to_indices[ring_key(all_admin_results[i].vertices)].push_back(i);
                        }

                        // Mark duplicates for removal — only dedup rings that have
                        // different name|level (from shared ways between relations).
                        // Keep the copy from the relation with MORE member ways
                        // (the "larger" relation, matching osmium's preference).
                        std::set<size_t> to_remove;
                        for (auto& [key, indices] : key_to_indices) {
                            if (indices.size() <= 1) continue;
                            // Check if these are from different relations (different name|level)
                            bool all_same = true;
                            for (size_t k = 1; k < indices.size(); k++) {
                                if (all_admin_results[indices[k]].name != all_admin_results[indices[0]].name ||
                                    all_admin_results[indices[k]].admin_level != all_admin_results[indices[0]].admin_level) {
                                    all_same = false;
                                    break;
                                }
                            }
                            if (all_same) continue; // Same relation produced duplicate rings — keep both

                            // Different relations — keep the last copy (higher relation ID)
                            for (size_t k = 0; k + 1 < indices.size(); k++) {
                                to_remove.insert(indices[k]);
                            }
                        }

                        if (!to_remove.empty()) {
                            std::vector<AdminResult> deduped;
                            deduped.reserve(all_admin_results.size() - to_remove.size());
                            for (size_t i = 0; i < all_admin_results.size(); i++) {
                                if (!to_remove.count(i)) {
                                    deduped.push_back(std::move(all_admin_results[i]));
                                }
                            }
                            all_admin_results = std::move(deduped);
                            std::cerr << "  Deduplicated " << to_remove.size()
                                      << " cross-relation duplicate rings." << std::endl;
                        }
                    }

                    // Merge into ParsedData
                    uint64_t total_admin_rings = 0;
                    for (auto& ar : all_admin_results) {
                        const char* cc = ar.country_code.empty() ? nullptr : ar.country_code.c_str();
                        add_admin_polygon(data, ar.vertices, ar.name.c_str(),
                                          ar.admin_level, cc, &admin_pool);
                        total_admin_rings++;
                    }

                    // Merge thread-local admin results into ParsedData (sequential, thread-safe)
                    uint64_t total_admin_rings = 0;
                    for (auto& local_results : thread_admin_results) {
                        for (auto& ar : local_results) {
                            const char* cc = ar.country_code.empty() ? nullptr : ar.country_code.c_str();
                            add_admin_polygon(data, ar.vertices, ar.name.c_str(),
                                              ar.admin_level, cc, &admin_pool);
                            total_admin_rings++;
                        }
                    }