sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 'mkdir -p ~/patches-stable && for f in ~/test-stable-new/*.bin; do b=$(basename $f); old=~/test-stable-old/$b; if [ -f "$old" ]; then zstd --patch-from="$old" "$f" -o ~/patches-stable/${b}.patch -f 2>&1; fi; done'

Generate zstd patches between stable-ordered builds


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
long mode automatically triggered
/home/michtest/test-stable-new/addr_entries.bin : 57.00%   (   377 MiB =>    215 MiB, /home/michtest/patches-stable/addr_entries.bin.patch)
long mode automatically triggered
/home/michtest/test-stable-new/addr_points.bin : 60.53%   (  1.28 GiB =>    794 MiB, /home/michtest/patches-stable/addr_points.bin.patch)
long mode automatically triggered
/home/michtest/test-stable-new/admin_cells.bin : 27.72%   (  1.61 MiB =>    458 KiB, /home/michtest/patches-stable/admin_cells.bin.patch)
long mode automatically triggered
/home/michtest/test-stable-new/admin_entries.bin : 40.92%   (  5.84 MiB =>   2.39 MiB, /home/michtest/patches-stable/admin_entries.bin.patch)
long mode automatically triggered
/home/michtest/test-stable-new/admin_polygons.bin : 45.39%   (  10.3 MiB =>   4.68 MiB, /home/michtest/patches-stable/admin_polygons.bin.patch)
long mode automatically triggered
/home/michtest/test-stable-new/admin_vertices.bin :  4.75%   (   278 MiB =>   13.2 MiB, /home/michtest/patches-stable/admin_vertices.bin.patch)
long mode automatically triggered
/home/michtest/test-stable-new/geo_cells.bin : 23.68%   (  1.56 GiB =>    378 MiB, /home/michtest/patches-stable/geo_cells.bin.patch)
long mode automatically triggered
/home/michtest/test-stable-new/interp_entries.bin : 28.16%   (   156 KiB =>   43.8 KiB, /home/michtest/patches-stable/interp_entries.bin.patch)
long mode automatically triggered
/home/michtest/test-stable-new/interp_nodes.bin :  4.62%   (   708 KiB =>   32.7 KiB, /home/michtest/patches-stable/interp_nodes.bin.patch)
long mode automatically triggered
/home/michtest/test-stable-new/interp_ways.bin : 25.55%   (   328 KiB =>   83.7 KiB, /home/michtest/patches-stable/interp_ways.bin.patch)
long mode automatically triggered
/home/michtest/test-stable-new/street_entries.bin : 25.20%   (   559 MiB =>    141 MiB, /home/michtest/patches-stable/street_entries.bin.patch)
long mode automatically triggered
/home/michtest/test-stable-new/street_nodes.bin :  7.74%   (  1.35 GiB =>    107 MiB, /home/michtest/patches-stable/street_nodes.bin.patch)
long mode automatically triggered
/home/michtest/test-stable-new/street_ways.bin : 49.66%   (   223 MiB =>    111 MiB, /home/michtest/patches-stable/street_ways.bin.patch)
long mode automatically triggered
/home/michtest/test-stable-new/strings.bin :  0.09%   (  78.7 MiB =>   69.6 KiB, /home/michtest/patches-stable/strings.bin.patch)


Interesting results. Let me compare the numbers properly:

| File | Unstable Patch | Stable Patch | Change |
|------|---------------|-------------|--------|
| strings.bin | 33 MiB (40%) | 70 KB (0.09%) | **massive win** |
| street_entries | 176 MiB (30%) | 141 MiB (25%) | better |
| addr_entries | 298 MiB (75%) | 215 MiB (57%) | better |
| street_nodes | 6.5 MiB (0.4%) | 107 MiB (7.7%) | **much worse** |
| admin_vertices | 3.2 MiB (0.2%) | 13.2 MiB (4.8%) | **worse** |
| addr_points | 883 MiB (67%) | 794 MiB (60%) | slightly better |
| **Total** | **~1.89 GiB** | **~1.77 GiB** | **-6%** |

The overall improvement is only 6%. The problem is twofold:

1. **Node reordering hurt**: By rearranging street_nodes to follow way sort order, nodes that were stable in PBF order got shuffled. street_nodes went from 0.4% to 7.7%.

2. **String pool sorting is a double-edged sword**: strings.bin itself patches at 0.09% (great!), but it changed all string offsets in every record, so the 8 bytes of string refs in every 16-byte addr_point are now different between builds. That's why addr_points barely improved.

The root cause: **string offsets are non-deterministic across builds regardless of sorting.** Two builds from different PBFs will have different string sets → different offsets. No ordering scheme fixes this — we need stable string offsets.

Let me try a different approach: **content-addressed string pool** where each string's position is determined by its hash, not by insertion order.


     1→#pragma once
     2→
     3→#include <cstdint>
     4→#include <string>
     5→#include <unordered_map>
     6→#include <vector>
     7→
     8→class StringPool {
     9→public:
    10→    uint32_t intern(const std::string& s) {
    11→        auto it = index_.find(s);
    12→        if (it != index_.end()) {
    13→            return it->second;
    14→        }
    15→        uint32_t offset = static_cast<uint32_t>(data_.size());
    16→        index_[s] = offset;
    17→        data_.insert(data_.end(), s.begin(), s.end());
    18→        data_.push_back('\0');
    19→        return offset;
    20→    }
    21→
    22→    const std::vector<char>& data() const { return data_; }
    23→    std::vector<char>& mutable_data() { return data_; }
    24→
    25→private:
    26→    std::unordered_map<std::string, uint32_t> index_;
    27→    std::vector<char> data_;
    28→};
    29→

#pragma once

#include <cstdint>
#include <string>
#include <unordered_map>
#include <vector>

class StringPool {
public:
    uint32_t intern(const std::string& s) {
        auto it = index_.find(s);
        if (it != index_.end()) {
            return it->second;
        }
        uint32_t offset = static_cast<uint32_t>(data_.size());
        index_[s] = offset;
        data_.insert(data_.end(), s.begin(), s.end());
        data_.push_back('\0');
        return offset;
    }

    const std::vector<char>& data() const { return data_; }
    std::vector<char>& mutable_data() { return data_; }

private:
    std::unordered_map<std::string, uint32_t> index_;
    std::vector<char> data_;
};


Let me think about this more carefully. The ~30s overhead (3% of build) is fine, but the results showed the stable ordering approach has a fundamental limitation: **string offset instability**. No record ordering can fix this.

Let me try a different approach — a **previous-build-based string pool** where strings that existed in the last build keep their exact offsets. This is the key to making ALL data files patch well.


Write

#pragma once

#include <cstdint>
#include <cstring>
#include <fstream>
#include <string>
#include <unordered_map>
#include <vector>

class StringPool {
public:
    uint32_t intern(const std::string& s) {
        auto it = index_.find(s);
        if (it != index_.end()) {
            return it->second;
        }
        uint32_t offset = static_cast<uint32_t>(data_.size());
        index_[s] = offset;
        data_.insert(data_.end(), s.begin(), s.end());
        data_.push_back('\0');
        return offset;
    }

    // Load a previous build's string pool to preserve offsets.
    // Strings that exist in the old pool keep their original offset.
    // New strings get appended at the end.
    // This makes string offsets stable across builds, enabling efficient binary patches.
    bool load_previous(const std::string& path) {
        std::ifstream f(path, std::ios::binary | std::ios::ate);
        if (!f.is_open()) return false;
        size_t size = f.tellg();
        if (size == 0) return false;
        f.seekg(0);
        data_.resize(size);
        f.read(data_.data(), size);

        // Build index from old pool
        size_t pos = 0;
        while (pos < data_.size()) {
            const char* str = data_.data() + pos;
            size_t len = strlen(str);
            if (len > 0) {
                index_[std::string(str, len)] = static_cast<uint32_t>(pos);
            }
            pos += len + 1;
        }
        return true;
    }

    size_t previous_size() const { return previous_size_; }
    void mark_previous_loaded() { previous_size_ = data_.size(); }

    const std::vector<char>& data() const { return data_; }
    std::vector<char>& mutable_data() { return data_; }

private:
    std::unordered_map<std::string, uint32_t> index_;
    std::vector<char> data_;
    size_t previous_size_ = 0;
};


The file /root/traccar-geocoder/builder/src/string_pool.h has been updated successfully.

#pragma once

#include <cstdint>
#include <cstring>
#include <fstream>
#include <string>
#include <unordered_map>
#include <vector>

class StringPool {
public:
    uint32_t intern(const std::string& s) {
        auto it = index_.find(s);
        if (it != index_.end()) {
            return it->second;
        }
        uint32_t offset = static_cast<uint32_t>(data_.size());
        index_[s] = offset;
        data_.insert(data_.end(), s.begin(), s.end());
        data_.push_back('\0');
        return offset;
    }

    // Load a previous build's string pool to preserve offsets.
    // Strings that exist in the old pool keep their original offset.
    // New strings get appended at the end.
    // This makes string offsets stable across builds, enabling efficient binary patches.
    bool load_previous(const std::string& path) {
        std::ifstream f(path, std::ios::binary | std::ios::ate);
        if (!f.is_open()) return false;
        size_t size = f.tellg();
        if (size == 0) return false;
        f.seekg(0);
        data_.resize(size);
        f.read(data_.data(), size);

        // Build index from old pool
        size_t pos = 0;
        while (pos < data_.size()) {
            const char* str = data_.data() + pos;
            size_t len = strlen(str);
            if (len > 0) {
                index_[std::string(str, len)] = static_cast<uint32_t>(pos);
            }
            pos += len + 1;
        }
        return true;
    }

    size_t previous_size() const { return previous_size_; }
    void mark_previous_loaded() { previous_size_ = data_.size(); }

    const std::vector<char>& data() const { return data_; }
    std::vector<char>& mutable_data() { return data_; }

private:
    std::unordered_map<std::string, uint32_t> index_;
    std::vector<char> data_;
    size_t previous_size_ = 0;
};

#pragma once
#include <cstdint>
+#include <cstring>
+#include <fstream>
#include <string>
#include <unordered_map>
#include <vector>

+    // Load a previous build's string pool to preserve offsets.

+    // Strings that exist in the old pool keep their original offset.

+    // New strings get appended at the end.

+    // This makes string offsets stable across builds, enabling efficient binary patches.

+    bool load_previous(const std::string& path) {

+        std::ifstream f(path, std::ios::binary | std::ios::ate);

+        if (!f.is_open()) return false;

+            const char* str = data_.data() + pos;

+                index_[std::string(str, len)] = static_cast<uint32_t>(pos);

+    size_t previous_size() const { return previous_size_; }

+    void mark_previous_loaded() { previous_size_ = data_.size(); }

#pragma once

#include <cstdint>
#include <string>
#include <unordered_map>
#include <vector>

class StringPool {
public:
    uint32_t intern(const std::string& s) {
        auto it = index_.find(s);
        if (it != index_.end()) {
            return it->second;
        }
        uint32_t offset = static_cast<uint32_t>(data_.size());
        index_[s] = offset;
        data_.insert(data_.end(), s.begin(), s.end());
        data_.push_back('\0');
        return offset;
    }

    const std::vector<char>& data() const { return data_; }
    std::vector<char>& mutable_data() { return data_; }

private:
    std::unordered_map<std::string, uint32_t> index_;
    std::vector<char> data_;
};


Now I need to add the `--previous-build` option to the builder and use it to load the old string pool. Let me also remove the alphabetical string sorting from the stabilization (it hurt more than it helped) and instead use the previous-build approach.


    64→    std::string output_dir = argv[1];
    65→    std::vector<std::string> input_files;
    66→    std::string save_cache_path;
    67→    std::string load_cache_path;
    68→    bool multi_output = false;
    69→    bool generate_continents = false;
    70→    IndexMode mode = IndexMode::Full;
    71→    SimplifyMode simplify_mode = SimplifyMode::MaxVertices;
    72→    double simplify_epsilon_override = 0; // 0 = use per-level defaults
    73→    bool multi_quality = false;
    74→    std::vector<double> quality_scales;
    75→
    76→    for (int i = 2; i < argc; i++) {
    77→        std::string arg = argv[i];
    78→        if (arg == "--street-level" && i + 1 < argc) {

    std::string output_dir = argv[1];
    std::vector<std::string> input_files;
    std::string save_cache_path;
    std::string load_cache_path;
    bool multi_output = false;
    bool generate_continents = false;
    IndexMode mode = IndexMode::Full;
    SimplifyMode simplify_mode = SimplifyMode::MaxVertices;
    double simplify_epsilon_override = 0; // 0 = use per-level defaults
    bool multi_quality = false;
    std::vector<double> quality_scales;

    for (int i = 2; i < argc; i++) {
        std::string arg = argv[i];
        if (arg == "--street-level" && i + 1 < argc) {


Edit

    std::string output_dir = argv[1];
    std::vector<std::string> input_files;
    std::string save_cache_path;
    std::string load_cache_path;
    bool multi_output = false;
    bool generate_continents = false;
    IndexMode mode = IndexMode::Full;
    SimplifyMode simplify_mode = SimplifyMode::MaxVertices;
    double simplify_epsilon_override = 0; // 0 = use per-level defaults
    bool multi_quality = false;
    std::vector<double> quality_scales;

    std::string output_dir = argv[1];
    std::vector<std::string> input_files;
    std::string save_cache_path;
    std::string load_cache_path;
    std::string previous_build_path;
    bool multi_output = false;
    bool generate_continents = false;
    IndexMode mode = IndexMode::Full;
    SimplifyMode simplify_mode = SimplifyMode::MaxVertices;
    double simplify_epsilon_override = 0; // 0 = use per-level defaults
    bool multi_quality = false;
    std::vector<double> quality_scales;