#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <functional>
#include <iomanip>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>
#include <zlib.h>

// geocoder-diff: Compare two canonical geocoder indexes and produce a compact patch.
//
// Strategy:
// 1. Build string remap (old string offsets → new string offsets)
// 2. For data files with string refs: apply remap to old file in memory, then binary diff
// 3. For coordinate files: direct binary diff (no string refs)
// 4. For small derived files: full replacement (compressed)
//
// The binary diff uses a simple block-based COPY/INSERT encoder:
// - Compare 64-byte blocks between (remapped) old and new
// - COPY for matching blocks, INSERT for different blocks
// - Handles size differences (new may be larger)
//
// Usage: geocoder-diff <old-canonical-dir> <new-canonical-dir> -o <patch-file>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>
#include <zlib.h>

#include "patch_format.h"

// --- zlib helpers ---

static std::vector<char> zlib_compress(const char* data, size_t size) {
    if (size == 0) return {};
    uLongf bound = compressBound(size);
    std::vector<char> out(bound);
    int ret = compress2(reinterpret_cast<Bytef*>(out.data()), &bound,
                        reinterpret_cast<const Bytef*>(data), size, 9);
    if (ret != Z_OK) return {};
    out.resize(bound);
    return out;
}

// --- String remap ---

static std::unordered_map<uint32_t, uint32_t> build_string_remap(
    const std::vector<char>& old_pool, const std::vector<char>& new_pool)
{
    // Index new pool
    std::unordered_map<std::string, uint32_t> new_index;
    size_t pos = 0;
    while (pos < new_pool.size()) {
        const char* s = new_pool.data() + pos;
        size_t len = strlen(s);
        new_index[std::string(s, len)] = static_cast<uint32_t>(pos);
        pos += len + 1;
    }

    // Map old offsets → new offsets
    std::unordered_map<uint32_t, uint32_t> remap;
    pos = 0;
    while (pos < old_pool.size()) {
        const char* s = old_pool.data() + pos;
        size_t len = strlen(s);
        auto it = new_index.find(std::string(s, len));
        if (it != new_index.end()) {
            remap[static_cast<uint32_t>(pos)] = it->second;
        }
        pos += len + 1;
    }
    return remap;
}

// Apply string remap to specific fields in a data buffer
static void remap_addr_points(std::vector<char>& data,
                                const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / 16;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * 16;
        // housenumber_id at offset 8, street_id at offset 12
        for (size_t off : {8, 12}) {
            uint32_t old_id;
            memcpy(&old_id, p + off, 4);
            auto it = remap.find(old_id);
            if (it != remap.end()) {
                memcpy(p + off, &it->second, 4);
            }
        }
    }
}

static void remap_ways(std::vector<char>& data, size_t stride,
                        const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / stride;
    size_t name_off = (stride == 12) ? 8 : 5;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * stride;
        uint32_t old_id;
        memcpy(&old_id, p + name_off, 4);
        auto it = remap.find(old_id);
        if (it != remap.end()) {
            memcpy(p + name_off, &it->second, 4);
        }
    }
}

static void remap_interps(std::vector<char>& data, size_t stride,
                           const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / stride;
    size_t street_off = (stride == 20) ? 8 : 5;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * stride;
        uint32_t old_id;
        memcpy(&old_id, p + street_off, 4);
        auto it = remap.find(old_id);
        if (it != remap.end()) {
            memcpy(p + street_off, &it->second, 4);
        }
    }
}

static void remap_admin_polygons(std::vector<char>& data, size_t stride,
                                   const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / stride;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * stride;
        uint32_t old_id;
        memcpy(&old_id, p + 8, 4); // name_id at offset 8
        auto it = remap.find(old_id);
        if (it != remap.end()) {
            memcpy(p + 8, &it->second, 4);
        }
    }
}

// --- Block-based COPY/INSERT binary diff ---
// Compares (remapped) old vs new, produces a compact delta.
// Block size of 64 bytes balances granularity vs overhead.

static constexpr size_t DIFF_BLOCK = 64;

static std::vector<char> compute_block_diff(const std::vector<char>& old_data,
                                              const std::vector<char>& new_data) {
    std::vector<char> ops;
    size_t old_pos = 0, new_pos = 0;

    while (new_pos < new_data.size()) {
        // Check if current block matches
        size_t remaining_new = new_data.size() - new_pos;
        size_t remaining_old = (old_pos < old_data.size()) ? old_data.size() - old_pos : 0;
        size_t block = std::min(DIFF_BLOCK, remaining_new);

        if (remaining_old >= block &&
            memcmp(old_data.data() + old_pos, new_data.data() + new_pos, block) == 0) {
            // Find full extent of match
            size_t match_len = block;
            while (old_pos + match_len < old_data.size() &&
                   new_pos + match_len < new_data.size() &&
                   old_data[old_pos + match_len] == new_data[new_pos + match_len]) {
                match_len++;
            }
            // Emit COPY: tag(1) + offset(8) + length(4) = 13 bytes
            ops.push_back(OP_COPY);
            uint64_t off = old_pos;
            uint32_t len = static_cast<uint32_t>(match_len);
            ops.insert(ops.end(), reinterpret_cast<const char*>(&off),
                      reinterpret_cast<const char*>(&off) + 8);
            ops.insert(ops.end(), reinterpret_cast<const char*>(&len),
                      reinterpret_cast<const char*>(&len) + 4);
            old_pos += match_len;
            new_pos += match_len;
        } else {
            // Find extent of non-match (up to next matching block)
            size_t insert_start = new_pos;
            size_t old_scan = old_pos;

            // Advance by one block
            new_pos += block;
            old_scan += std::min(block, remaining_old);

            // Keep scanning until we find a matching block again
            while (new_pos < new_data.size()) {
                size_t rem_new = new_data.size() - new_pos;
                size_t rem_old = (old_scan < old_data.size()) ? old_data.size() - old_scan : 0;
                size_t b = std::min(DIFF_BLOCK, rem_new);
                if (rem_old >= b &&
                    memcmp(old_data.data() + old_scan, new_data.data() + new_pos, b) == 0) {
                    break; // Found match
                }
                new_pos += b;
                old_scan += std::min(b, rem_old);
            }

            // Emit INSERT: tag(1) + length(4) + data
            size_t insert_len = new_pos - insert_start;
            ops.push_back(OP_INSERT);
            uint32_t len = static_cast<uint32_t>(insert_len);
            ops.insert(ops.end(), reinterpret_cast<const char*>(&len),
                      reinterpret_cast<const char*>(&len) + 4);
            ops.insert(ops.end(), new_data.data() + insert_start,
                      new_data.data() + insert_start + insert_len);
            old_pos = old_scan;
        }
    }
    return ops;
}

// --- Write helpers ---
static void write_u32(std::ofstream& f, uint32_t v) { f.write(reinterpret_cast<const char*>(&v), 4); }
static void write_u64(std::ofstream& f, uint64_t v) { f.write(reinterpret_cast<const char*>(&v), 8); }

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>" << std::endl;
        return 1;
    }

    std::string old_dir = argv[1];
    std::string new_dir = argv[2];
    std::string patch_path = argv[4];

    std::cerr << "Loading string pools..." << std::endl;
    auto old_strings = read_file(old_dir + "/strings.bin");
    auto new_strings = read_file(new_dir + "/strings.bin");
    auto remap = build_string_remap(old_strings, new_strings);
    std::cerr << "  Remap: " << remap.size() << " strings mapped" << std::endl;

    std::ofstream pf(patch_path, std::ios::binary);
    pf.write(GCPATCH_MAGIC, 8);
    write_u32(pf, GCPATCH_VERSION);
    write_u32(pf, 0);

    // Section: strings.bin (full replacement)
    {
        std::cerr << "  strings.bin: full replacement..." << std::endl;
        auto compressed = zlib_compress(new_strings.data(), new_strings.size());
        write_u32(pf, static_cast<uint32_t>(PatchFileId::STRINGS));
        write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));
        write_u64(pf, new_strings.size());
        write_u64(pf, compressed.size());
        pf.write(compressed.data(), compressed.size());
        std::cerr << "    " << new_strings.size() << " -> " << compressed.size() << std::endl;
    }

    // Helper: diff a file with optional string remapping
    auto diff_file = [&](PatchFileId id, const std::string& name,
                          std::function<void(std::vector<char>&)> remap_fn) {
        std::cerr << "  " << name << ": ";
        auto old_data = read_file(old_dir + "/" + name);
        auto new_data = read_file(new_dir + "/" + name);

        // Apply string remap to old data (makes it comparable to new)
        if (remap_fn) {
            remap_fn(old_data);
        }

        // Compute block diff
        auto ops = compute_block_diff(old_data, new_data);

        // Compress the ops stream
        auto compressed = zlib_compress(ops.data(), ops.size());

        // Also compute full replacement size for comparison
        auto full_compressed = zlib_compress(new_data.data(), new_data.size());

        // Use whichever is smaller
        bool use_delta = compressed.size() < full_compressed.size();

        write_u32(pf, static_cast<uint32_t>(id));
        if (use_delta) {
            write_u32(pf, static_cast<uint32_t>(PatchEncoding::COPY_INSERT));
            write_u64(pf, old_data.size()); // old size
            write_u64(pf, new_data.size()); // new size
            write_u64(pf, compressed.size());
            pf.write(compressed.data(), compressed.size());
            std::cerr << "delta " << compressed.size() << " bytes";
        } else {
            write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));
            write_u64(pf, new_data.size());
            write_u64(pf, full_compressed.size());
            pf.write(full_compressed.data(), full_compressed.size());
            std::cerr << "full " << full_compressed.size() << " bytes";
        }
        double pct = (new_data.size() > 0) ?
            (use_delta ? compressed.size() : full_compressed.size()) * 100.0 / new_data.size() : 0;
        std::cerr << " (" << std::fixed << std::setprecision(1) << pct << "% of "
                  << new_data.size() / 1024 / 1024 << " MiB)" << std::endl;
    };

    // Detect strides for struct files
    size_t way_stride = detect_way_stride(old_dir + "/street_ways.bin");
    auto old_interp_data = read_file(old_dir + "/interp_ways.bin");
    size_t interp_stride = (old_interp_data.size() % 20 == 0 && old_interp_data.size() > 0) ? 20 : 18;
    auto old_admin_data = read_file(old_dir + "/admin_polygons.bin");
    size_t admin_stride = 24;
    for (size_t s : {24, 20, 19}) {
        if (old_admin_data.size() % s == 0 && old_admin_data.size() > 0) { admin_stride = s; break; }
    }

    // Data files with string remapping
    diff_file(PatchFileId::ADDR_POINTS, "addr_points.bin",
        [&](std::vector<char>& d) { remap_addr_points(d, remap); });

    diff_file(PatchFileId::STREET_WAYS, "street_ways.bin",
        [&, way_stride](std::vector<char>& d) { remap_ways(d, way_stride, remap); });

    diff_file(PatchFileId::INTERP_WAYS, "interp_ways.bin",
        [&, interp_stride](std::vector<char>& d) { remap_interps(d, interp_stride, remap); });

    diff_file(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin",
        [&, admin_stride](std::vector<char>& d) { remap_admin_polygons(d, admin_stride, remap); });

    // Coordinate files (no string refs, direct diff)
    std::function<void(std::vector<char>&)> no_remap;
    diff_file(PatchFileId::STREET_NODES, "street_nodes.bin", no_remap);
    diff_file(PatchFileId::INTERP_NODES, "interp_nodes.bin", no_remap);
    diff_file(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", no_remap);

    // Derived/index files (no remap needed, direct diff)
    diff_file(PatchFileId::GEO_CELLS, "geo_cells.bin", no_remap);
    diff_file(PatchFileId::STREET_ENTRIES, "street_entries.bin", no_remap);
    diff_file(PatchFileId::ADDR_ENTRIES, "addr_entries.bin", no_remap);
    diff_file(PatchFileId::INTERP_ENTRIES, "interp_entries.bin", no_remap);
    diff_file(PatchFileId::ADMIN_CELLS, "admin_cells.bin", no_remap);
    diff_file(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin", no_remap);

    write_u32(pf, 0xFFFFFFFF); // end marker
    auto total = pf.tellp();
    std::cerr << "\nTotal patch: " << total << " bytes (" << total / 1024 / 1024 << " MiB)" << std::endl;
    return 0;
}

#include <cstdint>
#include <cstring>
#include <fstream>
+#include <functional>
+#include <iomanip>
#include <iostream>
#include <string>
#include <unordered_map>


Bash

cmake --build build -j$(nproc) --target geocoder-diff 2>&1 | tail -5

Build diff tool


[ 50%] Building CXX object CMakeFiles/geocoder-diff.dir/tools/geocoder_diff.cpp.o
[100%] Linking CXX executable geocoder-diff
[100%] Built target geocoder-diff


Now deploy both tools and test the full pipeline:


sshpass -p 'elon-is-insane' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" /root/traccar-geocoder/builder/build/geocoder-diff /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:~/ 2>&1

Deploy diff and patch tools


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.


sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
# Generate patch: canonical-old → canonical-new
time ~/geocoder-diff ~/canonical-old ~/canonical-new -o ~/europe-v2.gcpatch 2>&1
echo "==="
ls -lh ~/europe-v2.gcpatch
' 2>&1

Generate diff-based patch between canonical builds


Command running in background with ID: bipwzf03w. Output is being written to: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bipwzf03w.output


The diff is running — it needs to load both canonical builds (~14 GiB total) and compute block diffs with string remapping. Let me save a memory note about the current state while waiting.


Write

---
name: patch_system_design
description: Custom geocoder patch system - canonicalize/diff/patch tools for incremental index updates
type: project
---

Building a 3-tool patch system for incremental geocoder index updates:

1. **geocoder-canonicalize** — normalizes build output to deterministic byte order (strings sorted alphabetically, records sorted by content fingerprint, nodes grouped by parent record, entries derived from canonical data). ~5 min for Europe.

2. **geocoder-diff** — compares two canonical builds, produces .gcpatch file. Uses string remapping (old string offsets → new) before binary diffing data files. COPY/INSERT block encoding for delta sections, full replacement for small files.

3. **geocoder-patch** — applies .gcpatch to old canonical build, produces new canonical build.

**Why:** Sequential patching works because canonical output is deterministic — same semantic data always produces same bytes.

**Status:** All 3 tools compile and run. Canonicalize tested on Europe (5 min, produces 7 GiB canonical output). Diff tool running first real test.

**Key challenge:** String offsets differ between independent builds. The diff tool remaps old file's string offsets to match new build's pool before computing binary diff. This makes ~99.7% of record bytes match (only truly changed records differ).

**Files:**
- builder/tools/patch_format.h — shared format definitions
- builder/tools/geocoder_canonicalize.cpp
- builder/tools/geocoder_diff.cpp
- builder/tools/geocoder_patch.cpp
- builder/CMakeLists.txt — 3 new targets added

**How to apply:** The build_index.cpp was reverted to its original state (no --previous-build, no stabilization). Patching is entirely post-build.


File created successfully at: /root/.claude/projects/-root-traccar-geocoder/memory/project_patch_system.md

---
name: patch_system_design
description: Custom geocoder patch system - canonicalize/diff/patch tools for incremental index updates
type: project
---

Building a 3-tool patch system for incremental geocoder index updates:

1. **geocoder-canonicalize** — normalizes build output to deterministic byte order (strings sorted alphabetically, records sorted by content fingerprint, nodes grouped by parent record, entries derived from canonical data). ~5 min for Europe.

2. **geocoder-diff** — compares two canonical builds, produces .gcpatch file. Uses string remapping (old string offsets → new) before binary diffing data files. COPY/INSERT block encoding for delta sections, full replacement for small files.

3. **geocoder-patch** — applies .gcpatch to old canonical build, produces new canonical build.

**Why:** Sequential patching works because canonical output is deterministic — same semantic data always produces same bytes.

**Status:** All 3 tools compile and run. Canonicalize tested on Europe (5 min, produces 7 GiB canonical output). Diff tool running first real test.

**Key challenge:** String offsets differ between independent builds. The diff tool remaps old file's string offsets to match new build's pool before computing binary diff. This makes ~99.7% of record bytes match (only truly changed records differ).

**Files:**
- builder/tools/patch_format.h — shared format definitions
- builder/tools/geocoder_canonicalize.cpp
- builder/tools/geocoder_diff.cpp
- builder/tools/geocoder_patch.cpp
- builder/CMakeLists.txt — 3 new targets added

**How to apply:** The build_index.cpp was reverted to its original state (no --previous-build, no stabilization). Patching is entirely post-build.


Edit

- [project_pending_work.md](project_pending_work.md) — Pending tasks when node 3 comes back online (Europe revalidation, git push)

- [project_pending_work.md](project_pending_work.md) — Pending tasks when node 3 comes back online (Europe revalidation, git push)
- [project_patch_system.md](project_patch_system.md) — Custom geocoder patch system: canonicalize/diff/patch tools for incremental updates


<tool_use_error>File has not been read yet. Read it first before writing to it.</tool_use_error>

Error: File has not been read yet. Read it first before writing to it.


<system-reminder>This memory is 5 days old. Memories are point-in-time observations, not live state — claims about code behavior or file:line citations may be outdated. Verify against current code before asserting as fact.</system-reminder>
     1→# Memory Index
     2→
     3→## Project
     4→- [project_string_interleave_bug.md](project_string_interleave_bug.md) — Critical bugs found: string interleaving, address corruption, ring stitcher fixes
     5→- [project_validation_state.md](project_validation_state.md) — Current Europe validation results: 100% match on ways/addrs/interps, 4 remaining admin gaps
     6→
     7→- [project_pending_work.md](project_pending_work.md) — Pending tasks when node 3 comes back online (Europe revalidation, git push)
     8→
     9→## Reference
    10→- [reference_node_access.md](reference_node_access.md) — SSH access details for build nodes (node 3, 2a, 2b) via Tailscale
    11→

# Memory Index

## Project
- [project_string_interleave_bug.md](project_string_interleave_bug.md) — Critical bugs found: string interleaving, address corruption, ring stitcher fixes
- [project_validation_state.md](project_validation_state.md) — Current Europe validation results: 100% match on ways/addrs/interps, 4 remaining admin gaps

- [project_pending_work.md](project_pending_work.md) — Pending tasks when node 3 comes back online (Europe revalidation, git push)

## Reference
- [reference_node_access.md](reference_node_access.md) — SSH access details for build nodes (node 3, 2a, 2b) via Tailscale


- [project_pending_work.md](project_pending_work.md) — Pending tasks when node 3 comes back online (Europe revalidation, git push)

- [project_pending_work.md](project_pending_work.md) — Pending tasks when node 3 comes back online (Europe revalidation, git push)
- [project_patch_system.md](project_patch_system.md) — Custom geocoder patch system: canonicalize/diff/patch tools for incremental updates


The file /root/.claude/projects/-root-traccar-geocoder/memory/MEMORY.md has been updated successfully.

- [project_pending_work.md](project_pending_work.md) — Pending tasks when node 3 comes back online (Europe revalidation, git push)
- [project_patch_system.md](project_patch_system.md) — Custom geocoder patch system: canonicalize/diff/patch tools for incremental updates

# Memory Index

## Project
- [project_string_interleave_bug.md](project_string_interleave_bug.md) — Critical bugs found: string interleaving, address corruption, ring stitcher fixes
- [project_validation_state.md](project_validation_state.md) — Current Europe validation results: 100% match on ways/addrs/interps, 4 remaining admin gaps

- [project_pending_work.md](project_pending_work.md) — Pending tasks when node 3 comes back online (Europe revalidation, git push)

## Reference
- [reference_node_access.md](reference_node_access.md) — SSH access details for build nodes (node 3, 2a, 2b) via Tailscale

+- [project_patch_system.md](project_patch_system.md) — Custom geocoder patch system: canonicalize/diff/patch tools for incremental updates


Now let me check on the diff tool:


     1→Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
     2→Loading string pools...
     3→  Remap: 4980078 strings mapped
     4→  strings.bin: full replacement...
     5→    82573397 -> 21688674
     6→  addr_points.bin: delta 689067063 bytes (50.1% of 1312 MiB)
     7→  street_ways.bin: delta 68382693 bytes (29.3% of 222 MiB)
     8→  interp_ways.bin: full 109142 bytes (32.5% of 0 MiB)
     9→  admin_polygons.bin: full 5372585 bytes (49.7% of 10 MiB)
    10→  street_nodes.bin: delta 859609609 bytes (59.5% of 1378 MiB)
    11→  interp_nodes.bin: full 165160 bytes (2.7% of 5 MiB)
    12→  admin_vertices.bin: delta 791917646 bytes (57.4% of 1315 MiB)
    13→  geo_cells.bin:

Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Loading string pools...
  Remap: 4980078 strings mapped
  strings.bin: full replacement...
    82573397 -> 21688674
  addr_points.bin: delta 689067063 bytes (50.1% of 1312 MiB)
  street_ways.bin: delta 68382693 bytes (29.3% of 222 MiB)
  interp_ways.bin: full 109142 bytes (32.5% of 0 MiB)
  admin_polygons.bin: full 5372585 bytes (49.7% of 10 MiB)
  street_nodes.bin: delta 859609609 bytes (59.5% of 1378 MiB)
  interp_nodes.bin: full 165160 bytes (2.7% of 5 MiB)
  admin_vertices.bin: delta 791917646 bytes (57.4% of 1315 MiB)
  geo_cells.bin:


The diff is running but the results are poor — my block-based diff doesn't handle insertions (it compares at the same offset, not with sliding window). The coordinate files show 57-59% when zstd --patch-from gets 0.2-0.6%.

The fix: apply string remap to old files, write to temp, then use zstd --patch-from on the remapped files. Let me test this theory first:


Write

// Quick test: apply string remap to old canonical file, then compare with zstd.
// Usage: remap_test <old-dir> <new-dir> <output-dir>
// Writes remapped copies of data files to output-dir, then you can run
// zstd --patch-from on them.

#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>

#include "patch_format.h"

int main(int argc, char* argv[]) {
    if (argc < 4) {
        std::cerr << "Usage: remap_test <old-dir> <new-dir> <output-dir>" << std::endl;
        return 1;
    }
    std::string old_dir = argv[1], new_dir = argv[2], out_dir = argv[3];
    ensure_dir(out_dir);

    // Build string remap
    auto old_pool = read_file(old_dir + "/strings.bin");
    auto new_pool = read_file(new_dir + "/strings.bin");

    std::unordered_map<std::string, uint32_t> new_index;
    {
        size_t pos = 0;
        while (pos < new_pool.size()) {
            const char* s = new_pool.data() + pos;
            size_t len = strlen(s);
            new_index[std::string(s, len)] = static_cast<uint32_t>(pos);
            pos += len + 1;
        }
    }
    std::unordered_map<uint32_t, uint32_t> remap;
    {
        size_t pos = 0;
        while (pos < old_pool.size()) {
            const char* s = old_pool.data() + pos;
            size_t len = strlen(s);
            auto it = new_index.find(std::string(s, len));
            if (it != new_index.end())
                remap[static_cast<uint32_t>(pos)] = it->second;
            pos += len + 1;
        }
    }
    std::cerr << "Remap: " << remap.size() << " strings" << std::endl;

    // Remap addr_points.bin
    {
        auto data = read_file(old_dir + "/addr_points.bin");
        size_t n = data.size() / 16;
        for (size_t i = 0; i < n; i++) {
            char* p = data.data() + i * 16;
            for (size_t off : {8, 12}) {
                uint32_t old_id;
                memcpy(&old_id, p + off, 4);
                auto it = remap.find(old_id);
                if (it != remap.end()) memcpy(p + off, &it->second, 4);
            }
        }
        write_file(out_dir + "/addr_points.bin", data);
        std::cerr << "addr_points: " << n << " records remapped" << std::endl;
    }

    // Remap street_ways.bin
    {
        auto data = read_file(old_dir + "/street_ways.bin");
        size_t stride = (data.size() % 12 == 0) ? 12 : 9;
        size_t name_off = (stride == 12) ? 8 : 5;
        size_t n = data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = data.data() + i * stride;
            uint32_t old_id;
            memcpy(&old_id, p + name_off, 4);
            auto it = remap.find(old_id);
            if (it != remap.end()) memcpy(p + name_off, &it->second, 4);
        }
        write_file(out_dir + "/street_ways.bin", data);
        std::cerr << "street_ways: " << n << " records remapped" << std::endl;
    }

    // Copy files that don't need remapping (coord files, entries, cells)
    for (auto name : {"street_nodes.bin", "interp_nodes.bin", "admin_vertices.bin",
                       "geo_cells.bin", "street_entries.bin", "addr_entries.bin",
                       "interp_entries.bin", "admin_cells.bin", "admin_entries.bin"}) {
        auto data = read_file(old_dir + "/" + std::string(name));
        write_file(out_dir + "/" + std::string(name), data);
    }

    // Remap interp_ways
    {
        auto data = read_file(old_dir + "/interp_ways.bin");
        size_t stride = (data.size() % 20 == 0) ? 20 : 18;
        size_t street_off = (stride == 20) ? 8 : 5;
        size_t n = data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = data.data() + i * stride;
            uint32_t old_id;
            memcpy(&old_id, p + street_off, 4);
            auto it = remap.find(old_id);
            if (it != remap.end()) memcpy(p + street_off, &it->second, 4);
        }
        write_file(out_dir + "/interp_ways.bin", data);
        std::cerr << "interp_ways: " << n << " records remapped" << std::endl;
    }

    // Remap admin_polygons
    {
        auto data = read_file(old_dir + "/admin_polygons.bin");
        size_t stride = 24;
        for (size_t s : {24, 20, 19}) {
            if (data.size() % s == 0 && data.size() > 0) { stride = s; break; }
        }
        size_t n = data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = data.data() + i * stride;
            uint32_t old_id;
            memcpy(&old_id, p + 8, 4);
            auto it = remap.find(old_id);
            if (it != remap.end()) memcpy(p + 8, &it->second, 4);
        }
        write_file(out_dir + "/admin_polygons.bin", data);
        std::cerr << "admin_polygons: " << n << " records remapped" << std::endl;
    }

    // Copy strings (use new strings for the remapped build)
    write_file(out_dir + "/strings.bin", new_pool);

    std::cerr << "Done. Remapped files in " << out_dir << std::endl;
    return 0;
}


File created successfully at: /root/traccar-geocoder/builder/tools/remap_test.cpp

// Quick test: apply string remap to old canonical file, then compare with zstd.
// Usage: remap_test <old-dir> <new-dir> <output-dir>
// Writes remapped copies of data files to output-dir, then you can run
// zstd --patch-from on them.

#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>

#include "patch_format.h"

int main(int argc, char* argv[]) {
    if (argc < 4) {
        std::cerr << "Usage: remap_test <old-dir> <new-dir> <output-dir>" << std::endl;
        return 1;
    }
    std::string old_dir = argv[1], new_dir = argv[2], out_dir = argv[3];
    ensure_dir(out_dir);

    // Build string remap
    auto old_pool = read_file(old_dir + "/strings.bin");
    auto new_pool = read_file(new_dir + "/strings.bin");

    std::unordered_map<std::string, uint32_t> new_index;
    {
        size_t pos = 0;
        while (pos < new_pool.size()) {
            const char* s = new_pool.data() + pos;
            size_t len = strlen(s);
            new_index[std::string(s, len)] = static_cast<uint32_t>(pos);
            pos += len + 1;
        }
    }
    std::unordered_map<uint32_t, uint32_t> remap;
    {
        size_t pos = 0;
        while (pos < old_pool.size()) {
            const char* s = old_pool.data() + pos;
            size_t len = strlen(s);
            auto it = new_index.find(std::string(s, len));
            if (it != new_index.end())
                remap[static_cast<uint32_t>(pos)] = it->second;
            pos += len + 1;
        }
    }
    std::cerr << "Remap: " << remap.size() << " strings" << std::endl;

    // Remap addr_points.bin
    {
        auto data = read_file(old_dir + "/addr_points.bin");
        size_t n = data.size() / 16;
        for (size_t i = 0; i < n; i++) {
            char* p = data.data() + i * 16;
            for (size_t off : {8, 12}) {
                uint32_t old_id;
                memcpy(&old_id, p + off, 4);
                auto it = remap.find(old_id);
                if (it != remap.end()) memcpy(p + off, &it->second, 4);
            }
        }
        write_file(out_dir + "/addr_points.bin", data);
        std::cerr << "addr_points: " << n << " records remapped" << std::endl;
    }

    // Remap street_ways.bin
    {
        auto data = read_file(old_dir + "/street_ways.bin");
        size_t stride = (data.size() % 12 == 0) ? 12 : 9;
        size_t name_off = (stride == 12) ? 8 : 5;
        size_t n = data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = data.data() + i * stride;
            uint32_t old_id;
            memcpy(&old_id, p + name_off, 4);
            auto it = remap.find(old_id);
            if (it != remap.end()) memcpy(p + name_off, &it->second, 4);
        }
        write_file(out_dir + "/street_ways.bin", data);
        std::cerr << "street_ways: " << n << " records remapped" << std::endl;
    }

    // Copy files that don't need remapping (coord files, entries, cells)
    for (auto name : {"street_nodes.bin", "interp_nodes.bin", "admin_vertices.bin",
                       "geo_cells.bin", "street_entries.bin", "addr_entries.bin",
                       "interp_entries.bin", "admin_cells.bin", "admin_entries.bin"}) {
        auto data = read_file(old_dir + "/" + std::string(name));
        write_file(out_dir + "/" + std::string(name), data);
    }

    // Remap interp_ways
    {
        auto data = read_file(old_dir + "/interp_ways.bin");
        size_t stride = (data.size() % 20 == 0) ? 20 : 18;
        size_t street_off = (stride == 20) ? 8 : 5;
        size_t n = data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = data.data() + i * stride;
            uint32_t old_id;
            memcpy(&old_id, p + street_off, 4);
            auto it = remap.find(old_id);
            if (it != remap.end()) memcpy(p + street_off, &it->second, 4);
        }
        write_file(out_dir + "/interp_ways.bin", data);
        std::cerr << "interp_ways: " << n << " records remapped" << std::endl;
    }

    // Remap admin_polygons
    {
        auto data = read_file(old_dir + "/admin_polygons.bin");
        size_t stride = 24;
        for (size_t s : {24, 20, 19}) {
            if (data.size() % s == 0 && data.size() > 0) { stride = s; break; }
        }
        size_t n = data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = data.data() + i * stride;
            uint32_t old_id;
            memcpy(&old_id, p + 8, 4);
            auto it = remap.find(old_id);
            if (it != remap.end()) memcpy(p + 8, &it->second, 4);
        }
        write_file(out_dir + "/admin_polygons.bin", data);
        std::cerr << "admin_polygons: " << n << " records remapped" << std::endl;
    }

    // Copy strings (use new strings for the remapped build)
    write_file(out_dir + "/strings.bin", new_pool);

    std::cerr << "Done. Remapped files in " << out_dir << std::endl;
    return 0;
}


Edit

add_executable(geocoder-patch tools/geocoder_patch.cpp)
target_link_libraries(geocoder-patch PRIVATE ZLIB::ZLIB Threads::Threads)
target_include_directories(geocoder-patch PRIVATE tools)

add_executable(geocoder-patch tools/geocoder_patch.cpp)
target_link_libraries(geocoder-patch PRIVATE ZLIB::ZLIB Threads::Threads)
target_include_directories(geocoder-patch PRIVATE tools)

add_executable(remap-test tools/remap_test.cpp)
target_link_libraries(remap-test PRIVATE Threads::Threads)
target_include_directories(remap-test PRIVATE tools)


The file /root/traccar-geocoder/builder/CMakeLists.txt has been updated successfully.

add_executable(geocoder-patch tools/geocoder_patch.cpp)
target_link_libraries(geocoder-patch PRIVATE ZLIB::ZLIB Threads::Threads)
target_include_directories(geocoder-patch PRIVATE tools)

add_executable(remap-test tools/remap_test.cpp)
target_link_libraries(remap-test PRIVATE Threads::Threads)
target_include_directories(remap-test PRIVATE tools)

cmake_minimum_required(VERSION 3.16)
project(traccar-geocoder-build LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

find_package(Threads REQUIRED)
find_package(ZLIB REQUIRED)
find_package(BZip2 REQUIRED)
find_package(EXPAT REQUIRED)
find_package(s2 QUIET)

add_executable(build-index
    src/build_index.cpp
    src/pbf_reader.cpp
    src/s2_helpers.cpp
    src/ring_assembly.cpp
    src/cache.cpp
    src/continent_filter.cpp
    src/continent_boundaries.cpp
    src/cell_index.cpp
    src/interpolation.cpp
)

# Patch tools (no S2 dependency)
add_executable(geocoder-canonicalize tools/geocoder_canonicalize.cpp)
target_link_libraries(geocoder-canonicalize PRIVATE Threads::Threads)
target_include_directories(geocoder-canonicalize PRIVATE tools)

add_executable(geocoder-diff tools/geocoder_diff.cpp)
target_link_libraries(geocoder-diff PRIVATE ZLIB::ZLIB Threads::Threads)
target_include_directories(geocoder-diff PRIVATE tools)

add_executable(geocoder-patch tools/geocoder_patch.cpp)
target_link_libraries(geocoder-patch PRIVATE ZLIB::ZLIB Threads::Threads)
target_include_directories(geocoder-patch PRIVATE tools)

if(EXISTS /opt/homebrew/include)
    target_include_directories(build-index PRIVATE /opt/homebrew/include)
endif()

if(s2_FOUND)
    target_link_libraries(build-index PRIVATE
        s2::s2
        ZLIB::ZLIB
        BZip2::BZip2
        EXPAT::EXPAT
        Threads::Threads
    )
else()
    target_include_directories(build-index PRIVATE /usr/include/s2)
    target_link_libraries(build-index PRIVATE
        s2
        ZLIB::ZLIB
        BZip2::BZip2
        EXPAT::EXPAT
        Threads::Threads
    )
endif()


Bash

cmake -B build . 2>&1 | tail -2 && cmake --build build -j$(nproc) --target remap-test 2>&1 | tail -5

Build remap test tool


-- Generating done (0.0s)
-- Build files have been written to: /root/traccar-geocoder/builder/build
[ 50%] Building CXX object CMakeFiles/remap-test.dir/tools/remap_test.cpp.o
[100%] Linking CXX executable remap-test
[100%] Built target remap-test


sshpass -p 'elon-is-insane' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" /root/traccar-geocoder/builder/build/remap-test michtest@100.113.88.121:~/ 2>&1

Deploy remap test tool


sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
# 1. Remap old canonical files to use new string offsets
rm -rf ~/remapped-old
time ~/remap-test ~/canonical-old ~/canonical-new ~/remapped-old 2>&1
echo "==="
# 2. Now zstd --patch-from the remapped old vs new canonical
mkdir -p ~/patches-remapped
total=0
for f in ~/canonical-new/*.bin; do
    b=$(basename $f)
    old=~/remapped-old/$b
    if [ -f "$old" ]; then
        result=$(zstd --patch-from="$old" "$f" -o ~/patches-remapped/${b}.patch -f 2>&1)
        patchsize=$(stat -c%s ~/patches-remapped/${b}.patch 2>/dev/null || echo 0)
        newsize=$(stat -c%s $f)
        ratio=$(echo "scale=1; $patchsize*100/$newsize" | bc 2>/dev/null || echo "?")
        total=$((total + patchsize))
        printf "  %-25s %10s -> %10s (%s%%)\n" "$b" "$(numfmt --to=iec $newsize)" "$(numfmt --to=iec $patchsize)" "$ratio"
    fi
done
echo "  TOTAL: $(numfmt --to=iec $total)"
' 2>&1

Test string remap + zstd --patch-from approach


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Remap: 4980078 strings
addr_points: 85891830 records remapped
street_ways: 19452480 records remapped
interp_ways: 16785 records remapped
admin_polygons: 450907 records remapped
Done. Remapped files in /home/michtest/remapped-old
===

real	0m9.766s
user	0m6.429s
sys	0m3.336s
  addr_entries.bin                377M ->       299M (79.2%)
  addr_points.bin                 1.3G ->       3.8M (.2%)
  admin_cells.bin                 1.7M ->       459K (27.7%)
  admin_entries.bin               6.2M ->       2.5M (39.7%)
  admin_polygons.bin               11M ->       5.3M (50.4%)
  admin_vertices.bin              1.3G ->       2.9M (.2%)
  geo_cells.bin                   1.6G ->       379M (23.6%)
  interp_entries.bin              156K ->        66K (42.3%)
  interp_nodes.bin                5.9M ->       139K (2.3%)
  interp_ways.bin                 328K ->       108K (32.6%)
  street_entries.bin              560M ->       172M (30.6%)
  street_nodes.bin                1.4G ->       9.0M (.6%)
  street_ways.bin                 223M ->        69M (30.6%)
  strings.bin                      79M ->       9.8K (0%)
  TOTAL: 941M


**addr_points.bin went from 710 MiB to 3.8 MiB (0.2%)!** The string remap approach works perfectly. Total is now 941 MiB, down from 1.7 GiB. The remaining large items are the index/entry files (geo_cells 379 MiB, addr_entries 299 MiB, street_entries 172 MiB) which still have cascading offset shifts.

The data files are now excellent:
- addr_points: **0.2%** (was 54%)
- street_nodes: **0.6%** (unchanged, already good)
- admin_vertices: **0.2%** (unchanged)
- strings: **0.0%** (essentially zero)
- street_ways: **30.6%** (still has node_offset changes from reordering)

The remaining problem is the derived files (entries + cells). These change because record IDs shift when records are inserted/deleted. To fix those, the canonical format would need stable record IDs — which requires the previous-build approach the user rejected.

However, 941 MiB for a 7 GiB dataset (13.4%) is already a huge improvement. For planet it'd be ~12 GiB patches. Still not the 5-15 MiB target though.

The entry/cell files are the last frontier. They need a semantic-level diff (per-cell change tracking) rather than byte-level. Let me save this progress and note where we are — the remap+zstd approach is proven, and the next step is implementing the semantic entry diff.


Edit

**Status:** All 3 tools compile and run. Canonicalize tested on Europe (5 min, produces 7 GiB canonical output). Diff tool running first real test.

**Status:** Canonicalize works (5 min Europe). String remap + zstd proven: data files go from 50-67% to 0.2% patch ratios! Total patch 941 MiB (13.4% of 7 GiB). Remaining bottleneck: entry/cell files (geo_cells 379 MiB, addr_entries 299 MiB, street_entries 172 MiB) which cascade from record ID shifts. Need semantic per-cell diff for these.


The file /root/.claude/projects/-root-traccar-geocoder/memory/project_patch_system.md has been updated successfully.

---
name: patch_system_design
description: Custom geocoder patch system - canonicalize/diff/patch tools for incremental index updates
type: project
---

Building a 3-tool patch system for incremental geocoder index updates:

1. **geocoder-canonicalize** — normalizes build output to deterministic byte order (strings sorted alphabetically, records sorted by content fingerprint, nodes grouped by parent record, entries derived from canonical data). ~5 min for Europe.

2. **geocoder-diff** — compares two canonical builds, produces .gcpatch file. Uses string remapping (old string offsets → new) before binary diffing data files. COPY/INSERT block encoding for delta sections, full replacement for small files.

3. **geocoder-patch** — applies .gcpatch to old canonical build, produces new canonical build.

**Why:** Sequential patching works because canonical output is deterministic — same semantic data always produces same bytes.

**Status:** All 3 tools compile and run. Canonicalize tested on Europe (5 min, produces 7 GiB canonical output). Diff tool running first real test.

**Key challenge:** String offsets differ between independent builds. The diff tool remaps old file's string offsets to match new build's pool before computing binary diff. This makes ~99.7% of record bytes match (only truly changed records differ).

**Files:**
- builder/tools/patch_format.h — shared format definitions
- builder/tools/geocoder_canonicalize.cpp
- builder/tools/geocoder_diff.cpp
- builder/tools/geocoder_patch.cpp
- builder/CMakeLists.txt — 3 new targets added

**How to apply:** The build_index.cpp was reverted to its original state (no --previous-build, no stabilization). Patching is entirely post-build.

-**Status:** All 3 tools compile and run. Canonicalize tested on Europe (5 min, produces 7 GiB canonical output). Diff tool running first real test.

+**Status:** Canonicalize works (5 min Europe). String remap + zstd proven: data files go from 50-67% to 0.2% patch ratios! Total patch 941 MiB (13.4% of 7 GiB). Remaining bottleneck: entry/cell files (geo_cells 379 MiB, addr_entries 299 MiB, street_entries 172 MiB) which cascade from record ID shifts. Need semantic per-cell diff for these.


The string remap approach is a breakthrough — data files patch at **0.2%**. The remaining 941 MiB is almost entirely from entry/cell files where record ID shifts cascade.

**Results so far:**
| File | No remap | With remap | Notes |
|------|---------|-----------|-------|
| addr_points (1.3G) | 54% | **0.2%** | String remap eliminates byte diffs |
| street_nodes (1.4G) | 0.6% | 0.6% | Already good (coords don't change) |
| admin_vertices (1.3G) | 0.2% | 0.2% | Already good |
| strings (79M) | 0.09% | 0% | Already good |
| **geo_cells (1.6G)** | **24%** | **24%** | Entry offsets cascade |
| **addr_entries (377M)** | **79%** | **79%** | Record IDs shift |
| **street_entries (560M)** | **31%** | **31%** | Record IDs shift |
| street_ways (223M) | 31% | 31% | node_offset diffs from reordering |

The entry/cell files account for 850 MiB of the 941 MiB total. These need a semantic per-cell diff where the tool understands the cell→ID mapping structure. The user is away, so I'll continue implementing the custom entry diff to bring the total patch closer to the 5-15 MiB target.


<task-notification>
<task-id>bipwzf03w</task-id>
<tool-use-id>toolu_01FQnUigjoCKswXKE6uDiTYJ</tool-use-id>
<output-file>/tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bipwzf03w.output</output-file>
<status>completed</status>
<summary>Background command "Generate diff-based patch between canonical builds" completed (exit code 0)</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/bipwzf03w.output


     1→Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
     2→Loading string pools...
     3→  Remap: 4980078 strings mapped
     4→  strings.bin: full replacement...
     5→    82573397 -> 21688674
     6→  addr_points.bin: delta 689067063 bytes (50.1% of 1312 MiB)
     7→  street_ways.bin: delta 68382693 bytes (29.3% of 222 MiB)
     8→  interp_ways.bin: full 109142 bytes (32.5% of 0 MiB)
     9→  admin_polygons.bin: full 5372585 bytes (49.7% of 10 MiB)
    10→  street_nodes.bin: delta 859609609 bytes (59.5% of 1378 MiB)
    11→  interp_nodes.bin: full 165160 bytes (2.7% of 5 MiB)
    12→  admin_vertices.bin: delta 791917646 bytes (57.4% of 1315 MiB)
    13→  geo_cells.bin: delta 400426977 bytes (23.9% of 1596 MiB)
    14→  street_entries.bin: full 154113524 bytes (26.3% of 559 MiB)
    15→  addr_entries.bin: delta 223596715 bytes (56.6% of 376 MiB)
    16→  interp_entries.bin: full 65753 bytes (41.3% of 0 MiB)
    17→  admin_cells.bin: delta 588464 bytes (34.7% of 1 MiB)
    18→  admin_entries.bin: delta 2311926 bytes (36.0% of 6 MiB)
    19→
    20→Total patch: 3217416351 bytes (3068 MiB)
    21→===
    22→
    23→real	11m53.937s
    24→user	11m46.351s
    25→sys	0m7.519s
    26→-rw-rw-r-- 1 michtest michtest 3.0G Mar 27 00:49 /home/michtest/europe-v2.gcpatch
    27→

Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Loading string pools...
  Remap: 4980078 strings mapped
  strings.bin: full replacement...
    82573397 -> 21688674
  addr_points.bin: delta 689067063 bytes (50.1% of 1312 MiB)
  street_ways.bin: delta 68382693 bytes (29.3% of 222 MiB)
  interp_ways.bin: full 109142 bytes (32.5% of 0 MiB)
  admin_polygons.bin: full 5372585 bytes (49.7% of 10 MiB)
  street_nodes.bin: delta 859609609 bytes (59.5% of 1378 MiB)
  interp_nodes.bin: full 165160 bytes (2.7% of 5 MiB)
  admin_vertices.bin: delta 791917646 bytes (57.4% of 1315 MiB)
  geo_cells.bin: delta 400426977 bytes (23.9% of 1596 MiB)
  street_entries.bin: full 154113524 bytes (26.3% of 559 MiB)
  addr_entries.bin: delta 223596715 bytes (56.6% of 376 MiB)
  interp_entries.bin: full 65753 bytes (41.3% of 0 MiB)
  admin_cells.bin: delta 588464 bytes (34.7% of 1 MiB)
  admin_entries.bin: delta 2311926 bytes (36.0% of 6 MiB)

Total patch: 3217416351 bytes (3068 MiB)
===

real	11m53.937s
user	11m46.351s
sys	0m7.519s
-rw-rw-r-- 1 michtest michtest 3.0G Mar 27 00:49 /home/michtest/europe-v2.gcpatch


That was the earlier diff run with the naive block-based diff (3 GiB). Already superseded by the string remap + zstd approach which got us to 941 MiB.

The proven path forward is clear: string remap makes data files patch at 0.2%, but entry/cell files still dominate at 850 MiB. These need a semantic per-cell diff. Let me implement that now — a tool that understands entry files can diff them at the cell level, sending only changed cells.


Bash

sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
# Analyze how many cells actually changed
# Compare old vs new (remapped) addr_entries by parsing geo_cells + entries
python3 -c "
import struct, sys

def read_cells(path):
    data = open(path, \"rb\").read()
    n = len(data) // 20
    cells = {}
    for i in range(n):
        cell_id, s_off, a_off, i_off = struct.unpack_from(\"<QIII\", data, i*20)
        cells[cell_id] = (s_off, a_off, i_off)
    return cells

def read_entry(data, offset):
    if offset == 0xFFFFFFFF or offset + 2 > len(data):
        return []
    count = struct.unpack_from(\"<H\", data, offset)[0]
    if offset + 2 + count * 4 > len(data):
        return []
    return list(struct.unpack_from(\"<\" + \"I\" * count, data, offset + 2))

old_cells = read_cells(\"/home/michtest/remapped-old/geo_cells.bin\")
new_cells = read_cells(\"/home/michtest/canonical-new/geo_cells.bin\")

old_entries = {
    \"s\": open(\"/home/michtest/remapped-old/street_entries.bin\", \"rb\").read(),
    \"a\": open(\"/home/michtest/remapped-old/addr_entries.bin\", \"rb\").read(),
}
new_entries = {
    \"s\": open(\"/home/michtest/canonical-new/street_entries.bin\", \"rb\").read(),
    \"a\": open(\"/home/michtest/canonical-new/addr_entries.bin\", \"rb\").read(),
}

# Count changed cells
all_cells = set(old_cells) | set(new_cells)
changed = 0
same = 0
added = 0
removed = 0
for cid in all_cells:
    if cid not in old_cells:
        added += 1; continue
    if cid not in new_cells:
        removed += 1; continue
    o = old_cells[cid]
    n = new_cells[cid]
    # Compare addr entries
    oe = read_entry(old_entries[\"a\"], o[1])
    ne = read_entry(new_entries[\"a\"], n[1])
    if oe != ne:
        changed += 1
    else:
        same += 1

print(f\"Total cells: {len(all_cells)}\")
print(f\"Same: {same}\")
print(f\"Changed: {changed}\")
print(f\"Added: {added}\")
print(f\"Removed: {removed}\")
print(f\"Changed ratio: {(changed+added+removed)*100.0/len(all_cells):.2f}%\")
"
' 2>&1

Analyze how many cells actually changed between canonical builds


Command running in background with ID: b3iraonsi. Output is being written to: /tmp/claude-0/-root-traccar-geocoder/af1c4baa-0f9a-4420-a8b3-ce20dc28c8b9/tasks/b3iraonsi.output


     1→Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
     2→


Still running — parsing 84M cells in Python is slow. While that runs, let me think about the entry file problem more concretely.

The entry files reference record IDs (positions in the data arrays). In canonical format, record IDs are assigned by sort position. When records are added/removed, IDs shift. This means **every cell's entry list changes** even if the cell's actual streets/addresses didn't change, because the record IDs they reference shifted.

The fix requires a **record ID remap** in the entry files too. After string remapping the old data files:
- Old record at position X has the same bytes as new record at position Y
- The mapping X→Y is the record-level remap
- Apply this remap to old entry files → remapped entries match new entries

Let me build this record-level remap by matching records between old (remapped) and new data files:


Write

// Full remap test: apply string remap to data files AND record ID remap to entry/cell files.
// This should make zstd --patch-from produce near-zero patches for ALL files.
//
// Usage: full_remap_test <old-canonical> <new-canonical> <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>

#include "patch_format.h"

// Build record ID remap by matching records between remapped-old and new data files.
// Both files are in canonical sort order, so matching records are at approximately
// the same position. We match by byte equality (after string remap).
static std::vector<uint32_t> build_record_remap(
    const std::vector<char>& old_data, const std::vector<char>& new_data,
    size_t record_size)
{
    size_t old_count = old_data.size() / record_size;
    size_t new_count = new_data.size() / record_size;

    // Build hash map of new records: record_bytes → new_id
    // For 86M records × 16 bytes this is ~1.4 GiB for the map. Manageable.
    struct RecordHash {
        const char* data;
        size_t size;
        size_t operator()(uint32_t id) const {
            // FNV-1a hash of record bytes
            uint64_t h = 14695981039346656037ULL;
            const char* p = data + id * size;
            for (size_t i = 0; i < size; i++) {
                h ^= static_cast<uint8_t>(p[i]);
                h *= 1099511628211ULL;
            }
            return h;
        }
    };

    // Map: hash(record_bytes) → list of new_ids
    std::unordered_multimap<uint64_t, uint32_t> new_record_map;
    new_record_map.reserve(new_count);
    RecordHash hasher{new_data.data(), record_size};
    for (uint32_t i = 0; i < new_count; i++) {
        new_record_map.emplace(hasher(i), i);
    }

    // For each old record, find matching new record
    std::vector<uint32_t> old_to_new(old_count, 0xFFFFFFFF);
    RecordHash old_hasher{old_data.data(), record_size};
    size_t matched = 0, unmatched = 0;
    for (uint32_t oi = 0; oi < old_count; oi++) {
        uint64_t h = old_hasher(oi);
        auto range = new_record_map.equal_range(h);
        bool found = false;
        for (auto it = range.first; it != range.second; ++it) {
            uint32_t ni = it->second;
            if (memcmp(old_data.data() + oi * record_size,
                       new_data.data() + ni * record_size, record_size) == 0) {
                old_to_new[oi] = ni;
                new_record_map.erase(it);
                matched++;
                found = true;
                break;
            }
        }
        if (!found) unmatched++;
    }
    std::cerr << "    matched=" << matched << " unmatched=" << unmatched
              << " new_only=" << new_record_map.size() << std::endl;
    return old_to_new;
}

// Apply record ID remap to entry data
static void remap_entries(std::vector<char>& entries_data,
                           const std::vector<char>& cells_data,
                           size_t cell_stride, // 20 for geo, 12 for admin
                           size_t offset_pos,  // byte offset of entry_offset within cell record
                           const std::vector<uint32_t>& id_remap,
                           bool has_flags = false)
{
    size_t n_cells = cells_data.size() / cell_stride;
    for (size_t i = 0; i < n_cells; i++) {
        uint32_t entry_off;
        memcpy(&entry_off, cells_data.data() + i * cell_stride + offset_pos, 4);
        if (entry_off == 0xFFFFFFFF || entry_off + 2 > entries_data.size()) continue;

        uint16_t count;
        memcpy(&count, entries_data.data() + entry_off, 2);
        for (uint16_t j = 0; j < count; j++) {
            size_t id_pos = entry_off + 2 + j * 4;
            if (id_pos + 4 > entries_data.size()) break;
            uint32_t old_id;
            memcpy(&old_id, entries_data.data() + id_pos, 4);

            uint32_t flags = has_flags ? (old_id & 0x80000000) : 0;
            uint32_t masked = old_id & 0x7FFFFFFF;

            if (masked < id_remap.size() && id_remap[masked] != 0xFFFFFFFF) {
                uint32_t new_id = id_remap[masked] | flags;
                memcpy(entries_data.data() + id_pos, &new_id, 4);
            }
        }
    }
}

int main(int argc, char* argv[]) {
    if (argc < 4) {
        std::cerr << "Usage: full_remap_test <old-canonical> <new-canonical> <output-dir>" << std::endl;
        return 1;
    }
    std::string old_dir = argv[1], new_dir = argv[2], out_dir = argv[3];
    ensure_dir(out_dir);

    // Step 1: Build string remap
    std::cerr << "Building string remap..." << std::endl;
    auto old_pool = read_file(old_dir + "/strings.bin");
    auto new_pool = read_file(new_dir + "/strings.bin");

    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        std::unordered_map<std::string, uint32_t> new_index;
        size_t pos = 0;
        while (pos < new_pool.size()) {
            const char* s = new_pool.data() + pos;
            size_t len = strlen(s);
            new_index[std::string(s, len)] = static_cast<uint32_t>(pos);
            pos += len + 1;
        }
        pos = 0;
        while (pos < old_pool.size()) {
            const char* s = old_pool.data() + pos;
            size_t len = strlen(s);
            auto it = new_index.find(std::string(s, len));
            if (it != new_index.end())
                str_remap[static_cast<uint32_t>(pos)] = it->second;
            pos += len + 1;
        }
    }
    std::cerr << "  " << str_remap.size() << " strings mapped" << std::endl;

    // Step 2: Remap and match addr_points
    std::cerr << "Remapping addr_points..." << std::endl;
    std::vector<uint32_t> addr_remap;
    {
        auto old_data = read_file(old_dir + "/addr_points.bin");
        auto new_data = read_file(new_dir + "/addr_points.bin");

        // Apply string remap to old
        size_t n = old_data.size() / 16;
        for (size_t i = 0; i < n; i++) {
            char* p = old_data.data() + i * 16;
            for (size_t off : {8, 12}) {
                uint32_t old_id; memcpy(&old_id, p + off, 4);
                auto it = str_remap.find(old_id);
                if (it != str_remap.end()) memcpy(p + off, &it->second, 4);
            }
        }
        write_file(out_dir + "/addr_points.bin", old_data);

        // Build record remap
        addr_remap = build_record_remap(old_data, new_data, 16);
    }

    // Step 3: Remap and match street_ways
    std::cerr << "Remapping street_ways..." << std::endl;
    std::vector<uint32_t> way_remap;
    {
        auto old_data = read_file(old_dir + "/street_ways.bin");
        auto new_data = read_file(new_dir + "/street_ways.bin");
        size_t stride = (old_data.size() % 12 == 0) ? 12 : 9;
        size_t name_off = (stride == 12) ? 8 : 5;

        size_t n = old_data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = old_data.data() + i * stride;
            uint32_t old_id; memcpy(&old_id, p + name_off, 4);
            auto it = str_remap.find(old_id);
            if (it != str_remap.end()) memcpy(p + name_off, &it->second, 4);
        }
        write_file(out_dir + "/street_ways.bin", old_data);
        way_remap = build_record_remap(old_data, new_data, stride);
    }

    // Step 4: Remap interp_ways
    std::cerr << "Remapping interp_ways..." << std::endl;
    std::vector<uint32_t> interp_remap;
    {
        auto old_data = read_file(old_dir + "/interp_ways.bin");
        auto new_data = read_file(new_dir + "/interp_ways.bin");
        size_t stride = (old_data.size() % 20 == 0 && old_data.size() > 0) ? 20 : 18;
        size_t street_off = (stride == 20) ? 8 : 5;

        size_t n = old_data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = old_data.data() + i * stride;
            uint32_t old_id; memcpy(&old_id, p + street_off, 4);
            auto it = str_remap.find(old_id);
            if (it != str_remap.end()) memcpy(p + street_off, &it->second, 4);
        }
        write_file(out_dir + "/interp_ways.bin", old_data);
        interp_remap = build_record_remap(old_data, new_data, stride);
    }

    // Step 5: Remap admin_polygons
    std::cerr << "Remapping admin_polygons..." << std::endl;
    std::vector<uint32_t> admin_remap;
    {
        auto old_data = read_file(old_dir + "/admin_polygons.bin");
        auto new_data = read_file(new_dir + "/admin_polygons.bin");
        size_t stride = 24;
        for (size_t s : {24, 20, 19}) {
            if (old_data.size() % s == 0 && old_data.size() > 0) { stride = s; break; }
        }
        size_t n = old_data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = old_data.data() + i * stride;
            uint32_t old_id; memcpy(&old_id, p + 8, 4);
            auto it = str_remap.find(old_id);
            if (it != str_remap.end()) memcpy(p + 8, &it->second, 4);
        }
        write_file(out_dir + "/admin_polygons.bin", old_data);
        admin_remap = build_record_remap(old_data, new_data, stride);
    }

    // Step 6: Copy coordinate files unchanged (they don't have string refs)
    for (auto name : {"street_nodes.bin", "interp_nodes.bin", "admin_vertices.bin"}) {
        auto data = read_file(old_dir + "/" + std::string(name));
        write_file(out_dir + "/" + std::string(name), data);
    }

    // Step 7: Remap entry files using record ID remaps
    std::cerr << "Remapping entry files..." << std::endl;
    {
        auto geo_cells = read_file(old_dir + "/geo_cells.bin");
        auto street_entries = read_file(old_dir + "/street_entries.bin");
        auto addr_entries = read_file(old_dir + "/addr_entries.bin");
        auto interp_entries = read_file(old_dir + "/interp_entries.bin");

        remap_entries(street_entries, geo_cells, 20, 8, way_remap);
        remap_entries(addr_entries, geo_cells, 20, 12, addr_remap);
        remap_entries(interp_entries, geo_cells, 20, 16, interp_remap);

        write_file(out_dir + "/geo_cells.bin", geo_cells);
        write_file(out_dir + "/street_entries.bin", street_entries);
        write_file(out_dir + "/addr_entries.bin", addr_entries);
        write_file(out_dir + "/interp_entries.bin", interp_entries);
    }
    {
        auto admin_cells = read_file(old_dir + "/admin_cells.bin");
        auto admin_entries = read_file(old_dir + "/admin_entries.bin");
        remap_entries(admin_entries, admin_cells, 12, 8, admin_remap, true);
        write_file(out_dir + "/admin_cells.bin", admin_cells);
        write_file(out_dir + "/admin_entries.bin", admin_entries);
    }

    // Copy strings (use new pool)
    write_file(out_dir + "/strings.bin", new_pool);

    std::cerr << "Done. Fully remapped files in " << out_dir << std::endl;
    return 0;
}


File created successfully at: /root/traccar-geocoder/builder/tools/full_remap_test.cpp

// Full remap test: apply string remap to data files AND record ID remap to entry/cell files.
// This should make zstd --patch-from produce near-zero patches for ALL files.
//
// Usage: full_remap_test <old-canonical> <new-canonical> <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>

#include "patch_format.h"

// Build record ID remap by matching records between remapped-old and new data files.
// Both files are in canonical sort order, so matching records are at approximately
// the same position. We match by byte equality (after string remap).
static std::vector<uint32_t> build_record_remap(
    const std::vector<char>& old_data, const std::vector<char>& new_data,
    size_t record_size)
{
    size_t old_count = old_data.size() / record_size;
    size_t new_count = new_data.size() / record_size;

    // Build hash map of new records: record_bytes → new_id
    // For 86M records × 16 bytes this is ~1.4 GiB for the map. Manageable.
    struct RecordHash {
        const char* data;
        size_t size;
        size_t operator()(uint32_t id) const {
            // FNV-1a hash of record bytes
            uint64_t h = 14695981039346656037ULL;
            const char* p = data + id * size;
            for (size_t i = 0; i < size; i++) {
                h ^= static_cast<uint8_t>(p[i]);
                h *= 1099511628211ULL;
            }
            return h;
        }
    };

    // Map: hash(record_bytes) → list of new_ids
    std::unordered_multimap<uint64_t, uint32_t> new_record_map;
    new_record_map.reserve(new_count);
    RecordHash hasher{new_data.data(), record_size};
    for (uint32_t i = 0; i < new_count; i++) {
        new_record_map.emplace(hasher(i), i);
    }

    // For each old record, find matching new record
    std::vector<uint32_t> old_to_new(old_count, 0xFFFFFFFF);
    RecordHash old_hasher{old_data.data(), record_size};
    size_t matched = 0, unmatched = 0;
    for (uint32_t oi = 0; oi < old_count; oi++) {
        uint64_t h = old_hasher(oi);
        auto range = new_record_map.equal_range(h);
        bool found = false;
        for (auto it = range.first; it != range.second; ++it) {
            uint32_t ni = it->second;
            if (memcmp(old_data.data() + oi * record_size,
                       new_data.data() + ni * record_size, record_size) == 0) {
                old_to_new[oi] = ni;
                new_record_map.erase(it);
                matched++;
                found = true;
                break;
            }
        }
        if (!found) unmatched++;
    }
    std::cerr << "    matched=" << matched << " unmatched=" << unmatched
              << " new_only=" << new_record_map.size() << std::endl;
    return old_to_new;
}

// Apply record ID remap to entry data
static void remap_entries(std::vector<char>& entries_data,
                           const std::vector<char>& cells_data,
                           size_t cell_stride, // 20 for geo, 12 for admin
                           size_t offset_pos,  // byte offset of entry_offset within cell record
                           const std::vector<uint32_t>& id_remap,
                           bool has_flags = false)
{
    size_t n_cells = cells_data.size() / cell_stride;
    for (size_t i = 0; i < n_cells; i++) {
        uint32_t entry_off;
        memcpy(&entry_off, cells_data.data() + i * cell_stride + offset_pos, 4);
        if (entry_off == 0xFFFFFFFF || entry_off + 2 > entries_data.size()) continue;

        uint16_t count;
        memcpy(&count, entries_data.data() + entry_off, 2);
        for (uint16_t j = 0; j < count; j++) {
            size_t id_pos = entry_off + 2 + j * 4;
            if (id_pos + 4 > entries_data.size()) break;
            uint32_t old_id;
            memcpy(&old_id, entries_data.data() + id_pos, 4);

            uint32_t flags = has_flags ? (old_id & 0x80000000) : 0;
            uint32_t masked = old_id & 0x7FFFFFFF;

            if (masked < id_remap.size() && id_remap[masked] != 0xFFFFFFFF) {
                uint32_t new_id = id_remap[masked] | flags;
                memcpy(entries_data.data() + id_pos, &new_id, 4);
            }
        }
    }
}

int main(int argc, char* argv[]) {
    if (argc < 4) {
        std::cerr << "Usage: full_remap_test <old-canonical> <new-canonical> <output-dir>" << std::endl;
        return 1;
    }
    std::string old_dir = argv[1], new_dir = argv[2], out_dir = argv[3];
    ensure_dir(out_dir);

    // Step 1: Build string remap
    std::cerr << "Building string remap..." << std::endl;
    auto old_pool = read_file(old_dir + "/strings.bin");
    auto new_pool = read_file(new_dir + "/strings.bin");

    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        std::unordered_map<std::string, uint32_t> new_index;
        size_t pos = 0;
        while (pos < new_pool.size()) {
            const char* s = new_pool.data() + pos;
            size_t len = strlen(s);
            new_index[std::string(s, len)] = static_cast<uint32_t>(pos);
            pos += len + 1;
        }
        pos = 0;
        while (pos < old_pool.size()) {
            const char* s = old_pool.data() + pos;
            size_t len = strlen(s);
            auto it = new_index.find(std::string(s, len));
            if (it != new_index.end())
                str_remap[static_cast<uint32_t>(pos)] = it->second;
            pos += len + 1;
        }
    }
    std::cerr << "  " << str_remap.size() << " strings mapped" << std::endl;

    // Step 2: Remap and match addr_points
    std::cerr << "Remapping addr_points..." << std::endl;
    std::vector<uint32_t> addr_remap;
    {
        auto old_data = read_file(old_dir + "/addr_points.bin");
        auto new_data = read_file(new_dir + "/addr_points.bin");

        // Apply string remap to old
        size_t n = old_data.size() / 16;
        for (size_t i = 0; i < n; i++) {
            char* p = old_data.data() + i * 16;
            for (size_t off : {8, 12}) {
                uint32_t old_id; memcpy(&old_id, p + off, 4);
                auto it = str_remap.find(old_id);
                if (it != str_remap.end()) memcpy(p + off, &it->second, 4);
            }
        }
        write_file(out_dir + "/addr_points.bin", old_data);

        // Build record remap
        addr_remap = build_record_remap(old_data, new_data, 16);
    }

    // Step 3: Remap and match street_ways
    std::cerr << "Remapping street_ways..." << std::endl;
    std::vector<uint32_t> way_remap;
    {
        auto old_data = read_file(old_dir + "/street_ways.bin");
        auto new_data = read_file(new_dir + "/street_ways.bin");
        size_t stride = (old_data.size() % 12 == 0) ? 12 : 9;
        size_t name_off = (stride == 12) ? 8 : 5;

        size_t n = old_data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = old_data.data() + i * stride;
            uint32_t old_id; memcpy(&old_id, p + name_off, 4);
            auto it = str_remap.find(old_id);
            if (it != str_remap.end()) memcpy(p + name_off, &it->second, 4);
        }
        write_file(out_dir + "/street_ways.bin", old_data);
        way_remap = build_record_remap(old_data, new_data, stride);
    }

    // Step 4: Remap interp_ways
    std::cerr << "Remapping interp_ways..." << std::endl;
    std::vector<uint32_t> interp_remap;
    {
        auto old_data = read_file(old_dir + "/interp_ways.bin");
        auto new_data = read_file(new_dir + "/interp_ways.bin");
        size_t stride = (old_data.size() % 20 == 0 && old_data.size() > 0) ? 20 : 18;
        size_t street_off = (stride == 20) ? 8 : 5;

        size_t n = old_data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = old_data.data() + i * stride;
            uint32_t old_id; memcpy(&old_id, p + street_off, 4);
            auto it = str_remap.find(old_id);
            if (it != str_remap.end()) memcpy(p + street_off, &it->second, 4);
        }
        write_file(out_dir + "/interp_ways.bin", old_data);
        interp_remap = build_record_remap(old_data, new_data, stride);
    }

    // Step 5: Remap admin_polygons
    std::cerr << "Remapping admin_polygons..." << std::endl;
    std::vector<uint32_t> admin_remap;
    {
        auto old_data = read_file(old_dir + "/admin_polygons.bin");
        auto new_data = read_file(new_dir + "/admin_polygons.bin");
        size_t stride = 24;
        for (size_t s : {24, 20, 19}) {
            if (old_data.size() % s == 0 && old_data.size() > 0) { stride = s; break; }
        }
        size_t n = old_data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = old_data.data() + i * stride;
            uint32_t old_id; memcpy(&old_id, p + 8, 4);
            auto it = str_remap.find(old_id);
            if (it != str_remap.end()) memcpy(p + 8, &it->second, 4);
        }
        write_file(out_dir + "/admin_polygons.bin", old_data);
        admin_remap = build_record_remap(old_data, new_data, stride);
    }

    // Step 6: Copy coordinate files unchanged (they don't have string refs)
    for (auto name : {"street_nodes.bin", "interp_nodes.bin", "admin_vertices.bin"}) {
        auto data = read_file(old_dir + "/" + std::string(name));
        write_file(out_dir + "/" + std::string(name), data);
    }

    // Step 7: Remap entry files using record ID remaps
    std::cerr << "Remapping entry files..." << std::endl;
    {
        auto geo_cells = read_file(old_dir + "/geo_cells.bin");
        auto street_entries = read_file(old_dir + "/street_entries.bin");
        auto addr_entries = read_file(old_dir + "/addr_entries.bin");
        auto interp_entries = read_file(old_dir + "/interp_entries.bin");

        remap_entries(street_entries, geo_cells, 20, 8, way_remap);
        remap_entries(addr_entries, geo_cells, 20, 12, addr_remap);
        remap_entries(interp_entries, geo_cells, 20, 16, interp_remap);

        write_file(out_dir + "/geo_cells.bin", geo_cells);
        write_file(out_dir + "/street_entries.bin", street_entries);
        write_file(out_dir + "/addr_entries.bin", addr_entries);
        write_file(out_dir + "/interp_entries.bin", interp_entries);
    }
    {
        auto admin_cells = read_file(old_dir + "/admin_cells.bin");
        auto admin_entries = read_file(old_dir + "/admin_entries.bin");
        remap_entries(admin_entries, admin_cells, 12, 8, admin_remap, true);
        write_file(out_dir + "/admin_cells.bin", admin_cells);
        write_file(out_dir + "/admin_entries.bin", admin_entries);
    }

    // Copy strings (use new pool)
    write_file(out_dir + "/strings.bin", new_pool);

    std::cerr << "Done. Fully remapped files in " << out_dir << std::endl;
    return 0;
}


Edit

add_executable(remap-test tools/remap_test.cpp)
target_link_libraries(remap-test PRIVATE Threads::Threads)
target_include_directories(remap-test PRIVATE tools)

add_executable(remap-test tools/remap_test.cpp)
target_link_libraries(remap-test PRIVATE Threads::Threads)
target_include_directories(remap-test PRIVATE tools)

add_executable(full-remap-test tools/full_remap_test.cpp)
target_link_libraries(full-remap-test PRIVATE Threads::Threads)
target_include_directories(full-remap-test PRIVATE tools)


add_executable(remap-test tools/remap_test.cpp)
target_link_libraries(remap-test PRIVATE Threads::Threads)
target_include_directories(remap-test PRIVATE tools)

add_executable(full-remap-test tools/full_remap_test.cpp)
target_link_libraries(full-remap-test PRIVATE Threads::Threads)
target_include_directories(full-remap-test PRIVATE tools)

cmake_minimum_required(VERSION 3.16)
project(traccar-geocoder-build LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

find_package(Threads REQUIRED)
find_package(ZLIB REQUIRED)
find_package(BZip2 REQUIRED)
find_package(EXPAT REQUIRED)
find_package(s2 QUIET)

add_executable(build-index
    src/build_index.cpp
    src/pbf_reader.cpp
    src/s2_helpers.cpp
    src/ring_assembly.cpp
    src/cache.cpp
    src/continent_filter.cpp
    src/continent_boundaries.cpp
    src/cell_index.cpp
    src/interpolation.cpp
)

# Patch tools (no S2 dependency)
add_executable(geocoder-canonicalize tools/geocoder_canonicalize.cpp)
target_link_libraries(geocoder-canonicalize PRIVATE Threads::Threads)
target_include_directories(geocoder-canonicalize PRIVATE tools)

add_executable(geocoder-diff tools/geocoder_diff.cpp)
target_link_libraries(geocoder-diff PRIVATE ZLIB::ZLIB Threads::Threads)
target_include_directories(geocoder-diff PRIVATE tools)

add_executable(geocoder-patch tools/geocoder_patch.cpp)
target_link_libraries(geocoder-patch PRIVATE ZLIB::ZLIB Threads::Threads)
target_include_directories(geocoder-patch PRIVATE tools)

add_executable(remap-test tools/remap_test.cpp)
target_link_libraries(remap-test PRIVATE Threads::Threads)
target_include_directories(remap-test PRIVATE tools)

if(EXISTS /opt/homebrew/include)
    target_include_directories(build-index PRIVATE /opt/homebrew/include)
endif()

if(s2_FOUND)
    target_link_libraries(build-index PRIVATE
        s2::s2
        ZLIB::ZLIB
        BZip2::BZip2
        EXPAT::EXPAT
        Threads::Threads
    )
else()
    target_include_directories(build-index PRIVATE /usr/include/s2)
    target_link_libraries(build-index PRIVATE
        s2
        ZLIB::ZLIB
        BZip2::BZip2
        EXPAT::EXPAT
        Threads::Threads
    )
endif()


Bash

cmake -B build . 2>&1 | tail -2 && cmake --build build -j$(nproc) --target full-remap-test 2>&1 | tail -5

Build full remap test


-- Generating done (0.0s)
-- Build files have been written to: /root/traccar-geocoder/builder/build
[ 50%] Building CXX object CMakeFiles/full-remap-test.dir/tools/full_remap_test.cpp.o
[100%] Linking CXX executable full-remap-test
[100%] Built target full-remap-test


sshpass -p 'elon-is-insane' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" /root/traccar-geocoder/builder/build/full-remap-test michtest@100.113.88.121:~/ 2>&1

Deploy full remap test


sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
rm -rf ~/full-remapped
time ~/full-remap-test ~/canonical-old ~/canonical-new ~/full-remapped 2>&1
echo "=== ZSTD PATCHES ==="
mkdir -p ~/patches-full-remap
total=0
for f in ~/canonical-new/*.bin; do
    b=$(basename $f)
    old=~/full-remapped/$b
    if [ -f "$old" ]; then
        zstd --patch-from="$old" "$f" -o ~/patches-full-remap/${b}.patch -f 2>/dev/null
        patchsize=$(stat -c%s ~/patches-full-remap/${b}.patch 2>/dev/null || echo 0)
        newsize=$(stat -c%s $f)
        ratio=$(echo "scale=2; $patchsize*100/$newsize" | bc 2>/dev/null || echo "?")
        total=$((total + patchsize))
        printf "  %-25s %10s -> %10s (%s%%)\n" "$b" "$(numfmt --to=iec $newsize)" "$(numfmt --to=iec $patchsize)" "$ratio"
    fi
done
echo "  TOTAL: $(numfmt --to=iec $total)"
' 2>&1

Full remap test: string + record ID remap, then zstd patches


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Building string remap...
  4980078 strings mapped
Remapping addr_points...
    matched=85825938 unmatched=65892 new_only=164173
Remapping street_ways...
    matched=9166 unmatched=19443314 new_only=19459682
Remapping interp_ways...
    matched=0 unmatched=16785 new_only=16771
Remapping admin_polygons...
    matched=0 unmatched=450907 new_only=450864
Remapping entry files...
Done. Fully remapped files in /home/michtest/full-remapped
=== ZSTD PATCHES ===

real	1m10.260s
user	0m58.138s
sys	0m12.117s
  addr_entries.bin                377M ->       2.0M (.52%)
  addr_points.bin                 1.3G ->       3.8M (.28%)
  admin_cells.bin                 1.7M ->       459K (27.71%)
  admin_entries.bin               6.2M ->       2.5M (39.70%)
  admin_polygons.bin               11M ->       5.3M (50.46%)
  admin_vertices.bin              1.3G ->       2.9M (.21%)
  geo_cells.bin                   1.6G ->       379M (23.67%)
  interp_entries.bin              156K ->        66K (42.35%)
  interp_nodes.bin                5.9M ->       139K (2.30%)
  interp_ways.bin                 328K ->       108K (32.69%)
  street_entries.bin              560M ->       172M (30.60%)
  street_nodes.bin                1.4G ->       9.0M (.64%)
  street_ways.bin                 223M ->        69M (30.69%)
  strings.bin                      79M ->       9.8K (.01%)
  TOTAL: 644M


addr_entries improved from 299 MiB to 2 MiB! But street_ways and admin_polygons show 0% match because the `node_offset`/`vertex_offset` fields differ between builds. I need to match these records by CONTENT (name + coordinates) not by raw bytes.

Let me fix the matching to use content-based fingerprints:


Edit

    // Step 3: Remap and match street_ways
    std::cerr << "Remapping street_ways..." << std::endl;
    std::vector<uint32_t> way_remap;
    {
        auto old_data = read_file(old_dir + "/street_ways.bin");
        auto new_data = read_file(new_dir + "/street_ways.bin");
        size_t stride = (old_data.size() % 12 == 0) ? 12 : 9;
        size_t name_off = (stride == 12) ? 8 : 5;

        size_t n = old_data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = old_data.data() + i * stride;
            uint32_t old_id; memcpy(&old_id, p + name_off, 4);
            auto it = str_remap.find(old_id);
            if (it != str_remap.end()) memcpy(p + name_off, &it->second, 4);
        }
        write_file(out_dir + "/street_ways.bin", old_data);
        way_remap = build_record_remap(old_data, new_data, stride);
    }

    // Step 3: Remap and match street_ways (match by name + node coords, not raw bytes)
    std::cerr << "Remapping street_ways..." << std::endl;
    std::vector<uint32_t> way_remap;
    {
        auto old_data = read_file(old_dir + "/street_ways.bin");
        auto new_data = read_file(new_dir + "/street_ways.bin");
        auto old_nodes = read_file(old_dir + "/street_nodes.bin");
        auto new_nodes = read_file(new_dir + "/street_nodes.bin");
        size_t stride = (old_data.size() % 12 == 0) ? 12 : 9;
        size_t name_off = (stride == 12) ? 8 : 5;

        // Apply string remap to old ways
        size_t old_count = old_data.size() / stride;
        for (size_t i = 0; i < old_count; i++) {
            char* p = old_data.data() + i * stride;
            uint32_t old_id; memcpy(&old_id, p + name_off, 4);
            auto it = str_remap.find(old_id);
            if (it != str_remap.end()) memcpy(p + name_off, &it->second, 4);
        }
        write_file(out_dir + "/street_ways.bin", old_data);

        // Content-based matching: hash(name_id, node_count, node_coords)
        auto way_hash = [&](const char* way_data, const char* node_data, size_t stride, size_t name_off) -> uint64_t {
            uint32_t node_offset, name_id;
            uint8_t node_count;
            memcpy(&node_offset, way_data, 4);
            node_count = static_cast<uint8_t>(way_data[4]);
            memcpy(&name_id, way_data + name_off, 4);
            uint64_t h = 14695981039346656037ULL;
            auto mix = [&h](uint64_t v) { h ^= v; h *= 1099511628211ULL; };
            mix(name_id);
            mix(node_count);
            for (uint8_t j = 0; j < node_count; j++) {
                size_t noff = (node_offset + j) * 8;
                float lat, lng;
                memcpy(&lat, node_data + noff, 4);
                memcpy(&lng, node_data + noff + 4, 4);
                mix(to_grid(lat));
                mix(to_grid(lng));
            }
            return h;
        };

        size_t new_count = new_data.size() / stride;
        // Build hash map of new ways
        std::unordered_multimap<uint64_t, uint32_t> new_map;
        new_map.reserve(new_count);
        for (uint32_t i = 0; i < new_count; i++) {
            new_map.emplace(way_hash(new_data.data() + i * stride, new_nodes.data(), stride, name_off), i);
        }

        way_remap.assign(old_count, 0xFFFFFFFF);
        size_t matched = 0;
        for (uint32_t i = 0; i < old_count; i++) {
            uint64_t h = way_hash(old_data.data() + i * stride, old_nodes.data(), stride, name_off);
            auto it = new_map.find(h);
            if (it != new_map.end()) {
                way_remap[i] = it->second;
                new_map.erase(it);
                matched++;
            }
        }
        std::cerr << "    matched=" << matched << " unmatched=" << (old_count - matched)
                  << " new_only=" << new_map.size() << std::endl;
    }


The file /root/traccar-geocoder/builder/tools/full_remap_test.cpp has been updated successfully.

    // Step 3: Remap and match street_ways (match by name + node coords, not raw bytes)
    std::cerr << "Remapping street_ways..." << std::endl;
    std::vector<uint32_t> way_remap;
    {
        auto old_data = read_file(old_dir + "/street_ways.bin");
        auto new_data = read_file(new_dir + "/street_ways.bin");
        auto old_nodes = read_file(old_dir + "/street_nodes.bin");
        auto new_nodes = read_file(new_dir + "/street_nodes.bin");
        size_t stride = (old_data.size() % 12 == 0) ? 12 : 9;
        size_t name_off = (stride == 12) ? 8 : 5;

        // Apply string remap to old ways
        size_t old_count = old_data.size() / stride;
        for (size_t i = 0; i < old_count; i++) {
            char* p = old_data.data() + i * stride;
            uint32_t old_id; memcpy(&old_id, p + name_off, 4);
            auto it = str_remap.find(old_id);
            if (it != str_remap.end()) memcpy(p + name_off, &it->second, 4);
        }
        write_file(out_dir + "/street_ways.bin", old_data);

        // Content-based matching: hash(name_id, node_count, node_coords)
        auto way_hash = [&](const char* way_data, const char* node_data, size_t stride, size_t name_off) -> uint64_t {
            uint32_t node_offset, name_id;
            uint8_t node_count;
            memcpy(&node_offset, way_data, 4);
            node_count = static_cast<uint8_t>(way_data[4]);
            memcpy(&name_id, way_data + name_off, 4);
            uint64_t h = 14695981039346656037ULL;
            auto mix = [&h](uint64_t v) { h ^= v; h *= 1099511628211ULL; };
            mix(name_id);
            mix(node_count);
            for (uint8_t j = 0; j < node_count; j++) {
                size_t noff = (node_offset + j) * 8;
                float lat, lng;
                memcpy(&lat, node_data + noff, 4);
                memcpy(&lng, node_data + noff + 4, 4);
                mix(to_grid(lat));
                mix(to_grid(lng));
            }
            return h;
        };

        size_t new_count = new_data.size() / stride;
        // Build hash map of new ways
        std::unordered_multimap<uint64_t, uint32_t> new_map;
        new_map.reserve(new_count);
        for (uint32_t i = 0; i < new_count; i++) {
            new_map.emplace(way_hash(new_data.data() + i * stride, new_nodes.data(), stride, name_off), i);
        }

        way_remap.assign(old_count, 0xFFFFFFFF);
        size_t matched = 0;
        for (uint32_t i = 0; i < old_count; i++) {
            uint64_t h = way_hash(old_data.data() + i * stride, old_nodes.data(), stride, name_off);
            auto it = new_map.find(h);
            if (it != new_map.end()) {
                way_remap[i] = it->second;
                new_map.erase(it);
                matched++;
            }
        }
        std::cerr << "    matched=" << matched << " unmatched=" << (old_count - matched)
                  << " new_only=" << new_map.size() << std::endl;
    }

// Full remap test: apply string remap to data files AND record ID remap to entry/cell files.
// This should make zstd --patch-from produce near-zero patches for ALL files.
//
// Usage: full_remap_test <old-canonical> <new-canonical> <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>

#include "patch_format.h"

// Build record ID remap by matching records between remapped-old and new data files.
// Both files are in canonical sort order, so matching records are at approximately
// the same position. We match by byte equality (after string remap).
static std::vector<uint32_t> build_record_remap(
    const std::vector<char>& old_data, const std::vector<char>& new_data,
    size_t record_size)
{
    size_t old_count = old_data.size() / record_size;
    size_t new_count = new_data.size() / record_size;

    // Build hash map of new records: record_bytes → new_id
    // For 86M records × 16 bytes this is ~1.4 GiB for the map. Manageable.
    struct RecordHash {
        const char* data;
        size_t size;
        size_t operator()(uint32_t id) const {
            // FNV-1a hash of record bytes
            uint64_t h = 14695981039346656037ULL;
            const char* p = data + id * size;
            for (size_t i = 0; i < size; i++) {
                h ^= static_cast<uint8_t>(p[i]);
                h *= 1099511628211ULL;
            }
            return h;
        }
    };

    // Map: hash(record_bytes) → list of new_ids
    std::unordered_multimap<uint64_t, uint32_t> new_record_map;
    new_record_map.reserve(new_count);
    RecordHash hasher{new_data.data(), record_size};
    for (uint32_t i = 0; i < new_count; i++) {
        new_record_map.emplace(hasher(i), i);
    }

    // For each old record, find matching new record
    std::vector<uint32_t> old_to_new(old_count, 0xFFFFFFFF);
    RecordHash old_hasher{old_data.data(), record_size};
    size_t matched = 0, unmatched = 0;
    for (uint32_t oi = 0; oi < old_count; oi++) {
        uint64_t h = old_hasher(oi);
        auto range = new_record_map.equal_range(h);
        bool found = false;
        for (auto it = range.first; it != range.second; ++it) {
            uint32_t ni = it->second;
            if (memcmp(old_data.data() + oi * record_size,
                       new_data.data() + ni * record_size, record_size) == 0) {
                old_to_new[oi] = ni;
                new_record_map.erase(it);
                matched++;
                found = true;
                break;
            }
        }
        if (!found) unmatched++;
    }
    std::cerr << "    matched=" << matched << " unmatched=" << unmatched
              << " new_only=" << new_record_map.size() << std::endl;
    return old_to_new;
}

// Apply record ID remap to entry data
static void remap_entries(std::vector<char>& entries_data,
                           const std::vector<char>& cells_data,
                           size_t cell_stride, // 20 for geo, 12 for admin
                           size_t offset_pos,  // byte offset of entry_offset within cell record
                           const std::vector<uint32_t>& id_remap,
                           bool has_flags = false)
{
    size_t n_cells = cells_data.size() / cell_stride;
    for (size_t i = 0; i < n_cells; i++) {
        uint32_t entry_off;
        memcpy(&entry_off, cells_data.data() + i * cell_stride + offset_pos, 4);
        if (entry_off == 0xFFFFFFFF || entry_off + 2 > entries_data.size()) continue;

        uint16_t count;
        memcpy(&count, entries_data.data() + entry_off, 2);
        for (uint16_t j = 0; j < count; j++) {
            size_t id_pos = entry_off + 2 + j * 4;
            if (id_pos + 4 > entries_data.size()) break;
            uint32_t old_id;
            memcpy(&old_id, entries_data.data() + id_pos, 4);

            uint32_t flags = has_flags ? (old_id & 0x80000000) : 0;
            uint32_t masked = old_id & 0x7FFFFFFF;

            if (masked < id_remap.size() && id_remap[masked] != 0xFFFFFFFF) {
                uint32_t new_id = id_remap[masked] | flags;
                memcpy(entries_data.data() + id_pos, &new_id, 4);
            }
        }
    }
}

int main(int argc, char* argv[]) {
    if (argc < 4) {
        std::cerr << "Usage: full_remap_test <old-canonical> <new-canonical> <output-dir>" << std::endl;
        return 1;
    }
    std::string old_dir = argv[1], new_dir = argv[2], out_dir = argv[3];
    ensure_dir(out_dir);

    // Step 1: Build string remap
    std::cerr << "Building string remap..." << std::endl;
    auto old_pool = read_file(old_dir + "/strings.bin");
    auto new_pool = read_file(new_dir + "/strings.bin");

    std::unordered_map<uint32_t, uint32_t> str_remap;
    {
        std::unordered_map<std::string, uint32_t> new_index;
        size_t pos = 0;
        while (pos < new_pool.size()) {
            const char* s = new_pool.data() + pos;
            size_t len = strlen(s);
            new_index[std::string(s, len)] = static_cast<uint32_t>(pos);
            pos += len + 1;
        }
        pos = 0;
        while (pos < old_pool.size()) {
            const char* s = old_pool.data() + pos;
            size_t len = strlen(s);
            auto it = new_index.find(std::string(s, len));
            if (it != new_index.end())
                str_remap[static_cast<uint32_t>(pos)] = it->second;
            pos += len + 1;
        }
    }
    std::cerr << "  " << str_remap.size() << " strings mapped" << std::endl;

    // Step 2: Remap and match addr_points
    std::cerr << "Remapping addr_points..." << std::endl;
    std::vector<uint32_t> addr_remap;
    {
        auto old_data = read_file(old_dir + "/addr_points.bin");
        auto new_data = read_file(new_dir + "/addr_points.bin");

        // Apply string remap to old
        size_t n = old_data.size() / 16;
        for (size_t i = 0; i < n; i++) {
            char* p = old_data.data() + i * 16;
            for (size_t off : {8, 12}) {
                uint32_t old_id; memcpy(&old_id, p + off, 4);
                auto it = str_remap.find(old_id);
                if (it != str_remap.end()) memcpy(p + off, &it->second, 4);
            }
        }
        write_file(out_dir + "/addr_points.bin", old_data);

        // Build record remap
        addr_remap = build_record_remap(old_data, new_data, 16);
    }

    // Step 3: Remap and match street_ways
    std::cerr << "Remapping street_ways..." << std::endl;
    std::vector<uint32_t> way_remap;
    {
        auto old_data = read_file(old_dir + "/street_ways.bin");
        auto new_data = read_file(new_dir + "/street_ways.bin");
        size_t stride = (old_data.size() % 12 == 0) ? 12 : 9;
        size_t name_off = (stride == 12) ? 8 : 5;

        size_t n = old_data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = old_data.data() + i * stride;
            uint32_t old_id; memcpy(&old_id, p + name_off, 4);
            auto it = str_remap.find(old_id);
            if (it != str_remap.end()) memcpy(p + name_off, &it->second, 4);
        }
        write_file(out_dir + "/street_ways.bin", old_data);
        way_remap = build_record_remap(old_data, new_data, stride);
    }

    // Step 4: Remap interp_ways
    std::cerr << "Remapping interp_ways..." << std::endl;
    std::vector<uint32_t> interp_remap;
    {
        auto old_data = read_file(old_dir + "/interp_ways.bin");
        auto new_data = read_file(new_dir + "/interp_ways.bin");
        size_t stride = (old_data.size() % 20 == 0 && old_data.size() > 0) ? 20 : 18;
        size_t street_off = (stride == 20) ? 8 : 5;

        size_t n = old_data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = old_data.data() + i * stride;
            uint32_t old_id; memcpy(&old_id, p + street_off, 4);
            auto it = str_remap.find(old_id);
            if (it != str_remap.end()) memcpy(p + street_off, &it->second, 4);
        }
        write_file(out_dir + "/interp_ways.bin", old_data);
        interp_remap = build_record_remap(old_data, new_data, stride);
    }

    // Step 5: Remap admin_polygons
    std::cerr << "Remapping admin_polygons..." << std::endl;
    std::vector<uint32_t> admin_remap;
    {
        auto old_data = read_file(old_dir + "/admin_polygons.bin");
        auto new_data = read_file(new_dir + "/admin_polygons.bin");
        size_t stride = 24;
        for (size_t s : {24, 20, 19}) {
            if (old_data.size() % s == 0 && old_data.size() > 0) { stride = s; break; }
        }
        size_t n = old_data.size() / stride;
        for (size_t i = 0; i < n; i++) {
            char* p = old_data.data() + i * stride;
            uint32_t old_id; memcpy(&old_id, p + 8, 4);
            auto it = str_remap.find(old_id);
            if (it != str_remap.end()) memcpy(p + 8, &it->second, 4);
        }
        write_file(out_dir + "/admin_polygons.bin", old_data);
        admin_remap = build_record_remap(old_data, new_data, stride);
    }

    // Step 6: Copy coordinate files unchanged (they don't have string refs)
    for (auto name : {"street_nodes.bin", "interp_nodes.bin", "admin_vertices.bin"}) {
        auto data = read_file(old_dir + "/" + std::string(name));
        write_file(out_dir + "/" + std::string(name), data);
    }

    // Step 7: Remap entry files using record ID remaps
    std::cerr << "Remapping entry files..." << std::endl;
    {
        auto geo_cells = read_file(old_dir + "/geo_cells.bin");
        auto street_entries = read_file(old_dir + "/street_entries.bin");
        auto addr_entries = read_file(old_dir + "/addr_entries.bin");
        auto interp_entries = read_file(old_dir + "/interp_entries.bin");

        remap_entries(street_entries, geo_cells, 20, 8, way_remap);
        remap_entries(addr_entries, geo_cells, 20, 12, addr_remap);
        remap_entries(interp_entries, geo_cells, 20, 16, interp_remap);

        write_file(out_dir + "/geo_cells.bin", geo_cells);
        write_file(out_dir + "/street_entries.bin", street_entries);
        write_file(out_dir + "/addr_entries.bin", addr_entries);
        write_file(out_dir + "/interp_entries.bin", interp_entries);
    }
    {
        auto admin_cells = read_file(old_dir + "/admin_cells.bin");
        auto admin_entries = read_file(old_dir + "/admin_entries.bin");
        remap_entries(admin_entries, admin_cells, 12, 8, admin_remap, true);
        write_file(out_dir + "/admin_cells.bin", admin_cells);
        write_file(out_dir + "/admin_entries.bin", admin_entries);
    }

    // Copy strings (use new pool)
    write_file(out_dir + "/strings.bin", new_pool);

    std::cerr << "Done. Fully remapped files in " << out_dir << std::endl;
    return 0;
}

-    // Step 3: Remap and match street_ways

+    // Step 3: Remap and match street_ways (match by name + node coords, not raw bytes)

+        auto old_nodes = read_file(old_dir + "/street_nodes.bin");

+        auto new_nodes = read_file(new_dir + "/street_nodes.bin");

-        size_t n = old_data.size() / stride;

-        for (size_t i = 0; i < n; i++) {

+        // Apply string remap to old ways

+        size_t old_count = old_data.size() / stride;

+        for (size_t i = 0; i < old_count; i++) {

-        way_remap = build_record_remap(old_data, new_data, stride);

+        // Content-based matching: hash(name_id, node_count, node_coords)

+        auto way_hash = [&](const char* way_data, const char* node_data, size_t stride, size_t name_off) -> uint64_t {

+            uint32_t node_offset, name_id;

+            memcpy(&node_offset, way_data, 4);

+            node_count = static_cast<uint8_t>(way_data[4]);

+            memcpy(&name_id, way_data + name_off, 4);

+            for (uint8_t j = 0; j < node_count; j++) {

+                size_t noff = (node_offset + j) * 8;

+                memcpy(&lat, node_data + noff, 4);

+                memcpy(&lng, node_data + noff + 4, 4);

+        size_t new_count = new_data.size() / stride;

+        std::unordered_multimap<uint64_t, uint32_t> new_map;

+        for (uint32_t i = 0; i < new_count; i++) {

+            new_map.emplace(way_hash(new_data.data() + i * stride, new_nodes.data(), stride, name_off), i);

+        way_remap.assign(old_count, 0xFFFFFFFF);

+        for (uint32_t i = 0; i < old_count; i++) {

+            uint64_t h = way_hash(old_data.data() + i * stride, old_nodes.data(), stride, name_off);

+                way_remap[i] = it->second;

+        std::cerr << "    matched=" << matched << " unmatched=" << (old_count - matched)

+                  << " new_only=" << new_map.size() << std::endl;


Now do the same for interp_ways and admin_polygons: