The file /root/traccar-geocoder/builder/tools/patch_format.h has been updated successfully.

// Read WayHeader array — detect stride from file size vs known struct sizes
inline std::vector<PatchWayHeader> read_ways(const std::string& path) {
    auto data = read_file(path);
    // Determine stride: builder writes sizeof(WayHeader) which may be 9 or 12
    // depending on compiler padding. Try 12 first (common), then 9.
    size_t stride = 12;
    if (data.size() % 12 != 0 || data.size() / 12 == 0) {
        stride = 9;
    }
    size_t count = data.size() / stride;
    std::vector<PatchWayHeader> result(count);
    for (size_t i = 0; i < count; i++) {
        const char* p = data.data() + i * stride;
        memcpy(&result[i].node_offset, p, 4);
        result[i].node_count = static_cast<uint8_t>(p[4]);
        // name_id is at offset 5 (packed) or 8 (padded to 4-byte alignment)
        size_t name_off = (stride == 12) ? 8 : 5;
        memcpy(&result[i].name_id, p + name_off, 4);
    }
    return result;
}

inline size_t detect_way_stride(const std::string& path) {
    auto data = read_file(path);
    return (data.size() % 12 == 0 && data.size() / 12 > 0) ? 12 : 9;
}

#pragma once
// Shared definitions for the geocoder patch system.
// Used by geocoder-canonicalize, geocoder-diff, and geocoder-patch.

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>

// --- Binary record structs (must match types.h and server) ---

#pragma pack(push, 1)
struct PatchWayHeader {
    uint32_t node_offset;
    uint8_t node_count;
    uint32_t name_id;
};
static_assert(sizeof(PatchWayHeader) == 9, "WayHeader must be 9 bytes packed");

struct PatchAddrPoint {
    float lat;
    float lng;
    uint32_t housenumber_id;
    uint32_t street_id;
};
static_assert(sizeof(PatchAddrPoint) == 16, "AddrPoint must be 16 bytes");

struct PatchNodeCoord {
    float lat;
    float lng;
};
static_assert(sizeof(PatchNodeCoord) == 8, "NodeCoord must be 8 bytes");
#pragma pack(pop)

// InterpWay and AdminPolygon have compiler-dependent padding.
// Read them field-by-field instead of struct-casting.

struct PatchInterpWay {
    uint32_t node_offset;
    uint8_t node_count;
    uint32_t street_id;
    uint32_t start_number;
    uint32_t end_number;
    uint8_t interpolation;
};

struct PatchAdminPolygon {
    uint32_t vertex_offset;
    uint32_t vertex_count;
    uint32_t name_id;
    uint8_t admin_level;
    float area;
    uint16_t country_code;
};

// --- File I/O helpers ---

inline std::vector<char> read_file(const std::string& path) {
    std::ifstream f(path, std::ios::binary | std::ios::ate);
    if (!f) return {};
    auto size = f.tellg();
    f.seekg(0);
    std::vector<char> data(size);
    f.read(data.data(), size);
    return data;
}

inline bool write_file(const std::string& path, const char* data, size_t size) {
    std::ofstream f(path, std::ios::binary);
    if (!f) return false;
    f.write(data, size);
    return f.good();
}

inline bool write_file(const std::string& path, const std::vector<char>& data) {
    return write_file(path, data.data(), data.size());
}

inline const char* get_string(const std::vector<char>& pool, uint32_t offset) {
    if (offset >= pool.size()) return "";
    return pool.data() + offset;
}

// Read packed WayHeader array (9 bytes each, no padding)
inline std::vector<PatchWayHeader> read_ways(const std::string& path) {
    auto data = read_file(path);
    size_t count = data.size() / 9;
    std::vector<PatchWayHeader> result(count);
    for (size_t i = 0; i < count; i++) {
        memcpy(&result[i], data.data() + i * 9, 9);
    }
    return result;
}

template<typename T>
inline std::vector<T> read_structs(const std::string& path) {
    auto data = read_file(path);
    size_t count = data.size() / sizeof(T);
    std::vector<T> result(count);
    if (count > 0) memcpy(result.data(), data.data(), count * sizeof(T));
    return result;
}

// Read InterpWay with field-by-field parsing to handle padding
inline std::vector<PatchInterpWay> read_interps(const std::string& path) {
    auto data = read_file(path);
    // Determine record size from file: try sizeof(InterpWay) as compiled by the builder
    // The builder uses the same struct, so sizeof should match.
    size_t rec_size = sizeof(PatchInterpWay);
    // But the builder's InterpWay has padding. Read with the builder's size.
    // From types.h the struct is: u32 + u8 + u32 + u32 + u32 + u8 = 18 bytes raw
    // but compiler pads to 20 bytes typically. Check by trying both.
    size_t count_20 = data.size() / 20;
    size_t count_18 = data.size() / 18;
    // Use whichever divides evenly
    size_t stride = (data.size() % 20 == 0 && count_20 > 0) ? 20 :
                    (data.size() % 18 == 0 && count_18 > 0) ? 18 : 20;
    size_t count = data.size() / stride;
    std::vector<PatchInterpWay> result(count);
    for (size_t i = 0; i < count; i++) {
        const char* p = data.data() + i * stride;
        memcpy(&result[i].node_offset, p, 4); p += 4;
        result[i].node_count = *reinterpret_cast<const uint8_t*>(p); p += 1;
        // Skip padding to align street_id
        if (stride == 20) p += 3; // 3 bytes padding after node_count
        memcpy(&result[i].street_id, p, 4); p += 4;
        memcpy(&result[i].start_number, p, 4); p += 4;
        memcpy(&result[i].end_number, p, 4); p += 4;
        result[i].interpolation = *reinterpret_cast<const uint8_t*>(p);
    }
    return result;
}

// Read AdminPolygon with field-by-field parsing
inline std::vector<PatchAdminPolygon> read_admin_polygons(const std::string& path) {
    auto data = read_file(path);
    // AdminPolygon: u32 + u32 + u32 + u8 + f32 + u16 = 19 raw bytes
    // Compiler likely pads to 20 or 24 bytes. Try common sizes.
    size_t stride = 0;
    for (size_t s : {24, 20, 19}) {
        if (data.size() % s == 0 && data.size() / s > 0) { stride = s; break; }
    }
    if (stride == 0) stride = 24; // fallback
    size_t count = data.size() / stride;
    std::vector<PatchAdminPolygon> result(count);
    for (size_t i = 0; i < count; i++) {
        const char* p = data.data() + i * stride;
        memcpy(&result[i].vertex_offset, p, 4); p += 4;
        memcpy(&result[i].vertex_count, p, 4); p += 4;
        memcpy(&result[i].name_id, p, 4); p += 4;
        result[i].admin_level = *reinterpret_cast<const uint8_t*>(p); p += 1;
        // padding for area alignment
        if (stride >= 20) {
            size_t skip = (stride == 24) ? 3 : (stride == 20 ? 3 : 0);
            p += skip;
        }
        memcpy(&result[i].area, p, 4); p += 4;
        memcpy(&result[i].country_code, p, 2);
    }
    return result;
}

// --- Patch file format ---

static constexpr char GCPATCH_MAGIC[8] = {'G','C','P','A','T','C','H','\0'};
static constexpr uint32_t GCPATCH_VERSION = 1;

enum class PatchFileId : uint32_t {
    STRINGS = 0,
    STREET_WAYS = 1,
    STREET_NODES = 2,
    ADDR_POINTS = 3,
    INTERP_WAYS = 4,
    INTERP_NODES = 5,
    ADMIN_POLYGONS = 6,
    ADMIN_VERTICES = 7,
    GEO_CELLS = 8,
    STREET_ENTRIES = 9,
    ADDR_ENTRIES = 10,
    INTERP_ENTRIES = 11,
    ADMIN_CELLS = 12,
    ADMIN_ENTRIES = 13,
    COUNT = 14
};

static const char* patch_file_names[] = {
    "strings.bin", "street_ways.bin", "street_nodes.bin", "addr_points.bin",
    "interp_ways.bin", "interp_nodes.bin", "admin_polygons.bin", "admin_vertices.bin",
    "geo_cells.bin", "street_entries.bin", "addr_entries.bin", "interp_entries.bin",
    "admin_cells.bin", "admin_entries.bin"
};

// Encoding types for each section
enum class PatchEncoding : uint32_t {
    RAW_REPLACE = 0,  // Full replacement (zlib compressed)
    COPY_INSERT = 1,  // COPY/INSERT byte stream (zlib compressed)
};

// Patch file header
struct PatchHeader {
    char magic[8];
    uint32_t version;
    uint32_t flags;
    // Per-file sections follow
};

// Section header (one per file)
struct PatchSection {
    uint32_t file_id;       // PatchFileId
    uint32_t encoding;      // PatchEncoding
    uint64_t old_size;      // Expected size of old file
    uint64_t new_size;      // Size of reconstructed file
    uint64_t data_size;     // Size of compressed section data
};

// COPY_INSERT opcodes
enum : uint8_t {
    OP_COPY = 0,    // Copy from old file: [u8 tag][u64 offset][u32 length]
    OP_INSERT = 1,  // Insert new data:    [u8 tag][u32 length][data...]
};

// --- Grid coordinate for fingerprinting ---

inline int to_grid(float v) {
    return (int)(v * 1e5f + (v >= 0 ? 0.5f : -0.5f));
}

// --- Directory creation ---
inline void ensure_dir(const std::string& path) {
    std::string cmd = "mkdir -p '" + path + "'";
    system(cmd.c_str());
}

-// Read packed WayHeader array (9 bytes each, no padding)

+// Read WayHeader array — detect stride from file size vs known struct sizes

+    // Determine stride: builder writes sizeof(WayHeader) which may be 9 or 12

+    // depending on compiler padding. Try 12 first (common), then 9.

+    size_t count = data.size() / stride;

-        memcpy(&result[i], data.data() + i * 9, 9);

+        const char* p = data.data() + i * stride;

+        memcpy(&result[i].node_offset, p, 4);

+        result[i].node_count = static_cast<uint8_t>(p[4]);

+        // name_id is at offset 5 (packed) or 8 (padded to 4-byte alignment)

+        size_t name_off = (stride == 12) ? 8 : 5;

+        memcpy(&result[i].name_id, p + name_off, 4);

+inline size_t detect_way_stride(const std::string& path) {


Now update the canonicalize tool to write ways with the correct stride:


Edit

        // Write sorted ways + nodes
        std::vector<char> ways_buf;
        ways_buf.reserve(n * 9);
        std::vector<PatchNodeCoord> sorted_nodes;
        sorted_nodes.reserve(nodes.size());

        for (uint32_t i = 0; i < n; i++) {
            PatchWayHeader w = ways[order[i]];
            uint32_t old_offset = w.node_offset;
            w.node_offset = static_cast<uint32_t>(sorted_nodes.size());
            w.name_id = pool.map(w.name_id);
            ways_buf.insert(ways_buf.end(), reinterpret_cast<const char*>(&w),
                           reinterpret_cast<const char*>(&w) + 9);
            for (uint8_t j = 0; j < w.node_count; j++) {
                if (old_offset + j < nodes.size())
                    sorted_nodes.push_back(nodes[old_offset + j]);
            }
        }
        write_file(output_dir + "/street_ways.bin", ways_buf);

        // Write sorted ways + nodes, preserving original stride
        auto orig_way_data = read_file(input_dir + "/street_ways.bin");
        size_t way_stride = (orig_way_data.size() % 12 == 0 && n > 0) ? 12 : 9;

        std::vector<char> ways_buf;
        ways_buf.reserve(n * way_stride);
        std::vector<PatchNodeCoord> sorted_nodes;
        sorted_nodes.reserve(nodes.size());

        for (uint32_t i = 0; i < n; i++) {
            const char* src = orig_way_data.data() + order[i] * way_stride;
            char record[12] = {};
            memcpy(record, src, way_stride);

            // Update node_offset (offset 0)
            uint32_t new_node_offset = static_cast<uint32_t>(sorted_nodes.size());
            memcpy(record, &new_node_offset, 4);

            // Update name_id
            size_t name_off = (way_stride == 12) ? 8 : 5;
            uint32_t old_name_id;
            memcpy(&old_name_id, record + name_off, 4);
            uint32_t new_name_id = pool.map(old_name_id);
            memcpy(record + name_off, &new_name_id, 4);

            ways_buf.insert(ways_buf.end(), record, record + way_stride);

            PatchWayHeader w = ways[order[i]];
            for (uint8_t j = 0; j < w.node_count; j++) {
                if (w.node_offset + j < nodes.size())
                    sorted_nodes.push_back(nodes[w.node_offset + j]);
            }
        }
        write_file(output_dir + "/street_ways.bin", ways_buf);


The file /root/traccar-geocoder/builder/tools/geocoder_canonicalize.cpp has been updated successfully.

        // Write sorted ways + nodes, preserving original stride
        auto orig_way_data = read_file(input_dir + "/street_ways.bin");
        size_t way_stride = (orig_way_data.size() % 12 == 0 && n > 0) ? 12 : 9;

        std::vector<char> ways_buf;
        ways_buf.reserve(n * way_stride);
        std::vector<PatchNodeCoord> sorted_nodes;
        sorted_nodes.reserve(nodes.size());

        for (uint32_t i = 0; i < n; i++) {
            const char* src = orig_way_data.data() + order[i] * way_stride;
            char record[12] = {};
            memcpy(record, src, way_stride);

            // Update node_offset (offset 0)
            uint32_t new_node_offset = static_cast<uint32_t>(sorted_nodes.size());
            memcpy(record, &new_node_offset, 4);

            // Update name_id
            size_t name_off = (way_stride == 12) ? 8 : 5;
            uint32_t old_name_id;
            memcpy(&old_name_id, record + name_off, 4);
            uint32_t new_name_id = pool.map(old_name_id);
            memcpy(record + name_off, &new_name_id, 4);

            ways_buf.insert(ways_buf.end(), record, record + way_stride);

            PatchWayHeader w = ways[order[i]];
            for (uint8_t j = 0; j < w.node_count; j++) {
                if (w.node_offset + j < nodes.size())
                    sorted_nodes.push_back(nodes[w.node_offset + j]);
            }
        }
        write_file(output_dir + "/street_ways.bin", ways_buf);

// geocoder-canonicalize: Converts a geocoder index to canonical (deterministic) format.
//
// Canonical format properties:
// - strings.bin: strings sorted alphabetically
// - Data files: records sorted by content fingerprint
// - Node/vertex files: grouped by parent record in canonical order
// - Entries/cells: derived from canonical data ordering + original cell assignments
//
// Usage: geocoder-canonicalize <input-dir> <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <functional>
#include <iostream>
#include <numeric>
#include <string>
#include <unordered_map>
#include <vector>

#include "patch_format.h"

// --- Canonical string pool ---
// Sort all strings alphabetically, build old_offset -> new_offset remap.

struct CanonicalStringPool {
    std::vector<char> data;
    std::unordered_map<uint32_t, uint32_t> remap; // old_offset -> new_offset

    void build(const std::vector<char>& old_pool) {
        // Extract all strings with their offsets
        std::vector<std::pair<uint32_t, const char*>> strings;
        size_t pos = 0;
        while (pos < old_pool.size()) {
            const char* s = old_pool.data() + pos;
            strings.emplace_back(static_cast<uint32_t>(pos), s);
            pos += strlen(s) + 1;
        }

        // Sort alphabetically
        std::sort(strings.begin(), strings.end(), [](const auto& a, const auto& b) {
            return strcmp(a.second, b.second) < 0;
        });

        // Build new pool + remap
        data.clear();
        data.reserve(old_pool.size());
        remap.reserve(strings.size());
        for (auto& [old_off, str] : strings) {
            uint32_t new_off = static_cast<uint32_t>(data.size());
            remap[old_off] = new_off;
            size_t len = strlen(str);
            data.insert(data.end(), str, str + len + 1);
        }
    }

    uint32_t map(uint32_t old_off) const {
        auto it = remap.find(old_off);
        return (it != remap.end()) ? it->second : 0;
    }
};

// --- Cell assignment extraction ---
// Parse geo_cells.bin + entries files to extract cell -> [item_id] mappings.

struct GeoCellEntry {
    uint64_t cell_id;
    uint32_t street_offset;
    uint32_t addr_offset;
    uint32_t interp_offset;
};

struct CellAssignments {
    // For each item type: cell_id -> sorted list of item_ids
    std::unordered_map<uint64_t, std::vector<uint32_t>> street_cells;
    std::unordered_map<uint64_t, std::vector<uint32_t>> addr_cells;
    std::unordered_map<uint64_t, std::vector<uint32_t>> interp_cells;
    std::unordered_map<uint64_t, std::vector<uint32_t>> admin_cells; // with flags
};

static std::vector<uint32_t> parse_entry(const std::vector<char>& entries, uint32_t offset) {
    if (offset == 0xFFFFFFFF || offset + 2 > entries.size()) return {};
    uint16_t count;
    memcpy(&count, entries.data() + offset, 2);
    std::vector<uint32_t> ids(count);
    if (offset + 2 + count * 4 > entries.size()) return {};
    memcpy(ids.data(), entries.data() + offset + 2, count * 4);
    return ids;
}

static CellAssignments extract_cell_assignments(const std::string& dir) {
    CellAssignments ca;

    // Geo cells
    auto geo_data = read_file(dir + "/geo_cells.bin");
    auto street_entries = read_file(dir + "/street_entries.bin");
    auto addr_entries = read_file(dir + "/addr_entries.bin");
    auto interp_entries = read_file(dir + "/interp_entries.bin");

    size_t n_geo = geo_data.size() / 20;
    for (size_t i = 0; i < n_geo; i++) {
        GeoCellEntry e;
        memcpy(&e.cell_id, geo_data.data() + i * 20, 8);
        memcpy(&e.street_offset, geo_data.data() + i * 20 + 8, 4);
        memcpy(&e.addr_offset, geo_data.data() + i * 20 + 12, 4);
        memcpy(&e.interp_offset, geo_data.data() + i * 20 + 16, 4);

        auto s_ids = parse_entry(street_entries, e.street_offset);
        if (!s_ids.empty()) ca.street_cells[e.cell_id] = std::move(s_ids);

        auto a_ids = parse_entry(addr_entries, e.addr_offset);
        if (!a_ids.empty()) ca.addr_cells[e.cell_id] = std::move(a_ids);

        auto i_ids = parse_entry(interp_entries, e.interp_offset);
        if (!i_ids.empty()) ca.interp_cells[e.cell_id] = std::move(i_ids);
    }

    // Admin cells
    auto admin_cell_data = read_file(dir + "/admin_cells.bin");
    auto admin_entry_data = read_file(dir + "/admin_entries.bin");
    size_t n_admin = admin_cell_data.size() / 12;
    for (size_t i = 0; i < n_admin; i++) {
        uint64_t cell_id;
        uint32_t offset;
        memcpy(&cell_id, admin_cell_data.data() + i * 12, 8);
        memcpy(&offset, admin_cell_data.data() + i * 12 + 8, 4);
        auto ids = parse_entry(admin_entry_data, offset);
        if (!ids.empty()) ca.admin_cells[cell_id] = std::move(ids);
    }

    return ca;
}

// --- Remap cell assignments with old->new ID permutation ---
static void remap_cells(std::unordered_map<uint64_t, std::vector<uint32_t>>& cells,
                         const std::vector<uint32_t>& old_to_new) {
    for (auto& [cell_id, ids] : cells) {
        for (auto& id : ids) {
            uint32_t masked = id & 0x7FFFFFFF;
            uint32_t flags = id & 0x80000000;
            if (masked < old_to_new.size()) {
                id = old_to_new[masked] | flags;
            }
        }
        std::sort(ids.begin(), ids.end());
    }
}

// --- Write entries + cells files ---
static void write_entries_and_cells(
    const std::string& cells_path,
    const std::string& entries_path,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& cell_map,
    size_t cell_entry_size) // 20 for geo, 12 for admin
{
    // Collect and sort cells
    std::vector<uint64_t> sorted_cells;
    sorted_cells.reserve(cell_map.size());
    for (auto& [id, _] : cell_map) sorted_cells.push_back(id);
    std::sort(sorted_cells.begin(), sorted_cells.end());

    // Write entries file
    std::vector<char> entries_buf;
    std::unordered_map<uint64_t, uint32_t> cell_offsets;
    for (uint64_t cell_id : sorted_cells) {
        auto it = cell_map.find(cell_id);
        if (it == cell_map.end() || it->second.empty()) continue;
        cell_offsets[cell_id] = static_cast<uint32_t>(entries_buf.size());
        uint16_t count = static_cast<uint16_t>(it->second.size());
        entries_buf.insert(entries_buf.end(), reinterpret_cast<const char*>(&count),
                          reinterpret_cast<const char*>(&count) + 2);
        entries_buf.insert(entries_buf.end(),
                          reinterpret_cast<const char*>(it->second.data()),
                          reinterpret_cast<const char*>(it->second.data()) + it->second.size() * 4);
    }
    write_file(entries_path, entries_buf);

    // Write cells file
    std::vector<char> cells_buf;
    uint32_t no_data = 0xFFFFFFFF;
    for (uint64_t cell_id : sorted_cells) {
        cells_buf.insert(cells_buf.end(), reinterpret_cast<const char*>(&cell_id),
                        reinterpret_cast<const char*>(&cell_id) + 8);
        auto it = cell_offsets.find(cell_id);
        uint32_t off = (it != cell_offsets.end()) ? it->second : no_data;
        cells_buf.insert(cells_buf.end(), reinterpret_cast<const char*>(&off),
                        reinterpret_cast<const char*>(&off) + 4);
    }
    write_file(cells_path, cells_buf);
}

// Write merged geo_cells with 3 offset columns
static void write_geo_cells_and_entries(
    const std::string& dir,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& street_cells,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& addr_cells,
    const std::unordered_map<uint64_t, std::vector<uint32_t>>& interp_cells)
{
    // Collect all unique cell IDs
    std::vector<uint64_t> all_cells;
    auto add = [&](const auto& m) { for (auto& [id, _] : m) all_cells.push_back(id); };
    add(street_cells); add(addr_cells); add(interp_cells);
    std::sort(all_cells.begin(), all_cells.end());
    all_cells.erase(std::unique(all_cells.begin(), all_cells.end()), all_cells.end());

    // Write each entry file and collect offsets
    auto write_entries = [&](const std::string& path,
                              const std::unordered_map<uint64_t, std::vector<uint32_t>>& cells)
        -> std::unordered_map<uint64_t, uint32_t>
    {
        std::vector<char> buf;
        std::unordered_map<uint64_t, uint32_t> offsets;
        for (uint64_t cell_id : all_cells) {
            auto it = cells.find(cell_id);
            if (it == cells.end() || it->second.empty()) continue;
            offsets[cell_id] = static_cast<uint32_t>(buf.size());
            uint16_t count = static_cast<uint16_t>(it->second.size());
            buf.insert(buf.end(), reinterpret_cast<const char*>(&count),
                      reinterpret_cast<const char*>(&count) + 2);
            buf.insert(buf.end(), reinterpret_cast<const char*>(it->second.data()),
                      reinterpret_cast<const char*>(it->second.data()) + it->second.size() * 4);
        }
        write_file(path, buf);
        return offsets;
    };

    auto s_off = write_entries(dir + "/street_entries.bin", street_cells);
    auto a_off = write_entries(dir + "/addr_entries.bin", addr_cells);
    auto i_off = write_entries(dir + "/interp_entries.bin", interp_cells);

    // Write geo_cells.bin
    uint32_t no_data = 0xFFFFFFFF;
    std::vector<char> buf;
    buf.reserve(all_cells.size() * 20);
    for (uint64_t cell_id : all_cells) {
        buf.insert(buf.end(), reinterpret_cast<const char*>(&cell_id),
                  reinterpret_cast<const char*>(&cell_id) + 8);
        auto get = [&](const auto& m) -> uint32_t {
            auto it = m.find(cell_id);
            return (it != m.end()) ? it->second : no_data;
        };
        uint32_t so = get(s_off), ao = get(a_off), io = get(i_off);
        buf.insert(buf.end(), reinterpret_cast<const char*>(&so), reinterpret_cast<const char*>(&so) + 4);
        buf.insert(buf.end(), reinterpret_cast<const char*>(&ao), reinterpret_cast<const char*>(&ao) + 4);
        buf.insert(buf.end(), reinterpret_cast<const char*>(&io), reinterpret_cast<const char*>(&io) + 4);
    }
    write_file(dir + "/geo_cells.bin", buf);
}

// --- Main ---

int main(int argc, char* argv[]) {
    if (argc < 3) {
        std::cerr << "Usage: geocoder-canonicalize <input-dir> <output-dir>" << std::endl;
        return 1;
    }

    std::string input_dir = argv[1];
    std::string output_dir = argv[2];
    ensure_dir(output_dir);

    std::cerr << "Loading index from " << input_dir << "..." << std::endl;

    // 1. Sort string pool
    std::cerr << "  Sorting string pool..." << std::endl;
    auto old_strings = read_file(input_dir + "/strings.bin");
    CanonicalStringPool pool;
    pool.build(old_strings);
    write_file(output_dir + "/strings.bin", pool.data);
    std::cerr << "    " << pool.remap.size() << " strings" << std::endl;

    // 2. Extract cell assignments before reordering
    std::cerr << "  Extracting cell assignments..." << std::endl;
    auto ca = extract_cell_assignments(input_dir);

    // 3. Canonicalize addr_points
    std::cerr << "  Sorting addr_points..." << std::endl;
    {
        auto points = read_structs<PatchAddrPoint>(input_dir + "/addr_points.bin");
        size_t n = points.size();
        std::vector<uint32_t> order(n);
        std::iota(order.begin(), order.end(), 0);

        // Sort by canonical key: (street, housenumber, lat_grid, lng_grid)
        std::sort(order.begin(), order.end(), [&](uint32_t a, uint32_t b) {
            const auto& pa = points[a];
            const auto& pb = points[b];
            // Compare by remapped street name
            uint32_t sa = pool.map(pa.street_id), sb = pool.map(pb.street_id);
            int c = strcmp(pool.data.data() + sa, pool.data.data() + sb);
            if (c != 0) return c < 0;
            // Then housenumber
            uint32_t ha = pool.map(pa.housenumber_id), hb = pool.map(pb.housenumber_id);
            c = strcmp(pool.data.data() + ha, pool.data.data() + hb);
            if (c != 0) return c < 0;
            // Then coordinates
            int la = to_grid(pa.lat), lb = to_grid(pb.lat);
            if (la != lb) return la < lb;
            return to_grid(pa.lng) < to_grid(pb.lng);
        });

        // Build old->new mapping and write
        std::vector<uint32_t> old_to_new(n);
        for (uint32_t i = 0; i < n; i++) old_to_new[order[i]] = i;

        std::vector<PatchAddrPoint> sorted(n);
        for (uint32_t i = 0; i < n; i++) {
            sorted[i] = points[order[i]];
            sorted[i].housenumber_id = pool.map(sorted[i].housenumber_id);
            sorted[i].street_id = pool.map(sorted[i].street_id);
        }
        write_file(output_dir + "/addr_points.bin",
                   reinterpret_cast<const char*>(sorted.data()), n * sizeof(PatchAddrPoint));
        remap_cells(ca.addr_cells, old_to_new);
        std::cerr << "    " << n << " addr points" << std::endl;
    }

    // 4. Canonicalize street_ways + street_nodes
    std::cerr << "  Sorting street_ways..." << std::endl;
    {
        auto ways = read_ways(input_dir + "/street_ways.bin");
        auto nodes = read_structs<PatchNodeCoord>(input_dir + "/street_nodes.bin");
        size_t n = ways.size();
        std::vector<uint32_t> order(n);
        std::iota(order.begin(), order.end(), 0);

        // Sort by (name, node_count, first_node_lat, first_node_lng)
        std::sort(order.begin(), order.end(), [&](uint32_t a, uint32_t b) {
            const auto& wa = ways[a];
            const auto& wb = ways[b];
            uint32_t na = pool.map(wa.name_id), nb = pool.map(wb.name_id);
            int c = strcmp(pool.data.data() + na, pool.data.data() + nb);
            if (c != 0) return c < 0;
            if (wa.node_count != wb.node_count) return wa.node_count < wb.node_count;
            // Compare first node
            if (wa.node_count > 0 && wb.node_count > 0) {
                int la = to_grid(nodes[wa.node_offset].lat);
                int lb = to_grid(nodes[wb.node_offset].lat);
                if (la != lb) return la < lb;
                int lga = to_grid(nodes[wa.node_offset].lng);
                int lgb = to_grid(nodes[wb.node_offset].lng);
                if (lga != lgb) return lga < lgb;
            }
            return false;
        });

        std::vector<uint32_t> old_to_new(n);
        for (uint32_t i = 0; i < n; i++) old_to_new[order[i]] = i;

        // Write sorted ways + nodes
        std::vector<char> ways_buf;
        ways_buf.reserve(n * 9);
        std::vector<PatchNodeCoord> sorted_nodes;
        sorted_nodes.reserve(nodes.size());

        for (uint32_t i = 0; i < n; i++) {
            PatchWayHeader w = ways[order[i]];
            uint32_t old_offset = w.node_offset;
            w.node_offset = static_cast<uint32_t>(sorted_nodes.size());
            w.name_id = pool.map(w.name_id);
            ways_buf.insert(ways_buf.end(), reinterpret_cast<const char*>(&w),
                           reinterpret_cast<const char*>(&w) + 9);
            for (uint8_t j = 0; j < w.node_count; j++) {
                if (old_offset + j < nodes.size())
                    sorted_nodes.push_back(nodes[old_offset + j]);
            }
        }
        write_file(output_dir + "/street_ways.bin", ways_buf);
        write_file(output_dir + "/street_nodes.bin",
                   reinterpret_cast<const char*>(sorted_nodes.data()),
                   sorted_nodes.size() * sizeof(PatchNodeCoord));
        remap_cells(ca.street_cells, old_to_new);
        std::cerr << "    " << n << " ways, " << sorted_nodes.size() << " nodes" << std::endl;
    }

    // 5. Canonicalize interp_ways + interp_nodes
    std::cerr << "  Sorting interp_ways..." << std::endl;
    {
        auto interps = read_interps(input_dir + "/interp_ways.bin");
        auto nodes = read_structs<PatchNodeCoord>(input_dir + "/interp_nodes.bin");
        size_t n = interps.size();
        std::vector<uint32_t> order(n);
        std::iota(order.begin(), order.end(), 0);

        std::sort(order.begin(), order.end(), [&](uint32_t a, uint32_t b) {
            const auto& ia = interps[a];
            const auto& ib = interps[b];
            uint32_t sa = pool.map(ia.street_id), sb = pool.map(ib.street_id);
            int c = strcmp(pool.data.data() + sa, pool.data.data() + sb);
            if (c != 0) return c < 0;
            if (ia.start_number != ib.start_number) return ia.start_number < ib.start_number;
            if (ia.end_number != ib.end_number) return ia.end_number < ib.end_number;
            return ia.interpolation < ib.interpolation;
        });

        std::vector<uint32_t> old_to_new(n);
        for (uint32_t i = 0; i < n; i++) old_to_new[order[i]] = i;

        // Write sorted interps + nodes using builder's native struct size
        auto orig_data = read_file(input_dir + "/interp_ways.bin");
        size_t stride = (orig_data.size() % 20 == 0 && n > 0) ? 20 :
                        (orig_data.size() % 18 == 0 && n > 0) ? 18 : 20;

        std::vector<char> interp_buf;
        interp_buf.reserve(n * stride);
        std::vector<PatchNodeCoord> sorted_nodes;
        sorted_nodes.reserve(nodes.size());

        for (uint32_t i = 0; i < n; i++) {
            // Copy the raw bytes from original file for correct padding
            const char* src = orig_data.data() + order[i] * stride;
            char record[24] = {};
            memcpy(record, src, stride);

            // Update node_offset
            uint32_t new_node_offset = static_cast<uint32_t>(sorted_nodes.size());
            memcpy(record, &new_node_offset, 4);

            // Update street_id (at offset 5 or 8 depending on padding)
            size_t street_off = (stride == 20) ? 8 : 5;
            uint32_t old_street_id;
            memcpy(&old_street_id, record + street_off, 4);
            uint32_t new_street_id = pool.map(old_street_id);
            memcpy(record + street_off, &new_street_id, 4);

            interp_buf.insert(interp_buf.end(), record, record + stride);

            // Copy nodes
            const auto& iw = interps[order[i]];
            for (uint8_t j = 0; j < iw.node_count; j++) {
                if (iw.node_offset + j < nodes.size())
                    sorted_nodes.push_back(nodes[iw.node_offset + j]);
            }
        }
        write_file(output_dir + "/interp_ways.bin", interp_buf);
        write_file(output_dir + "/interp_nodes.bin",
                   reinterpret_cast<const char*>(sorted_nodes.data()),
                   sorted_nodes.size() * sizeof(PatchNodeCoord));
        remap_cells(ca.interp_cells, old_to_new);
        std::cerr << "    " << n << " interps, " << sorted_nodes.size() << " nodes" << std::endl;
    }

    // 6. Canonicalize admin_polygons + admin_vertices
    std::cerr << "  Sorting admin_polygons..." << std::endl;
    {
        auto polygons = read_admin_polygons(input_dir + "/admin_polygons.bin");
        auto vertices = read_structs<PatchNodeCoord>(input_dir + "/admin_vertices.bin");
        size_t n = polygons.size();
        std::vector<uint32_t> order(n);
        std::iota(order.begin(), order.end(), 0);

        std::sort(order.begin(), order.end(), [&](uint32_t a, uint32_t b) {
            const auto& pa = polygons[a];
            const auto& pb = polygons[b];
            uint32_t na = pool.map(pa.name_id), nb = pool.map(pb.name_id);
            int c = strcmp(pool.data.data() + na, pool.data.data() + nb);
            if (c != 0) return c < 0;
            if (pa.admin_level != pb.admin_level) return pa.admin_level < pb.admin_level;
            if (pa.country_code != pb.country_code) return pa.country_code < pb.country_code;
            return pa.vertex_count < pb.vertex_count;
        });

        std::vector<uint32_t> old_to_new(n);
        for (uint32_t i = 0; i < n; i++) old_to_new[order[i]] = i;

        // Write sorted polygons + vertices using builder's native struct size
        auto orig_data = read_file(input_dir + "/admin_polygons.bin");
        size_t stride = 0;
        for (size_t s : {24, 20, 19}) {
            if (orig_data.size() % s == 0 && n > 0) { stride = s; break; }
        }
        if (stride == 0) stride = 24;

        std::vector<char> poly_buf;
        poly_buf.reserve(n * stride);
        std::vector<PatchNodeCoord> sorted_verts;
        sorted_verts.reserve(vertices.size());

        for (uint32_t i = 0; i < n; i++) {
            const char* src = orig_data.data() + order[i] * stride;
            char record[32] = {};
            memcpy(record, src, stride);

            // Update vertex_offset
            uint32_t new_vert_offset = static_cast<uint32_t>(sorted_verts.size());
            memcpy(record, &new_vert_offset, 4);

            // Update name_id (at offset 8)
            uint32_t old_name_id;
            memcpy(&old_name_id, record + 8, 4);
            uint32_t new_name_id = pool.map(old_name_id);
            memcpy(record + 8, &new_name_id, 4);

            // Recompute area for canonical output
            // (area depends on vertex order which hasn't changed, so copy as-is)

            poly_buf.insert(poly_buf.end(), record, record + stride);

            // Copy vertices
            const auto& p = polygons[order[i]];
            for (uint32_t j = 0; j < p.vertex_count; j++) {
                if (p.vertex_offset + j < vertices.size())
                    sorted_verts.push_back(vertices[p.vertex_offset + j]);
            }
        }
        write_file(output_dir + "/admin_polygons.bin", poly_buf);
        write_file(output_dir + "/admin_vertices.bin",
                   reinterpret_cast<const char*>(sorted_verts.data()),
                   sorted_verts.size() * sizeof(PatchNodeCoord));
        remap_cells(ca.admin_cells, old_to_new);
        std::cerr << "    " << n << " polygons, " << sorted_verts.size() << " vertices" << std::endl;
    }

    // 7. Write canonical entries and cell index files
    std::cerr << "  Writing cell indexes..." << std::endl;
    write_geo_cells_and_entries(output_dir, ca.street_cells, ca.addr_cells, ca.interp_cells);
    write_entries_and_cells(output_dir + "/admin_cells.bin", output_dir + "/admin_entries.bin",
                           ca.admin_cells, 12);

    std::cerr << "Done. Canonical output in " << output_dir << std::endl;
    return 0;
}

+        // Write sorted ways + nodes, preserving original stride

+        auto orig_way_data = read_file(input_dir + "/street_ways.bin");

+        size_t way_stride = (orig_way_data.size() % 12 == 0 && n > 0) ? 12 : 9;

+        ways_buf.reserve(n * way_stride);

+            const char* src = orig_way_data.data() + order[i] * way_stride;

+            memcpy(record, src, way_stride);

+            // Update node_offset (offset 0)

+            uint32_t new_node_offset = static_cast<uint32_t>(sorted_nodes.size());

+            memcpy(record, &new_node_offset, 4);

+            size_t name_off = (way_stride == 12) ? 8 : 5;

+            memcpy(&old_name_id, record + name_off, 4);

+            uint32_t new_name_id = pool.map(old_name_id);

+            memcpy(record + name_off, &new_name_id, 4);

+            ways_buf.insert(ways_buf.end(), record, record + way_stride);

-            uint32_t old_offset = w.node_offset;

-            w.node_offset = static_cast<uint32_t>(sorted_nodes.size());

-            w.name_id = pool.map(w.name_id);

-            ways_buf.insert(ways_buf.end(), reinterpret_cast<const char*>(&w),

-                           reinterpret_cast<const char*>(&w) + 9);

-                if (old_offset + j < nodes.size())

-                    sorted_nodes.push_back(nodes[old_offset + j]);

+                if (w.node_offset + j < nodes.size())

+                    sorted_nodes.push_back(nodes[w.node_offset + j]);


Bash

cmake --build build -j$(nproc) --target geocoder-canonicalize 2>&1 | tail -5

Rebuild canonicalize tool


[ 50%] Building CXX object CMakeFiles/geocoder-canonicalize.dir/tools/geocoder_canonicalize.cpp.o
[100%] Linking CXX executable geocoder-canonicalize
[100%] Built target geocoder-canonicalize


sshpass -p 'elon-is-insane' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" /root/traccar-geocoder/builder/build/geocoder-canonicalize michtest@100.113.88.121:~/geocoder-canonicalize 2>&1

Deploy fixed canonicalize tool


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.


sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
rm -rf ~/canonical-old ~/canonical-new
time ~/geocoder-canonicalize ~/test-europe-old/full ~/canonical-old 2>&1
' 2>&1

Run canonicalize on old Europe build


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Loading index from /home/michtest/test-europe-old/full...
  Sorting string pool...
    4982052 strings
  Extracting cell assignments...
  Sorting addr_points...
    85891830 addr points
  Sorting street_ways...
    19452480 ways, 180557381 nodes
  Sorting interp_ways...
    16785 interps, 761965 nodes
  Sorting admin_polygons...
    450907 polygons, 172367816 vertices
  Writing cell indexes...
Done. Canonical output in /home/michtest/canonical-old

real	5m4.267s
user	4m53.989s
sys	0m10.139s


Canonicalize works. 5 minutes for Europe. Now canonicalize the new build and generate zstd patches between the two canonical outputs:


sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
time ~/geocoder-canonicalize ~/test-europe-new/full ~/canonical-new 2>&1
' 2>&1

Canonicalize new Europe build


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Loading index from /home/michtest/test-europe-new/full...
  Sorting string pool...
    4983588 strings
  Extracting cell assignments...
  Sorting addr_points...
    85990111 addr points
  Sorting street_ways...
    19468848 ways, 180700045 nodes
  Sorting interp_ways...
    16771 interps, 768852 nodes
  Sorting admin_polygons...
    450864 polygons, 172397114 vertices
  Writing cell indexes...
Done. Canonical output in /home/michtest/canonical-new

real	5m6.971s
user	4m48.389s
sys	0m18.111s


Both canonical builds done. Now the big test — generate zstd patches between canonical outputs:


sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
mkdir -p ~/patches-canonical
for f in ~/canonical-new/*.bin; do
    b=$(basename $f)
    old=~/canonical-old/$b
    if [ -f "$old" ]; then
        zstd --patch-from="$old" "$f" -o ~/patches-canonical/${b}.patch -f 2>&1
    fi
done
echo "=== SUMMARY ==="
total=0
for f in ~/canonical-new/*.bin; do
    b=$(basename $f)
    old=~/canonical-old/$b
    patch=~/patches-canonical/${b}.patch
    newsize=$(stat -c%s $f)
    patchsize=$(stat -c%s $patch 2>/dev/null || echo 0)
    ratio=$(echo "scale=1; $patchsize*100/$newsize" | bc 2>/dev/null || echo "?")
    total=$((total + patchsize))
    printf "  %-25s %10s -> %10s (%s%%)\n" "$b" "$(numfmt --to=iec $newsize)" "$(numfmt --to=iec $patchsize)" "$ratio"
done
echo "  TOTAL PATCH SIZE: $(numfmt --to=iec $total)"
'

Generate zstd patches between canonical builds


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
long mode automatically triggered
/home/michtest/canonical-new/addr_entries.bin : 79.23%   (   377 MiB =>    298 MiB, /home/michtest/patches-canonical/addr_entries.bin.patch)
long mode automatically triggered
/home/michtest/canonical-new/addr_points.bin : 54.14%   (  1.28 GiB =>    710 MiB, /home/michtest/patches-canonical/addr_points.bin.patch)
long mode automatically triggered
/home/michtest/canonical-new/admin_cells.bin : 27.71%   (  1.62 MiB =>    458 KiB, /home/michtest/patches-canonical/admin_cells.bin.patch)
long mode automatically triggered
/home/michtest/canonical-new/admin_entries.bin : 39.70%   (  6.13 MiB =>   2.43 MiB, /home/michtest/patches-canonical/admin_entries.bin.patch)
long mode automatically triggered
/home/michtest/canonical-new/admin_polygons.bin : 50.46%   (  10.3 MiB =>   5.21 MiB, /home/michtest/patches-canonical/admin_polygons.bin.patch)
long mode automatically triggered
/home/michtest/canonical-new/admin_vertices.bin :  0.22%   (  1.28 GiB =>   2.88 MiB, /home/michtest/patches-canonical/admin_vertices.bin.patch)
long mode automatically triggered
/home/michtest/canonical-new/geo_cells.bin : 23.68%   (  1.56 GiB =>    378 MiB, /home/michtest/patches-canonical/geo_cells.bin.patch)
long mode automatically triggered
/home/michtest/canonical-new/interp_entries.bin : 42.36%   (   156 KiB =>   65.9 KiB, /home/michtest/patches-canonical/interp_entries.bin.patch)
long mode automatically triggered
/home/michtest/canonical-new/interp_nodes.bin :  2.31%   (  5.87 MiB =>    138 KiB, /home/michtest/patches-canonical/interp_nodes.bin.patch)
long mode automatically triggered
/home/michtest/canonical-new/interp_ways.bin : 32.99%   (   328 KiB =>    108 KiB, /home/michtest/patches-canonical/interp_ways.bin.patch)
long mode automatically triggered
/home/michtest/canonical-new/street_entries.bin : 30.61%   (   559 MiB =>    171 MiB, /home/michtest/patches-canonical/street_entries.bin.patch)
long mode automatically triggered
/home/michtest/canonical-new/street_nodes.bin :  0.65%   (  1.35 GiB =>   8.94 MiB, /home/michtest/patches-canonical/street_nodes.bin.patch)
long mode automatically triggered
/home/michtest/canonical-new/street_ways.bin : 30.70%   (   223 MiB =>   68.4 MiB, /home/michtest/patches-canonical/street_ways.bin.patch)
long mode automatically triggered
/home/michtest/canonical-new/strings.bin :  0.09%   (  78.7 MiB =>   69.6 KiB, /home/michtest/patches-canonical/strings.bin.patch)
=== SUMMARY ===
  addr_entries.bin                377M ->       299M (79.2%)
  addr_points.bin                 1.3G ->       711M (54.1%)
  admin_cells.bin                 1.7M ->       459K (27.7%)
  admin_entries.bin               6.2M ->       2.5M (39.7%)
  admin_polygons.bin               11M ->       5.3M (50.4%)
  admin_vertices.bin              1.3G ->       2.9M (.2%)
  geo_cells.bin                   1.6G ->       379M (23.6%)
  interp_entries.bin              156K ->        66K (42.3%)
  interp_nodes.bin                5.9M ->       139K (2.3%)
  interp_ways.bin                 328K ->       109K (32.9%)
  street_entries.bin              560M ->       172M (30.6%)
  street_nodes.bin                1.4G ->       9.0M (.6%)
  street_ways.bin                 223M ->        69M (30.6%)
  strings.bin                      79M ->        70K (0%)
  TOTAL PATCH SIZE: 1.7G


The zstd patches against canonical builds are still 1.7 GiB — similar to before. This confirms that canonical ordering alone isn't enough; we need the custom diff tool that understands string remapping.

The key insight: both canonical builds have records in the same order, but string offsets differ. A custom diff that applies string remapping before comparing would see ~99.7% identical bytes.

Let me implement the diff and patch tools:


Write

// geocoder-diff: Compare two canonical geocoder indexes and produce a compact patch.
//
// The patch encodes:
// 1. New strings.bin (full replacement, zlib compressed)
// 2. String remap table (old_offset → new_offset for every string)
// 3. Per data file: record-level diff (matched records copied with string fixup,
//    new records inserted, removed records skipped)
//
// Usage: geocoder-diff <old-canonical-dir> <new-canonical-dir> -o <patch-file>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>
#include <zlib.h>

#include "patch_format.h"

// --- zlib compression helpers ---

static std::vector<char> zlib_compress(const char* data, size_t size) {
    uLongf bound = compressBound(size);
    std::vector<char> out(bound);
    int ret = compress2(reinterpret_cast<Bytef*>(out.data()), &bound,
                        reinterpret_cast<const Bytef*>(data), size, 9);
    if (ret != Z_OK) { std::cerr << "zlib compress failed" << std::endl; return {}; }
    out.resize(bound);
    return out;
}

// --- Build string remap ---
// Maps every string offset in old pool to its offset in new pool.

struct StringRemap {
    std::vector<std::pair<uint32_t, uint32_t>> entries; // (old_off, new_off)
    std::unordered_map<uint32_t, uint32_t> map; // old_off → new_off

    void build(const std::vector<char>& old_pool, const std::vector<char>& new_pool) {
        // Index new pool: string content → offset
        std::unordered_map<std::string, uint32_t> new_index;
        {
            size_t pos = 0;
            while (pos < new_pool.size()) {
                const char* s = new_pool.data() + pos;
                size_t len = strlen(s);
                new_index[std::string(s, len)] = static_cast<uint32_t>(pos);
                pos += len + 1;
            }
        }

        // Map each old string to new offset
        size_t pos = 0;
        while (pos < old_pool.size()) {
            const char* s = old_pool.data() + pos;
            size_t len = strlen(s);
            uint32_t old_off = static_cast<uint32_t>(pos);
            auto it = new_index.find(std::string(s, len));
            uint32_t new_off = (it != new_index.end()) ? it->second : 0xFFFFFFFF;
            entries.push_back({old_off, new_off});
            map[old_off] = new_off;
            pos += len + 1;
        }
    }

    uint32_t remap(uint32_t old_off) const {
        auto it = map.find(old_off);
        return (it != map.end()) ? it->second : 0;
    }
};

// --- Record-level diff for fixed-stride files ---
// Walks both files record by record. For matching records (same fingerprint),
// emits COPY. For new/removed records, emits INSERT/skip.

struct RecordDiff {
    // Operations to reconstruct new file from old + patch data
    struct Op {
        enum Type { COPY_REMAP, INSERT };
        Type type;
        uint32_t old_index;  // for COPY_REMAP: index in old file
        // For INSERT: data is in insert_data at insert_offset
        uint32_t insert_offset;
        uint32_t insert_length;
    };
    std::vector<Op> ops;
    std::vector<char> insert_data; // concatenated new record bytes
};

// Fingerprint an addr_point using resolved strings
static uint64_t addr_hash(const PatchAddrPoint& p, const std::vector<char>& pool) {
    // Simple hash of grid coords + string content
    uint64_t h = 14695981039346656037ULL;
    auto mix = [&h](uint64_t v) { h ^= v; h *= 1099511628211ULL; };
    mix(to_grid(p.lat));
    mix(to_grid(p.lng));
    const char* hn = get_string(pool, p.housenumber_id);
    while (*hn) { mix(*hn); hn++; }
    mix(0xFF);
    const char* st = get_string(pool, p.street_id);
    while (*st) { mix(*st); st++; }
    return h;
}

static uint64_t way_hash(const PatchWayHeader& w, const PatchNodeCoord* nodes,
                          const std::vector<char>& pool) {
    uint64_t h = 14695981039346656037ULL;
    auto mix = [&h](uint64_t v) { h ^= v; h *= 1099511628211ULL; };
    const char* name = get_string(pool, w.name_id);
    while (*name) { mix(*name); name++; }
    mix(0xFF);
    mix(w.node_count);
    for (uint8_t i = 0; i < w.node_count; i++) {
        mix(to_grid(nodes[w.node_offset + i].lat));
        mix(to_grid(nodes[w.node_offset + i].lng));
    }
    return h;
}

// --- Write patch file ---

static void write_u32(std::ofstream& f, uint32_t v) { f.write(reinterpret_cast<const char*>(&v), 4); }
static void write_u64(std::ofstream& f, uint64_t v) { f.write(reinterpret_cast<const char*>(&v), 8); }

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>" << std::endl;
        return 1;
    }

    std::string old_dir = argv[1];
    std::string new_dir = argv[2];
    std::string patch_path = argv[4];

    std::cerr << "Loading indexes..." << std::endl;

    // Load string pools and build remap
    auto old_strings = read_file(old_dir + "/strings.bin");
    auto new_strings = read_file(new_dir + "/strings.bin");
    StringRemap remap;
    remap.build(old_strings, new_strings);
    std::cerr << "  String remap: " << remap.entries.size() << " entries" << std::endl;

    // Open patch file
    std::ofstream pf(patch_path, std::ios::binary);
    if (!pf) { std::cerr << "Cannot open " << patch_path << std::endl; return 1; }

    // Write header
    pf.write(GCPATCH_MAGIC, 8);
    write_u32(pf, GCPATCH_VERSION);
    write_u32(pf, 0); // flags

    // Section 1: New strings.bin (full replacement, compressed)
    std::cerr << "  Writing strings section..." << std::endl;
    {
        auto compressed = zlib_compress(new_strings.data(), new_strings.size());
        write_u32(pf, static_cast<uint32_t>(PatchFileId::STRINGS));
        write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));
        write_u64(pf, new_strings.size()); // uncompressed size
        write_u64(pf, compressed.size());  // compressed size
        pf.write(compressed.data(), compressed.size());
        std::cerr << "    " << new_strings.size() << " -> " << compressed.size() << " bytes" << std::endl;
    }

    // Section 2: String remap table
    std::cerr << "  Writing string remap..." << std::endl;
    {
        // Only write remaps where old_off != new_off
        std::vector<std::pair<uint32_t, uint32_t>> changed;
        for (auto& [old_off, new_off] : remap.entries) {
            if (old_off != new_off && new_off != 0xFFFFFFFF) {
                changed.push_back({old_off, new_off});
            }
        }
        write_u32(pf, 0xFFFFFFFE); // special marker for remap section
        write_u32(pf, static_cast<uint32_t>(changed.size()));
        for (auto& [o, n] : changed) {
            write_u32(pf, o);
            write_u32(pf, n);
        }
        std::cerr << "    " << changed.size() << " changed offsets (of " << remap.entries.size() << " total)" << std::endl;
    }

    // Helper: diff a data file where records have fixed stride
    // Strategy: for each file, send the full new file compressed.
    // The applicator just decompresses and writes it.
    // This is simple and correct. For files where zlib compresses well
    // against their size, this is good enough.
    auto write_file_section = [&](PatchFileId id, const std::string& name) {
        std::cerr << "  Writing " << name << "..." << std::endl;
        auto new_data = read_file(new_dir + "/" + name);
        auto compressed = zlib_compress(new_data.data(), new_data.size());
        write_u32(pf, static_cast<uint32_t>(id));
        write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));
        write_u64(pf, new_data.size());
        write_u64(pf, compressed.size());
        pf.write(compressed.data(), compressed.size());
        std::cerr << "    " << new_data.size() << " -> " << compressed.size()
                  << " (" << (new_data.size() > 0 ? compressed.size() * 100 / new_data.size() : 0) << "%)" << std::endl;
    };

    // For now: send ALL files as full replacement (compressed).
    // This gives us correct patches that we can test end-to-end.
    // Optimization: use COPY_INSERT encoding for large files later.
    write_file_section(PatchFileId::STREET_WAYS, "street_ways.bin");
    write_file_section(PatchFileId::STREET_NODES, "street_nodes.bin");
    write_file_section(PatchFileId::ADDR_POINTS, "addr_points.bin");
    write_file_section(PatchFileId::INTERP_WAYS, "interp_ways.bin");
    write_file_section(PatchFileId::INTERP_NODES, "interp_nodes.bin");
    write_file_section(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin");
    write_file_section(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin");
    write_file_section(PatchFileId::GEO_CELLS, "geo_cells.bin");
    write_file_section(PatchFileId::STREET_ENTRIES, "street_entries.bin");
    write_file_section(PatchFileId::ADDR_ENTRIES, "addr_entries.bin");
    write_file_section(PatchFileId::INTERP_ENTRIES, "interp_entries.bin");
    write_file_section(PatchFileId::ADMIN_CELLS, "admin_cells.bin");
    write_file_section(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin");

    // Write end marker
    write_u32(pf, 0xFFFFFFFF);

    std::cerr << "Patch written to " << patch_path << " (" << pf.tellp() << " bytes)" << std::endl;
    return 0;
}


File created successfully at: /root/traccar-geocoder/builder/tools/geocoder_diff.cpp

// geocoder-diff: Compare two canonical geocoder indexes and produce a compact patch.
//
// The patch encodes:
// 1. New strings.bin (full replacement, zlib compressed)
// 2. String remap table (old_offset → new_offset for every string)
// 3. Per data file: record-level diff (matched records copied with string fixup,
//    new records inserted, removed records skipped)
//
// Usage: geocoder-diff <old-canonical-dir> <new-canonical-dir> -o <patch-file>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>
#include <zlib.h>

#include "patch_format.h"

// --- zlib compression helpers ---

static std::vector<char> zlib_compress(const char* data, size_t size) {
    uLongf bound = compressBound(size);
    std::vector<char> out(bound);
    int ret = compress2(reinterpret_cast<Bytef*>(out.data()), &bound,
                        reinterpret_cast<const Bytef*>(data), size, 9);
    if (ret != Z_OK) { std::cerr << "zlib compress failed" << std::endl; return {}; }
    out.resize(bound);
    return out;
}

// --- Build string remap ---
// Maps every string offset in old pool to its offset in new pool.

struct StringRemap {
    std::vector<std::pair<uint32_t, uint32_t>> entries; // (old_off, new_off)
    std::unordered_map<uint32_t, uint32_t> map; // old_off → new_off

    void build(const std::vector<char>& old_pool, const std::vector<char>& new_pool) {
        // Index new pool: string content → offset
        std::unordered_map<std::string, uint32_t> new_index;
        {
            size_t pos = 0;
            while (pos < new_pool.size()) {
                const char* s = new_pool.data() + pos;
                size_t len = strlen(s);
                new_index[std::string(s, len)] = static_cast<uint32_t>(pos);
                pos += len + 1;
            }
        }

        // Map each old string to new offset
        size_t pos = 0;
        while (pos < old_pool.size()) {
            const char* s = old_pool.data() + pos;
            size_t len = strlen(s);
            uint32_t old_off = static_cast<uint32_t>(pos);
            auto it = new_index.find(std::string(s, len));
            uint32_t new_off = (it != new_index.end()) ? it->second : 0xFFFFFFFF;
            entries.push_back({old_off, new_off});
            map[old_off] = new_off;
            pos += len + 1;
        }
    }

    uint32_t remap(uint32_t old_off) const {
        auto it = map.find(old_off);
        return (it != map.end()) ? it->second : 0;
    }
};

// --- Record-level diff for fixed-stride files ---
// Walks both files record by record. For matching records (same fingerprint),
// emits COPY. For new/removed records, emits INSERT/skip.

struct RecordDiff {
    // Operations to reconstruct new file from old + patch data
    struct Op {
        enum Type { COPY_REMAP, INSERT };
        Type type;
        uint32_t old_index;  // for COPY_REMAP: index in old file
        // For INSERT: data is in insert_data at insert_offset
        uint32_t insert_offset;
        uint32_t insert_length;
    };
    std::vector<Op> ops;
    std::vector<char> insert_data; // concatenated new record bytes
};

// Fingerprint an addr_point using resolved strings
static uint64_t addr_hash(const PatchAddrPoint& p, const std::vector<char>& pool) {
    // Simple hash of grid coords + string content
    uint64_t h = 14695981039346656037ULL;
    auto mix = [&h](uint64_t v) { h ^= v; h *= 1099511628211ULL; };
    mix(to_grid(p.lat));
    mix(to_grid(p.lng));
    const char* hn = get_string(pool, p.housenumber_id);
    while (*hn) { mix(*hn); hn++; }
    mix(0xFF);
    const char* st = get_string(pool, p.street_id);
    while (*st) { mix(*st); st++; }
    return h;
}

static uint64_t way_hash(const PatchWayHeader& w, const PatchNodeCoord* nodes,
                          const std::vector<char>& pool) {
    uint64_t h = 14695981039346656037ULL;
    auto mix = [&h](uint64_t v) { h ^= v; h *= 1099511628211ULL; };
    const char* name = get_string(pool, w.name_id);
    while (*name) { mix(*name); name++; }
    mix(0xFF);
    mix(w.node_count);
    for (uint8_t i = 0; i < w.node_count; i++) {
        mix(to_grid(nodes[w.node_offset + i].lat));
        mix(to_grid(nodes[w.node_offset + i].lng));
    }
    return h;
}

// --- Write patch file ---

static void write_u32(std::ofstream& f, uint32_t v) { f.write(reinterpret_cast<const char*>(&v), 4); }
static void write_u64(std::ofstream& f, uint64_t v) { f.write(reinterpret_cast<const char*>(&v), 8); }

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>" << std::endl;
        return 1;
    }

    std::string old_dir = argv[1];
    std::string new_dir = argv[2];
    std::string patch_path = argv[4];

    std::cerr << "Loading indexes..." << std::endl;

    // Load string pools and build remap
    auto old_strings = read_file(old_dir + "/strings.bin");
    auto new_strings = read_file(new_dir + "/strings.bin");
    StringRemap remap;
    remap.build(old_strings, new_strings);
    std::cerr << "  String remap: " << remap.entries.size() << " entries" << std::endl;

    // Open patch file
    std::ofstream pf(patch_path, std::ios::binary);
    if (!pf) { std::cerr << "Cannot open " << patch_path << std::endl; return 1; }

    // Write header
    pf.write(GCPATCH_MAGIC, 8);
    write_u32(pf, GCPATCH_VERSION);
    write_u32(pf, 0); // flags

    // Section 1: New strings.bin (full replacement, compressed)
    std::cerr << "  Writing strings section..." << std::endl;
    {
        auto compressed = zlib_compress(new_strings.data(), new_strings.size());
        write_u32(pf, static_cast<uint32_t>(PatchFileId::STRINGS));
        write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));
        write_u64(pf, new_strings.size()); // uncompressed size
        write_u64(pf, compressed.size());  // compressed size
        pf.write(compressed.data(), compressed.size());
        std::cerr << "    " << new_strings.size() << " -> " << compressed.size() << " bytes" << std::endl;
    }

    // Section 2: String remap table
    std::cerr << "  Writing string remap..." << std::endl;
    {
        // Only write remaps where old_off != new_off
        std::vector<std::pair<uint32_t, uint32_t>> changed;
        for (auto& [old_off, new_off] : remap.entries) {
            if (old_off != new_off && new_off != 0xFFFFFFFF) {
                changed.push_back({old_off, new_off});
            }
        }
        write_u32(pf, 0xFFFFFFFE); // special marker for remap section
        write_u32(pf, static_cast<uint32_t>(changed.size()));
        for (auto& [o, n] : changed) {
            write_u32(pf, o);
            write_u32(pf, n);
        }
        std::cerr << "    " << changed.size() << " changed offsets (of " << remap.entries.size() << " total)" << std::endl;
    }

    // Helper: diff a data file where records have fixed stride
    // Strategy: for each file, send the full new file compressed.
    // The applicator just decompresses and writes it.
    // This is simple and correct. For files where zlib compresses well
    // against their size, this is good enough.
    auto write_file_section = [&](PatchFileId id, const std::string& name) {
        std::cerr << "  Writing " << name << "..." << std::endl;
        auto new_data = read_file(new_dir + "/" + name);
        auto compressed = zlib_compress(new_data.data(), new_data.size());
        write_u32(pf, static_cast<uint32_t>(id));
        write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));
        write_u64(pf, new_data.size());
        write_u64(pf, compressed.size());
        pf.write(compressed.data(), compressed.size());
        std::cerr << "    " << new_data.size() << " -> " << compressed.size()
                  << " (" << (new_data.size() > 0 ? compressed.size() * 100 / new_data.size() : 0) << "%)" << std::endl;
    };

    // For now: send ALL files as full replacement (compressed).
    // This gives us correct patches that we can test end-to-end.
    // Optimization: use COPY_INSERT encoding for large files later.
    write_file_section(PatchFileId::STREET_WAYS, "street_ways.bin");
    write_file_section(PatchFileId::STREET_NODES, "street_nodes.bin");
    write_file_section(PatchFileId::ADDR_POINTS, "addr_points.bin");
    write_file_section(PatchFileId::INTERP_WAYS, "interp_ways.bin");
    write_file_section(PatchFileId::INTERP_NODES, "interp_nodes.bin");
    write_file_section(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin");
    write_file_section(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin");
    write_file_section(PatchFileId::GEO_CELLS, "geo_cells.bin");
    write_file_section(PatchFileId::STREET_ENTRIES, "street_entries.bin");
    write_file_section(PatchFileId::ADDR_ENTRIES, "addr_entries.bin");
    write_file_section(PatchFileId::INTERP_ENTRIES, "interp_entries.bin");
    write_file_section(PatchFileId::ADMIN_CELLS, "admin_cells.bin");
    write_file_section(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin");

    // Write end marker
    write_u32(pf, 0xFFFFFFFF);

    std::cerr << "Patch written to " << patch_path << " (" << pf.tellp() << " bytes)" << std::endl;
    return 0;
}


// geocoder-patch: Apply a .gcpatch file to a canonical geocoder index.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
#include <zlib.h>

#include "patch_format.h"

static uint32_t read_u32(std::ifstream& f) {
    uint32_t v; f.read(reinterpret_cast<char*>(&v), 4); return v;
}
static uint64_t read_u64(std::ifstream& f) {
    uint64_t v; f.read(reinterpret_cast<char*>(&v), 8); return v;
}

static std::vector<char> zlib_decompress(const char* data, size_t compressed_size, size_t uncompressed_size) {
    std::vector<char> out(uncompressed_size);
    uLongf dest_len = uncompressed_size;
    int ret = uncompress(reinterpret_cast<Bytef*>(out.data()), &dest_len,
                         reinterpret_cast<const Bytef*>(data), compressed_size);
    if (ret != Z_OK) {
        std::cerr << "zlib decompress failed: " << ret << std::endl;
        return {};
    }
    out.resize(dest_len);
    return out;
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }

    std::string current_dir = argv[1];
    std::string patch_path = argv[2];
    std::string output_dir = argv[4];
    ensure_dir(output_dir);

    // Open patch file
    std::ifstream pf(patch_path, std::ios::binary);
    if (!pf) { std::cerr << "Cannot open " << patch_path << std::endl; return 1; }

    // Read and verify header
    char magic[8];
    pf.read(magic, 8);
    if (memcmp(magic, GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch file (bad magic)" << std::endl;
        return 1;
    }
    uint32_t version = read_u32(pf);
    if (version != GCPATCH_VERSION) {
        std::cerr << "Unsupported patch version: " << version << std::endl;
        return 1;
    }
    uint32_t flags = read_u32(pf);
    (void)flags;

    std::cerr << "Applying patch from " << patch_path << "..." << std::endl;

    // Process sections
    while (pf) {
        uint32_t section_id = read_u32(pf);
        if (!pf || section_id == 0xFFFFFFFF) break; // end marker

        // String remap section (special)
        if (section_id == 0xFFFFFFFE) {
            uint32_t count = read_u32(pf);
            // Skip remap entries (not needed for RAW_REPLACE mode)
            pf.seekg(count * 8, std::ios::cur);
            std::cerr << "  Skipping string remap (" << count << " entries)" << std::endl;
            continue;
        }

        uint32_t encoding = read_u32(pf);
        uint64_t uncompressed_size = read_u64(pf);
        uint64_t compressed_size = read_u64(pf);

        if (section_id >= static_cast<uint32_t>(PatchFileId::COUNT)) {
            std::cerr << "  Unknown section " << section_id << ", skipping " << compressed_size << " bytes" << std::endl;
            pf.seekg(compressed_size, std::ios::cur);
            continue;
        }

        const char* filename = patch_file_names[section_id];

        if (encoding == static_cast<uint32_t>(PatchEncoding::RAW_REPLACE)) {
            // Read compressed data
            std::vector<char> compressed(compressed_size);
            pf.read(compressed.data(), compressed_size);

            // Decompress
            auto data = zlib_decompress(compressed.data(), compressed_size, uncompressed_size);
            if (data.empty() && uncompressed_size > 0) {
                std::cerr << "  FAILED to decompress " << filename << std::endl;
                return 1;
            }

            // Write output file
            std::string out_path = output_dir + "/" + filename;
            write_file(out_path, data);
            std::cerr << "  " << filename << ": " << compressed_size << " -> " << data.size() << " bytes" << std::endl;
        } else {
            std::cerr << "  Unknown encoding " << encoding << " for " << filename << std::endl;
            pf.seekg(compressed_size, std::ios::cur);
        }
    }

    std::cerr << "Patch applied. Output in " << output_dir << std::endl;
    return 0;
}


File created successfully at: /root/traccar-geocoder/builder/tools/geocoder_patch.cpp

// geocoder-patch: Apply a .gcpatch file to a canonical geocoder index.
//
// Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
#include <zlib.h>

#include "patch_format.h"

static uint32_t read_u32(std::ifstream& f) {
    uint32_t v; f.read(reinterpret_cast<char*>(&v), 4); return v;
}
static uint64_t read_u64(std::ifstream& f) {
    uint64_t v; f.read(reinterpret_cast<char*>(&v), 8); return v;
}

static std::vector<char> zlib_decompress(const char* data, size_t compressed_size, size_t uncompressed_size) {
    std::vector<char> out(uncompressed_size);
    uLongf dest_len = uncompressed_size;
    int ret = uncompress(reinterpret_cast<Bytef*>(out.data()), &dest_len,
                         reinterpret_cast<const Bytef*>(data), compressed_size);
    if (ret != Z_OK) {
        std::cerr << "zlib decompress failed: " << ret << std::endl;
        return {};
    }
    out.resize(dest_len);
    return out;
}

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-patch <current-dir> <patch-file> -o <output-dir>" << std::endl;
        return 1;
    }

    std::string current_dir = argv[1];
    std::string patch_path = argv[2];
    std::string output_dir = argv[4];
    ensure_dir(output_dir);

    // Open patch file
    std::ifstream pf(patch_path, std::ios::binary);
    if (!pf) { std::cerr << "Cannot open " << patch_path << std::endl; return 1; }

    // Read and verify header
    char magic[8];
    pf.read(magic, 8);
    if (memcmp(magic, GCPATCH_MAGIC, 8) != 0) {
        std::cerr << "Invalid patch file (bad magic)" << std::endl;
        return 1;
    }
    uint32_t version = read_u32(pf);
    if (version != GCPATCH_VERSION) {
        std::cerr << "Unsupported patch version: " << version << std::endl;
        return 1;
    }
    uint32_t flags = read_u32(pf);
    (void)flags;

    std::cerr << "Applying patch from " << patch_path << "..." << std::endl;

    // Process sections
    while (pf) {
        uint32_t section_id = read_u32(pf);
        if (!pf || section_id == 0xFFFFFFFF) break; // end marker

        // String remap section (special)
        if (section_id == 0xFFFFFFFE) {
            uint32_t count = read_u32(pf);
            // Skip remap entries (not needed for RAW_REPLACE mode)
            pf.seekg(count * 8, std::ios::cur);
            std::cerr << "  Skipping string remap (" << count << " entries)" << std::endl;
            continue;
        }

        uint32_t encoding = read_u32(pf);
        uint64_t uncompressed_size = read_u64(pf);
        uint64_t compressed_size = read_u64(pf);

        if (section_id >= static_cast<uint32_t>(PatchFileId::COUNT)) {
            std::cerr << "  Unknown section " << section_id << ", skipping " << compressed_size << " bytes" << std::endl;
            pf.seekg(compressed_size, std::ios::cur);
            continue;
        }

        const char* filename = patch_file_names[section_id];

        if (encoding == static_cast<uint32_t>(PatchEncoding::RAW_REPLACE)) {
            // Read compressed data
            std::vector<char> compressed(compressed_size);
            pf.read(compressed.data(), compressed_size);

            // Decompress
            auto data = zlib_decompress(compressed.data(), compressed_size, uncompressed_size);
            if (data.empty() && uncompressed_size > 0) {
                std::cerr << "  FAILED to decompress " << filename << std::endl;
                return 1;
            }

            // Write output file
            std::string out_path = output_dir + "/" + filename;
            write_file(out_path, data);
            std::cerr << "  " << filename << ": " << compressed_size << " -> " << data.size() << " bytes" << std::endl;
        } else {
            std::cerr << "  Unknown encoding " << encoding << " for " << filename << std::endl;
            pf.seekg(compressed_size, std::ios::cur);
        }
    }

    std::cerr << "Patch applied. Output in " << output_dir << std::endl;
    return 0;
}


Edit

# Patch tools (no S2 dependency)
add_executable(geocoder-canonicalize tools/geocoder_canonicalize.cpp)
target_link_libraries(geocoder-canonicalize PRIVATE Threads::Threads)
target_include_directories(geocoder-canonicalize PRIVATE tools)

# Patch tools (no S2 dependency)
add_executable(geocoder-canonicalize tools/geocoder_canonicalize.cpp)
target_link_libraries(geocoder-canonicalize PRIVATE Threads::Threads)
target_include_directories(geocoder-canonicalize PRIVATE tools)

add_executable(geocoder-diff tools/geocoder_diff.cpp)
target_link_libraries(geocoder-diff PRIVATE ZLIB::ZLIB Threads::Threads)
target_include_directories(geocoder-diff PRIVATE tools)

add_executable(geocoder-patch tools/geocoder_patch.cpp)
target_link_libraries(geocoder-patch PRIVATE ZLIB::ZLIB Threads::Threads)
target_include_directories(geocoder-patch PRIVATE tools)


The file /root/traccar-geocoder/builder/CMakeLists.txt has been updated successfully.

# Patch tools (no S2 dependency)
add_executable(geocoder-canonicalize tools/geocoder_canonicalize.cpp)
target_link_libraries(geocoder-canonicalize PRIVATE Threads::Threads)
target_include_directories(geocoder-canonicalize PRIVATE tools)

add_executable(geocoder-diff tools/geocoder_diff.cpp)
target_link_libraries(geocoder-diff PRIVATE ZLIB::ZLIB Threads::Threads)
target_include_directories(geocoder-diff PRIVATE tools)

add_executable(geocoder-patch tools/geocoder_patch.cpp)
target_link_libraries(geocoder-patch PRIVATE ZLIB::ZLIB Threads::Threads)
target_include_directories(geocoder-patch PRIVATE tools)

cmake_minimum_required(VERSION 3.16)
project(traccar-geocoder-build LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

find_package(Threads REQUIRED)
find_package(ZLIB REQUIRED)
find_package(BZip2 REQUIRED)
find_package(EXPAT REQUIRED)
find_package(s2 QUIET)

add_executable(build-index
    src/build_index.cpp
    src/pbf_reader.cpp
    src/s2_helpers.cpp
    src/ring_assembly.cpp
    src/cache.cpp
    src/continent_filter.cpp
    src/continent_boundaries.cpp
    src/cell_index.cpp
    src/interpolation.cpp
)

# Patch tools (no S2 dependency)
add_executable(geocoder-canonicalize tools/geocoder_canonicalize.cpp)
target_link_libraries(geocoder-canonicalize PRIVATE Threads::Threads)
target_include_directories(geocoder-canonicalize PRIVATE tools)

if(EXISTS /opt/homebrew/include)
    target_include_directories(build-index PRIVATE /opt/homebrew/include)
endif()

if(s2_FOUND)
    target_link_libraries(build-index PRIVATE
        s2::s2
        ZLIB::ZLIB
        BZip2::BZip2
        EXPAT::EXPAT
        Threads::Threads
    )
else()
    target_include_directories(build-index PRIVATE /usr/include/s2)
    target_link_libraries(build-index PRIVATE
        s2
        ZLIB::ZLIB
        BZip2::BZip2
        EXPAT::EXPAT
        Threads::Threads
    )
endif()

+target_link_libraries(geocoder-diff PRIVATE ZLIB::ZLIB Threads::Threads)

+target_link_libraries(geocoder-patch PRIVATE ZLIB::ZLIB Threads::Threads)


Bash

cmake -B build . 2>&1 | tail -3 && cmake --build build -j$(nproc) 2>&1 | tail -10

Build all tools


-- Configuring done (0.0s)
-- Generating done (0.0s)
-- Build files have been written to: /root/traccar-geocoder/builder/build
[  6%] Building CXX object CMakeFiles/geocoder-patch.dir/tools/geocoder_patch.cpp.o
[ 12%] Building CXX object CMakeFiles/geocoder-diff.dir/tools/geocoder_diff.cpp.o
[ 25%] Built target geocoder-canonicalize
[ 87%] Built target build-index
[ 93%] Linking CXX executable geocoder-patch
[ 93%] Built target geocoder-patch
[100%] Linking CXX executable geocoder-diff
[100%] Built target geocoder-diff


All three tools build. Now let me deploy and test the full pipeline: canonicalize → diff → patch → verify.


sshpass -p 'elon-is-insane' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" /root/traccar-geocoder/builder/build/geocoder-diff /root/traccar-geocoder/builder/build/geocoder-patch michtest@100.113.88.121:~/ 2>&1

Deploy diff and patch tools to node 3


sshpass -p 'elon-is-insane' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="tailscale nc %h %p" michtest@100.113.88.121 '
# Generate patch from canonical old → canonical new
time ~/geocoder-diff ~/canonical-old ~/canonical-new -o ~/europe.gcpatch 2>&1
echo "==="
ls -lh ~/europe.gcpatch
' 2>&1

Generate patch between canonical builds


Warning: Permanently added '100.113.88.121' (ED25519) to the list of known hosts.
Loading indexes...
  String remap: 4982052 entries
  Writing strings section...
    82573397 -> 21688674 bytes
  Writing string remap...
    4980054 changed offsets (of 4982052 total)
  Writing street_ways.bin...
    233626176 -> 68385499 (29%)
  Writing street_nodes.bin...
    1445600360 -> 859633960 (59%)
  Writing addr_points.bin...
    1375841776 -> 689169135 (50%)
  Writing interp_ways.bin...
    335420 -> 109142 (32%)
  Writing interp_nodes.bin...
    6150816 -> 165160 (2%)
  Writing admin_polygons.bin...
    10820736 -> 5372585 (49%)
  Writing admin_vertices.bin...
    1379176912 -> 792489660 (57%)
  Writing geo_cells.bin...
    1674295440 -> 400456262 (23%)
  Writing street_entries.bin...
    586571766 -> 154113524 (26%)
  Writing addr_entries.bin...
    395005236 -> 223596813 (56%)
  Writing interp_entries.bin...
    159386 -> 65753 (41%)
  Writing admin_cells.bin...
    1693932 -> 602894 (35%)
  Writing admin_entries.bin...
    6425798 -> 2311928 (35%)
Patch written to /home/michtest/europe.gcpatch (3258001785 bytes)
===

real	6m0.112s
user	5m57.589s
sys	0m2.492s
-rw-rw-r-- 1 michtest michtest 3.1G Mar 27 00:34 /home/michtest/europe.gcpatch


3.1 GiB — that's just zlib compression of the full files, not a delta. I need to implement actual binary diffing with string remapping. Let me rewrite the diff tool properly:


Write

// geocoder-diff: Compare two canonical geocoder indexes and produce a compact patch.
//
// Strategy:
// 1. Build string remap (old string offsets → new string offsets)
// 2. For data files with string refs: apply remap to old file in memory, then binary diff
// 3. For coordinate files: direct binary diff (no string refs)
// 4. For small derived files: full replacement (compressed)
//
// The binary diff uses a simple block-based COPY/INSERT encoder:
// - Compare 64-byte blocks between (remapped) old and new
// - COPY for matching blocks, INSERT for different blocks
// - Handles size differences (new may be larger)
//
// Usage: geocoder-diff <old-canonical-dir> <new-canonical-dir> -o <patch-file>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>
#include <zlib.h>

#include "patch_format.h"

// --- zlib helpers ---

static std::vector<char> zlib_compress(const char* data, size_t size) {
    if (size == 0) return {};
    uLongf bound = compressBound(size);
    std::vector<char> out(bound);
    int ret = compress2(reinterpret_cast<Bytef*>(out.data()), &bound,
                        reinterpret_cast<const Bytef*>(data), size, 9);
    if (ret != Z_OK) return {};
    out.resize(bound);
    return out;
}

// --- String remap ---

static std::unordered_map<uint32_t, uint32_t> build_string_remap(
    const std::vector<char>& old_pool, const std::vector<char>& new_pool)
{
    // Index new pool
    std::unordered_map<std::string, uint32_t> new_index;
    size_t pos = 0;
    while (pos < new_pool.size()) {
        const char* s = new_pool.data() + pos;
        size_t len = strlen(s);
        new_index[std::string(s, len)] = static_cast<uint32_t>(pos);
        pos += len + 1;
    }

    // Map old offsets → new offsets
    std::unordered_map<uint32_t, uint32_t> remap;
    pos = 0;
    while (pos < old_pool.size()) {
        const char* s = old_pool.data() + pos;
        size_t len = strlen(s);
        auto it = new_index.find(std::string(s, len));
        if (it != new_index.end()) {
            remap[static_cast<uint32_t>(pos)] = it->second;
        }
        pos += len + 1;
    }
    return remap;
}

// Apply string remap to specific fields in a data buffer
static void remap_addr_points(std::vector<char>& data,
                                const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / 16;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * 16;
        // housenumber_id at offset 8, street_id at offset 12
        for (size_t off : {8, 12}) {
            uint32_t old_id;
            memcpy(&old_id, p + off, 4);
            auto it = remap.find(old_id);
            if (it != remap.end()) {
                memcpy(p + off, &it->second, 4);
            }
        }
    }
}

static void remap_ways(std::vector<char>& data, size_t stride,
                        const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / stride;
    size_t name_off = (stride == 12) ? 8 : 5;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * stride;
        uint32_t old_id;
        memcpy(&old_id, p + name_off, 4);
        auto it = remap.find(old_id);
        if (it != remap.end()) {
            memcpy(p + name_off, &it->second, 4);
        }
    }
}

static void remap_interps(std::vector<char>& data, size_t stride,
                           const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / stride;
    size_t street_off = (stride == 20) ? 8 : 5;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * stride;
        uint32_t old_id;
        memcpy(&old_id, p + street_off, 4);
        auto it = remap.find(old_id);
        if (it != remap.end()) {
            memcpy(p + street_off, &it->second, 4);
        }
    }
}

static void remap_admin_polygons(std::vector<char>& data, size_t stride,
                                   const std::unordered_map<uint32_t, uint32_t>& remap) {
    size_t n = data.size() / stride;
    for (size_t i = 0; i < n; i++) {
        char* p = data.data() + i * stride;
        uint32_t old_id;
        memcpy(&old_id, p + 8, 4); // name_id at offset 8
        auto it = remap.find(old_id);
        if (it != remap.end()) {
            memcpy(p + 8, &it->second, 4);
        }
    }
}

// --- Block-based COPY/INSERT binary diff ---
// Compares (remapped) old vs new, produces a compact delta.
// Block size of 64 bytes balances granularity vs overhead.

static constexpr size_t DIFF_BLOCK = 64;

static std::vector<char> compute_block_diff(const std::vector<char>& old_data,
                                              const std::vector<char>& new_data) {
    std::vector<char> ops;
    size_t old_pos = 0, new_pos = 0;

    while (new_pos < new_data.size()) {
        // Check if current block matches
        size_t remaining_new = new_data.size() - new_pos;
        size_t remaining_old = (old_pos < old_data.size()) ? old_data.size() - old_pos : 0;
        size_t block = std::min(DIFF_BLOCK, remaining_new);

        if (remaining_old >= block &&
            memcmp(old_data.data() + old_pos, new_data.data() + new_pos, block) == 0) {
            // Find full extent of match
            size_t match_len = block;
            while (old_pos + match_len < old_data.size() &&
                   new_pos + match_len < new_data.size() &&
                   old_data[old_pos + match_len] == new_data[new_pos + match_len]) {
                match_len++;
            }
            // Emit COPY: tag(1) + offset(8) + length(4) = 13 bytes
            ops.push_back(OP_COPY);
            uint64_t off = old_pos;
            uint32_t len = static_cast<uint32_t>(match_len);
            ops.insert(ops.end(), reinterpret_cast<const char*>(&off),
                      reinterpret_cast<const char*>(&off) + 8);
            ops.insert(ops.end(), reinterpret_cast<const char*>(&len),
                      reinterpret_cast<const char*>(&len) + 4);
            old_pos += match_len;
            new_pos += match_len;
        } else {
            // Find extent of non-match (up to next matching block)
            size_t insert_start = new_pos;
            size_t old_scan = old_pos;

            // Advance by one block
            new_pos += block;
            old_scan += std::min(block, remaining_old);

            // Keep scanning until we find a matching block again
            while (new_pos < new_data.size()) {
                size_t rem_new = new_data.size() - new_pos;
                size_t rem_old = (old_scan < old_data.size()) ? old_data.size() - old_scan : 0;
                size_t b = std::min(DIFF_BLOCK, rem_new);
                if (rem_old >= b &&
                    memcmp(old_data.data() + old_scan, new_data.data() + new_pos, b) == 0) {
                    break; // Found match
                }
                new_pos += b;
                old_scan += std::min(b, rem_old);
            }

            // Emit INSERT: tag(1) + length(4) + data
            size_t insert_len = new_pos - insert_start;
            ops.push_back(OP_INSERT);
            uint32_t len = static_cast<uint32_t>(insert_len);
            ops.insert(ops.end(), reinterpret_cast<const char*>(&len),
                      reinterpret_cast<const char*>(&len) + 4);
            ops.insert(ops.end(), new_data.data() + insert_start,
                      new_data.data() + insert_start + insert_len);
            old_pos = old_scan;
        }
    }
    return ops;
}

// --- Write helpers ---
static void write_u32(std::ofstream& f, uint32_t v) { f.write(reinterpret_cast<const char*>(&v), 4); }
static void write_u64(std::ofstream& f, uint64_t v) { f.write(reinterpret_cast<const char*>(&v), 8); }

int main(int argc, char* argv[]) {
    if (argc < 5 || std::string(argv[3]) != "-o") {
        std::cerr << "Usage: geocoder-diff <old-dir> <new-dir> -o <patch-file>" << std::endl;
        return 1;
    }

    std::string old_dir = argv[1];
    std::string new_dir = argv[2];
    std::string patch_path = argv[4];

    std::cerr << "Loading string pools..." << std::endl;
    auto old_strings = read_file(old_dir + "/strings.bin");
    auto new_strings = read_file(new_dir + "/strings.bin");
    auto remap = build_string_remap(old_strings, new_strings);
    std::cerr << "  Remap: " << remap.size() << " strings mapped" << std::endl;

    std::ofstream pf(patch_path, std::ios::binary);
    pf.write(GCPATCH_MAGIC, 8);
    write_u32(pf, GCPATCH_VERSION);
    write_u32(pf, 0);

    // Section: strings.bin (full replacement)
    {
        std::cerr << "  strings.bin: full replacement..." << std::endl;
        auto compressed = zlib_compress(new_strings.data(), new_strings.size());
        write_u32(pf, static_cast<uint32_t>(PatchFileId::STRINGS));
        write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));
        write_u64(pf, new_strings.size());
        write_u64(pf, compressed.size());
        pf.write(compressed.data(), compressed.size());
        std::cerr << "    " << new_strings.size() << " -> " << compressed.size() << std::endl;
    }

    // Helper: diff a file with optional string remapping
    auto diff_file = [&](PatchFileId id, const std::string& name,
                          std::function<void(std::vector<char>&)> remap_fn) {
        std::cerr << "  " << name << ": ";
        auto old_data = read_file(old_dir + "/" + name);
        auto new_data = read_file(new_dir + "/" + name);

        // Apply string remap to old data (makes it comparable to new)
        if (remap_fn) {
            remap_fn(old_data);
        }

        // Compute block diff
        auto ops = compute_block_diff(old_data, new_data);

        // Compress the ops stream
        auto compressed = zlib_compress(ops.data(), ops.size());

        // Also compute full replacement size for comparison
        auto full_compressed = zlib_compress(new_data.data(), new_data.size());

        // Use whichever is smaller
        bool use_delta = compressed.size() < full_compressed.size();

        write_u32(pf, static_cast<uint32_t>(id));
        if (use_delta) {
            write_u32(pf, static_cast<uint32_t>(PatchEncoding::COPY_INSERT));
            write_u64(pf, old_data.size()); // old size
            write_u64(pf, new_data.size()); // new size
            write_u64(pf, compressed.size());
            pf.write(compressed.data(), compressed.size());
            std::cerr << "delta " << compressed.size() << " bytes";
        } else {
            write_u32(pf, static_cast<uint32_t>(PatchEncoding::RAW_REPLACE));
            write_u64(pf, new_data.size());
            write_u64(pf, full_compressed.size());
            pf.write(full_compressed.data(), full_compressed.size());
            std::cerr << "full " << full_compressed.size() << " bytes";
        }
        double pct = (new_data.size() > 0) ?
            (use_delta ? compressed.size() : full_compressed.size()) * 100.0 / new_data.size() : 0;
        std::cerr << " (" << std::fixed << std::setprecision(1) << pct << "% of "
                  << new_data.size() / 1024 / 1024 << " MiB)" << std::endl;
    };

    // Detect strides for struct files
    size_t way_stride = detect_way_stride(old_dir + "/street_ways.bin");
    auto old_interp_data = read_file(old_dir + "/interp_ways.bin");
    size_t interp_stride = (old_interp_data.size() % 20 == 0 && old_interp_data.size() > 0) ? 20 : 18;
    auto old_admin_data = read_file(old_dir + "/admin_polygons.bin");
    size_t admin_stride = 24;
    for (size_t s : {24, 20, 19}) {
        if (old_admin_data.size() % s == 0 && old_admin_data.size() > 0) { admin_stride = s; break; }
    }

    // Data files with string remapping
    diff_file(PatchFileId::ADDR_POINTS, "addr_points.bin",
        [&](std::vector<char>& d) { remap_addr_points(d, remap); });

    diff_file(PatchFileId::STREET_WAYS, "street_ways.bin",
        [&, way_stride](std::vector<char>& d) { remap_ways(d, way_stride, remap); });

    diff_file(PatchFileId::INTERP_WAYS, "interp_ways.bin",
        [&, interp_stride](std::vector<char>& d) { remap_interps(d, interp_stride, remap); });

    diff_file(PatchFileId::ADMIN_POLYGONS, "admin_polygons.bin",
        [&, admin_stride](std::vector<char>& d) { remap_admin_polygons(d, admin_stride, remap); });

    // Coordinate files (no string refs, direct diff)
    diff_file(PatchFileId::STREET_NODES, "street_nodes.bin", nullptr);
    diff_file(PatchFileId::INTERP_NODES, "interp_nodes.bin", nullptr);
    diff_file(PatchFileId::ADMIN_VERTICES, "admin_vertices.bin", nullptr);

    // Derived/index files (no remap needed, direct diff)
    diff_file(PatchFileId::GEO_CELLS, "geo_cells.bin", nullptr);
    diff_file(PatchFileId::STREET_ENTRIES, "street_entries.bin", nullptr);
    diff_file(PatchFileId::ADDR_ENTRIES, "addr_entries.bin", nullptr);
    diff_file(PatchFileId::INTERP_ENTRIES, "interp_entries.bin", nullptr);
    diff_file(PatchFileId::ADMIN_CELLS, "admin_cells.bin", nullptr);
    diff_file(PatchFileId::ADMIN_ENTRIES, "admin_entries.bin", nullptr);

    write_u32(pf, 0xFFFFFFFF); // end marker
    auto total = pf.tellp();
    std::cerr << "\nTotal patch: " << total << " bytes (" << total / 1024 / 1024 << " MiB)" << std::endl;
    return 0;
}