From 56f4fce4a2d31321a6b1dd63d2ceb8ce887e50eb Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 21 Aug 2026 12:36:49 +0100 Subject: [PATCH 1/5] Add tiled tree recovery API Restore TiledTree from serialized tree state and a caller-validated tile boundary. Harden tree deserialization and replace untrusted suffix tiles during resumed growth. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 455a0fc3-10c9-4831-bf66-f06ff952f607 --- README.md | 24 ++- doc/tiles-guide.rst | 49 ++++-- merklecpp.h | 104 +++++++++--- merklecpp_tiles.h | 222 ++++++++++++++++++++---- test/CMakeLists.txt | 1 + test/tiles_resume.cpp | 382 ++++++++++++++++++++++++++++++++++++++++++ test/unit_tests.cpp | 55 ++++++ 7 files changed, 769 insertions(+), 68 deletions(-) create mode 100644 test/tiles_resume.cpp diff --git a/README.md b/README.md index f2dfe0c..1807815 100644 --- a/README.md +++ b/README.md @@ -61,12 +61,24 @@ tile-derived inclusion proof is byte-identical to one from log.consistency_proof(/*m=*/log.size() / 2, /*n=*/log.size()); } -`TiledTree` creates a new tiled tree. The configured prefix may exist, but the -default alias requires `/sha256-256w/tile` not to exist, even as an -empty directory. Construction atomically claims that tile namespace and rejects -an existing one because tile files alone do not identify or restore the tree -that produced them. Applications with externally persisted tree state can use -the lower-level `TileStore` and `TileWriter` APIs to resume a store. +`TiledTree` constructors create a new tiled tree. The configured prefix may +exist, but the default alias requires `/sha256-256w/tile` not to exist, +even as an empty directory. Construction atomically claims that tile namespace. + +Applications that persist the matching tree state and full-tile boundary can +resume an existing namespace directly: + + auto log = merkle::tiles::TiledTree::resume( + cfg, + "sha256", + serialised_tree, + full_tile_boundary); + +The boundary must cover a complete, durable tile prefix at every required level +and overlap the resident portion of the serialized tree. Tile files beyond it +are treated as untrusted and replaced when a later flush reaches them. The +application remains responsible for establishing namespace ownership and +matching the serialized tree to the trusted tiles. See the [tiled storage guide](doc/tiles-guide.rst) for a how-to covering flushing, compaction, rollback, proofs, and the lower-level building blocks, diff --git a/doc/tiles-guide.rst b/doc/tiles-guide.rst index d11300e..45926b6 100644 --- a/doc/tiles-guide.rst +++ b/doc/tiles-guide.rst @@ -95,17 +95,44 @@ construction keeps its writer bound to the destination tree's tile store. A relative prefix therefore binds to the working directory at that moment; later working-directory changes do not move the tile store. -``TiledTree`` always creates a new tiled tree. The configured prefix may already -exist, but the algorithm-qualified tile namespace must not: the default alias -atomically creates ``/sha256-256w/tile`` and rejects it whenever it already -exists, even if it is empty. Construction does not adopt existing tiles because -those files do not identify the tree that produced them or contain enough state -to restore its size and root. If your application persists and validates that -state separately, use the lower-level ``TileStore`` and ``TileWriter`` APIs; -``TileWriter`` intentionally resumes existing full tiles and therefore trusts the -caller to supply the same tree and hash function. A fresh writer scans the -requested range in order, stopping at the first missing or malformed file, so -an interior hole is rewritten rather than hidden by later files. +``TiledTree`` constructors always create a new tiled tree. The configured prefix +may already exist, but the algorithm-qualified tile namespace must not: the +default alias atomically creates ``/sha256-256w/tile`` and rejects it +whenever it already exists, even if it is empty. + +Resuming an externally checkpointed tree +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Tile files do not identify the tree that produced them or contain enough state +to restore its size and root. An application that separately persists and +validates that state can reopen an existing namespace with ``resume()``: + +.. code:: cpp + + auto log = merkle::tiles::TiledTree::resume( + cfg, + "sha256", + serialised_tree, + full_tile_boundary); + +``serialised_tree`` is the existing ``merkle::Tree`` serialization. +``full_tile_boundary`` is a leaf count, must be a multiple of ``TILE_WIDTH``, and +must identify a complete, durable tile prefix at every required level. The +factory deserializes the tree, rejects trailing bytes, and requires its resident +range to satisfy ``Config::retention_margin``. It also compares the last leaf in +the prefix with the matching resident tree leaf, catching an incorrect boundary +or namespace at the hand-off point. + +The application remains responsible for establishing namespace ownership and +validating that every tile below the supplied boundary belongs to the same +Merkle history. The boundary cannot be inferred from the tree's ``min_index()`` +or from the files on disk. Existing files beyond it are excluded from proof +reads and replaced from the restored tree when a later ``flush()`` reaches them. + +The lower-level ``TileWriter`` still intentionally resumes existing full tiles +and trusts the caller to supply the same tree and hash function. A fresh writer +scans the requested range in order, stopping at the first missing or malformed +file, so an interior hole is rewritten rather than hidden by later files. ``flush()`` is incremental: each call writes only the full tiles that became complete since the previous call. Full tiles are immutable: written once after diff --git a/merklecpp.h b/merklecpp.h index 0306f2e..f395e5d 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -593,7 +593,14 @@ namespace merkle HashT& out)> class TreeT { + static_assert(HASH_SIZE > 0, "tree hash size must be greater than zero"); + protected: + static constexpr size_t maximum_num_leaves() + { + return size_t{1} << (std::numeric_limits::digits - 1); + } + /// @brief The structure of tree nodes struct Node { @@ -829,6 +836,10 @@ namespace merkle MERKLECPP_TRACE( MERKLECPP_TOUT << "> insert " << hash.to_string(TRACE_HASH_SIZE) << std::endl;); + if (num_leaves() >= maximum_num_leaves()) + { + throw std::runtime_error("maximum tree size exceeded"); + } uninserted_leaf_nodes.push_back(Node::make(hash)); statistics.num_insert++; } @@ -1521,35 +1532,76 @@ namespace merkle { MERKLECPP_TRACE(MERKLECPP_TOUT << "> deserialise " << std::endl;); - clear(); + size_t cursor = position; + const uint64_t serialised_num_leaf_nodes = + deserialise_uint64_t(bytes, cursor); + const uint64_t serialised_num_flushed = + deserialise_uint64_t(bytes, cursor); + if ( + serialised_num_leaf_nodes > std::numeric_limits::max() || + serialised_num_flushed > std::numeric_limits::max()) + { + throw std::runtime_error("serialised tree size exceeds size_t"); + } + + const size_t restored_num_leaf_nodes = + static_cast(serialised_num_leaf_nodes); + const size_t restored_num_flushed = + static_cast(serialised_num_flushed); + if (restored_num_leaf_nodes == 0 && restored_num_flushed != 0) + { + throw std::runtime_error( + "serialised tree has flushed leaves but no resident leaf"); + } - size_t num_leaf_nodes = deserialise_uint64_t(bytes, position); - num_flushed = deserialise_uint64_t(bytes, position); + constexpr size_t size_digits = std::numeric_limits::digits; + constexpr size_t max_tree_leaves = maximum_num_leaves(); + if ( + restored_num_leaf_nodes > max_tree_leaves || + restored_num_flushed > max_tree_leaves - restored_num_leaf_nodes) + { + throw std::runtime_error("serialised tree is too large"); + } + if ( + cursor > bytes.size() || + restored_num_leaf_nodes > (bytes.size() - cursor) / HASH_SIZE) + { + throw std::runtime_error("not enough bytes for serialised tree leaves"); + } - leaf_nodes.reserve(num_leaf_nodes); - for (size_t i = 0; i < num_leaf_nodes; i++) + std::vector restored_leaf_nodes; + restored_leaf_nodes.reserve(restored_num_leaf_nodes); + std::vector> level; + level.reserve(restored_num_leaf_nodes); + for (size_t i = 0; i < restored_num_leaf_nodes; i++) { - Node* n = Node::make(bytes.data() + position); - position += HASH_SIZE; - leaf_nodes.push_back(n); + Hash h(bytes, cursor); + auto n = std::unique_ptr(Node::make(h)); + restored_leaf_nodes.push_back(n.get()); + level.push_back(std::move(n)); } - std::vector level = leaf_nodes; - std::vector next_level; - size_t it = num_flushed; - uint8_t level_no = 0; + std::vector> next_level; + size_t it = restored_num_flushed; + size_t level_no = 0; while (it != 0 || level.size() > 1) { // Restore extra hashes on the left edge of the tree if ((it & 0x01) != 0U) { - Hash h(bytes, position); + Hash h(bytes, cursor); MERKLECPP_TRACE(MERKLECPP_TOUT << "+";); - auto n = Node::make(h); - n->height = level_no + 1; - n->size = (1 << n->height) - 1; + auto n = std::unique_ptr(Node::make(h)); + if (level_no >= size_digits) + { + throw std::runtime_error("serialised tree height exceeds size_t"); + } + n->height = static_cast(level_no + 1); + n->size = n->height == size_digits ? + std::numeric_limits::max() : + (size_t{1} << n->height) - 1; assert(n->invariant()); - level.insert(level.begin(), n); + level.insert(level.begin(), std::move(n)); } MERKLECPP_TRACE( @@ -1558,15 +1610,20 @@ namespace merkle MERKLECPP_TOUT << std::endl;); // Rebuild the level + next_level.reserve((level.size() + 1) / 2); for (size_t i = 0; i < level.size(); i += 2) { if (i + 1 >= level.size()) { - next_level.push_back(level.at(i)); + next_level.push_back(std::move(level.at(i))); } else { - next_level.push_back(Node::make(level.at(i), level.at(i + 1))); + auto parent = std::unique_ptr( + Node::make(level.at(i).get(), level.at(i + 1).get())); + (void)level.at(i).release(); + (void)level.at(i + 1).release(); + next_level.push_back(std::move(parent)); } } @@ -1581,9 +1638,14 @@ namespace merkle if (level.size() == 1) { - _root = level.at(0); - assert(_root->invariant()); + assert(level.at(0)->invariant()); } + + clear(); + leaf_nodes = std::move(restored_leaf_nodes); + num_flushed = restored_num_flushed; + _root = level.empty() ? nullptr : level.at(0).release(); + position = cursor; } /// @brief Operator to serialise the tree diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index dedc47c..616c944 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -168,6 +168,13 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) uint8_t TILE_HEIGHT_VALUE = DEFAULT_TILE_HEIGHT> class EntryBundleWriterT; + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&), + uint8_t TILE_HEIGHT_VALUE> + class TiledTreeT; + /// @brief Reads and writes tlog-tiles tile files on a local filesystem. /// @tparam HASH_SIZE Size of each hash in bytes /// @tparam HASH_FUNCTION The tree's node hash function (carried for use by @@ -880,6 +887,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) uint8_t TILE_HEIGHT_VALUE> class TileWriterT { + friend class TiledTreeT; + public: /// @brief The type of hashes stored in tiles. using Hash = HashT; @@ -944,24 +953,23 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) if (cursor_inited[level] == 0) { - next_full[level] = full_prefix_length(level, full_tiles); + next_full[level] = + trust_existing_tiles ? full_prefix_length(level, full_tiles) : 0; cursor_inited[level] = 1; } for (uint64_t n = next_full[level]; n < full_tiles; n++) { - if (store.confirm_full_tile(level, n)) + if (trust_existing_tiles && store.confirm_full_tile(level, n)) { - continue; // immutable: never rewrite an existing full tile + next_full[level] = n + 1; + continue; } store.write_tile( TileRef{level, n}, collect(level, n * TILE_WIDTH, TILE_WIDTH, leaf_at)); stats.full_written++; - } - if (full_tiles > next_full[level]) - { - next_full[level] = full_tiles; + next_full[level] = n + 1; } } @@ -979,6 +987,39 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief Per-level flag indicating next_full has been initialised. std::vector cursor_inited; + /// @brief Whether correctly-sized files at and beyond next_full may be + /// adopted rather than replaced. + bool trust_existing_tiles = true; + + /// @brief Constructs a writer after a caller-validated tile prefix. + /// Files beyond @p trusted_size are replaced before they become readable. + TileWriterT(Store& store, uint64_t trusted_size) : + store(store), trust_existing_tiles(false) + { + for (uint8_t level = 0; level <= MAX_TILE_LEVEL; level++) + { + const uint64_t entries = entries_at_level(trusted_size, level); + if (entries == 0) + { + break; + } + ensure_level(level); + next_full[level] = entries / TILE_WIDTH; + cursor_inited[level] = 1; + } + } + + /// @brief Rebinds a moved writer to its destination store. + TileWriterT(Store& store, TileWriterT&& other) noexcept : + store(store), trust_existing_tiles(other.trust_existing_tiles) + { + if (!trust_existing_tiles) + { + next_full = std::move(other.next_full); + cursor_inited = std::move(other.cursor_inited); + } + } + /// @brief Number of complete level-@p level entries for a tree of @p /// size. static uint64_t entries_at_level(uint64_t size, uint8_t level) @@ -1603,11 +1644,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// Config::compact_on_flush, or call compact() explicitly; it never drops /// the un-tiled frontier. Proofs are served from the combination of the /// resident tree (frontier) and the full tiles (compacted past). - /// @note TiledTree creates a new tiled tree and cannot reopen one from tile - /// files alone. Construction atomically claims a previously absent tile - /// namespace because the files do not identify their tree or record enough - /// state to restore it. Use TileWriter directly only when the caller owns - /// and restores that state. + /// @note TiledTree constructors create a new tiled tree and atomically + /// claim a previously absent tile namespace. The resume() factory reopens + /// an existing namespace only when the caller supplies the matching + /// serialized tree state and validated full-tile boundary; tile files alone + /// do not identify or restore their tree. /// @warning No internal synchronization is provided. Callers must serialize /// all access to a shared tree, including proof operations. template < @@ -1645,9 +1686,9 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief Configuration for a tiled tree. struct Config { - /// @brief Root directory for a new tiled tree. - /// @note The directory itself may exist, but its algorithm-qualified - /// tile subdirectory must be absent. + /// @brief Root directory for the tiled tree. + /// @note Fresh construction requires the algorithm-qualified tile + /// subdirectory to be absent; resume() requires it to exist. std::filesystem::path prefix; /// @brief Number of most-recent leaves to keep resident when @@ -1663,6 +1704,34 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) bool compact_on_flush = false; }; + /// @brief Resumes a tiled tree from serialized tree state and an existing + /// tile namespace. + /// @param config Runtime configuration, including the tile prefix + /// @param hash_algorithm_short_name Lowercase hash algorithm namespace + /// @param serialised_tree Serialized merkle::TreeT state + /// @param full_tile_boundary Caller-validated number of leaves covered by + /// a complete, durable tile prefix + /// @return A tiled tree whose tile reads are capped at + /// @p full_tile_boundary + /// @note The serialized tree is deserialized by this call. At every tile + /// level, all full tiles below @p full_tile_boundary must already exist, + /// be durable, and belong to the same tree. Existing files beyond that + /// boundary are not trusted and are replaced when a later flush reaches + /// them. + [[nodiscard]] static TiledTreeT resume( + Config config, + const std::string& hash_algorithm_short_name, + const std::vector& serialised_tree, + size_t full_tile_boundary) + { + return TiledTreeT( + ResumeTag{}, + std::move(config), + hash_algorithm_short_name, + serialised_tree, + full_tile_boundary); + } + explicit TiledTreeT(Config config) : config(std::move(config)), store(this->config.prefix), writer(store) { @@ -1687,7 +1756,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) TiledTreeT(TiledTreeT&& other) noexcept : config(std::move(other.config)), store(std::move(other.store)), - writer(store), + writer(store, std::move(other.writer)), tree(std::move(other.tree)), tiles_size(std::exchange(other.tiles_size, 0)), sealed_size(std::exchange(other.sealed_size, 0)) @@ -1743,9 +1812,9 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } /// @brief Access to the underlying tile store. - /// @warning Files written or changed through this reference are trusted - /// by later flushes without checking that their hashes match this tree. - /// Mismatched files can silently invalidate proofs after compaction. + /// @warning Files changed within flushed_size() are used by proofs without + /// checking that their hashes match this tree. Mismatched files can + /// silently invalidate proofs after compaction. Store& store_ref() { return store; @@ -1805,17 +1874,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) size_t compact() { const size_t covered = (tiles_size / TILE_WIDTH) * TILE_WIDTH; - size_t target = covered > config.retention_margin ? - covered - config.retention_margin : - 0; - target = (target / TILE_WIDTH) * TILE_WIDTH; - // TreeT cannot retract below min_index(). Keep the final tiled leaf - // resident so rollback to a size of exactly immutable_size() remains - // representable after compaction. - if (covered > 0 && target == covered) - { - target--; - } + const size_t target = compaction_target(covered); if (target > tree.min_index()) { tree.flush_to(target); @@ -1891,6 +1950,9 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } protected: + struct ResumeTag + {}; + Config config; Store store; Writer writer; @@ -1898,6 +1960,106 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) size_t tiles_size = 0; size_t sealed_size = 0; + TiledTreeT( + ResumeTag, + Config config, + const std::string& hash_algorithm_short_name, + const std::vector& serialised_tree, + size_t full_tile_boundary) : + config(std::move(config)), + store(this->config.prefix, hash_algorithm_short_name), + writer(store, static_cast(full_tile_boundary)), + tree(deserialise_tree(serialised_tree)), + tiles_size(full_tile_boundary), + sealed_size(full_tile_boundary) + { + validate_resume_state(full_tile_boundary); + } + + static Tree deserialise_tree(const std::vector& serialised_tree) + { + size_t position = 0; + Tree restored(serialised_tree, position); + if (position != serialised_tree.size()) + { + throw std::runtime_error( + "TiledTree::resume: trailing bytes in serialized tree"); + } + return restored; + } + + void validate_resume_state(size_t full_tile_boundary) + { + const auto tile_root = store.root() / "tile"; + std::error_code ec; + if (!std::filesystem::is_directory(tile_root, ec) || ec) + { + throw std::runtime_error(std::format( + "TiledTree::resume: tile namespace does not exist: {}", + tile_root.string())); + } + if (full_tile_boundary % TILE_WIDTH != 0) + { + throw std::runtime_error( + "TiledTree::resume: full tile boundary is not tile-aligned"); + } + if (full_tile_boundary > tree.num_leaves()) + { + throw std::runtime_error( + "TiledTree::resume: full tile boundary exceeds tree size"); + } + if (!tree.invariant()) + { + throw std::runtime_error( + "TiledTree::resume: deserialized tree invariant failed"); + } + if (full_tile_boundary == 0) + { + if (tree.min_index() != 0) + { + throw std::runtime_error( + "TiledTree::resume: compacted tree has no tile coverage"); + } + return; + } + const size_t required_resident_leaves = + std::min(config.retention_margin, full_tile_boundary); + const size_t latest_resident_start = + full_tile_boundary - + std::max(required_resident_leaves, size_t{1}); + if (tree.min_index() > latest_resident_start) + { + throw std::runtime_error( + "TiledTree::resume: serialized tree does not satisfy the configured " + "retention margin"); + } + + const uint64_t last_tile = + static_cast(full_tile_boundary / TILE_WIDTH - 1); + const auto hashes = store.read_tile(TileRef{0, last_tile}); + if (hashes.back() != tree.leaf(full_tile_boundary - 1)) + { + throw std::runtime_error( + "TiledTree::resume: tile prefix does not match serialized tree"); + } + } + + [[nodiscard]] size_t compaction_target(size_t covered) const + { + size_t target = covered > config.retention_margin ? + covered - config.retention_margin : + 0; + target = (target / TILE_WIDTH) * TILE_WIDTH; + // TreeT cannot retract below min_index(). Keep the final tiled leaf + // resident so rollback to a size of exactly immutable_size() remains + // representable after compaction. + if (covered > 0 && target == covered) + { + target--; + } + return target; + } + void claim_tile_namespace() const { const auto tile_root = store.root() / "tile"; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a8c5a68..c75a2e8 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -41,6 +41,7 @@ add_merklecpp_test(tiles_store tiles_store.cpp) add_merklecpp_test(tiles_writer tiles_writer.cpp) add_merklecpp_test(tiles_proofs tiles_proofs.cpp) add_merklecpp_test(tiles_tree tiles_tree.cpp) +add_merklecpp_test(tiles_resume tiles_resume.cpp) add_merklecpp_test(tiles_docs tiles_docs.cpp) add_merklecpp_test(tiles_entries tiles_entries.cpp) add_merklecpp_test(tiles_geometry tiles_geometry.cpp) diff --git a/test/tiles_resume.cpp b/test/tiles_resume.cpp new file mode 100644 index 0000000..c16ba79 --- /dev/null +++ b/test/tiles_resume.cpp @@ -0,0 +1,382 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "tiles_test_util.h" +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TileRef; + +static void ccf_hash(const Hash& left, const Hash& right, Hash& out) +{ + merkle::Tree::hash_function(left, right, out); +} + +using CcfTiledTree = merkle::tiles::TiledTreeT; +using SmallCcfTiledTree = + merkle::tiles::TiledTreeT; + +static void expect(bool condition, const std::string& what) +{ + if (!condition) + { + throw std::runtime_error("check failed: " + what); + } +} + +template +static void expect_throws(Fn&& fn, const std::string& what) +{ + try + { + std::forward(fn)(); + } + catch (const std::exception&) + { + return; + } + throw std::runtime_error("expected exception: " + what); +} + +static std::vector serialise(CcfTiledTree::Tree& tree) +{ + std::vector bytes; + tree.serialise(bytes); + return bytes; +} + +int main() +{ + const TemporaryDirectory temporary_directory("merklecpp_tiles_resume"); + const fs::path& base = temporary_directory.path(); + const auto hashes = make_hashes(1024); + constexpr auto hash_namespace = "ccf-sha256"; + + try + { + // Resume a compacted frontier, retain the trust boundary through a move, + // and replace a correctly-sized but untrusted suffix tile. + { + CcfTiledTree::Config config; + config.prefix = base / "round_trip"; + config.compact_on_flush = true; + + std::vector serialised_tree; + size_t full_tile_boundary = 0; + Hash root_at_700; + { + CcfTiledTree source(config, hash_namespace); + for (size_t i = 0; i < 700; i++) + { + source.append(hashes[i]); + } + source.flush(); + full_tile_boundary = source.flushed_size(); + expect(full_tile_boundary == 512, "source boundary"); + expect(source.tree_ref().min_index() == 511, "source compacted"); + root_at_700 = source.root(); + serialised_tree = serialise(source.tree_ref()); + + source.store_ref().write_tile( + TileRef{0, 2}, std::vector(CcfTiledTree::TILE_WIDTH)); + } + + auto resumed = CcfTiledTree::resume( + config, hash_namespace, serialised_tree, full_tile_boundary); + expect(resumed.size() == 700, "resumed size"); + expect(resumed.root() == root_at_700, "resumed root"); + expect(resumed.flushed_size() == 512, "resumed flushed boundary"); + expect(resumed.immutable_size() == 512, "resumed immutable boundary"); + expect(resumed.flush().full_written == 0, "trusted prefix reused"); + expect( + resumed.inclusion_proof(0, resumed.size())->verify(root_at_700), + "resumed tiled proof"); + expect( + resumed.inclusion_proof(699, resumed.size())->verify(root_at_700), + "resumed frontier proof"); + + CcfTiledTree moved(std::move(resumed)); + for (size_t i = 700; i < 768; i++) + { + moved.append(hashes[i]); + } + expect(moved.flush().full_written == 1, "untrusted tile replaced"); + expect(moved.flushed_size() == 768, "extended boundary"); + + const std::vector expected_tile( + hashes.begin() + 512, hashes.begin() + 768); + expect( + moved.store_ref().read_tile(TileRef{0, 2}) == expected_tile, + "replacement tile contents"); + + CcfTiledTree::Tree reference; + for (size_t i = 0; i < 768; i++) + { + reference.insert(hashes[i]); + } + const Hash expected_root = reference.root(); + expect(moved.root() == expected_root, "extended root"); + expect( + moved.inclusion_proof(600, moved.size())->verify(expected_root), + "extended proof"); + } + + // An existing empty namespace and an empty serialized tree can be resumed. + std::vector empty_tree; + CcfTiledTree::Config empty_config; + empty_config.prefix = base / "empty"; + { + CcfTiledTree fresh(empty_config, hash_namespace); + empty_tree = serialise(fresh.tree_ref()); + } + { + const auto empty = + CcfTiledTree::resume(empty_config, hash_namespace, empty_tree, 0); + expect(empty.size() == 0, "empty resumed size"); + expect(empty.flushed_size() == 0, "empty resumed boundary"); + } + + const fs::path missing_prefix = base / "missing"; + expect_throws( + [&]() { + CcfTiledTree::Config missing; + missing.prefix = missing_prefix; + (void)CcfTiledTree::resume(missing, hash_namespace, empty_tree, 0); + }, + "missing namespace"); + expect( + !fs::exists(missing_prefix), "failed resume does not create namespace"); + + expect_throws( + [&]() { + auto trailing = empty_tree; + trailing.push_back(0); + (void)CcfTiledTree::resume(empty_config, hash_namespace, trailing, 0); + }, + "trailing serialized bytes"); + + expect_throws( + [&]() { + auto truncated = empty_tree; + truncated.pop_back(); + (void)CcfTiledTree::resume(empty_config, hash_namespace, truncated, 0); + }, + "truncated serialized tree"); + + expect_throws( + [&]() { + std::vector truncated_leaf; + merkle::serialise_uint64_t(1, truncated_leaf); + merkle::serialise_uint64_t(0, truncated_leaf); + (void)CcfTiledTree::resume( + empty_config, hash_namespace, truncated_leaf, 0); + }, + "truncated serialized leaf"); + + // A sub-tile frontier can resume at boundary zero and later publish its + // first tile. + { + CcfTiledTree::Config config; + config.prefix = base / "sub_tile"; + std::vector serialised_tree; + { + CcfTiledTree source(config, hash_namespace); + for (size_t i = 0; i < 100; i++) + { + source.append(hashes[i]); + } + serialised_tree = serialise(source.tree_ref()); + } + + auto resumed = + CcfTiledTree::resume(config, hash_namespace, serialised_tree, 0); + for (size_t i = 100; i < CcfTiledTree::TILE_WIDTH; i++) + { + resumed.append(hashes[i]); + } + expect(resumed.flush().full_written == 1, "first tile after resume"); + expect( + resumed.flushed_size() == CcfTiledTree::TILE_WIDTH, + "first resumed boundary"); + } + + // A compacted tree must retain the configured overlap with its tile prefix. + { + CcfTiledTree::Config config; + config.prefix = base / "coverage_gap"; + { + CcfTiledTree source(config, hash_namespace); + for (size_t i = 0; i < 600; i++) + { + source.append(hashes[i]); + } + source.flush(); + } + + CcfTiledTree::Tree compacted; + for (size_t i = 0; i < 600; i++) + { + compacted.insert(hashes[i]); + } + compacted.flush_to(512); + const auto compacted_tree = serialise(compacted); + + expect_throws( + [&]() { + (void)CcfTiledTree::resume( + config, hash_namespace, compacted_tree, 512); + }, + "missing boundary leaf"); + expect_throws( + [&]() { + (void)CcfTiledTree::resume( + config, hash_namespace, compacted_tree, 513); + }, + "unaligned boundary"); + expect_throws( + [&]() { + (void)CcfTiledTree::resume( + config, hash_namespace, compacted_tree, 768); + }, + "boundary beyond tree"); + } + + // Resume preserves the configured minimum resident range. + { + CcfTiledTree::Config config; + config.prefix = base / "retention"; + config.compact_on_flush = true; + std::vector serialised_tree; + { + CcfTiledTree source(config, hash_namespace); + for (size_t i = 0; i < 700; i++) + { + source.append(hashes[i]); + } + source.flush(); + serialised_tree = serialise(source.tree_ref()); + } + + config.retention_margin = 300; + expect_throws( + [&]() { + (void)CcfTiledTree::resume( + config, hash_namespace, serialised_tree, 512); + }, + "retention margin"); + } + + // Any state retaining the configured margin is valid, even when its + // compaction point is not tile-aligned. + { + CcfTiledTree::Config config; + config.prefix = base / "exact_retention"; + config.retention_margin = 300; + { + CcfTiledTree source(config, hash_namespace); + for (size_t i = 0; i < 900; i++) + { + source.append(hashes[i]); + } + source.flush(); + } + + CcfTiledTree::Tree compacted; + for (size_t i = 0; i < 900; i++) + { + compacted.insert(hashes[i]); + } + compacted.flush_to(468); + const auto serialised_tree = serialise(compacted); + auto resumed = + CcfTiledTree::resume(config, hash_namespace, serialised_tree, 768); + expect(resumed.tree_ref().min_index() == 468, "exact retention accepted"); + } + + // The trusted boundary applies independently at every tile level. + { + SmallCcfTiledTree::Config config; + config.prefix = base / "all_levels"; + std::vector serialised_tree; + { + SmallCcfTiledTree source(config, hash_namespace); + for (size_t i = 0; i < 24; i++) + { + source.append(hashes[i]); + } + source.flush(); + serialised_tree = serialise(source.tree_ref()); + + for (uint64_t index = 6; index < 8; index++) + { + source.store_ref().write_tile( + TileRef{0, index}, + std::vector(SmallCcfTiledTree::TILE_WIDTH)); + } + source.store_ref().write_tile( + TileRef{1, 1}, std::vector(SmallCcfTiledTree::TILE_WIDTH)); + } + + auto resumed = + SmallCcfTiledTree::resume(config, hash_namespace, serialised_tree, 24); + for (size_t i = 24; i < 32; i++) + { + resumed.append(hashes[i]); + } + expect( + resumed.flush().full_written == 3, + "untrusted tiles replaced at every level"); + + const std::vector expected( + hashes.begin() + 24, hashes.begin() + 28); + expect( + resumed.store_ref().read_tile(TileRef{0, 6}) == expected, + "level-0 suffix replaced"); + } + + // The overlapping boundary leaf catches a mismatched tile namespace. + { + CcfTiledTree::Config config; + config.prefix = base / "mismatch"; + std::vector serialised_tree; + { + CcfTiledTree source(config, hash_namespace); + for (size_t i = 0; i < 600; i++) + { + source.append(hashes[i]); + } + source.flush(); + serialised_tree = serialise(source.tree_ref()); + source.store_ref().write_tile( + TileRef{0, 1}, std::vector(CcfTiledTree::TILE_WIDTH)); + } + + expect_throws( + [&]() { + (void)CcfTiledTree::resume( + config, hash_namespace, serialised_tree, 512); + }, + "mismatched boundary tile"); + } + + std::cout << "tiles_resume: OK" << '\n'; + } + catch (const std::exception& error) + { + std::cout << "Error: " << error.what() << '\n'; + return 1; + } + + return 0; +} diff --git a/test/unit_tests.cpp b/test/unit_tests.cpp index ce7ce8d..4606ffb 100644 --- a/test/unit_tests.cpp +++ b/test/unit_tests.cpp @@ -277,6 +277,61 @@ TEST_CASE("Empty tree") REQUIRE_NOTHROW(merkle::Tree dt(buffer)); // NOLINT(misc-const-correctness) } +TEST_CASE("Tree deserialisation validates input") +{ + merkle::Tree::Hash hash; + hash.bytes[0] = 1; + merkle::Tree existing(hash); + + std::vector truncated; + merkle::serialise_uint64_t(1, truncated); + merkle::serialise_uint64_t(0, truncated); + REQUIRE_THROWS(existing.deserialise(truncated)); + REQUIRE(existing.num_leaves() == 1); + REQUIRE(existing.root() == hash); + + std::vector oversized_leaf_count; + merkle::serialise_uint64_t(uint64_t{1} << 30, oversized_leaf_count); + merkle::serialise_uint64_t(0, oversized_leaf_count); + REQUIRE_THROWS(existing.deserialise(oversized_leaf_count)); + REQUIRE(existing.num_leaves() == 1); + REQUIRE(existing.root() == hash); + + std::vector missing_resident_leaf; + merkle::serialise_uint64_t(0, missing_resident_leaf); + merkle::serialise_uint64_t(1, missing_resident_leaf); + REQUIRE_THROWS(merkle::Tree(missing_resident_leaf)); + + std::vector tall_compacted_tree; + constexpr uint64_t flushed = uint64_t{1} << 30; + merkle::serialise_uint64_t(1, tall_compacted_tree); + merkle::serialise_uint64_t(flushed, tall_compacted_tree); + hash.serialise(tall_compacted_tree); + hash.serialise(tall_compacted_tree); + + merkle::Tree restored(tall_compacted_tree); + REQUIRE(restored.invariant()); + REQUIRE(restored.min_index() == flushed); + REQUIRE(restored.num_leaves() == flushed + 1); + + std::vector maximum_tree; + constexpr size_t size_digits = std::numeric_limits::digits; + constexpr size_t maximum_leaves = size_t{1} << (size_digits - 1); + merkle::serialise_uint64_t(1, maximum_tree); + merkle::serialise_uint64_t(maximum_leaves - 1, maximum_tree); + hash.serialise(maximum_tree); + for (size_t i = 0; i < size_digits - 1; i++) + { + hash.serialise(maximum_tree); + } + + merkle::Tree maximum(maximum_tree); + REQUIRE(maximum.invariant()); + REQUIRE(maximum.num_leaves() == maximum_leaves); + REQUIRE_THROWS(maximum.insert(hash)); + REQUIRE(maximum.num_leaves() == maximum_leaves); +} + TEST_CASE("One-node tree") { merkle::Tree::Hash h; From 922418c7445e6c25f8d18b78cec8a90e26d38a2d Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 21 Aug 2026 14:19:26 +0100 Subject: [PATCH 2/5] Add frontier-only tile recovery Restore logical tree state before tiles are ready, populate or repair the namespace with a detached writer, and adopt a quiesced durable prefix without trusting stale suffix files. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 455a0fc3-10c9-4831-bf66-f06ff952f607 --- README.md | 23 +++++ doc/tiles-guide.rst | 76 +++++++++++++++-- merklecpp_tiles.h | 194 +++++++++++++++++++++++++++++++++--------- test/tiles_resume.cpp | 99 +++++++++++++++++++++ test/tiles_writer.cpp | 76 ++++++++++++----- 5 files changed, 403 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 1807815..4f4fc08 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,29 @@ are treated as untrusted and replaced when a later flush reaches them. The application remains responsible for establishing namespace ownership and matching the serialized tree to the trusted tiles. +If tiles are not ready yet, restore the logical tree first and populate the +namespace independently. This does not claim or inspect the namespace; the +caller must establish exclusive ownership: + + auto log = merkle::tiles::TiledTree::from_frontier( + cfg, + "sha256", + serialised_tree); + + merkle::tiles::TileStore repair_store(cfg.prefix, "sha256"); + auto repair = merkle::tiles::TileWriter::repair( + repair_store, + trusted_full_tile_boundary); + repair.write_up_to(target_size, leaf_at); + + // After the repair writer is quiesced: + log.adopt_tile_prefix(target_full_tile_boundary); + +The independent writer can run in the background. The caller must serialize +writers for the namespace and quiesce them before adoption. Until the repaired +prefix overlaps the resident frontier, root computation and appends remain +available but tile-dependent proofs and flushes of non-resident history fail. + See the [tiled storage guide](doc/tiles-guide.rst) for a how-to covering flushing, compaction, rollback, proofs, and the lower-level building blocks, and the [illustrated walkthrough](doc/tiles-illustrated.rst) for the tile layout diff --git a/doc/tiles-guide.rst b/doc/tiles-guide.rst index 45926b6..e735c87 100644 --- a/doc/tiles-guide.rst +++ b/doc/tiles-guide.rst @@ -129,15 +129,77 @@ Merkle history. The boundary cannot be inferred from the tree's ``min_index()`` or from the files on disk. Existing files beyond it are excluded from proof reads and replaced from the restored tree when a later ``flush()`` reaches them. -The lower-level ``TileWriter`` still intentionally resumes existing full tiles -and trusts the caller to supply the same tree and hash function. A fresh writer -scans the requested range in order, stopping at the first missing or malformed -file, so an interior hole is rewritten rather than hidden by later files. +An ordinary lower-level ``TileWriter`` intentionally resumes existing full +tiles and trusts the caller to supply the same tree and hash function. A fresh +writer scans the requested range in order, stopping at the first missing or +malformed file, so an interior hole is rewritten rather than hidden by later +files. + +Restoring while tiles are populated or repaired +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +An application need not block logical tree recovery on tile reconstruction. +``from_frontier()`` restores only the serialized tree and binds the configured +namespace without inspecting, creating, or trusting it: + +.. code:: cpp + + auto log = merkle::tiles::TiledTree::from_frontier( + cfg, + "sha256", + serialised_tree); + +``root()``, ``size()``, and ``append()`` are immediately available. +``flushed_size()`` and ``immutable_size()`` start at zero. If the serialized +tree has already discarded old leaves, proofs requiring that history and +``flush()`` fail clearly until a complete tile prefix overlaps the resident +frontier. The application must establish exclusive ownership of the namespace; +unlike fresh construction, this factory does not claim it. + +Populate or repair the namespace with an independent store and a repair writer: + +.. code:: cpp + + merkle::tiles::TileStore repair_store(cfg.prefix, "sha256"); + auto repair = merkle::tiles::TileWriter::repair( + repair_store, + trusted_full_tile_boundary); + + repair.write_up_to( + target_size, + [&](uint64_t i) -> const merkle::Hash& { return authoritative_leaf(i); }); + +The trusted boundary must be tile-aligned. Tiles below it are preserved; every +tile at or beyond it is replaced from the supplied authoritative leaves as the +writer reaches that tile. This makes the same API suitable for an absent +namespace, a stale suffix, or a namespace whose untrusted portion needs repair. +The application remains responsible for validating the trusted prefix. + +The independent writer may run on a background thread because it owns a +separate ``TileStore`` and does not access the ``TiledTree``. Do not run it +concurrently with ``log.flush()`` or another writer for the namespace. Quiesce +the repair writer before making its output visible to the tree: + +.. code:: cpp + + log.adopt_tile_prefix(target_full_tile_boundary); + +Adoption is monotonic. It verifies alignment, namespace presence, tree size, +configured retention, and the overlapping boundary leaf, then updates both the +flushed and immutable boundaries and resets the live writer at that prefix. All +tile levels below the adopted boundary must already be complete and durable. +Files beyond it remain untrusted and are replaced by later live flushes. +Adoption does not compact; call ``compact()`` separately when desired. + +Persist the serialized frontier and its adopted boundary as one application +checkpoint. On restart, pass that pair to ``resume()``; during a rebuild, pass +the frontier alone to ``from_frontier()`` and adopt only after repair completes. ``flush()`` is incremental: each call writes only the full tiles that became -complete since the previous call. Full tiles are immutable: written once after -all 256 entries are final and never rewritten. The remaining frontier stays in -memory until it crosses the next full-tile boundary. +complete since the previous call. Tiles in the trusted prefix are immutable. +Files in an untrusted repair suffix may be replaced before that suffix is +adopted. The remaining frontier stays in memory until it crosses the next +full-tile boundary. Tile files are written through unique temporary files, synced, then published with an atomic replace. On POSIX, file contents are synced, each newly created diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 616c944..33ec775 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -872,14 +872,15 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @tparam TILE_HEIGHT_VALUE Number of tree levels represented by a tile /// @note Only balanced subtrees are tiled: a level-L entry is the root of a /// complete 2**(TILE_HEIGHT_VALUE*L)-leaf subtree. Only full tiles are - /// written; they are therefore immutable and written exactly once. - /// Entries beyond the last full-tile boundary remain in memory until a - /// later flush completes the next tile. + /// written. Tiles in a caller-validated prefix are immutable; repair() may + /// replace files beyond that prefix. Entries beyond the last full-tile + /// boundary remain in memory until a later flush completes the next tile. /// @warning No internal synchronization is provided. Callers must serialize /// access to a writer and its store. - /// @warning A writer trusts existing full tiles as output from the same - /// tree and hash function. Callers resuming a store must establish that - /// ownership and restore the matching tree state. + /// @warning An ordinary writer trusts existing full tiles as output from the + /// same tree and hash function. repair() trusts only its explicit prefix and + /// replaces later tiles. Callers must establish prefix ownership and restore + /// the matching tree state. template < size_t HASH_SIZE, void HASH_FUNCTION( @@ -919,15 +920,31 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief Constructs a writer over @p store. explicit TileWriterT(Store& store) : store(store) {} + /// @brief Constructs a writer that repairs a store after a trusted prefix. + /// @param store The store to populate or repair + /// @param trusted_size Caller-validated, tile-aligned leaf count below + /// which every required full tile is already durable and correct + /// @return A writer that preserves the trusted prefix and replaces every + /// existing full tile beyond it as write_up_to() reaches that tile + /// @note The returned writer owns no store. It may be used independently + /// of a TiledTree, including on a background thread, while the caller + /// serializes all writers sharing the namespace. + [[nodiscard]] static TileWriterT repair( + Store& store, uint64_t trusted_size) + { + return TileWriterT(store, trusted_size); + } + /// @brief Writes all newly-complete full tiles for a tree of @p size /// leaves. /// @param size The current tree size /// @param leaf_at Returns the level-0 leaf hash for a leaf index in /// [0, size); only ever queried for leaves of complete subtrees. /// @return Counts of tiles written - /// @note Incremental: full tiles already on disk are immutable and are - /// never rewritten once validated and confirmed durable. Malformed files - /// are replaced. Entries that do not complete a tile are not written. + /// @note Incremental: an ordinary writer reuses full tiles already on disk + /// once validated and confirmed durable. A repair writer preserves only + /// its trusted prefix and replaces later tiles. Malformed files are + /// replaced. Entries that do not complete a tile are not written. /// Tiles are always rolled up through MAX_TILE_LEVEL, so the on-disk set /// always contains the higher-level roll-ups that proof generation relies /// on. @@ -993,9 +1010,22 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief Constructs a writer after a caller-validated tile prefix. /// Files beyond @p trusted_size are replaced before they become readable. - TileWriterT(Store& store, uint64_t trusted_size) : - store(store), trust_existing_tiles(false) + TileWriterT(Store& store, uint64_t trusted_size) : store(store) { + reset_trusted_size(trusted_size); + } + + /// @brief Resets this writer after a caller-validated tile prefix. + void reset_trusted_size(uint64_t trusted_size) + { + if (trusted_size % TILE_WIDTH != 0) + { + throw std::runtime_error( + "TileWriter::repair: trusted size is not tile-aligned"); + } + + std::vector restored_next_full; + std::vector restored_cursor_inited; for (uint8_t level = 0; level <= MAX_TILE_LEVEL; level++) { const uint64_t entries = entries_at_level(trusted_size, level); @@ -1003,10 +1033,15 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) { break; } - ensure_level(level); - next_full[level] = entries / TILE_WIDTH; - cursor_inited[level] = 1; + const size_t needed = (size_t)level + 1; + restored_next_full.resize(needed, 0); + restored_cursor_inited.resize(needed, 0); + restored_next_full[level] = entries / TILE_WIDTH; + restored_cursor_inited[level] = 1; } + next_full = std::move(restored_next_full); + cursor_inited = std::move(restored_cursor_inited); + trust_existing_tiles = false; } /// @brief Rebinds a moved writer to its destination store. @@ -1644,11 +1679,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// Config::compact_on_flush, or call compact() explicitly; it never drops /// the un-tiled frontier. Proofs are served from the combination of the /// resident tree (frontier) and the full tiles (compacted past). - /// @note TiledTree constructors create a new tiled tree and atomically - /// claim a previously absent tile namespace. The resume() factory reopens - /// an existing namespace only when the caller supplies the matching - /// serialized tree state and validated full-tile boundary; tile files alone - /// do not identify or restore their tree. + /// @note TiledTree constructors create a new tiled tree and atomically claim + /// a previously absent tile namespace. from_frontier() restores logical tree + /// state while trusting no tiles, so a separate writer can populate or + /// repair the namespace before adopt_tile_prefix(). resume() combines those + /// steps when a validated tile prefix already exists. /// @warning No internal synchronization is provided. Callers must serialize /// all access to a shared tree, including proof operations. template < @@ -1689,6 +1724,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief Root directory for the tiled tree. /// @note Fresh construction requires the algorithm-qualified tile /// subdirectory to be absent; resume() requires it to exist. + /// from_frontier() does not inspect or create it. std::filesystem::path prefix; /// @brief Number of most-recent leaves to keep resident when @@ -1704,6 +1740,29 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) bool compact_on_flush = false; }; + /// @brief Restores logical tree state without trusting any tile files. + /// @param config Runtime configuration, including the tile prefix + /// @param hash_algorithm_short_name Lowercase hash algorithm namespace + /// @param serialised_tree Serialized merkle::TreeT state + /// @return A tree with flushed_size() and immutable_size() equal to zero + /// @note This does not inspect, create, or claim the tile namespace. + /// Existing files remain untrusted. Root computation and appends are + /// available immediately, but tile-dependent proofs and flushes of + /// non-resident history throw until a complete prefix is adopted. + /// @warning The caller must establish exclusive ownership of the + /// configured namespace. + [[nodiscard]] static TiledTreeT from_frontier( + Config config, + const std::string& hash_algorithm_short_name, + const std::vector& serialised_tree) + { + return TiledTreeT( + FrontierTag{}, + std::move(config), + hash_algorithm_short_name, + serialised_tree); + } + /// @brief Resumes a tiled tree from serialized tree state and an existing /// tile namespace. /// @param config Runtime configuration, including the tile prefix @@ -1820,6 +1879,27 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) return store; } + /// @brief Adopts a complete, durable tile prefix produced independently. + /// @param full_tile_boundary Tile-aligned number of covered leaves + /// @note The caller must quiesce every writer for this namespace before + /// calling. Every required full tile below the boundary must be durable, + /// correct for this tree, and use the same geometry and hash function. + /// Adoption is monotonic, does not compact, and keeps files beyond the new + /// boundary untrusted so later flushes replace them. + void adopt_tile_prefix(size_t full_tile_boundary) + { + if (full_tile_boundary < sealed_size) + { + throw std::runtime_error( + "TiledTree::adopt_tile_prefix: boundary precedes immutable tiles"); + } + validate_tile_prefix(full_tile_boundary); + writer.reset_trusted_size( + static_cast(full_tile_boundary)); + tiles_size = full_tile_boundary; + sealed_size = full_tile_boundary; + } + /// @brief Writes newly-complete full tiles to disk; compacts only if /// Config::compact_on_flush is set. /// @return Counts of the full tiles written by this flush @@ -1838,6 +1918,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } const size_t covered = (n / TILE_WIDTH) * TILE_WIDTH; + if (!has_complete_history()) + { + throw std::runtime_error( + "TiledTree::flush: tile prefix does not overlap the resident tree"); + } if (covered > sealed_size) { sealed_size = covered; @@ -1950,6 +2035,9 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } protected: + struct FrontierTag + {}; + struct ResumeTag {}; @@ -1960,20 +2048,32 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) size_t tiles_size = 0; size_t sealed_size = 0; + TiledTreeT( + FrontierTag, + Config config, + const std::string& hash_algorithm_short_name, + const std::vector& serialised_tree) : + config(std::move(config)), + store(this->config.prefix, hash_algorithm_short_name), + writer(store, 0), + tree(deserialise_tree(serialised_tree)) + { + validate_frontier(); + } + TiledTreeT( ResumeTag, Config config, const std::string& hash_algorithm_short_name, const std::vector& serialised_tree, size_t full_tile_boundary) : - config(std::move(config)), - store(this->config.prefix, hash_algorithm_short_name), - writer(store, static_cast(full_tile_boundary)), - tree(deserialise_tree(serialised_tree)), - tiles_size(full_tile_boundary), - sealed_size(full_tile_boundary) + TiledTreeT( + FrontierTag{}, + std::move(config), + hash_algorithm_short_name, + serialised_tree) { - validate_resume_state(full_tile_boundary); + adopt_tile_prefix(full_tile_boundary); } static Tree deserialise_tree(const std::vector& serialised_tree) @@ -1983,42 +2083,48 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) if (position != serialised_tree.size()) { throw std::runtime_error( - "TiledTree::resume: trailing bytes in serialized tree"); + "TiledTree: trailing bytes in serialized tree"); } return restored; } - void validate_resume_state(size_t full_tile_boundary) + void validate_frontier() { + if (!tree.invariant()) + { + throw std::runtime_error( + "TiledTree: deserialized tree invariant failed"); + } + } + + void validate_tile_prefix(size_t full_tile_boundary) + { + validate_frontier(); const auto tile_root = store.root() / "tile"; std::error_code ec; if (!std::filesystem::is_directory(tile_root, ec) || ec) { throw std::runtime_error(std::format( - "TiledTree::resume: tile namespace does not exist: {}", + "TiledTree::adopt_tile_prefix: tile namespace does not exist: {}", tile_root.string())); } if (full_tile_boundary % TILE_WIDTH != 0) { throw std::runtime_error( - "TiledTree::resume: full tile boundary is not tile-aligned"); + "TiledTree::adopt_tile_prefix: boundary is not tile-aligned"); } if (full_tile_boundary > tree.num_leaves()) { throw std::runtime_error( - "TiledTree::resume: full tile boundary exceeds tree size"); - } - if (!tree.invariant()) - { - throw std::runtime_error( - "TiledTree::resume: deserialized tree invariant failed"); + "TiledTree::adopt_tile_prefix: boundary exceeds tree size"); } if (full_tile_boundary == 0) { if (tree.min_index() != 0) { throw std::runtime_error( - "TiledTree::resume: compacted tree has no tile coverage"); + "TiledTree::adopt_tile_prefix: compacted tree has no tile " + "coverage"); } return; } @@ -2030,7 +2136,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) if (tree.min_index() > latest_resident_start) { throw std::runtime_error( - "TiledTree::resume: serialized tree does not satisfy the configured " + "TiledTree::adopt_tile_prefix: tree does not satisfy the configured " "retention margin"); } @@ -2040,10 +2146,15 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) if (hashes.back() != tree.leaf(full_tile_boundary - 1)) { throw std::runtime_error( - "TiledTree::resume: tile prefix does not match serialized tree"); + "TiledTree::adopt_tile_prefix: prefix does not match the tree"); } } + [[nodiscard]] bool has_complete_history() const + { + return tree.min_index() == 0 || tiles_size > tree.min_index(); + } + [[nodiscard]] size_t compaction_target(size_t covered) const { size_t target = covered > config.retention_margin ? @@ -2097,6 +2208,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) template auto with_engine(Fn fn) { + if (!has_complete_history()) + { + throw std::runtime_error( + "TiledTree: tile prefix does not overlap the resident tree"); + } MemoryHashSourceT mem(tree); TileHashSourceT tile_src( store, tiles_size); diff --git a/test/tiles_resume.cpp b/test/tiles_resume.cpp index c16ba79..2f3c5b5 100644 --- a/test/tiles_resume.cpp +++ b/test/tiles_resume.cpp @@ -211,6 +211,105 @@ int main() "first resumed boundary"); } + // A compacted frontier can run without tiles while an independent writer + // repairs the namespace. Once the repair is quiesced, adopting its complete + // prefix enables proofs and normal incremental flushing. + { + SmallCcfTiledTree::Config config; + config.prefix = base / "frontier_only"; + + SmallCcfTiledTree::Tree frontier; + for (size_t i = 0; i < 40; i++) + { + frontier.insert(hashes[i]); + } + const Hash frontier_root = frontier.root(); + frontier.flush_to(24); + const auto serialised_frontier = serialise(frontier); + + auto restored = SmallCcfTiledTree::from_frontier( + config, hash_namespace, serialised_frontier); + expect(restored.size() == 40, "frontier-only size"); + expect(restored.root() == frontier_root, "frontier-only root"); + expect(restored.flushed_size() == 0, "frontier-only tile boundary"); + expect( + restored.immutable_size() == 0, "frontier-only immutable boundary"); + expect( + !fs::exists(config.prefix), "frontier-only restore performs no I/O"); + expect_throws( + [&]() { (void)restored.inclusion_proof(0, restored.size()); }, + "proof before tile adoption"); + expect_throws( + [&]() { (void)restored.flush(); }, "flush before tile repair"); + expect( + restored.immutable_size() == 0, + "failed frontier-only flush seals nothing"); + expect( + !fs::exists(config.prefix), + "failed frontier-only flush performs no I/O"); + + SmallCcfTiledTree::Store repair_store(config.prefix, hash_namespace); + expect_throws( + [&]() { (void)SmallCcfTiledTree::Writer::repair(repair_store, 1); }, + "unaligned repair boundary"); + repair_store.write_tile( + TileRef{0, 0}, std::vector(SmallCcfTiledTree::TILE_WIDTH)); + + const auto leaf_at = [&](uint64_t index) -> const Hash& { + return hashes[static_cast(index)]; + }; + { + auto repair = SmallCcfTiledTree::Writer::repair(repair_store, 0); + expect( + repair.write_up_to(24, leaf_at).full_written == 7, + "initial background repair"); + } + const std::vector first_tile( + hashes.begin(), hashes.begin() + SmallCcfTiledTree::TILE_WIDTH); + expect( + repair_store.read_tile(TileRef{0, 0}) == first_tile, + "repair replaces untrusted tile"); + expect_throws( + [&]() { restored.adopt_tile_prefix(24); }, + "partial prefix does not reach resident frontier"); + + { + auto repair = SmallCcfTiledTree::Writer::repair(repair_store, 24); + expect( + repair.write_up_to(40, leaf_at).full_written == 5, + "continued background repair"); + } + repair_store.write_tile( + TileRef{0, 10}, std::vector(SmallCcfTiledTree::TILE_WIDTH)); + + restored.adopt_tile_prefix(40); + expect(restored.flushed_size() == 40, "adopted tile boundary"); + expect(restored.immutable_size() == 40, "adopted immutable boundary"); + expect( + restored.inclusion_proof(0, restored.size())->verify(frontier_root), + "proof after tile adoption"); + expect_throws( + [&]() { restored.adopt_tile_prefix(36); }, + "tile adoption cannot regress"); + + for (size_t i = 40; i < 44; i++) + { + restored.append(hashes[i]); + } + expect( + restored.flush().full_written == 1, + "normal flush replaces untrusted suffix"); + const std::vector next_tile( + hashes.begin() + 40, hashes.begin() + 44); + expect( + repair_store.read_tile(TileRef{0, 10}) == next_tile, + "normal flush publishes authoritative suffix"); + const Hash extended_root = restored.root(); + expect( + restored.inclusion_proof(0, restored.size())->verify(extended_root), + "proof after resumed growth"); + } + // A compacted tree must retain the configured overlap with its tile prefix. { CcfTiledTree::Config config; diff --git a/test/tiles_writer.cpp b/test/tiles_writer.cpp index b852652..4a8d5e8 100644 --- a/test/tiles_writer.cpp +++ b/test/tiles_writer.cpp @@ -47,9 +47,9 @@ static size_t tile_file_count(const Store& store) // Roll up a full level-0 tile and compare with a level-1 tile entry. static Hash rollup(const std::vector& leaves) { - return merkle::tiles::perfect_root< - merkle::Tree::Hash::size_bytes, - merkle::Tree::hash_function>(leaves); + return merkle::tiles:: + perfect_root( + leaves); } class TileWriterProbe : public TileWriter @@ -293,6 +293,48 @@ int main() std::cout << "F (interior recovery): OK" << '\n'; } + // ---- F2. A repair writer preserves an explicit trusted prefix and + // overwrites correctly-sized files beyond it. + { + const auto hashes = make_hashes(768); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + const fs::path dir = base / "f_repair"; + TileStore store(dir); + { + TileWriter writer(store); + expect( + writer.write_up_to(512, leaf_at).full_written == 2, + "F2 initial trusted prefix"); + } + + store.write_tile(TileRef{0, 2}, std::vector(TileStore::TILE_WIDTH)); + bool unaligned_threw = false; + try + { + (void)TileWriter::repair(store, 1); + } + catch (const std::runtime_error&) + { + unaligned_threw = true; + } + expect(unaligned_threw, "F2 rejects unaligned trusted prefix"); + + auto repair = TileWriter::repair(store, 512); + expect( + repair.write_up_to(768, leaf_at).full_written == 1, + "F2 replaces untrusted suffix"); + expect( + repair.write_up_to(768, leaf_at).full_written == 0, + "F2 repair is incremental"); + const std::vector expected(hashes.begin() + 512, hashes.end()); + expect( + store.read_tile(TileRef{0, 2}) == expected, + "F2 repaired suffix contents"); + expect(tile_file_count(store) == 3, "F2 exact tile file count"); + + std::cout << "F2 (trusted-prefix repair): OK" << '\n'; + } + // ---- G. Recovery is bounded by the requested tree size, so sparse files // at geometrically increasing indices cannot overflow its search. { @@ -315,8 +357,7 @@ int main() TileWriter writer(store); const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; expect( - writer.write_up_to(TileStore::TILE_WIDTH, leaf_at).full_written == - 0, + writer.write_up_to(TileStore::TILE_WIDTH, leaf_at).full_written == 0, "G bounded sparse recovery"); expect(store.has_full_tile(0, 0), "G requested tile remains valid"); @@ -343,8 +384,7 @@ int main() TileWriterProbe writer(store); writer.mark_level_complete( - 0, - (uint64_t)TileStore::TILE_WIDTH * TileStore::TILE_WIDTH); + 0, (uint64_t)TileStore::TILE_WIDTH * TileStore::TILE_WIDTH); writer.mark_level_complete(1, TileStore::TILE_WIDTH); bool leaf_requested = false; @@ -372,12 +412,10 @@ int main() // when producing higher-level tiles. { using Hash384 = merkle::Tree384::Hash; - using TileStore384 = merkle::tiles::TileStoreT< - Hash384::size_bytes, - merkle::Tree384::hash_function>; - using TileWriter384 = merkle::tiles::TileWriterT< - Hash384::size_bytes, - merkle::Tree384::hash_function>; + using TileStore384 = merkle::tiles:: + TileStoreT; + using TileWriter384 = merkle::tiles:: + TileWriterT; constexpr uint64_t size = (uint64_t)TileStore384::TILE_WIDTH * TileStore384::TILE_WIDTH; @@ -397,9 +435,9 @@ int main() hashes.begin(), hashes.begin() + TileStore384::TILE_WIDTH); expect( level1[0] == - merkle::tiles::perfect_root< - Hash384::size_bytes, - merkle::Tree384::hash_function>(first_tile), + merkle::tiles:: + perfect_root( + first_tile), "I first SHA-384 roll-up"); merkle::Tree384 tree; @@ -408,9 +446,9 @@ int main() tree.insert(hash); } expect( - merkle::tiles::perfect_root< - Hash384::size_bytes, - merkle::Tree384::hash_function>(level1) == tree.root(), + merkle::tiles:: + perfect_root( + level1) == tree.root(), "I SHA-384 tiled root"); expect(tile_file_count(store) == 257, "I exact tile file count"); From 737083ab5a625e2d5a58ff0c14ba4b4c5a3b6e60 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 21 Aug 2026 15:16:14 +0100 Subject: [PATCH 3/5] Fix Clang header analysis Make the traversal child invariant explicit and satisfy the public-header lint rules for deserialization and writer rebinding. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 455a0fc3-10c9-4831-bf66-f06ff952f607 --- merklecpp.h | 8 ++++++-- merklecpp_tiles.h | 19 ++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/merklecpp.h b/merklecpp.h index f395e5d..8964bc2 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -1126,6 +1126,10 @@ namespace merkle << std::endl;); if (cur->height == height) { + if (!cur->left || !cur->right) + { + throw std::runtime_error("unexpected null child node"); + } if (!f(cur, go_right)) { continue; @@ -1544,9 +1548,9 @@ namespace merkle throw std::runtime_error("serialised tree size exceeds size_t"); } - const size_t restored_num_leaf_nodes = + const auto restored_num_leaf_nodes = static_cast(serialised_num_leaf_nodes); - const size_t restored_num_flushed = + const auto restored_num_flushed = static_cast(serialised_num_flushed); if (restored_num_leaf_nodes == 0 && restored_num_flushed != 0) { diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 33ec775..69589ee 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -1045,13 +1045,18 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } /// @brief Rebinds a moved writer to its destination store. + // members are moved individually so the store reference can be rebound. + // NOLINTNEXTLINE(cppcoreguidelines-rvalue-reference-param-not-moved) TileWriterT(Store& store, TileWriterT&& other) noexcept : - store(store), trust_existing_tiles(other.trust_existing_tiles) + store(store), + next_full(std::move(other.next_full)), + cursor_inited(std::move(other.cursor_inited)), + trust_existing_tiles(other.trust_existing_tiles) { - if (!trust_existing_tiles) + if (trust_existing_tiles) { - next_full = std::move(other.next_full); - cursor_inited = std::move(other.cursor_inited); + next_full.clear(); + cursor_inited.clear(); } } @@ -2049,7 +2054,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) size_t sealed_size = 0; TiledTreeT( - FrontierTag, + [[maybe_unused]] FrontierTag tag, Config config, const std::string& hash_algorithm_short_name, const std::vector& serialised_tree) : @@ -2062,7 +2067,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } TiledTreeT( - ResumeTag, + [[maybe_unused]] ResumeTag tag, Config config, const std::string& hash_algorithm_short_name, const std::vector& serialised_tree, @@ -2140,7 +2145,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) "retention margin"); } - const uint64_t last_tile = + const auto last_tile = static_cast(full_tile_boundary / TILE_WIDTH - 1); const auto hashes = store.read_tile(TileRef{0, last_tile}); if (hashes.back() != tree.leaf(full_tile_boundary - 1)) From 23dc45452ccd45de3b76328b73d04a712db3b275 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 26 Aug 2026 12:00:44 +0100 Subject: [PATCH 4/5] Verify resumed tile integrity Validate every required tile and roll-up against the serialized frontier, and restore distinct flushed and immutable boundaries after interrupted flushes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 21 ++++-- doc/tiles-guide.rst | 61 ++++++++++----- merklecpp_tiles.h | 159 +++++++++++++++++++++++++++++++++----- test/tiles_resume.cpp | 172 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 369 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 4f4fc08..4a54252 100644 --- a/README.md +++ b/README.md @@ -74,11 +74,22 @@ resume an existing namespace directly: serialised_tree, full_tile_boundary); -The boundary must cover a complete, durable tile prefix at every required level -and overlap the resident portion of the serialized tree. Tile files beyond it -are treated as untrusted and replaced when a later flush reaches them. The -application remains responsible for establishing namespace ownership and -matching the serialized tree to the trusted tiles. +The boundary must cover a complete, durable tile prefix at every required +level and overlap the resident portion of the serialized tree. Recovery reads +every required tile, validates each stored roll-up, and compares the prefix +root with the serialized frontier. Tile files beyond the boundary are treated +as untrusted and replaced when a later flush reaches them. The application +remains responsible for establishing namespace ownership. + +After an interrupted flush, restore the last successful prefix and the +possibly larger rollback seal separately: + + auto log = merkle::tiles::TiledTree::resume( + cfg, + "sha256", + serialised_tree, + flushed_tile_boundary, + immutable_boundary); If tiles are not ready yet, restore the logical tree first and populate the namespace independently. This does not claim or inspect the namespace; the diff --git a/doc/tiles-guide.rst b/doc/tiles-guide.rst index e735c87..74d3157 100644 --- a/doc/tiles-guide.rst +++ b/doc/tiles-guide.rst @@ -104,8 +104,8 @@ Resuming an externally checkpointed tree ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Tile files do not identify the tree that produced them or contain enough state -to restore its size and root. An application that separately persists and -validates that state can reopen an existing namespace with ``resume()``: +to restore its size and root. An application that separately persists the tree +state and boundaries can reopen an existing namespace with ``resume()``: .. code:: cpp @@ -119,15 +119,38 @@ validates that state can reopen an existing namespace with ``resume()``: ``full_tile_boundary`` is a leaf count, must be a multiple of ``TILE_WIDTH``, and must identify a complete, durable tile prefix at every required level. The factory deserializes the tree, rejects trailing bytes, and requires its resident -range to satisfy ``Config::retention_margin``. It also compares the last leaf in -the prefix with the matching resident tree leaf, catching an incorrect boundary -or namespace at the hand-off point. +range to satisfy ``Config::retention_margin``. It reads every required tile, +checks every stored higher-level entry against the roll-up of its child tile, +and compares the resulting prefix root with the past root derived from the +serialized frontier. Missing, malformed, divergent, or incorrectly rolled-up +tiles reject recovery before the tree is exposed. This verification is linear +in the stored prefix and may perform substantial I/O for a large tree. + +The application remains responsible for establishing namespace ownership. The +boundary cannot be inferred from the tree's ``min_index()`` or from the files on +disk. Existing files beyond it are excluded from proof reads and replaced from +the restored tree when a later ``flush()`` reaches them. + +An interrupted flush may publish files and advance ``immutable_size()`` without +advancing ``flushed_size()``. Persist both values with the serialized frontier +and restore them with the five-argument overload: -The application remains responsible for establishing namespace ownership and -validating that every tile below the supplied boundary belongs to the same -Merkle history. The boundary cannot be inferred from the tree's ``min_index()`` -or from the files on disk. Existing files beyond it are excluded from proof -reads and replaced from the restored tree when a later ``flush()`` reaches them. +.. code:: cpp + + auto log = merkle::tiles::TiledTree::resume( + cfg, + "sha256", + serialised_tree, + flushed_tile_boundary, + immutable_boundary); + +The complete prefix through ``flushed_tile_boundary`` is verified and available +to proofs. ``immutable_boundary`` separately restores the rollback seal. Tiles +between the two boundaries remain untrusted and are replaced from the resident +frontier when ``flush()`` is retried. Both boundaries are tile-aligned, +``flushed_tile_boundary <= immutable_boundary``, and neither may exceed the +serialized tree's last complete tile boundary. When they are equal, the +four-argument overload above is equivalent. An ordinary lower-level ``TileWriter`` intentionally resumes existing full tiles and trusts the caller to supply the same tree and hash function. A fresh @@ -184,16 +207,16 @@ the repair writer before making its output visible to the tree: log.adopt_tile_prefix(target_full_tile_boundary); -Adoption is monotonic. It verifies alignment, namespace presence, tree size, -configured retention, and the overlapping boundary leaf, then updates both the -flushed and immutable boundaries and resets the live writer at that prefix. All -tile levels below the adopted boundary must already be complete and durable. -Files beyond it remain untrusted and are replaced by later live flushes. -Adoption does not compact; call ``compact()`` separately when desired. +Adoption is monotonic. It performs the same exhaustive tile, roll-up, and prefix +root verification as ``resume()``, then updates both the flushed and immutable +boundaries and resets the live writer at that prefix. Files beyond it remain +untrusted and are replaced by later live flushes. Adoption does not compact; +call ``compact()`` separately when desired. -Persist the serialized frontier and its adopted boundary as one application -checkpoint. On restart, pass that pair to ``resume()``; during a rebuild, pass -the frontier alone to ``from_frontier()`` and adopt only after repair completes. +Persist the serialized frontier, flushed boundary, and immutable boundary +atomically as one application checkpoint. On restart, pass them to ``resume()``; +during a rebuild, pass the frontier alone to ``from_frontier()`` and adopt only +after repair completes. ``flush()`` is incremental: each call writes only the full tiles that became complete since the previous call. Tiles in the trusted prefix are immutable. diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 69589ee..af728c6 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -201,6 +201,10 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) HASH_SIZE, HASH_FUNCTION, TILE_HEIGHT_VALUE>; + friend class TiledTreeT< + HASH_SIZE, + HASH_FUNCTION, + TILE_HEIGHT_VALUE>; public: /// @brief The type of hashes stored in tiles. @@ -1773,8 +1777,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @param config Runtime configuration, including the tile prefix /// @param hash_algorithm_short_name Lowercase hash algorithm namespace /// @param serialised_tree Serialized merkle::TreeT state - /// @param full_tile_boundary Caller-validated number of leaves covered by - /// a complete, durable tile prefix + /// @param full_tile_boundary Number of leaves covered by a complete, + /// durable tile prefix /// @return A tiled tree whose tile reads are capped at /// @p full_tile_boundary /// @note The serialized tree is deserialized by this call. At every tile @@ -1787,13 +1791,43 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const std::string& hash_algorithm_short_name, const std::vector& serialised_tree, size_t full_tile_boundary) + { + return resume( + std::move(config), + hash_algorithm_short_name, + serialised_tree, + full_tile_boundary, + full_tile_boundary); + } + + /// @brief Resumes a tiled tree after an interrupted flush. + /// @param config Runtime configuration, including the tile prefix + /// @param hash_algorithm_short_name Lowercase hash algorithm namespace + /// @param serialised_tree Serialized merkle::TreeT state + /// @param flushed_tile_boundary Number of leaves covered by the last + /// complete, durable tile prefix + /// @param immutable_boundary Rollback seal for tiles that may have been + /// published by an interrupted flush + /// @return A tiled tree whose proof reads are capped at + /// @p flushed_tile_boundary and whose rollback seal is + /// @p immutable_boundary + /// @note The complete prefix is verified against the serialized frontier. + /// Tiles between the two boundaries remain untrusted and are replaced by + /// the next flush. + [[nodiscard]] static TiledTreeT resume( + Config config, + const std::string& hash_algorithm_short_name, + const std::vector& serialised_tree, + size_t flushed_tile_boundary, + size_t immutable_boundary) { return TiledTreeT( ResumeTag{}, std::move(config), hash_algorithm_short_name, serialised_tree, - full_tile_boundary); + flushed_tile_boundary, + immutable_boundary); } explicit TiledTreeT(Config config) : @@ -1887,10 +1921,10 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief Adopts a complete, durable tile prefix produced independently. /// @param full_tile_boundary Tile-aligned number of covered leaves /// @note The caller must quiesce every writer for this namespace before - /// calling. Every required full tile below the boundary must be durable, - /// correct for this tree, and use the same geometry and hash function. - /// Adoption is monotonic, does not compact, and keeps files beyond the new - /// boundary untrusted so later flushes replace them. + /// calling. Every required full tile below the boundary is verified + /// against the serialized frontier. Adoption is monotonic, does not + /// compact, and keeps files beyond the new boundary untrusted so later + /// flushes replace them. void adopt_tile_prefix(size_t full_tile_boundary) { if (full_tile_boundary < sealed_size) @@ -2071,14 +2105,15 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) Config config, const std::string& hash_algorithm_short_name, const std::vector& serialised_tree, - size_t full_tile_boundary) : + size_t flushed_tile_boundary, + size_t immutable_boundary) : TiledTreeT( FrontierTag{}, std::move(config), hash_algorithm_short_name, serialised_tree) { - adopt_tile_prefix(full_tile_boundary); + restore_tile_boundaries(flushed_tile_boundary, immutable_boundary); } static Tree deserialise_tree(const std::vector& serialised_tree) @@ -2110,25 +2145,25 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) if (!std::filesystem::is_directory(tile_root, ec) || ec) { throw std::runtime_error(std::format( - "TiledTree::adopt_tile_prefix: tile namespace does not exist: {}", + "TiledTree: tile namespace does not exist: {}", tile_root.string())); } if (full_tile_boundary % TILE_WIDTH != 0) { throw std::runtime_error( - "TiledTree::adopt_tile_prefix: boundary is not tile-aligned"); + "TiledTree: tile boundary is not aligned"); } if (full_tile_boundary > tree.num_leaves()) { throw std::runtime_error( - "TiledTree::adopt_tile_prefix: boundary exceeds tree size"); + "TiledTree: tile boundary exceeds tree size"); } if (full_tile_boundary == 0) { if (tree.min_index() != 0) { throw std::runtime_error( - "TiledTree::adopt_tile_prefix: compacted tree has no tile " + "TiledTree: compacted tree has no tile " "coverage"); } return; @@ -2141,18 +2176,104 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) if (tree.min_index() > latest_resident_start) { throw std::runtime_error( - "TiledTree::adopt_tile_prefix: tree does not satisfy the configured " + "TiledTree: tree does not satisfy the configured " "retention margin"); } - const auto last_tile = - static_cast(full_tile_boundary / TILE_WIDTH - 1); - const auto hashes = store.read_tile(TileRef{0, last_tile}); - if (hashes.back() != tree.leaf(full_tile_boundary - 1)) + store.begin_write_attempt(); + for (uint8_t level = 0; level <= MAX_TILE_LEVEL; level++) + { + const uint64_t entries = Writer::entries_at_level( + static_cast(full_tile_boundary), level); + if (entries == 0) + { + break; + } + + const uint64_t full_tiles = entries / TILE_WIDTH; + for (uint64_t tile_index = 0; tile_index < full_tiles; tile_index++) + { + const auto hashes = read_required_tile(level, tile_index); + if (level == 0) + { + continue; + } + + const uint64_t first_child = tile_index * TILE_WIDTH; + for (uint64_t offset = 0; offset < TILE_WIDTH; offset++) + { + const auto child = + read_required_tile((uint8_t)(level - 1), first_child + offset); + const auto expected = + perfect_root(child); + if (hashes[(size_t)offset] != expected) + { + throw std::runtime_error(std::format( + "TiledTree: tile roll-up mismatch at " + "level {}, index {}, entry {}", + static_cast(level), + tile_index, + offset)); + } + } + } + } + + TileHashSourceT tile_source( + store, static_cast(full_tile_boundary)); + ProofEngineT engine(tile_source); + const Hash tile_root_hash = + engine.root(static_cast(full_tile_boundary)); + const Hash frontier_root_hash = + *tree.past_root(full_tile_boundary - 1); + if (tile_root_hash != frontier_root_hash) { throw std::runtime_error( - "TiledTree::adopt_tile_prefix: prefix does not match the tree"); + "TiledTree: tile prefix root does not match " + "the serialized frontier"); + } + } + + [[nodiscard]] std::vector read_required_tile( + uint8_t level, uint64_t index) + { + if (!store.confirm_full_tile(level, index)) + { + throw std::runtime_error(std::format( + "TiledTree: missing or malformed tile at " + "level {}, index {}", + static_cast(level), + index)); } + return store.read_tile(TileRef{level, index}); + } + + void restore_tile_boundaries( + size_t flushed_tile_boundary, size_t immutable_boundary) + { + if (immutable_boundary % TILE_WIDTH != 0) + { + throw std::runtime_error( + "TiledTree::resume: immutable boundary is not tile-aligned"); + } + if (immutable_boundary < flushed_tile_boundary) + { + throw std::runtime_error( + "TiledTree::resume: immutable boundary precedes flushed tiles"); + } + const size_t covered = + (tree.num_leaves() / TILE_WIDTH) * TILE_WIDTH; + if (immutable_boundary > covered) + { + throw std::runtime_error( + "TiledTree::resume: immutable boundary exceeds tree size"); + } + + validate_tile_prefix(flushed_tile_boundary); + writer.reset_trusted_size( + static_cast(flushed_tile_boundary)); + tiles_size = flushed_tile_boundary; + sealed_size = immutable_boundary; } [[nodiscard]] bool has_complete_history() const diff --git a/test/tiles_resume.cpp b/test/tiles_resume.cpp index 2f3c5b5..a5d4190 100644 --- a/test/tiles_resume.cpp +++ b/test/tiles_resume.cpp @@ -444,7 +444,7 @@ int main() "level-0 suffix replaced"); } - // The overlapping boundary leaf catches a mismatched tile namespace. + // Prefix-root verification catches a mismatched boundary tile. { CcfTiledTree::Config config; config.prefix = base / "mismatch"; @@ -469,6 +469,176 @@ int main() "mismatched boundary tile"); } + // Resume verifies every required tile and roll-up, not only the boundary + // leaf where the namespace overlaps the resident frontier. + { + SmallCcfTiledTree::Config config; + config.prefix = base / "integrity"; + std::vector serialised_tree; + { + SmallCcfTiledTree source(config, hash_namespace); + for (size_t i = 0; i < 24; i++) + { + source.append(hashes[i]); + } + source.flush(); + serialised_tree = serialise(source.tree_ref()); + } + + SmallCcfTiledTree::Store store(config.prefix, hash_namespace); + const auto level0_tile0 = store.read_tile(TileRef{0, 0}); + store.write_tile( + TileRef{0, 0}, std::vector(SmallCcfTiledTree::TILE_WIDTH)); + expect_throws( + [&]() { + (void)SmallCcfTiledTree::resume( + config, hash_namespace, serialised_tree, 24); + }, + "divergent earlier level-0 tile"); + store.write_tile(TileRef{0, 0}, level0_tile0); + + const auto level0_tile1 = store.read_tile(TileRef{0, 1}); + fs::resize_file(store.tile_path(TileRef{0, 1}), 1); + expect_throws( + [&]() { + (void)SmallCcfTiledTree::resume( + config, hash_namespace, serialised_tree, 24); + }, + "malformed earlier level-0 tile"); + store.write_tile(TileRef{0, 1}, level0_tile1); + + const auto level1_tile0 = store.read_tile(TileRef{1, 0}); + store.write_tile( + TileRef{1, 0}, std::vector(SmallCcfTiledTree::TILE_WIDTH)); + expect_throws( + [&]() { + (void)SmallCcfTiledTree::resume( + config, hash_namespace, serialised_tree, 24); + }, + "divergent higher-level roll-up"); + store.write_tile(TileRef{1, 0}, level1_tile0); + + fs::remove(store.tile_path(TileRef{1, 0})); + expect_throws( + [&]() { + (void)SmallCcfTiledTree::resume( + config, hash_namespace, serialised_tree, 24); + }, + "missing higher-level roll-up"); + } + + // Recovery preserves the rollback seal from an interrupted flush while + // trusting proof reads only through the last fully successful boundary. + { + CcfTiledTree::Config config; + config.prefix = base / "interrupted"; + std::vector serialised_tree; + fs::path blocker; + Hash expected_root; + { + CcfTiledTree source(config, hash_namespace); + for (size_t i = 0; i < 512; i++) + { + source.append(hashes[i]); + } + expected_root = source.root(); + blocker = source.store_ref().tile_path(TileRef{0, 1}); + fs::create_directories(blocker); + expect_throws([&]() { (void)source.flush(); }, "interrupted flush"); + expect(source.flushed_size() == 0, "interrupted flushed boundary"); + expect(source.immutable_size() == 512, "interrupted rollback seal"); + expect( + source.store_ref().has_full_tile(0, 0), + "interrupted flush published first tile"); + serialised_tree = serialise(source.tree_ref()); + } + + auto resumed = CcfTiledTree::resume( + config, hash_namespace, serialised_tree, 0, 512); + expect(resumed.flushed_size() == 0, "restored flushed boundary"); + expect(resumed.immutable_size() == 512, "restored rollback seal"); + expect_throws( + [&]() { resumed.retract_to(0); }, "restored rollback protection"); + + fs::remove(blocker); + expect( + resumed.flush().full_written == 2, + "interrupted suffix is replaced from trusted boundary"); + expect(resumed.flushed_size() == 512, "retry advances flushed boundary"); + expect(resumed.immutable_size() == 512, "retry retains rollback seal"); + expect(resumed.root() == expected_root, "retry preserves root"); + expect( + resumed.inclusion_proof(0, 512)->verify(expected_root), + "retry produces valid tiled proof"); + + expect_throws( + [&]() { + (void)CcfTiledTree::resume( + config, hash_namespace, serialised_tree, 0, 513); + }, + "unaligned immutable boundary"); + expect_throws( + [&]() { + (void)CcfTiledTree::resume( + config, hash_namespace, serialised_tree, 512, 256); + }, + "immutable boundary before flushed boundary"); + } + + // A non-empty trusted prefix remains untouched when recovery replaces the + // suffix covered by a larger interrupted-flush rollback seal. + { + CcfTiledTree::Config config; + config.prefix = base / "interrupted_after_prefix"; + std::vector serialised_tree; + std::vector trusted_tile; + fs::path blocker; + Hash expected_root; + { + CcfTiledTree source(config, hash_namespace); + for (size_t i = 0; i < 256; i++) + { + source.append(hashes[i]); + } + expect( + source.flush().full_written == 1, "initial trusted tile published"); + trusted_tile = source.store_ref().read_tile(TileRef{0, 0}); + + for (size_t i = 256; i < 512; i++) + { + source.append(hashes[i]); + } + expected_root = source.root(); + blocker = source.store_ref().tile_path(TileRef{0, 1}); + fs::create_directories(blocker); + expect_throws( + [&]() { (void)source.flush(); }, "interrupted suffix flush"); + expect(source.flushed_size() == 256, "trusted prefix retained"); + expect(source.immutable_size() == 512, "larger rollback seal retained"); + serialised_tree = serialise(source.tree_ref()); + } + + auto resumed = CcfTiledTree::resume( + config, hash_namespace, serialised_tree, 256, 512); + expect(resumed.flushed_size() == 256, "restored trusted prefix"); + expect(resumed.immutable_size() == 512, "restored larger rollback seal"); + expect( + resumed.store_ref().read_tile(TileRef{0, 0}) == trusted_tile, + "trusted tile validates before retry"); + + fs::remove(blocker); + expect( + resumed.flush().full_written == 1, + "retry writes only the untrusted suffix"); + expect( + resumed.store_ref().read_tile(TileRef{0, 0}) == trusted_tile, + "retry preserves trusted tile"); + expect(resumed.root() == expected_root, "suffix retry preserves root"); + expect( + resumed.inclusion_proof(0, 512)->verify(expected_root), + "suffix retry produces valid proof"); + } + std::cout << "tiles_resume: OK" << '\n'; } catch (const std::exception& error) From 0c14bad4996fa4e4c791e2a3f83fcc6874251923 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 26 Aug 2026 13:51:13 +0100 Subject: [PATCH 5/5] Add committed-prefix tile flushing Allow consensus-driven callers to write and seal only complete tiles within a committed leaf-count prefix while retaining a rollbackable logical suffix. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 2 ++ doc/tiles-guide.rst | 17 ++++++++++++++- merklecpp_tiles.h | 48 +++++++++++++++++++++++++++++------------ test/tiles_geometry.cpp | 46 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 4a54252..5c934d9 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,8 @@ tile-derived inclusion proof is byte-identical to one from // Write newly-complete tiles. With compaction enabled // this also drops from memory the leaves already covered by a full tile; // otherwise the tree keeps every leaf and you can call log.compact() later. + // Use flush_up_to(committed_leaf_count) instead to preserve a speculative + // suffix. log.flush(); // Proofs are served from tiles + the resident tree, even for flushed leaves. diff --git a/doc/tiles-guide.rst b/doc/tiles-guide.rst index 74d3157..3e97c60 100644 --- a/doc/tiles-guide.rst +++ b/doc/tiles-guide.rst @@ -86,7 +86,7 @@ from the ``tiles_docs`` test so the documented code is compiled and run. :dedent: 2 This example assumes ``batch`` is non-empty. ``root()`` throws on an empty tree; -``size()``, ``flush()``, and ``compact()`` are safe at size 0. +``size()``, ``flush()``, ``flush_up_to()``, and ``compact()`` are safe at size 0. ``TiledTree`` can be move-constructed, but it cannot be copied or assigned. Move construction keeps its writer bound to the destination tree's tile store. @@ -224,6 +224,21 @@ Files in an untrusted repair suffix may be replaced before that suffix is adopted. The remaining frontier stays in memory until it crosses the next full-tile boundary. +For a tree that contains a speculative or otherwise rollbackable suffix, flush +only a committed prefix: + +.. code:: cpp + + log.flush_up_to(committed_leaf_count); + +The argument is a **leaf count**, not the index of the final leaf. Only complete +tiles wholly contained in that prefix are written and sealed; later leaves stay +resident and rollbackable. ``flush()`` is equivalent to +``flush_up_to(log.size())``. A count beyond ``size()`` throws, while a count +whose full-tile boundary precedes ``flushed_size()`` is a monotonic no-op. +With ``compact_on_flush``, compaction is likewise limited to the resulting +flushed boundary. + Tile files are written through unique temporary files, synced, then published with an atomic replace. On POSIX, file contents are synced, each newly created directory is made durable by syncing its parent, and the destination directory diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index af728c6..33deb6e 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -1948,35 +1948,55 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// while flushed_size() does not; the tree remains resident and the flush /// can be retried without rewriting finalized tiles. Stats flush() + { + return flush_up_to(tree.num_leaves()); + } + + /// @brief Writes newly-complete full tiles within a committed prefix. + /// @param leaf_count Number of leaves in the prefix that may be made + /// immutable + /// @return Counts of the full tiles written by this flush + /// @note This never writes or seals a complete tile beyond + /// @p leaf_count, even when the logical tree contains more leaves. + /// Requests below flushed_size() are monotonic no-ops. Compaction, when + /// configured, only drops leaves covered by the resulting flushed_size(). + Stats flush_up_to(size_t leaf_count) { Stats stats; const size_t n = tree.num_leaves(); - if (n == 0) + if (leaf_count > n) { - return stats; + throw std::runtime_error( + "TiledTree::flush_up_to: leaf count exceeds tree size"); } - const size_t covered = (n / TILE_WIDTH) * TILE_WIDTH; + const size_t covered = (leaf_count / TILE_WIDTH) * TILE_WIDTH; + if (covered < tiles_size) + { + return stats; + } if (!has_complete_history()) { throw std::runtime_error( - "TiledTree::flush: tile prefix does not overlap the resident tree"); + "TiledTree::flush_up_to: tile prefix does not overlap the " + "resident tree"); } if (covered > sealed_size) { sealed_size = covered; } - stats = writer.write_up_to(n, [this](uint64_t i) -> const Hash& { - if (i < tree.min_index()) - { - throw std::runtime_error(std::format( - "TiledTree::flush: cannot regenerate a missing or malformed " - "tile from non-resident leaf {}", - i)); - } - return tree.leaf((size_t)i); - }); + stats = + writer.write_up_to(leaf_count, [this](uint64_t i) -> const Hash& { + if (i < tree.min_index()) + { + throw std::runtime_error(std::format( + "TiledTree::flush_up_to: cannot regenerate a missing or " + "malformed tile from non-resident leaf {}", + i)); + } + return tree.leaf((size_t)i); + }); tiles_size = covered; if (config.compact_on_flush) diff --git a/test/tiles_geometry.cpp b/test/tiles_geometry.cpp index 77a1b2d..89ef9a8 100644 --- a/test/tiles_geometry.cpp +++ b/test/tiles_geometry.cpp @@ -245,6 +245,52 @@ TEST_CASE("Alternate geometry drives tiled tree lifecycle boundaries") CHECK(tree.size() == 12); } +TEST_CASE("Committed-prefix flush leaves the logical suffix rollbackable") +{ + const TemporaryDirectory temporary_directory; + const auto hashes = make_hashes(40); + SmallTiledTree::Config config; + config.prefix = temporary_directory.path() / "committed_prefix"; + config.compact_on_flush = true; + SmallTiledTree tree(config); + merkle::Tree reference; + for (const auto& hash : hashes) + { + tree.append(hash); + reference.insert(hash); + } + + CHECK(tree.flush_up_to(26).full_written == 7); + CHECK(tree.flushed_size() == 24); + CHECK(tree.immutable_size() == 24); + CHECK(tree.tree_ref().min_index() == 23); + CHECK(tree.store_ref().has_full_tile(0, 5)); + CHECK_FALSE(tree.store_ref().has_full_tile(0, 6)); + CHECK(tree.store_ref().has_full_tile(1, 0)); + CHECK_FALSE(tree.store_ref().has_full_tile(1, 1)); + CHECK(tree.root() == reference.root()); + CHECK_THROWS_AS(tree.retract_to(22), std::runtime_error); + + CHECK_NOTHROW(tree.retract_to(27)); + CHECK(tree.size() == 28); + for (size_t i = 28; i < hashes.size(); i++) + { + tree.append(hashes[i]); + } + + CHECK(tree.flush_up_to(8).full_written == 0); + CHECK(tree.flushed_size() == 24); + CHECK(tree.immutable_size() == 24); + CHECK(tree.flush_up_to(40).full_written == 5); + CHECK(tree.flushed_size() == 40); + CHECK(tree.immutable_size() == 40); + CHECK(tree.store_ref().has_full_tile(0, 9)); + CHECK(tree.store_ref().has_full_tile(1, 1)); + CHECK(tree.root() == reference.root()); + CHECK(*tree.inclusion_proof(0, tree.size()) == *reference.path(0)); + CHECK_THROWS_AS(tree.flush_up_to(41), std::runtime_error); +} + TEST_CASE("Alternate geometry preserves interrupted flush seals") { const TemporaryDirectory temporary_directory;