From 9a1fe8536c74947a9bd48e98c4c176d1a3c71fa6 Mon Sep 17 00:00:00 2001 From: Arjan Zijderveld <5286904+arjanz@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:49:50 +0100 Subject: [PATCH] perf: reuse incremental Merkle trie snapshots for state roots Retain one materialized trie per StateStorage and update only changed paths using immutable nodes. Preserve state-view validation and fallback behavior for custom encoders and unsupported inputs. Add coverage for fork isolation, external writes, mutable buffers, rollback, finalization and historical roots. Validation: 129 focused tests and 11 subtests passed; 1,000 traces passed with 2,000 root comparisons against the existing builder. --- pyjamaz/incremental_trie.py | 137 ++++++++++ pyjamaz/state/storage.py | 52 +++- test/test_incremental_state_root.py | 383 ++++++++++++++++++++++++++++ test/test_incremental_trie.py | 204 +++++++++++++++ 4 files changed, 772 insertions(+), 4 deletions(-) create mode 100644 pyjamaz/incremental_trie.py create mode 100644 test/test_incremental_state_root.py create mode 100644 test/test_incremental_trie.py diff --git a/pyjamaz/incremental_trie.py b/pyjamaz/incremental_trie.py new file mode 100644 index 00000000..b2383508 --- /dev/null +++ b/pyjamaz/incremental_trie.py @@ -0,0 +1,137 @@ +"""Persistent Appendix D trie snapshots with cached hashes on immutable nodes.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass + +from pyjamaz.merkle import PatriciaMerkleTrie, _cached_trie_branch_hash + + +_EMPTY_HASH = bytes(32) + + +@dataclass(frozen=True, slots=True) +class _Leaf: + key: bytes + value: bytes + digest: bytes + + +@dataclass(frozen=True, slots=True) +class _Branch: + left: _Node + right: _Node + digest: bytes + + +_Node = _Leaf | _Branch | None + + +def _digest(node: _Node) -> bytes: + return _EMPTY_HASH if node is None else node.digest + + +def _leaf(key: bytes, value: bytes) -> _Leaf: + # Reuse the existing bounded leaf caches, including their admission limits. + return _Leaf(key, value, PatriciaMerkleTrie([(key, value)]).root()) + + +def _branch(left: _Node, right: _Node) -> _Node: + if left is None and (right is None or isinstance(right, _Leaf)): + return right + if right is None and isinstance(left, _Leaf): + return left + # A unary branch containing multiple leaves must retain its bit position. + return _Branch(left, right, _cached_trie_branch_hash(_digest(left), _digest(right))) + + +def _build(data: list[tuple[bytes, bytes]], index: int = 0) -> _Node: + if not data: + return None + if len(data) == 1: + return _leaf(*data[0]) + left, right = [], [] + for key, value in data: + if index >= len(key) * 8: + raise ValueError("Trie keys must not be prefixes of one another") + (right if PatriciaMerkleTrie._bit(key, index) else left).append((key, value)) + return _branch(_build(left, index + 1), _build(right, index + 1)) + + +def _replace(node: _Node, key: bytes, value: bytes | None, index: int) -> _Node: + if node is None: + return None if value is None else _leaf(key, value) + if isinstance(node, _Leaf): + if key == node.key: + if value is None: + return None + return node if value == node.value else _leaf(key, value) + if value is None: + return node + if key.startswith(node.key) or node.key.startswith(key): + raise ValueError("Trie keys must not be prefixes of one another") + old_right = PatriciaMerkleTrie._bit(node.key, index) + new_right = PatriciaMerkleTrie._bit(key, index) + if old_right != new_right: + other = _leaf(key, value) + return _branch(node, other) if new_right else _branch(other, node) + child = _replace(node, key, value, index + 1) + return _branch(None, child) if old_right else _branch(child, None) + if index >= len(key) * 8: + if value is None: + return node + raise ValueError("Trie keys must not be prefixes of one another") + if PatriciaMerkleTrie._bit(key, index): + right = _replace(node.right, key, value, index + 1) + return node if right is node.right else _branch(node.left, right) + left = _replace(node.left, key, value, index + 1) + return node if left is node.left else _branch(left, node.right) + + +def _validate(key: bytes, value: bytes | None) -> None: + if type(key) is not bytes or len(key) not in (31, 32): + raise ValueError("Trie keys must be immutable 31- or 32-byte bytes") + if value is not None and type(value) is not bytes: + raise TypeError("Trie values must be immutable bytes or None for deletion") + + +@dataclass(frozen=True, slots=True, init=False) +class IncrementalMerkleTrie: + """An immutable trie; updates share unchanged nodes with earlier snapshots. + + ``None`` deletes a key, while ``b\"\"`` is a stored value. Keys may be 31 or + 32 bytes, but a shorter key cannot prefix another key in the same snapshot. + """ + + _node: _Node + + def __init__(self, data: Iterable[tuple[bytes, bytes]] | Mapping[bytes, bytes] = ()): + values = {} + entries = data.items() if isinstance(data, Mapping) else data + for key, value in entries: + _validate(key, value) + if value is None: + raise TypeError("Initial trie values must be immutable bytes") + values[key] = value + object.__setattr__(self, "_node", _build(list(values.items()))) + + def root(self) -> bytes: + return _digest(self._node) + + def update(self, changes: Mapping[bytes, bytes | None]) -> IncrementalMerkleTrie: + node = self._node + for key, value in changes.items(): + _validate(key, value) + if value is None: + node = _replace(node, key, None, 0) + # Remove obsolete keys first so replacing a short key with a prefixed + # long key (or vice versa) does not create a transient prefix collision. + for key, value in changes.items(): + if value is not None: + node = _replace(node, key, value, 0) + if node is self._node: + return self + snapshot = object.__new__(type(self)) + object.__setattr__(snapshot, "_node", node) + return snapshot diff --git a/pyjamaz/state/storage.py b/pyjamaz/state/storage.py index f29f08a4..22cf5468 100644 --- a/pyjamaz/state/storage.py +++ b/pyjamaz/state/storage.py @@ -3,6 +3,8 @@ from enum import Enum from typing import Optional, Dict, List, Tuple +from pyjamaz import merkle +from pyjamaz.incremental_trie import IncrementalMerkleTrie from pyjamaz.merkle import PatriciaMerkleTrie from pyjamaz.models.block import AncestorReference, Header from pyjamaz.settings import DEBUG @@ -10,6 +12,13 @@ from pyjamaz.utils import format_hash, log_execution_time +_STATE_TRIE_TYPE = PatriciaMerkleTrie +_STATE_TRIE_METHODS = ( + PatriciaMerkleTrie._leaf, PatriciaMerkleTrie._branch, PatriciaMerkleTrie._bit, + PatriciaMerkleTrie.merkle, PatriciaMerkleTrie.root, +) + + class ItemStatus(Enum): deleted = 1 @@ -27,6 +36,7 @@ def __init__(self, storage_engine: StorageEngine): self.parents: Dict[bytes, Optional[bytes]] = {} # GP-0.8.0-eq:5.3 (A) self.ancestors: Dict[bytes, Header | AncestorReference] = {} + self._trie_snapshot = (None, IncrementalMerkleTrie()) def add_ancestor(self, header: Header | AncestorReference): self.ancestors[header.hash] = header @@ -170,13 +180,47 @@ def state_root(self) -> bytes: if len(self.transaction) > 0: raise ValueError(f"Pending transaction; commit or rollback first") - state_trie = PatriciaMerkleTrie(list(self.as_dict().items())) - - state_root = state_trie.root() + state_root = self._root_from_items(self.as_dict()) DEBUG and logging.debug(f"StateStorage: Calculated state root {format_hash(state_root)}") return state_root + def _root_from_items(self, items: Dict[bytes, bytes]) -> bytes: + # Keep the original path for custom encoders and non-state-trie input. + if (PatriciaMerkleTrie is not _STATE_TRIE_TYPE or ( + PatriciaMerkleTrie._leaf, PatriciaMerkleTrie._branch, PatriciaMerkleTrie._bit, + PatriciaMerkleTrie.merkle, PatriciaMerkleTrie.root, + ) != _STATE_TRIE_METHODS or merkle.blake2b_256_hash is not merkle._TRIE_HASH_FUNCTION): + return PatriciaMerkleTrie(list(items.items())).root() + + current = {} + for key, value in items.items(): + if (type(key) is not bytes or len(key) not in (31, 32) + or type(value) not in (bytes, bytearray)): + return PatriciaMerkleTrie(list(items.items())).root() + # Mutable buffers must not alias the previous materialized view. + current[key] = bytes(value) + + previous, trie = self._trie_snapshot + try: + if previous is None: + trie = IncrementalMerkleTrie(current) + else: + changes = {key: value for key, value in current.items() + if previous.get(key) != value} + changes.update((key, None) for key in previous if key not in current) + trie = trie.update(changes) + except ValueError: + # A 31-byte key may prefix a 32-byte key in malformed input. Let + # the original builder report its existing error for that case. + return PatriciaMerkleTrie(list(items.items())).root() + + # Retain only the latest view, rather than a separate trie per block. + # Full materialization above observes external base writes and validates + # ancestry even after a fork switch, clear(), or failed transaction. + self._trie_snapshot = (current, trie) + return trie.root() + def as_dict(self) -> Dict[bytes, bytes]: if len(self.transaction) > 0: raise ValueError(f"Pending transaction; commit or rollback first") @@ -249,7 +293,7 @@ def as_list_at(self, block_hash: bytes) -> List[Tuple[bytes, bytes]]: return sorted(self.as_dict_at(block_hash).items(), key=lambda item: item[0]) def state_root_at(self, block_hash: bytes) -> bytes: - return PatriciaMerkleTrie(list(self.as_dict_at(block_hash).items())).root() + return self._root_from_items(self.as_dict_at(block_hash)) def finalize(self, block_hash: bytes): diff --git a/test/test_incremental_state_root.py b/test/test_incremental_state_root.py new file mode 100644 index 00000000..2c6f9a49 --- /dev/null +++ b/test/test_incremental_state_root.py @@ -0,0 +1,383 @@ +"""State-root reuse must observe every materialized state and storage mutation.""" + +from contextlib import contextmanager +from copy import deepcopy + +import pytest + +from pyjamaz import merkle +from pyjamaz.merkle import PatriciaMerkleTrie +from pyjamaz.state import storage as state_storage_module +from pyjamaz.state.storage import StateStorage +from pyjamaz.storage import InMemoryStorageEngine + + +KEYS = [bytes([index]) * 31 for index in range(8)] +FINALIZED, PARENT, LEFT, RIGHT, CHILD = [bytes([index]) * 32 for index in range(1, 6)] + + +def reference_root(items): + # Freeze buffers independently of any snapshot retained by StateStorage. + return PatriciaMerkleTrie([(bytes(key), bytes(value)) for key, value in items.items()]).root() + + +def commit(storage, block_hash, parent_hash, changes): + storage.set_block_hash(block_hash, parent_hash) + for key, value in changes.items(): + storage.put(key, value) + storage.commit() + + +def assert_root(storage, expected): + root = reference_root(expected) + assert storage.as_dict() == expected + assert storage.state_root() == root + assert storage.state_root() == root + + +def assert_retained_root(storage, block_hash, expected): + selected = storage.block_hash + assert storage.state_root_at(block_hash) == reference_root(expected) + assert storage.block_hash == selected + + +@pytest.fixture +def forked_state(): + a, b, c, d, e, f, _, _ = KEYS + base = {a: b"base a", b: b"b" * 33, c: b""} + engine = InMemoryStorageEngine() + state_engine = engine.namespace(b"state") + for key, value in base.items(): + state_engine.put(key, value) + storage = StateStorage(state_engine) + storage.set_finalized_block_hash(FINALIZED) + commit(storage, PARENT, FINALIZED, {a: b"parent a", d: b"parent d"}) + commit(storage, LEFT, PARENT, {b: None, e: b"left e"}) + commit(storage, RIGHT, PARENT, {c: b"right c", f: b"right f"}) + commit(storage, CHILD, LEFT, {a: b"child a", d: None}) + expected = { + FINALIZED: base, + PARENT: {**base, a: b"parent a", d: b"parent d"}, + LEFT: {a: b"parent a", c: b"", d: b"parent d", e: b"left e"}, + RIGHT: {**base, a: b"parent a", c: b"right c", d: b"parent d", f: b"right f"}, + CHILD: {a: b"child a", c: b"", e: b"left e"}, + } + return storage, engine, expected + + +def test_switching_forks_and_historical_reads_keeps_the_selected_state(forked_state): + storage, _, expected = forked_state + for selected_hash in (CHILD, RIGHT, LEFT, PARENT, FINALIZED, CHILD): + storage.select_block_hash(selected_hash) + assert_root(storage, expected[selected_hash]) + for historical_hash in (RIGHT, FINALIZED, LEFT, CHILD): + assert_retained_root(storage, historical_hash, expected[historical_hash]) + assert_root(storage, expected[selected_hash]) + + +@pytest.mark.parametrize("mutation", ["engine", "alias", "transaction", "raw_dict"]) +def test_external_base_writes_are_observed_with_overlays(forked_state, mutation): + storage, engine, expected = forked_state + a, b, c, _, _, _, g, _ = KEYS + storage.select_block_hash(CHILD) + assert_root(storage, expected[CHILD]) + changes = {a: b"new base a", b: b"new base b", c: None, g: b"new base g"} + target = engine.namespace(b"state") if mutation == "alias" else storage.storage_engine + if mutation == "transaction": + with target.transaction() as tx: + for key, value in changes.items(): + tx.delete(key) if value is None else tx.put(key, value) + elif mutation == "raw_dict": + for key, value in changes.items(): + if value is None: + engine.storage.pop(b"state-" + key) + else: + engine.storage[b"state-" + key] = value + else: + for key, value in changes.items(): + target.delete(key) if value is None else target.put(key, value) + + updated_base = {a: b"new base a", b: b"new base b", g: b"new base g"} + updated_child = {a: b"child a", KEYS[4]: b"left e", g: b"new base g"} + assert_root(storage, updated_child) + assert_retained_root(storage, FINALIZED, updated_base) + assert_root(storage, updated_child) + + +@pytest.mark.parametrize("length", [0, 32, 33, 8192]) +def test_mutating_a_stored_buffer_cannot_alias_the_previous_snapshot(length): + value = bytearray(b"v" * length) + engine = InMemoryStorageEngine({KEYS[0]: value, KEYS[1]: b"unchanged"}) + storage = StateStorage(engine) + storage.set_finalized_block_hash(FINALIZED) + first = storage.state_root() + + value[:] = b"w" * max(1, length) + changed = reference_root(engine.as_dict()) + assert changed != first + assert_root(storage, engine.as_dict()) + assert_retained_root(storage, FINALIZED, engine.as_dict()) + + value[:] = b"v" * length + assert storage.state_root() == first + + +@pytest.mark.parametrize("reset_overlay", [False, True]) +def test_replacing_an_ancestor_overlay_updates_cached_descendant_views(forked_state, reset_overlay): + storage, _, expected = forked_state + for block_hash in (PARENT, LEFT, RIGHT, CHILD): + assert_retained_root(storage, block_hash, expected[block_hash]) + storage.select_block_hash(PARENT) + if reset_overlay: + storage.set_block_hash(PARENT, FINALIZED) + storage.put(KEYS[6], b"replacement") + storage.commit() + + base = expected[FINALIZED] + replacement_parent = {**base, KEYS[6]: b"replacement"} + replacement_left = {**replacement_parent, KEYS[4]: b"left e"} + replacement_left.pop(KEYS[1]) + replacement_right = {**replacement_parent, KEYS[2]: b"right c", KEYS[5]: b"right f"} + replacement_child = {**replacement_left, KEYS[0]: b"child a"} + for block_hash, items in ( + (PARENT, replacement_parent), (LEFT, replacement_left), + (RIGHT, replacement_right), (CHILD, replacement_child), + ): + storage.select_block_hash(block_hash) + assert_root(storage, items) + assert_retained_root(storage, block_hash, items) + + +def test_rollback_observes_the_parent_and_drops_the_retained_overlay(forked_state): + storage, _, expected = forked_state + storage.select_block_hash(CHILD) + assert_root(storage, expected[CHILD]) + storage.put(KEYS[7], b"pending") + storage.rollback() + + # The legacy rollback operation keeps the selection but removes its writes. + assert storage.block_hash == CHILD + assert_root(storage, expected[LEFT]) + with pytest.raises(KeyError, match="unavailable"): + storage.state_root_at(CHILD) + assert_retained_root(storage, LEFT, expected[LEFT]) + + +def test_discard_reselects_the_predecessor_after_a_warm_root(forked_state): + storage, _, expected = forked_state + storage.select_block_hash(CHILD) + assert_root(storage, expected[CHILD]) + storage.put(KEYS[7], b"pending") + storage.discard_candidate(CHILD, LEFT) + + assert storage.block_hash == LEFT + assert storage.transaction == {} + assert_root(storage, expected[LEFT]) + with pytest.raises(KeyError, match="unavailable"): + storage.state_root_at(CHILD) + + +def test_finalization_preserves_descendants_and_prunes_the_other_fork(forked_state): + storage, _, expected = forked_state + for block_hash, items in expected.items(): + assert_retained_root(storage, block_hash, items) + storage.select_block_hash(RIGHT) + storage.finalize(PARENT) + assert storage.block_hash == RIGHT + assert storage.storage_engine.as_dict() == expected[PARENT] + assert_root(storage, expected[RIGHT]) + for block_hash in (PARENT, LEFT, RIGHT, CHILD): + assert_retained_root(storage, block_hash, expected[block_hash]) + + storage.finalize(LEFT) + assert storage.block_hash == LEFT + assert storage.storage_engine.as_dict() == expected[LEFT] + assert_root(storage, expected[LEFT]) + assert_retained_root(storage, CHILD, expected[CHILD]) + for pruned_hash in (FINALIZED, PARENT, RIGHT): + with pytest.raises(KeyError, match="unavailable"): + storage.state_root_at(pruned_hash) + + +def test_failed_finalization_keeps_the_previous_root_and_storage(forked_state, monkeypatch): + storage, _, expected = forked_state + storage.select_block_hash(CHILD) + assert_root(storage, expected[CHILD]) + previous_changes = deepcopy(storage.change_sets) + previous_parents = dict(storage.parents) + transaction = storage.storage_engine.transaction + + @contextmanager + def failing_transaction(): + with transaction() as tx: + yield tx + raise RuntimeError("simulated base write failure") + + with monkeypatch.context() as patch: + patch.setattr(storage.storage_engine, "transaction", failing_transaction) + with pytest.raises(RuntimeError, match="simulated base write failure"): + storage.finalize(LEFT) + + assert storage.finalized_block_hash == FINALIZED + assert storage.block_hash == CHILD + assert storage.change_sets == previous_changes + assert storage.parents == previous_parents + assert storage.storage_engine.as_dict() == expected[FINALIZED] + assert_root(storage, expected[CHILD]) + storage.finalize(LEFT) + assert_root(storage, expected[CHILD]) + + +@pytest.mark.parametrize("clear_before_reload", [False, True]) +def test_clear_and_reload_can_reuse_hashes_with_independent_contents(forked_state, clear_before_reload): + storage, engine, expected = forked_state + assert_root(storage, expected[CHILD]) + if clear_before_reload: + storage.clear(keep_ancestors=True) + engine.storage.clear() + replacement = {KEYS[6]: b"independent pre-state", KEYS[7]: b""} + for key, value in replacement.items(): + engine.namespace(b"state").put(key, value) + if not clear_before_reload: + storage.clear() + storage.set_finalized_block_hash(FINALIZED) + assert_root(storage, replacement) + assert_retained_root(storage, FINALIZED, replacement) + commit(storage, CHILD, FINALIZED, {KEYS[6]: None}) + assert_root(storage, {KEYS[7]: b""}) + + storage.clear() + engine.storage.clear() + assert storage.state_root() == bytes(32) + + +def test_pending_writes_reject_current_roots_but_allow_historical_roots(forked_state): + storage, _, expected = forked_state + storage.select_block_hash(CHILD) + assert_root(storage, expected[CHILD]) + storage.put(KEYS[7], b"pending") + + for block_hash in (CHILD, LEFT, RIGHT, FINALIZED): + assert_retained_root(storage, block_hash, expected[block_hash]) + with pytest.raises(ValueError, match="Pending transaction"): + storage.state_root() + assert storage.transaction == {KEYS[7]: b"pending"} + storage.start_tx() + assert_root(storage, expected[CHILD]) + + +def test_warm_roots_do_not_hide_ancestry_cycles(forked_state): + storage, _, expected = forked_state + assert_root(storage, expected[CHILD]) + assert_retained_root(storage, CHILD, expected[CHILD]) + storage.parents[PARENT] = CHILD + + with pytest.raises(ValueError, match="Cycle detected"): + storage.state_root() + with pytest.raises(KeyError, match="cycle"): + storage.state_root_at(CHILD) + assert_retained_root(storage, FINALIZED, expected[FINALIZED]) + storage.parents[PARENT] = FINALIZED + assert_root(storage, expected[CHILD]) + + +@pytest.mark.parametrize("missing", ["overlay", "parent"]) +def test_warm_historical_roots_still_validate_retained_ancestry(forked_state, missing): + storage, _, expected = forked_state + assert_retained_root(storage, CHILD, expected[CHILD]) + if missing == "overlay": + storage.change_sets.pop(PARENT) + else: + storage.parents.pop(PARENT) + with pytest.raises(KeyError, match="unavailable"): + storage.state_root_at(CHILD) + # The current view historically permits missing overlays/parent links. + assert_root(storage, storage.as_dict()) + + +@pytest.mark.parametrize("invalid_hash", [bytes(32), bytes(31), bytes(33), b"z" * 32]) +def test_warm_historical_roots_do_not_accept_unavailable_hashes(forked_state, invalid_hash): + storage, _, expected = forked_state + assert_root(storage, expected[CHILD]) + with pytest.raises(KeyError): + storage.state_root_at(invalid_hash) + assert_root(storage, expected[CHILD]) + + +def test_legacy_short_keys_keep_the_existing_state_root_behavior(): + engine = InMemoryStorageEngine({b"timeslot": b"0", b"nochange": b"test"}) + storage = StateStorage(engine) + assert_root(storage, engine.as_dict()) + engine.put(b"timeslot", b"1") + assert_root(storage, engine.as_dict()) + engine.delete(b"nochange") + assert_root(storage, engine.as_dict()) + + +@pytest.mark.parametrize("hook", ["_leaf", "_branch", "_bit", "merkle", "root", "hash", "type"]) +def test_warm_roots_preserve_custom_trie_hooks_and_recover_afterward(monkeypatch, hook): + engine = InMemoryStorageEngine({KEYS[0]: b"first", KEYS[1]: b"second" * 8}) + storage = StateStorage(engine) + storage.set_finalized_block_hash(FINALIZED) + ordinary_root = storage.state_root() + original_hash = merkle.blake2b_256_hash + legacy_type = PatriciaMerkleTrie + + with monkeypatch.context() as patch: + if hook == "hash": + patch.setattr(merkle, "blake2b_256_hash", lambda data: original_hash(b"custom" + data)) + elif hook == "type": + original_leaf = PatriciaMerkleTrie._leaf + + class CustomTrie(PatriciaMerkleTrie): + @staticmethod + def _leaf(key, value): + return b"custom" + original_leaf(key, value) + + legacy_type = CustomTrie + patch.setattr(state_storage_module, "PatriciaMerkleTrie", CustomTrie) + else: + original = getattr(PatriciaMerkleTrie, hook) + if hook == "_bit": + replacement = staticmethod(lambda key, index: not original(key, index)) + elif hook.startswith("_"): + replacement = staticmethod(lambda *args: b"custom" + original(*args)) + else: + replacement = lambda *args, **kwargs: original_hash(b"custom" + original(*args, **kwargs)) + patch.setattr(PatriciaMerkleTrie, hook, replacement) + + expected = legacy_type(list(engine.as_dict().items())).root() + assert expected != ordinary_root + assert storage.state_root() == expected + assert storage.state_root_at(FINALIZED) == expected + engine.put(KEYS[0], b"changed while customized") + expected = legacy_type(list(engine.as_dict().items())).root() + assert storage.state_root() == expected + + assert_root(storage, engine.as_dict()) + assert_retained_root(storage, FINALIZED, engine.as_dict()) + + +@pytest.mark.parametrize("warm", [False, True]) +@pytest.mark.parametrize("key_length", [31, 32]) +def test_malformed_key_prefixes_preserve_legacy_errors_without_poisoning_roots(warm, key_length): + original, collision = bytes(key_length), bytes(63 - key_length) + engine = InMemoryStorageEngine({original: b"original", KEYS[1]: b"other"}) + storage = StateStorage(engine) + storage.set_finalized_block_hash(FINALIZED) + expected = reference_root(engine.as_dict()) + if warm: + assert storage.state_root() == expected + engine.put(collision, b"malformed") + + with pytest.raises(IndexError): + PatriciaMerkleTrie(list(engine.as_dict().items())).root() + with pytest.raises(IndexError): + storage.state_root() + with pytest.raises(IndexError): + storage.state_root_at(FINALIZED) + + engine.delete(collision) + assert storage.state_root() == expected + engine.put(original, b"recovered and changed") + assert_root(storage, engine.as_dict()) diff --git a/test/test_incremental_trie.py b/test/test_incremental_trie.py new file mode 100644 index 00000000..f30cc8ea --- /dev/null +++ b/test/test_incremental_trie.py @@ -0,0 +1,204 @@ +import random +from dataclasses import FrozenInstanceError + +import pytest + +from pyjamaz import merkle +from pyjamaz.incremental_trie import IncrementalMerkleTrie +from pyjamaz.merkle import PatriciaMerkleTrie + + +def reference_root(data): + return PatriciaMerkleTrie(list(data.items())).root() + + +@pytest.fixture(autouse=True) +def clear_hash_caches(): + caches = ( + merkle._cached_trie_leaf_hash, + merkle._cached_trie_large_leaf_hash, + merkle._cached_trie_branch_hash, + ) + for cache in caches: + cache.cache_clear() + yield + for cache in caches: + cache.cache_clear() + + +@pytest.mark.parametrize("key_length", [31, 32]) +def test_random_update_batches_match_full_rebuild(key_length): + rng = random.Random(1847) + keys = [rng.randbytes(key_length) for _ in range(100)] + data = {key: rng.randbytes(rng.randrange(80)) for key in keys[:40]} + trie = IncrementalMerkleTrie(data.items()) + assert trie.root() == reference_root(data) + + for _ in range(150): + changes = { + rng.choice(keys): None if rng.randrange(3) == 0 else rng.randbytes(rng.randrange(80)) + for _ in range(rng.randrange(1, 10)) + } + previous, previous_root = trie, trie.root() + trie = trie.update(changes) + for key, value in changes.items(): + if value is None: + data.pop(key, None) + else: + data[key] = value + assert trie.root() == reference_root(data) + assert previous.root() == previous_root + + +@pytest.mark.parametrize("key_length", [31, 32]) +def test_shared_prefix_branches_survive_until_only_one_leaf_remains(key_length): + first = bytes(key_length) + second = bytes(key_length - 1) + b"\x01" + outside = b"\x80" + bytes(key_length - 1) + data = {first: b"first", second: b"second", outside: b"outside"} + trie = IncrementalMerkleTrie(data) + + # Removing the other half leaves unary branches above two deep leaves. + trie = trie.update({outside: None}) + del data[outside] + assert trie.root() == reference_root(data) + assert trie._node.right is None + assert trie._node.left is not None + + trie = trie.update({second: None}) + del data[second] + assert trie.root() == reference_root(data) + assert trie._node.key == first + assert trie.update({first: None}).root() == bytes(32) + + +def test_empty_values_are_distinct_from_missing_entries(): + key = bytes(31) + empty = IncrementalMerkleTrie() + stored = empty.update({key: b""}) + assert stored.root() == reference_root({key: b""}) + assert stored.root() != empty.root() + assert stored.update({key: None}).root() == empty.root() + + +def test_no_op_changes_preserve_snapshot_identity(): + key, absent = bytes(31), b"\x80" + bytes(30) + trie = IncrementalMerkleTrie({key: b"value"}) + assert trie.update({}) is trie + assert trie.update({key: b"value"}) is trie + assert trie.update({absent: None}) is trie + assert trie.update({key: b"value", absent: None}) is trie + + +def test_forks_and_constructor_inputs_are_isolated(): + left, right = bytes(31), b"\x80" + bytes(30) + data = {left: b"left", right: b"right"} + base = IncrementalMerkleTrie(data) + base_root = base.root() + first = base.update({left: b"first"}) + second = base.update({right: None}) + data.clear() + + assert base.root() == base_root + assert first.root() == reference_root({left: b"first", right: b"right"}) + assert second.root() == reference_root({left: b"left"}) + with pytest.raises(FrozenInstanceError): + base._node = None + with pytest.raises(FrozenInstanceError): + base._node.left.value = b"mutated" + + +def test_update_shares_untouched_subtrees_and_hashes_only_changed_path(): + keys = [bytes([head]) + bytes(30) for head in (0x00, 0x40, 0x80, 0xC0)] + trie = IncrementalMerkleTrie({key: bytes([index]) for index, key in enumerate(keys)}) + leaf_before = merkle._cached_trie_leaf_hash.cache_info() + branch_before = merkle._cached_trie_branch_hash.cache_info() + + changed = trie.update({keys[0]: b"changed"}) + + assert changed._node is not trie._node + assert changed._node.right is trie._node.right + assert changed._node.left.right is trie._node.left.right + leaf_after = merkle._cached_trie_leaf_hash.cache_info() + branch_after = merkle._cached_trie_branch_hash.cache_info() + assert leaf_after.misses - leaf_before.misses == 1 + assert branch_after.misses - branch_before.misses == 2 + assert leaf_after.hits == leaf_before.hits + assert branch_after.hits == branch_before.hits + + for _ in range(3): + changed.root() + trie.root() + assert changed.update({keys[0]: b"changed", bytes([0xE0]) + bytes(30): None}) is changed + assert merkle._cached_trie_leaf_hash.cache_info() == leaf_after + assert merkle._cached_trie_branch_hash.cache_info() == branch_after + + +@pytest.mark.parametrize("key", [b"", bytes(30), bytes(33), bytearray(31), "x" * 31]) +def test_invalid_keys_are_rejected(key): + with pytest.raises(ValueError, match="31- or 32-byte"): + IncrementalMerkleTrie([(key, b"value")]) + if not isinstance(key, bytearray): + with pytest.raises(ValueError, match="31- or 32-byte"): + IncrementalMerkleTrie().update({key: b"value"}) + + +@pytest.mark.parametrize("value", [bytearray(b"value"), memoryview(b"value"), "value", None]) +def test_non_bytes_initial_values_are_rejected(value): + with pytest.raises(TypeError, match="bytes"): + IncrementalMerkleTrie([(bytes(31), value)]) + if value is not None: + with pytest.raises(TypeError, match="bytes"): + IncrementalMerkleTrie().update({bytes(31): value}) + + +@pytest.mark.parametrize("reverse", [False, True]) +def test_prefix_collisions_are_rejected_without_changing_the_snapshot(reverse): + keys = [bytes(31), bytes(32)] + if reverse: + keys.reverse() + with pytest.raises(ValueError, match="prefixes"): + IncrementalMerkleTrie([(key, b"value") for key in keys]) + trie = IncrementalMerkleTrie({keys[0]: b"original"}) + root = trie.root() + with pytest.raises(ValueError, match="prefixes"): + trie.update({keys[1]: b"new"}) + assert trie.root() == root + assert trie.update({keys[1]: None}) is trie + + +def test_short_key_collision_with_an_existing_branch_is_rejected(): + trie = IncrementalMerkleTrie({bytes(32): b"first", bytes(31) + b"\x01": b"second"}) + with pytest.raises(ValueError, match="prefixes"): + trie.update({bytes(31): b"new"}) + assert trie.update({bytes(31): None}) is trie + + +@pytest.mark.parametrize("reverse", [False, True]) +def test_replacing_key_lengths_in_a_batch_ignores_mapping_order(reverse): + previous, replacement = bytes(31), bytes(32) + if reverse: + previous, replacement = replacement, previous + trie = IncrementalMerkleTrie({previous: b"old"}) + changed = trie.update({replacement: b"new", previous: None}) + assert changed.root() == reference_root({replacement: b"new"}) + assert trie.root() == reference_root({previous: b"old"}) + + +def test_mixed_key_lengths_work_when_no_key_prefixes_another(): + data = {bytes(31): b"short", b"\x80" + bytes(31): b"long"} + trie = IncrementalMerkleTrie(data) + assert trie.root() == reference_root(data) + data[bytes(31)] = b"changed" + assert trie.update({bytes(31): b"changed"}).root() == reference_root(data) + + +@pytest.mark.parametrize("size", [32, 33, 8192, 256 * 1024]) +def test_value_sizes_use_existing_bounded_leaf_caches(size): + key, value = bytes(31), b"v" * size + trie = IncrementalMerkleTrie({key: value}) + assert trie.root() == reference_root({key: value}) + small = merkle._cached_trie_leaf_hash.cache_info() + large = merkle._cached_trie_large_leaf_hash.cache_info() + assert small.currsize == (1 if len(key) + size <= 4096 else 0) + assert large.currsize == (1 if 4096 < len(key) + size <= 256 * 1024 else 0)