Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 137 additions & 0 deletions pyjamaz/incremental_trie.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
"""Persistent Appendix D trie snapshots with cached hashes on immutable nodes."""

from __future__ import annotations

from collections.abc import Iterable, Mapping
from dataclasses import dataclass

from pyjamaz.merkle import PatriciaMerkleTrie, _cached_trie_branch_hash


_EMPTY_HASH = bytes(32)


@dataclass(frozen=True, slots=True)
class _Leaf:
key: bytes
value: bytes
digest: bytes


@dataclass(frozen=True, slots=True)
class _Branch:
left: _Node
right: _Node
digest: bytes


_Node = _Leaf | _Branch | None


def _digest(node: _Node) -> bytes:
return _EMPTY_HASH if node is None else node.digest


def _leaf(key: bytes, value: bytes) -> _Leaf:
# Reuse the existing bounded leaf caches, including their admission limits.
return _Leaf(key, value, PatriciaMerkleTrie([(key, value)]).root())


def _branch(left: _Node, right: _Node) -> _Node:
if left is None and (right is None or isinstance(right, _Leaf)):
return right
if right is None and isinstance(left, _Leaf):
return left
# A unary branch containing multiple leaves must retain its bit position.
return _Branch(left, right, _cached_trie_branch_hash(_digest(left), _digest(right)))


def _build(data: list[tuple[bytes, bytes]], index: int = 0) -> _Node:
if not data:
return None
if len(data) == 1:
return _leaf(*data[0])
left, right = [], []
for key, value in data:
if index >= len(key) * 8:
raise ValueError("Trie keys must not be prefixes of one another")
(right if PatriciaMerkleTrie._bit(key, index) else left).append((key, value))
return _branch(_build(left, index + 1), _build(right, index + 1))


def _replace(node: _Node, key: bytes, value: bytes | None, index: int) -> _Node:
if node is None:
return None if value is None else _leaf(key, value)
if isinstance(node, _Leaf):
if key == node.key:
if value is None:
return None
return node if value == node.value else _leaf(key, value)
if value is None:
return node
if key.startswith(node.key) or node.key.startswith(key):
raise ValueError("Trie keys must not be prefixes of one another")
old_right = PatriciaMerkleTrie._bit(node.key, index)
new_right = PatriciaMerkleTrie._bit(key, index)
if old_right != new_right:
other = _leaf(key, value)
return _branch(node, other) if new_right else _branch(other, node)
child = _replace(node, key, value, index + 1)
return _branch(None, child) if old_right else _branch(child, None)
if index >= len(key) * 8:
if value is None:
return node
raise ValueError("Trie keys must not be prefixes of one another")
if PatriciaMerkleTrie._bit(key, index):
right = _replace(node.right, key, value, index + 1)
return node if right is node.right else _branch(node.left, right)
left = _replace(node.left, key, value, index + 1)
return node if left is node.left else _branch(left, node.right)


def _validate(key: bytes, value: bytes | None) -> None:
if type(key) is not bytes or len(key) not in (31, 32):
raise ValueError("Trie keys must be immutable 31- or 32-byte bytes")
if value is not None and type(value) is not bytes:
raise TypeError("Trie values must be immutable bytes or None for deletion")


@dataclass(frozen=True, slots=True, init=False)
class IncrementalMerkleTrie:
"""An immutable trie; updates share unchanged nodes with earlier snapshots.

``None`` deletes a key, while ``b\"\"`` is a stored value. Keys may be 31 or
32 bytes, but a shorter key cannot prefix another key in the same snapshot.
"""

_node: _Node

def __init__(self, data: Iterable[tuple[bytes, bytes]] | Mapping[bytes, bytes] = ()):
values = {}
entries = data.items() if isinstance(data, Mapping) else data
for key, value in entries:
_validate(key, value)
if value is None:
raise TypeError("Initial trie values must be immutable bytes")
values[key] = value
object.__setattr__(self, "_node", _build(list(values.items())))

def root(self) -> bytes:
return _digest(self._node)

def update(self, changes: Mapping[bytes, bytes | None]) -> IncrementalMerkleTrie:
node = self._node
for key, value in changes.items():
_validate(key, value)
if value is None:
node = _replace(node, key, None, 0)
# Remove obsolete keys first so replacing a short key with a prefixed
# long key (or vice versa) does not create a transient prefix collision.
for key, value in changes.items():
if value is not None:
node = _replace(node, key, value, 0)
if node is self._node:
return self
snapshot = object.__new__(type(self))
object.__setattr__(snapshot, "_node", node)
return snapshot
52 changes: 48 additions & 4 deletions pyjamaz/state/storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,22 @@
from enum import Enum
from typing import Optional, Dict, List, Tuple

from pyjamaz import merkle
from pyjamaz.incremental_trie import IncrementalMerkleTrie
from pyjamaz.merkle import PatriciaMerkleTrie
from pyjamaz.models.block import AncestorReference, Header
from pyjamaz.settings import DEBUG
from pyjamaz.storage import StorageEngine
from pyjamaz.utils import format_hash, log_execution_time


_STATE_TRIE_TYPE = PatriciaMerkleTrie
_STATE_TRIE_METHODS = (
PatriciaMerkleTrie._leaf, PatriciaMerkleTrie._branch, PatriciaMerkleTrie._bit,
PatriciaMerkleTrie.merkle, PatriciaMerkleTrie.root,
)


class ItemStatus(Enum):
deleted = 1

Expand All @@ -27,6 +36,7 @@ def __init__(self, storage_engine: StorageEngine):
self.parents: Dict[bytes, Optional[bytes]] = {}
# GP-0.8.0-eq:5.3 (A)
self.ancestors: Dict[bytes, Header | AncestorReference] = {}
self._trie_snapshot = (None, IncrementalMerkleTrie())

def add_ancestor(self, header: Header | AncestorReference):
self.ancestors[header.hash] = header
Expand Down Expand Up @@ -170,13 +180,47 @@ def state_root(self) -> bytes:
if len(self.transaction) > 0:
raise ValueError(f"Pending transaction; commit or rollback first")

state_trie = PatriciaMerkleTrie(list(self.as_dict().items()))

state_root = state_trie.root()
state_root = self._root_from_items(self.as_dict())
DEBUG and logging.debug(f"StateStorage: Calculated state root {format_hash(state_root)}")

return state_root

def _root_from_items(self, items: Dict[bytes, bytes]) -> bytes:
# Keep the original path for custom encoders and non-state-trie input.
if (PatriciaMerkleTrie is not _STATE_TRIE_TYPE or (
PatriciaMerkleTrie._leaf, PatriciaMerkleTrie._branch, PatriciaMerkleTrie._bit,
PatriciaMerkleTrie.merkle, PatriciaMerkleTrie.root,
) != _STATE_TRIE_METHODS or merkle.blake2b_256_hash is not merkle._TRIE_HASH_FUNCTION):
return PatriciaMerkleTrie(list(items.items())).root()

current = {}
for key, value in items.items():
if (type(key) is not bytes or len(key) not in (31, 32)
or type(value) not in (bytes, bytearray)):
return PatriciaMerkleTrie(list(items.items())).root()
# Mutable buffers must not alias the previous materialized view.
current[key] = bytes(value)

previous, trie = self._trie_snapshot
try:
if previous is None:
trie = IncrementalMerkleTrie(current)
else:
changes = {key: value for key, value in current.items()
if previous.get(key) != value}
changes.update((key, None) for key in previous if key not in current)
trie = trie.update(changes)
except ValueError:
# A 31-byte key may prefix a 32-byte key in malformed input. Let
# the original builder report its existing error for that case.
return PatriciaMerkleTrie(list(items.items())).root()

# Retain only the latest view, rather than a separate trie per block.
# Full materialization above observes external base writes and validates
# ancestry even after a fork switch, clear(), or failed transaction.
self._trie_snapshot = (current, trie)
return trie.root()

def as_dict(self) -> Dict[bytes, bytes]:
if len(self.transaction) > 0:
raise ValueError(f"Pending transaction; commit or rollback first")
Expand Down Expand Up @@ -249,7 +293,7 @@ def as_list_at(self, block_hash: bytes) -> List[Tuple[bytes, bytes]]:
return sorted(self.as_dict_at(block_hash).items(), key=lambda item: item[0])

def state_root_at(self, block_hash: bytes) -> bytes:
return PatriciaMerkleTrie(list(self.as_dict_at(block_hash).items())).root()
return self._root_from_items(self.as_dict_at(block_hash))


def finalize(self, block_hash: bytes):
Expand Down
Loading
Loading