feat(storage): diff-layer state storage with an in-memory LRU cache

CLAUDE.md

@@ -366,24 +366,33 @@ cargo test -p ethlambda-blockchain --test forkchoice_spectests -- --test-threads
   finalized boundary, signatures are pruned (`prune_old_block_signatures`) while
   headers and bodies are kept forever. `get_signed_block` returns `None` for a
   pruned finalized block
+- States are stored as parent-linked diffs (`StateDiffs`, never pruned) plus
+  full-state snapshots (`States`) written only at 1024-slot anchors (and the
+  bootstrap). Neither is ever pruned. `get_state` returns an anchor snapshot or
+  reconstructs by walking diffs back to the nearest anchor; results are memoized
+  in an in-memory LRU (`STATE_CACHE_CAPACITY`) so recent reads stay hot
 - `LiveChain` table provides fast `(slot||root) → parent_root` index for fork choice
 - Storage uses trait-based API: `StorageBackend` → `StorageReadView` (reads) + `StorageWriteBatch` (atomic writes)
 
-### Storage Tables (10)
+### Storage Tables (7)
+
+These are the variants of the `Table` enum (`crates/storage/src/api/tables.rs`).
 
 | Table | Key → Value | Purpose |
 |-------|-------------|---------|
 | `BlockHeaders` | H256 → BlockHeader | Block headers by root |
 | `BlockBodies` | H256 → BlockBody | Block bodies (empty for genesis) |
-| `BlockSignatures` | H256 → BlockSignatures | Signatures (absent for genesis) |
-| `States` | H256 → State | Beacon states by root |
-| `LatestKnownAttestations` | u64 → AttestationData | Fork-choice-active attestations |
-| `LatestNewAttestations` | u64 → AttestationData | Pending (pre-promotion) attestations |
-| `GossipSignatures` | SignatureKey → ValidatorSignature | Individual validator signatures |
-| `AggregatedPayloads` | SignatureKey → Vec\<AggregatedSignatureProof\> | Aggregated proofs |
+| `BlockSignatures` | (slot\|\|root) → BlockSignatures | Type-2 proof blob; keyed slot\|\|root so pruning scans in slot order and stops early; absent for genesis, pruned below finalized |
+| `States` | H256 → State | Full-state snapshots; bootstrap + 1024-slot anchors only; never pruned |
+| `StateDiffs` | H256 → StateDiff | Parent-linked state diff per non-genesis state; never pruned |
 | `Metadata` | string → various | Store state (head, config, checkpoints) |
 | `LiveChain` | (slot\|\|root) → parent\_root | Fast fork choice traversal index |
 
+Attestations and gossip signatures are **not** persisted tables; they live in
+in-memory `Store` buffers (`new_payloads`, `known_payloads`, `gossip_signatures`)
+and are consumed during the tick pipeline (promotion at intervals 0/4,
+aggregation at interval 2).
+
 ### State Root Computation
 - Always computed via `tree_hash_root()` after full state transition
 - Must match proposer's pre-computed `block.state_root`

Cargo.toml

@@ -69,6 +69,7 @@ vergen-git2 = { version = "9", features = ["rustc"] }
 
 rayon = "1.11"
 rand = "0.10"
+lru = "0.16"
 rocksdb = "0.24"
 libc = "0.2"
 reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }

crates/blockchain/src/store.rs

@@ -1265,8 +1265,20 @@ mod tests {
 
         // Head state justified `a` (slot 1), which lies on the head's chain.
         let head_justified = Checkpoint { root: a, slot: 1 };
-        let mut head_state = State::from_genesis(1000, vec![]);
+        // Persist `b`'s post-state via the diff API, diffed against the genesis
+        // anchor. Build it as a valid direct child of genesis (the STF appends the
+        // parent block root to historical_block_hashes), with the head's justified
+        // checkpoint set; `insert_state` reads the base from
+        // `latest_block_header.parent_root`, and `get_state(b)` then returns it
+        // from the cache.
+        let genesis_state = store.get_state(&genesis).expect("genesis state");
+        let mut head_state = genesis_state.clone();
+        head_state.slot = genesis_state.slot + 1;
         head_state.latest_justified = head_justified;
+        head_state.latest_block_header.parent_root = genesis;
+        let mut hbh = genesis_state.historical_block_hashes.to_vec();
+        hbh.push(genesis);
+        head_state.historical_block_hashes = hbh.try_into().expect("within limit");
         store
             .insert_state(b, head_state)
             .expect("insert head state should succeed");

crates/storage/Cargo.toml

@@ -18,6 +18,9 @@ thiserror.workspace = true
 
 libssz.workspace = true
 libssz-derive.workspace = true
+libssz-types.workspace = true
+
+lru.workspace = true
 
 [dev-dependencies]
 tempfile = "3"

crates/storage/src/api/tables.rs

@@ -11,7 +11,16 @@ pub enum Table {
     /// All other blocks must have an entry in this table.
     BlockSignatures,
     /// State storage: H256 -> State
+    ///
+    /// Holds full-state snapshots only: the bootstrap anchor plus one anchor per
+    /// 1024-slot window. Never pruned. Non-anchor states live in `StateDiffs` and
+    /// are reconstructed on demand (memoized by an in-memory cache).
     States,
+    /// State diffs: H256 -> StateDiff
+    ///
+    /// Parent-linked diff written for every non-genesis state. Never pruned, so
+    /// it preserves full state history. See `get_state` for reconstruction.
+    StateDiffs,
     /// Metadata: string keys -> various scalar values
     Metadata,
     /// Live chain index: (slot || root) -> parent_root
@@ -23,11 +32,12 @@ pub enum Table {
 }
 
 /// All table variants.
-pub const ALL_TABLES: [Table; 6] = [
+pub const ALL_TABLES: [Table; 7] = [
     Table::BlockHeaders,
     Table::BlockBodies,
     Table::BlockSignatures,
     Table::States,
+    Table::StateDiffs,
     Table::Metadata,
     Table::LiveChain,
 ];
@@ -40,6 +50,7 @@ impl Table {
             Table::BlockBodies => "block_bodies",
             Table::BlockSignatures => "block_signatures",
             Table::States => "states",
+            Table::StateDiffs => "state_diffs",
             Table::Metadata => "metadata",
             Table::LiveChain => "live_chain",
         }

crates/storage/src/backend/rocksdb.rs

@@ -11,15 +11,11 @@ use std::path::Path;
 use std::sync::Arc;
 
 /// Returns the column family name for a table.
+///
+/// Delegates to [`Table::name`] so the CF name and the metrics label share a
+/// single source of truth (and a new table only needs one mapping).
 fn cf_name(table: Table) -> &'static str {
-    match table {
-        Table::BlockHeaders => "block_headers",
-        Table::BlockBodies => "block_bodies",
-        Table::BlockSignatures => "block_signatures",
-        Table::States => "states",
-        Table::Metadata => "metadata",
-        Table::LiveChain => "live_chain",
-    }
+    table.name()
 }
 
 /// RocksDB storage backend.

crates/storage/src/lib.rs

@@ -1,6 +1,7 @@
 mod api;
 pub mod backend;
 mod error;
+mod state_diff;
 mod store;
 
 pub use api::{ALL_TABLES, StorageBackend, StorageReadView, StorageWriteBatch, Table};