From a8d7dfee658cf1fb22848fb23656921d850a03db Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 22 Aug 2026 08:55:19 +0700 Subject: [PATCH 1/4] fix(costs): report append-only write churn as replacement, not new storage (#822) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BulkAppendTree / CommitmentTree / PrivateDocumentStore issued every data put with `cost_info: None`, so the commit path billed key + value as NEW storage for dense-buffer slots rewritten from epoch 2 on, the compaction blob that supersedes the buffer it was built from (~630 KB on one append per epoch at chunk_power 11) and the frontier rewritten on every append — about 2x the bytes that persist. From GROVE_V4 (new gates `bulk_append_tree_versions.cost. append_storage_accounting` and `commitment_tree_versions.cost. frontier_save_storage_accounting`; V1..V3 locked at the shipped figures) an append charges each entry's permanent bytes once — its chunk-blob share, as added_bytes at its own append — and reports the rest as replacement: slot rewrites (growth added, shrink not credited, key not charged), the compaction blob (entry bytes replaced, framing added) and the frontier rewrite (replaces the frontier loaded at open). Stored bytes, roots and proofs are identical. Mechanism: `SlotWriteAccounting::AgainstCommitted` in the dense tree reads the slot's committed value and attaches `KeyValueStorageCost::for_in_place_value_rewrite`; `MmrStore` takes a `LeafValueStorageCost::PartlyPrepaid` policy fed by `chunk_blob_entry_bytes`; appends report `prepaid_chunk_bytes` for the caller to bill; `commit_mmr` / `CommitmentTree::save` take the grove version. The transactional `StorageContext::put` now completes the prefixed key cost for a new-node cost_info, as the `Batch` impls do. The shared V4 CommitmentTreeInsert estimator moves the epoch x entry term from added to replaced; PDS and BulkAppend estimators gain matching replaced terms. The #813 estimator >= actual bound tests stay green. Co-Authored-By: Claude Fable 5 --- costs/src/storage_cost/key_value_cost.rs | 73 +- docs/crates/costs.md | 35 + grovedb-bulk-append-tree/src/chunk.rs | 88 +++ grovedb-bulk-append-tree/src/cost/mod.rs | 71 +- grovedb-bulk-append-tree/src/cost/v0.rs | 20 +- grovedb-bulk-append-tree/src/cost/v1.rs | 27 +- grovedb-bulk-append-tree/src/lib.rs | 4 +- grovedb-bulk-append-tree/src/test_utils.rs | 8 +- grovedb-bulk-append-tree/src/tree/append.rs | 52 +- grovedb-bulk-append-tree/src/tree/mod.rs | 22 + .../src/tree/storage_accounting_tests.rs | 238 ++++++ .../src/commitment_tree/cost/mod.rs | 105 +++ .../src/commitment_tree/cost/v0.rs | 12 + .../src/commitment_tree/cost/v1.rs | 21 + .../src/commitment_tree/mod.rs | 52 +- .../src/commitment_tree/tests.rs | 28 +- grovedb-commitment-tree/src/error.rs | 5 + .../src/lib.rs | 2 + .../src/test_utils.rs | 7 +- .../src/tests.rs | 62 ++ .../src/tree.rs | 107 ++- grovedb-merkle-mountain-range/Cargo.toml | 3 +- grovedb-merkle-mountain-range/src/lib.rs | 2 +- .../src/storage_adapter.rs | 86 ++- grovedb-private-document-store/src/store.rs | 39 +- .../src/version/bulk_append_tree_versions.rs | 18 + .../src/version/commitment_tree_versions.rs | 30 + grovedb-version/src/version/mod.rs | 5 +- grovedb-version/src/version/v1.rs | 14 + grovedb-version/src/version/v2.rs | 14 + grovedb-version/src/version/v3.rs | 14 + grovedb-version/src/version/v4.rs | 32 + .../estimated_costs/average_case_costs.rs | 32 +- grovedb/src/batch/estimated_costs/mod.rs | 59 +- .../batch/estimated_costs/worst_case_costs.rs | 38 +- grovedb/src/operations/bulk_append_tree.rs | 20 +- grovedb/src/operations/commitment_tree.rs | 14 +- .../src/operations/private_document_store.rs | 7 +- .../tests/append_storage_accounting_tests.rs | 709 ++++++++++++++++++ .../tests/commitment_tree_cost_bound_tests.rs | 12 +- grovedb/src/tests/mod.rs | 1 + .../storage_context/context_tx.rs | 24 +- storage/src/rocksdb_storage/tests.rs | 85 +++ 43 files changed, 2184 insertions(+), 113 deletions(-) create mode 100644 grovedb-bulk-append-tree/src/tree/storage_accounting_tests.rs create mode 100644 grovedb-commitment-tree/src/commitment_tree/cost/mod.rs create mode 100644 grovedb-commitment-tree/src/commitment_tree/cost/v0.rs create mode 100644 grovedb-commitment-tree/src/commitment_tree/cost/v1.rs create mode 100644 grovedb-version/src/version/commitment_tree_versions.rs create mode 100644 grovedb/src/tests/append_storage_accounting_tests.rs diff --git a/costs/src/storage_cost/key_value_cost.rs b/costs/src/storage_cost/key_value_cost.rs index e62dc60f8..f624074ca 100644 --- a/costs/src/storage_cost/key_value_cost.rs +++ b/costs/src/storage_cost/key_value_cost.rs @@ -39,7 +39,7 @@ use crate::{ }; /// Storage only operation costs separated by key and value -#[derive(PartialEq, Clone, Eq, Default)] +#[derive(Debug, PartialEq, Clone, Eq, Default)] pub struct KeyValueStorageCost { /// Key storage_cost costs pub key_storage_cost: StorageCost, @@ -110,6 +110,33 @@ impl KeyValueStorageCost { } } + /// Cost of rewriting an existing value in place — `previous_len` bytes + /// stored under a key that already exists, overwritten with `new_len` + /// bytes — with no refund semantics. + /// + /// The paid size of a value is its length plus the varint encoding that + /// length, which is what the commit path verifies `added + replaced` + /// against. The key is not charged (it exists); `replaced_bytes` is the + /// smaller of the previous and the new paid size; `added_bytes` is the + /// growth beyond the previous size, if any; a shrink is NOT credited as + /// removed bytes — used for rolling data (a reused buffer slot, a + /// frontier rewritten on every append) where a refund would mean paying + /// someone back for bytes nobody owns. + pub fn for_in_place_value_rewrite(previous_len: u32, new_len: u32) -> Self { + let paid_previous = previous_len + previous_len.required_space() as u32; + let paid_new = new_len + new_len.required_space() as u32; + KeyValueStorageCost { + key_storage_cost: StorageCost::default(), + value_storage_cost: StorageCost { + added_bytes: paid_new.saturating_sub(paid_previous), + replaced_bytes: paid_new.min(paid_previous), + removed_bytes: NoStorageRemoval, + }, + new_node: false, + needs_value_verification: true, + } + } + /// Returns the total removed bytes between the key removed bytes and the /// value removed bytes pub fn combined_removed_bytes(self) -> StorageRemovedBytes { @@ -138,3 +165,47 @@ impl AddAssign for KeyValueStorageCost { self.needs_value_verification &= rhs.needs_value_verification; } } + +#[cfg(test)] +mod in_place_rewrite_tests { + use super::*; + + #[test] + fn same_size_is_fully_replaced() { + let c = KeyValueStorageCost::for_in_place_value_rewrite(312, 312); + assert_eq!(c.value_storage_cost.replaced_bytes, 314); + assert_eq!(c.value_storage_cost.added_bytes, 0); + assert_eq!(c.value_storage_cost.removed_bytes, NoStorageRemoval); + assert_eq!(c.key_storage_cost, StorageCost::default()); + assert!(!c.new_node); + assert!(c.needs_value_verification); + // Verifies against the paid size of the new value. + assert!(c.value_storage_cost.verify(314).is_ok()); + } + + #[test] + fn growth_is_added_on_top_of_the_previous_size() { + // 74 -> 106 bytes (a frontier gaining one ommer). + let c = KeyValueStorageCost::for_in_place_value_rewrite(74, 106); + assert_eq!(c.value_storage_cost.replaced_bytes, 75); + assert_eq!(c.value_storage_cost.added_bytes, 32); + assert!(c.value_storage_cost.verify(107).is_ok()); + } + + #[test] + fn shrink_is_replaced_at_the_new_size_and_not_credited() { + let c = KeyValueStorageCost::for_in_place_value_rewrite(1066, 74); + assert_eq!(c.value_storage_cost.replaced_bytes, 75); + assert_eq!(c.value_storage_cost.added_bytes, 0); + assert_eq!(c.value_storage_cost.removed_bytes, NoStorageRemoval); + assert!(c.value_storage_cost.verify(75).is_ok()); + } + + #[test] + fn varint_width_change_counts_as_growth() { + // 127 -> 128 bytes crosses a varint boundary: paid 128 -> 130. + let c = KeyValueStorageCost::for_in_place_value_rewrite(127, 128); + assert_eq!(c.value_storage_cost.replaced_bytes, 128); + assert_eq!(c.value_storage_cost.added_bytes, 2); + } +} diff --git a/docs/crates/costs.md b/docs/crates/costs.md index 21b874889..d2d4c7dcc 100644 --- a/docs/crates/costs.md +++ b/docs/crates/costs.md @@ -117,6 +117,41 @@ Count trees add overhead for maintaining element counts: - Additional 8 bytes for count storage - Propagation costs through parent trees +#### Append-only trees (BulkAppendTree, CommitmentTree, PrivateDocumentStore) + +The append-only family writes three kinds of data rows: dense-buffer slots +(one per position, keys reused every epoch), the chunk blob an epoch is +compacted into (plus the MMR internal nodes), and — for the commitment +tree — the frontier, one value rewritten on every append. How those writes +are reported to the fee layer is version-gated +(`bulk_append_tree_versions.cost.append_storage_accounting`, +`commitment_tree_versions.cost.frontier_save_storage_accounting`): + +| write | GROVE_V1..V3 (v0) | GROVE_V4 (v1, issue #822) | +|---|---|---| +| buffer slot, first epoch | key + value `added` | key + value `added` | +| buffer slot, epoch ≥ 2 | key + value `added` | value `replaced` (growth `added`, shrink not credited); key not charged | +| entry's chunk-blob share | — (blob charged at compaction) | entry bytes `added` at the entry's own append | +| compaction blob | key + whole blob `added` | entry bytes `replaced`, framing + key `added` | +| MMR internal nodes | `added` | `added` | +| frontier rewrite | key + value `added` every save | value `replaced`, growth `added`; first save fully `added` | + +Under v0 every note is billed roughly twice over its life (slot + blob) and +the whole blob (≈ 630 KB at `chunk_power` 11) lands on one append per +epoch. Under v1 each entry's permanent bytes are charged once, at its own +append, and the rest of the churn is replacement; the sum of `added_bytes` +over an epoch matches the database growth. `removed_bytes` stays +`NoStorageRemoval` in both — a rolling buffer refunds nobody. Stored bytes, +roots and proofs are identical under both versions. + +Mechanically: the dense tree's `SlotWriteAccounting::AgainstCommitted` +reads the slot's committed value before the write (the read is billed) and +attaches `KeyValueStorageCost::for_in_place_value_rewrite`; the MMR +`MmrStore` takes a `LeafValueStorageCost::PartlyPrepaid` policy that the +bulk tree feeds `chunk_blob_entry_bytes`; `BulkAppendTree` appends report +`prepaid_chunk_bytes` for the caller to bill (already included in the cost +of the `CostResult`-returning `append_deferred_roots`). + ## Cost Context The `CostContext` trait provides functional combinators for cost-aware operations: diff --git a/grovedb-bulk-append-tree/src/chunk.rs b/grovedb-bulk-append-tree/src/chunk.rs index e4d8dce95..711108df1 100644 --- a/grovedb-bulk-append-tree/src/chunk.rs +++ b/grovedb-bulk-append-tree/src/chunk.rs @@ -47,6 +47,50 @@ pub fn serialize_chunk_blob(entries: &[Vec]) -> Result, BulkAppendEr } } +/// The number of entry bytes a chunk blob carries — the sum of the lengths of +/// its entries, excluding the format byte, the headers and the per-entry +/// length prefixes. +/// +/// This is the part of a compaction blob the bulk-append tree has already +/// charged as added storage, one entry at a time, by the time the blob is +/// written; the storage adapter reports it as replaced. A blob that does not +/// parse yields `0`, which charges the whole write as new storage — the safe +/// direction — rather than failing a flush over a cost figure. +pub fn chunk_blob_entry_bytes(blob: &[u8]) -> u32 { + let Some((&format, data)) = blob.split_first() else { + return 0; + }; + match format { + FORMAT_FIXED => { + // [count: u32 BE] [entry_size: u32 BE] [entries...] + if data.len() < 8 { + return 0; + } + let payload = data.len() - 8; + u32::try_from(payload).unwrap_or(u32::MAX) + } + FORMAT_VARIABLE => { + // [len: u32 BE] [entry] repeated. + let mut entry_bytes: u64 = 0; + let mut offset = 0; + while offset < data.len() { + let Some(len_bytes) = data.get(offset..offset + 4) else { + return 0; + }; + let len = u32::from_be_bytes(len_bytes.try_into().expect("4 bytes")) as usize; + offset += 4; + if offset + len > data.len() { + return 0; + } + entry_bytes += len as u64; + offset += len; + } + u32::try_from(entry_bytes).unwrap_or(u32::MAX) + } + _ => 0, + } +} + /// Deserialize a chunk blob into individual entries. /// /// Handles both fixed-size and variable-size formats based on the leading @@ -336,3 +380,47 @@ mod tests { assert!(matches!(err, BulkAppendError::CorruptedData(_))); } } + +#[cfg(test)] +mod entry_bytes_tests { + use super::*; + + #[test] + fn fixed_format_entry_bytes_is_count_times_size() { + let blob = serialize_chunk_blob(&[vec![1u8; 8], vec![2u8; 8], vec![3u8; 8]]).unwrap(); + assert_eq!(blob[0], FORMAT_FIXED); + assert_eq!(chunk_blob_entry_bytes(&blob), 24); + assert_eq!(blob.len(), 9 + 24); + } + + #[test] + fn variable_format_entry_bytes_excludes_length_prefixes() { + let blob = serialize_chunk_blob(&[vec![1u8; 8], vec![2u8; 16], vec![3u8; 4]]).unwrap(); + assert_eq!(blob[0], FORMAT_VARIABLE); + assert_eq!(chunk_blob_entry_bytes(&blob), 28); + assert_eq!(blob.len(), 1 + 3 * 4 + 28); + } + + #[test] + fn malformed_blobs_prepay_nothing() { + assert_eq!(chunk_blob_entry_bytes(&[]), 0); + assert_eq!( + chunk_blob_entry_bytes(&[0x7f, 1, 2, 3]), + 0, + "unknown format" + ); + assert_eq!( + chunk_blob_entry_bytes(&[FORMAT_FIXED, 0, 0, 0]), + 0, + "short header" + ); + let mut truncated = serialize_chunk_blob(&[vec![1u8; 8], vec![2u8; 16]]).unwrap(); + truncated.truncate(truncated.len() - 3); + assert_eq!(chunk_blob_entry_bytes(&truncated), 0, "truncated entry"); + assert_eq!( + chunk_blob_entry_bytes(&[FORMAT_VARIABLE, 0, 0]), + 0, + "truncated prefix" + ); + } +} diff --git a/grovedb-bulk-append-tree/src/cost/mod.rs b/grovedb-bulk-append-tree/src/cost/mod.rs index 7b3a412db..144d00c82 100644 --- a/grovedb-bulk-append-tree/src/cost/mod.rs +++ b/grovedb-bulk-append-tree/src/cost/mod.rs @@ -1,18 +1,81 @@ //! Versioned cost accounting for the bulk-append tree. //! -//! Only the reported hash count is versioned; chunk bytes, roots and stored -//! state are identical under every version. This gate reaches a live fee — +//! Only cost reporting is versioned; chunk bytes, roots and stored state are +//! identical under every version. These gates reach live fees — //! `CommitmentTree` adds the compaction hash count straight into its own -//! `hash_node_calls` — so the corrected figure arrives as a new version rather -//! than replacing the old one. +//! `hash_node_calls`, and every data put's cost information becomes storage +//! fees at commit — so corrected figures arrive as new versions rather than +//! replacing the old ones. +//! +//! Two gates live here: +//! +//! - `compaction_hash_count` — the hashes a compacting append reports. +//! - `append_storage_accounting` — how an append's data-storage writes are +//! reported to the storage cost layer (issue #822). v0 issues every put +//! with no cost information, so each is charged as new storage: a buffer +//! slot rewritten in epoch 2+, and the chunk blob that supersedes the +//! buffer, are both billed as permanent growth. v1 charges each entry's +//! permanent bytes once — its chunk-blob share, at its own append — and +//! reports the slot rewrite and the compaction blob as replacements of +//! the bytes they supersede. mod v0; mod v1; +use grovedb_dense_fixed_sized_merkle_tree::SlotWriteAccounting; +use grovedb_merkle_mountain_range::LeafValueStorageCost; use grovedb_version::{error::GroveVersionError, version::GroveVersion}; use crate::BulkAppendError; +/// How an append's data-storage writes are reported to the storage cost +/// layer. Selected per grove version by [`append_storage_accounting`]. +#[derive(Clone, Copy)] +pub(crate) struct AppendStorageAccounting { + /// How the dense-buffer slot write is reported. + pub slot_write: SlotWriteAccounting, + /// How the chunk-blob leaf is reported when the MMR overlay is flushed. + pub chunk_leaf: LeafValueStorageCost, + /// Whether the entry's chunk-blob share is charged as added storage at + /// its own append (so the later blob write can be a replacement). + prepay_chunk_share: bool, +} + +impl AppendStorageAccounting { + /// The entry's chunk-blob share to charge as added storage at its append: + /// its own bytes, or nothing when the blob is charged in full at + /// compaction instead. + pub fn prepaid_chunk_bytes(&self, value_len: usize) -> u32 { + if self.prepay_chunk_share { + u32::try_from(value_len).unwrap_or(u32::MAX) + } else { + 0 + } + } +} + +/// The storage accounting an append uses under `grove_version`. +pub(crate) fn append_storage_accounting( + grove_version: &GroveVersion, +) -> Result { + match grove_version + .bulk_append_tree_versions + .cost + .append_storage_accounting + { + 0 => Ok(v0::append_storage_accounting()), + 1 => Ok(v1::append_storage_accounting()), + version => Err(BulkAppendError::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "BulkAppendTree append storage accounting".to_string(), + known_versions: vec![0, 1], + received: version, + } + .to_string(), + )), + } +} + /// Hashes to report for a compacting append. /// /// `leaf_count` is the MMR leaf count BEFORE the push (what diff --git a/grovedb-bulk-append-tree/src/cost/v0.rs b/grovedb-bulk-append-tree/src/cost/v0.rs index b66a16456..576d55385 100644 --- a/grovedb-bulk-append-tree/src/cost/v0.rs +++ b/grovedb-bulk-append-tree/src/cost/v0.rs @@ -7,8 +7,26 @@ //! Locked: GROVE_V1..V3 are released and CommitmentTree has been billing this //! figure on mainnet. -use grovedb_merkle_mountain_range::hash_count_for_push; +use grovedb_dense_fixed_sized_merkle_tree::SlotWriteAccounting; +use grovedb_merkle_mountain_range::{hash_count_for_push, LeafValueStorageCost}; + +use super::AppendStorageAccounting; pub(super) fn compaction_hash_count(leaf_count: u64) -> u32 { hash_count_for_push(leaf_count) } + +/// Shipped storage accounting: every data put is issued with no cost +/// information, so the commit path bills key + value as new storage — the +/// slot rewrite, the chunk blob and the MMR nodes alike — and nothing is +/// charged at append time beyond the slot put itself. +/// +/// Locked: GROVE_V1..V3 are released and the shielded pool has been billed +/// this way on mainnet. +pub(super) fn append_storage_accounting() -> AppendStorageAccounting { + AppendStorageAccounting { + slot_write: SlotWriteAccounting::AsNew, + chunk_leaf: LeafValueStorageCost::New, + prepay_chunk_share: false, + } +} diff --git a/grovedb-bulk-append-tree/src/cost/v1.rs b/grovedb-bulk-append-tree/src/cost/v1.rs index d7862b44d..a3472c2a8 100644 --- a/grovedb-bulk-append-tree/src/cost/v1.rs +++ b/grovedb-bulk-append-tree/src/cost/v1.rs @@ -2,7 +2,13 @@ //! //! Adds the peak-bagging merges [`v0`](super::v0) omitted. Used from GROVE_V4. -use grovedb_merkle_mountain_range::{hash_count_for_push, hash_count_for_root_bagging}; +use grovedb_dense_fixed_sized_merkle_tree::SlotWriteAccounting; +use grovedb_merkle_mountain_range::{ + hash_count_for_push, hash_count_for_root_bagging, LeafValueStorageCost, +}; + +use super::AppendStorageAccounting; +use crate::chunk::chunk_blob_entry_bytes; pub(super) fn compaction_hash_count(leaf_count: u64, mmr_size_after_push: u64) -> u32 { // Derived from the MMR shape rather than read back out of the accumulated @@ -11,3 +17,22 @@ pub(super) fn compaction_hash_count(leaf_count: u64, mmr_size_after_push: u64) - // two gates are independent. hash_count_for_push(leaf_count).saturating_add(hash_count_for_root_bagging(mmr_size_after_push)) } + +/// Churn-as-replacement storage accounting (issue #822). Used from GROVE_V4. +/// +/// - The entry's chunk-blob share (its own bytes) is charged as added +/// storage at its append — these are the bytes that persist. +/// - The buffer slot write is sized against the value the slot already holds +/// in committed storage: a rewrite (epoch 2 onward) is replaced, growth is +/// added, shrink is not credited; a slot written for the first time stays +/// fully added. +/// - The compaction blob is reported as a replacement of the entry bytes it +/// supersedes (all prepaid), leaving only its framing — and the MMR +/// internal nodes — as added storage. +pub(super) fn append_storage_accounting() -> AppendStorageAccounting { + AppendStorageAccounting { + slot_write: SlotWriteAccounting::AgainstCommitted, + chunk_leaf: LeafValueStorageCost::PartlyPrepaid(chunk_blob_entry_bytes), + prepay_chunk_share: true, + } +} diff --git a/grovedb-bulk-append-tree/src/lib.rs b/grovedb-bulk-append-tree/src/lib.rs index 0100c00a2..8a81f7654 100644 --- a/grovedb-bulk-append-tree/src/lib.rs +++ b/grovedb-bulk-append-tree/src/lib.rs @@ -18,7 +18,7 @@ mod tree; pub mod test_utils; // Re-export main types -pub use chunk::{deserialize_chunk_blob, serialize_chunk_blob}; +pub use chunk::{chunk_blob_entry_bytes, deserialize_chunk_blob, serialize_chunk_blob}; pub use error::BulkAppendError; pub use grovedb_dense_fixed_sized_merkle_tree::{DenseFixedSizedMerkleTree, DenseTreeProof}; #[cfg(feature = "storage")] @@ -26,4 +26,4 @@ pub use grovedb_merkle_mountain_range::{MmrKeySize, MmrStore}; pub use proof::{BulkAppendTreeProof, BulkAppendTreeProofResult}; pub use tree::{hash::compute_state_root, leaf_count_to_mmr_size, BulkAppendTree}; #[cfg(feature = "storage")] -pub use tree::{AppendResult, BufferQueryResult, ChunkQueryResult}; +pub use tree::{AppendNoStateRootResult, AppendResult, BufferQueryResult, ChunkQueryResult}; diff --git a/grovedb-bulk-append-tree/src/test_utils.rs b/grovedb-bulk-append-tree/src/test_utils.rs index d68915dbe..561a3d772 100644 --- a/grovedb-bulk-append-tree/src/test_utils.rs +++ b/grovedb-bulk-append-tree/src/test_utils.rs @@ -26,6 +26,9 @@ pub struct MemStorageContext { pub data: RefCell, Vec>>, pub fail_get: Cell, pub fail_put: Cell, + /// Every data `put` in order, with the cost information it carried — + /// what a real storage context would hand the commit path to bill. + pub puts: RefCell, Option)>>, } impl MemStorageContext { @@ -69,7 +72,7 @@ impl<'db> StorageContext<'db> for MemStorageContext { key: K, value: &[u8], _children_sizes: ChildrenSizesWithIsSumTree, - _cost_info: Option, + cost_info: Option, ) -> CostResult<(), grovedb_storage::Error> { if self.fail_put.get() { return Err(grovedb_storage::Error::StorageError( @@ -77,6 +80,9 @@ impl<'db> StorageContext<'db> for MemStorageContext { )) .wrap_with_cost(OperationCost::default()); } + self.puts + .borrow_mut() + .push((key.as_ref().to_vec(), cost_info)); self.data .borrow_mut() .insert(key.as_ref().to_vec(), value.to_vec()); diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index 89819e2a1..db2049c6d 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -9,7 +9,11 @@ use super::{ capacity_for_height, hash::compute_state_root, AppendNoStateRootResult, AppendResult, BulkAppendTree, }; -use crate::{chunk::serialize_chunk_blob, cost::compaction_hash_count, BulkAppendError}; +use crate::{ + chunk::serialize_chunk_blob, + cost::{append_storage_accounting, compaction_hash_count}, + BulkAppendError, +}; impl<'db, S: StorageContext<'db>> BulkAppendTree { /// Create a new empty tree. @@ -74,6 +78,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { // +1 for the blake3 state-root computation we just did. hash_count: r.hash_count.saturating_add(1), compacted: r.compacted, + prepaid_chunk_bytes: r.prepaid_chunk_bytes, }) } @@ -89,8 +94,10 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// [`CommitmentTree::append_many_raw`]: ../../grovedb_commitment_tree/struct.CommitmentTree.html#method.append_many_raw /// /// Stored bytes, chunks and roots are identical under every grove - /// version; only the reported `hash_count` differs, and only for an append - /// that compacts. + /// version; what the version selects is the reported `hash_count` (for an + /// append that compacts) and the storage accounting of the data writes — + /// the cost information attached to the slot put, and the + /// `prepaid_chunk_bytes` the caller bills as added storage. pub fn append_no_state_root( &mut self, value: &[u8], @@ -98,11 +105,16 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { ) -> Result { let mut hash_count: u32 = 0; let global_position = self.total_count; + let accounting = append_storage_accounting(grove_version)?; // 1. Try to insert into the dense tree buffer. - let try_result = self.dense_tree.try_insert(value).unwrap().map_err(|e| { - BulkAppendError::StorageError(format!("dense tree insert failed: {}", e)) - })?; + let try_result = self + .dense_tree + .try_insert_with_accounting(value, accounting.slot_write) + .unwrap() + .map_err(|e| { + BulkAppendError::StorageError(format!("dense tree insert failed: {}", e)) + })?; let compacted = match try_result { Some((_dense_root, _position)) => { @@ -134,6 +146,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { global_position, hash_count, compacted, + prepaid_chunk_bytes: accounting.prepaid_chunk_bytes(value.len()), }) } @@ -161,10 +174,14 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { ) -> CostResult { let mut cost = OperationCost::default(); let global_position = self.total_count; + let accounting = match append_storage_accounting(grove_version) { + Ok(a) => a, + Err(e) => return Err(e).wrap_with_cost(cost), + }; let try_result = match self .dense_tree - .try_insert_no_root(value) + .try_insert_no_root_with_accounting(value, accounting.slot_write) .unwrap_add_cost(&mut cost) { Ok(r) => r, @@ -214,10 +231,19 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { // `hash_node_calls` and changing it would move a released cost. let hash_count = cost.hash_node_calls; + // The entry's chunk-blob share is billed here, in the returned cost; + // the mirror field is informational for this path (see its doc). + let prepaid_chunk_bytes = accounting.prepaid_chunk_bytes(value.len()); + cost.storage_cost.added_bytes = cost + .storage_cost + .added_bytes + .saturating_add(prepaid_chunk_bytes); + Ok(AppendNoStateRootResult { global_position, hash_count, compacted, + prepaid_chunk_bytes, }) .wrap_with_cost(cost) } @@ -454,12 +480,18 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// /// Cost tracking is intentionally omitted at this boundary: /// BulkAppendTree returns plain `Result`, not `CostResult`. Storage - /// I/O costs are captured by the caller's `commit_multi_context_batch`. - pub fn commit_mmr(&mut self) -> Result<(), BulkAppendError> { + /// I/O costs are captured by the caller's `commit_multi_context_batch`, + /// from the cost information attached to each node's put — which is what + /// `grove_version` selects: under the shipped accounting every node is + /// new storage; from GROVE_V4 the chunk blob is reported as a + /// replacement of the entry bytes the appends already charged. + pub fn commit_mmr(&mut self, grove_version: &GroveVersion) -> Result<(), BulkAppendError> { if self.mmr_overlay.is_empty() { return Ok(()); } - let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32); + let accounting = append_storage_accounting(grove_version)?; + let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32) + .with_leaf_value_storage_cost(accounting.chunk_leaf); let mut mmr = MMR::new_with_overlay( self.mmr_size(), &mmr_store, diff --git a/grovedb-bulk-append-tree/src/tree/mod.rs b/grovedb-bulk-append-tree/src/tree/mod.rs index bcea9a521..f8dfe3e47 100644 --- a/grovedb-bulk-append-tree/src/tree/mod.rs +++ b/grovedb-bulk-append-tree/src/tree/mod.rs @@ -17,6 +17,8 @@ mod fetch; #[cfg(feature = "storage")] pub use fetch::{BufferQueryResult, ChunkQueryResult}; +#[cfg(all(test, feature = "storage"))] +mod storage_accounting_tests; #[cfg(all(test, feature = "storage"))] mod tests; @@ -38,6 +40,10 @@ pub struct AppendResult { pub hash_count: u32, /// Whether compaction (epoch flush) occurred. pub compacted: bool, + /// The entry's chunk-blob share, which the caller bills as + /// `storage_cost.added_bytes`. See + /// [`AppendNoStateRootResult::prepaid_chunk_bytes`]. + pub prepaid_chunk_bytes: u32, } /// Result returned by [`BulkAppendTree::append_no_state_root`]. @@ -55,6 +61,22 @@ pub struct AppendNoStateRootResult { pub hash_count: u32, /// Whether compaction (epoch flush) occurred. pub compacted: bool, + /// The entry's share of the chunk blob it will eventually be compacted + /// into — its own bytes — to be charged as `storage_cost.added_bytes` at + /// this append, so that the blob written at compaction can be reported as + /// a replacement of bytes already paid for (issue #822). `0` under the + /// shipped accounting (GROVE_V1..V3), where the blob is charged in full + /// at compaction instead. + /// + /// Like `hash_count`, this follows the "caller bills" convention of the + /// `Result`-returning appends ([`append`](BulkAppendTree::append), + /// [`append_no_state_root`](BulkAppendTree::append_no_state_root)): + /// add it to the operation's `storage_cost.added_bytes`. The + /// `CostResult`-returning + /// [`append_deferred_roots`](BulkAppendTree::append_deferred_roots) + /// already includes it in the returned cost; there the field is a + /// mirror for information only — do not bill it twice. + pub prepaid_chunk_bytes: u32, } /// Compute MMR size from leaf count: `2 * n - popcount(n)`. diff --git a/grovedb-bulk-append-tree/src/tree/storage_accounting_tests.rs b/grovedb-bulk-append-tree/src/tree/storage_accounting_tests.rs new file mode 100644 index 000000000..f01e9d65e --- /dev/null +++ b/grovedb-bulk-append-tree/src/tree/storage_accounting_tests.rs @@ -0,0 +1,238 @@ +//! The storage accounting of an append's data writes (issue #822), pinned at +//! the level of the cost information each put carries. +//! +//! The in-memory context records every `put` with its `cost_info`, which is +//! exactly what a real storage context hands the commit path to bill. +//! Note that the in-memory context is immediate — reads see earlier writes +//! of the same session — so the "slot holds a committed value" cases here +//! stand in for a slot written in an earlier, committed session; the +//! batch-dedup behaviour of a real transactional context is pinned by the +//! GroveDB-level tests. + +use grovedb_costs::storage_cost::{ + key_value_cost::KeyValueStorageCost, removal::StorageRemovedBytes::NoStorageRemoval, +}; +use grovedb_version::version::{v1::GROVE_V1, v3::GROVE_V3, v4::GROVE_V4, GroveVersion}; + +use crate::{test_utils::MemStorageContext, BulkAppendError, BulkAppendTree}; + +fn varint_len(mut n: u32) -> u32 { + let mut len = 1; + while n >= 0x80 { + n >>= 7; + len += 1; + } + len +} + +/// The paid size of a stored value: its length plus the varint of it. +fn paid(len: u32) -> u32 { + len + varint_len(len) +} + +/// Dense-buffer slot puts (2-byte position keys), in order. +fn slot_puts(tree: &BulkAppendTree) -> Vec> { + tree.dense_tree + .storage + .puts + .borrow() + .iter() + .filter(|(k, _)| k.len() == 2) + .map(|(_, c)| c.clone()) + .collect() +} + +/// MMR node puts (4-byte position keys), in order. +fn mmr_puts(tree: &BulkAppendTree) -> Vec> { + tree.dense_tree + .storage + .puts + .borrow() + .iter() + .filter(|(k, _)| k.len() == 4) + .map(|(_, c)| c.clone()) + .collect() +} + +/// Twelve appends at `chunk_power` 2 (epoch 4): a fixed-size first epoch, +/// a variable-size second epoch, and a compacting third. +const VALUES: [&[u8]; 12] = [ + &[1; 8], &[2; 8], &[3; 8], &[4; 8], // compaction 1 (fixed-format blob) + &[5; 8], // slot 0: same size + &[6; 16], // slot 1: grows + &[7; 4], // slot 2: shrinks + &[8; 8], // compaction 2 (variable-format blob), one merge + &[9; 8], // slot 0: same size + &[10; 8], // slot 1: shrinks back + &[11; 8], // slot 2: grows back + &[12; 8], // compaction 3 +]; + +fn run(version: &GroveVersion) -> (BulkAppendTree, Vec, [u8; 32]) { + let mut tree = BulkAppendTree::new(2, MemStorageContext::new()).expect("new"); + let mut prepaid = Vec::new(); + for v in VALUES { + let r = tree.append_no_state_root(v, version).expect("append"); + prepaid.push(r.prepaid_chunk_bytes); + } + let root = tree.compute_current_state_root().expect("root"); + tree.commit_mmr(version).expect("commit"); + (tree, prepaid, root) +} + +/// v0 (GROVE_V1..V3): every data put — slot, blob, MMR node — is issued +/// with no cost information, so the commit path bills each as new storage, +/// and nothing is prepaid at append time. +#[test] +fn v0_issues_every_put_without_cost_info_and_prepays_nothing() { + for version in [&GROVE_V1, &GROVE_V3] { + let (tree, prepaid, _) = run(version); + assert!(prepaid.iter().all(|&p| p == 0), "{prepaid:?}"); + let slots = slot_puts(&tree); + assert_eq!(slots.len(), 9, "three slots per epoch, three epochs"); + assert!(slots.iter().all(Option::is_none), "{slots:?}"); + let nodes = mmr_puts(&tree); + // 3 leaves + 1 merge (leaf count 1 -> 2) = 4 nodes; the third leaf + // (count 2 -> 3) collapses nothing. + assert_eq!(nodes.len(), 4); + assert!(nodes.iter().all(Option::is_none), "{nodes:?}"); + } +} + +/// v1 (GROVE_V4): a slot written for the first time is new storage; a slot +/// that already holds a value is a replacement — growth added, shrink not +/// credited, key not charged; every append prepays its own bytes. +#[test] +fn v1_slot_rewrites_are_replacements_and_every_append_prepays_its_bytes() { + let (tree, prepaid, _) = run(&GROVE_V4); + let expected_prepaid: Vec = VALUES.iter().map(|v| v.len() as u32).collect(); + assert_eq!(prepaid, expected_prepaid); + + let slots = slot_puts(&tree); + assert_eq!(slots.len(), 9); + // Epoch 1: fresh slots. + assert!(slots[..3].iter().all(Option::is_none), "{slots:?}"); + + let rewrite = |previous: u32, new: u32| KeyValueStorageCost { + key_storage_cost: Default::default(), + value_storage_cost: grovedb_costs::storage_cost::StorageCost { + added_bytes: paid(new).saturating_sub(paid(previous)), + replaced_bytes: paid(new).min(paid(previous)), + removed_bytes: NoStorageRemoval, + }, + new_node: false, + needs_value_verification: true, + }; + // Epoch 2 over epoch 1's 8-byte values. + assert_eq!(slots[3], Some(rewrite(8, 8)), "same size: fully replaced"); + assert_eq!( + slots[4], + Some(rewrite(8, 16)), + "growth: 9 replaced, 8 added" + ); + assert_eq!( + slots[5], + Some(rewrite(8, 4)), + "shrink: 5 replaced, nothing credited" + ); + assert_eq!(slots[4].as_ref().unwrap().value_storage_cost.added_bytes, 8); + assert_eq!(slots[5].as_ref().unwrap().value_storage_cost.added_bytes, 0); + // Epoch 3 over epoch 2's values. + assert_eq!(slots[6], Some(rewrite(8, 8))); + assert_eq!(slots[7], Some(rewrite(16, 8))); + assert_eq!(slots[8], Some(rewrite(4, 8))); +} + +/// v1: the compaction blob is reported as a replacement of the entry bytes +/// it supersedes — all prepaid — with only its framing added; internal MMR +/// nodes are new storage. +#[test] +fn v1_commit_mmr_reports_blob_as_replacement_of_prepaid_entry_bytes() { + let (tree, _, _) = run(&GROVE_V4); + let nodes = mmr_puts(&tree); + assert_eq!(nodes.len(), 4, "{nodes:?}"); + + let leaf = |entry_bytes: u32, blob_len: u32| { + // MmrNode leaf envelope: flag (1) + hash (32) + length (4) + blob. + let node_len = 37 + blob_len; + KeyValueStorageCost { + key_storage_cost: Default::default(), + value_storage_cost: grovedb_costs::storage_cost::StorageCost { + added_bytes: paid(node_len) - entry_bytes, + replaced_bytes: entry_bytes, + removed_bytes: NoStorageRemoval, + }, + new_node: true, + needs_value_verification: true, + } + }; + // Blob 1: four 8-byte entries, fixed format: 9-byte header + 32. + assert_eq!(nodes[0], Some(leaf(32, 9 + 32))); + // Blob 2: 8, 16, 4, 8 -> variable format: 1 + sum(4 + len) = 53, 36 entry bytes. + assert_eq!(nodes[1], Some(leaf(36, 53))); + // The merge of leaves 1 and 2 is an internal node: new storage. + assert_eq!(nodes[2], None); + // Blob 3: fixed again. + assert_eq!(nodes[3], Some(leaf(32, 9 + 32))); +} + +/// The tree itself must not depend on the accounting version. +#[test] +fn stored_state_and_roots_are_identical_across_accounting_versions() { + let (t3, _, root3) = run(&GROVE_V3); + let (t4, _, root4) = run(&GROVE_V4); + assert_eq!(root3, root4); + assert_eq!( + *t3.dense_tree.storage.data.borrow(), + *t4.dense_tree.storage.data.borrow(), + "byte-identical storage" + ); +} + +/// The cost-propagating append bills the prepaid share in its returned cost +/// and mirrors it in the result; the plain appends leave billing to the +/// caller. +#[test] +fn append_deferred_roots_bills_prepaid_share_in_cost() { + let mut tree = BulkAppendTree::new(2, MemStorageContext::new()).expect("new"); + let ctx = tree.append_deferred_roots(&[7u8; 20], &GROVE_V4); + let r = ctx.value.expect("append"); + assert_eq!(r.prepaid_chunk_bytes, 20); + assert_eq!(ctx.cost.storage_cost.added_bytes, 20); + assert_eq!(ctx.cost.storage_cost.replaced_bytes, 0); + + let mut legacy = BulkAppendTree::new(2, MemStorageContext::new()).expect("new"); + let ctx = legacy.append_deferred_roots(&[7u8; 20], &GROVE_V3); + assert_eq!(ctx.value.expect("append").prepaid_chunk_bytes, 0); + assert_eq!(ctx.cost.storage_cost.added_bytes, 0); +} + +/// An unknown accounting version is rejected at every entry that consults +/// it, never silently treated as one of the implemented ones. +#[test] +fn unknown_storage_accounting_version_is_rejected() { + let mut bad = GROVE_V4.clone(); + bad.bulk_append_tree_versions.cost.append_storage_accounting = 99; + + let mut tree = BulkAppendTree::new(2, MemStorageContext::new()).expect("new"); + assert!(matches!( + tree.append_no_state_root(&[1; 8], &bad), + Err(BulkAppendError::VersionError(_)) + )); + assert!(matches!( + tree.append_deferred_roots(&[1; 8], &bad).value, + Err(BulkAppendError::VersionError(_)) + )); + // A flush with staged nodes consults the gate too. + for v in &VALUES[..4] { + tree.append_no_state_root(v, &GROVE_V4).expect("append"); + } + assert!(matches!( + tree.commit_mmr(&bad), + Err(BulkAppendError::VersionError(_)) + )); + // The overlay survives the rejected flush. + tree.commit_mmr(&GROVE_V4) + .expect("flush with a known version"); + assert_eq!(mmr_puts(&tree).len(), 1); +} diff --git a/grovedb-commitment-tree/src/commitment_tree/cost/mod.rs b/grovedb-commitment-tree/src/commitment_tree/cost/mod.rs new file mode 100644 index 000000000..cfb4f410f --- /dev/null +++ b/grovedb-commitment-tree/src/commitment_tree/cost/mod.rs @@ -0,0 +1,105 @@ +//! Versioned cost accounting for the commitment tree. +//! +//! Only cost reporting is versioned; the frontier bytes, anchors and state +//! roots are identical under every version. The gate reaches a live fee — +//! every shielded append rewrites the frontier — so the corrected figure +//! arrives as a new version rather than replacing the old one. +//! +//! - `frontier_save_storage_accounting` — how [`CommitmentTree::save`] reports +//! the `__ct_data__` rewrite (issue #822). v0 issues the put with no cost +//! information, so the key and the whole frontier are charged as new +//! storage on every append. v1 reports it as a replacement of the bytes +//! loaded at open, with only growth added. +//! +//! [`CommitmentTree::save`]: super::CommitmentTree::save + +mod v0; +mod v1; + +use grovedb_costs::storage_cost::key_value_cost::KeyValueStorageCost; +use grovedb_version::{error::GroveVersionError, version::GroveVersion}; + +use crate::CommitmentTreeError; + +/// Cost information to attach to the frontier put. +/// +/// `persisted_len` is the serialized size of the frontier as loaded at open +/// (`None` when the key did not exist), `new_len` the size being written. +pub(crate) fn frontier_save_cost_info( + persisted_len: Option, + new_len: u32, + grove_version: &GroveVersion, +) -> Result, CommitmentTreeError> { + match grove_version + .commitment_tree_versions + .cost + .frontier_save_storage_accounting + { + 0 => Ok(v0::frontier_save_cost_info()), + 1 => Ok(v1::frontier_save_cost_info(persisted_len, new_len)), + version => Err(CommitmentTreeError::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "CommitmentTree frontier save storage accounting".to_string(), + known_versions: vec![0, 1], + received: version, + } + .to_string(), + )), + } +} + +#[cfg(test)] +mod tests { + use grovedb_version::version::{v1::GROVE_V1, v3::GROVE_V3, v4::GROVE_V4}; + + use super::*; + + #[test] + fn v0_attaches_no_cost_info() { + for version in [&GROVE_V1, &GROVE_V3] { + assert!(frontier_save_cost_info(None, 74, version) + .unwrap() + .is_none()); + assert!(frontier_save_cost_info(Some(74), 106, version) + .unwrap() + .is_none()); + } + } + + #[test] + fn v1_first_save_is_new_storage_and_rewrites_replace_with_growth_added() { + // No committed frontier: the key is created, everything is added. + assert!(frontier_save_cost_info(None, 74, &GROVE_V4) + .unwrap() + .is_none()); + // 74 -> 106 bytes (one more ommer): paid 75 replaced, 32 added. + let c = frontier_save_cost_info(Some(74), 106, &GROVE_V4) + .unwrap() + .unwrap(); + assert_eq!(c.value_storage_cost.replaced_bytes, 75); + assert_eq!(c.value_storage_cost.added_bytes, 32); + assert!(!c.new_node); + // 1066 -> 74 bytes (position 2^k): replaced at the new size, nothing credited. + let c = frontier_save_cost_info(Some(1066), 74, &GROVE_V4) + .unwrap() + .unwrap(); + assert_eq!(c.value_storage_cost.replaced_bytes, 75); + assert_eq!(c.value_storage_cost.added_bytes, 0); + assert_eq!( + c.value_storage_cost.removed_bytes, + grovedb_costs::storage_cost::removal::StorageRemovedBytes::NoStorageRemoval + ); + } + + #[test] + fn unknown_version_is_rejected() { + let mut bad = GROVE_V4.clone(); + bad.commitment_tree_versions + .cost + .frontier_save_storage_accounting = 99; + assert!(matches!( + frontier_save_cost_info(Some(74), 74, &bad), + Err(CommitmentTreeError::VersionError(_)) + )); + } +} diff --git a/grovedb-commitment-tree/src/commitment_tree/cost/v0.rs b/grovedb-commitment-tree/src/commitment_tree/cost/v0.rs new file mode 100644 index 000000000..436390cc7 --- /dev/null +++ b/grovedb-commitment-tree/src/commitment_tree/cost/v0.rs @@ -0,0 +1,12 @@ +//! Shipped frontier-save accounting: no cost information, so the commit path +//! bills the key and the whole serialized frontier as new storage on every +//! append. +//! +//! Locked: GROVE_V1..V3 are released and the shielded pool has been billed +//! this way on mainnet. + +use grovedb_costs::storage_cost::key_value_cost::KeyValueStorageCost; + +pub(super) fn frontier_save_cost_info() -> Option { + None +} diff --git a/grovedb-commitment-tree/src/commitment_tree/cost/v1.rs b/grovedb-commitment-tree/src/commitment_tree/cost/v1.rs new file mode 100644 index 000000000..2e2414ebb --- /dev/null +++ b/grovedb-commitment-tree/src/commitment_tree/cost/v1.rs @@ -0,0 +1,21 @@ +//! Churn-as-replacement frontier-save accounting (issue #822). Used from +//! GROVE_V4. +//! +//! The frontier is one value rewritten in place on every append. When a +//! committed frontier was loaded at open, the rewrite is reported as a +//! replacement of it — `replaced_bytes` is the smaller of the previous and +//! the new paid size, `added_bytes` is growth only, and shrink (a position +//! with fewer ommers) is not credited — and the key, which exists, is not +//! charged. The very first save of a tree creates the key and stays fully +//! added. + +use grovedb_costs::storage_cost::key_value_cost::KeyValueStorageCost; + +pub(super) fn frontier_save_cost_info( + persisted_len: Option, + new_len: u32, +) -> Option { + // The same "replace what was there, add the growth" shape the + // bulk-append tree uses for a rewritten buffer slot. + persisted_len.map(|previous| KeyValueStorageCost::for_in_place_value_rewrite(previous, new_len)) +} diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index 9f674e583..9802c3011 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -23,6 +23,7 @@ use orchard::{ use crate::{compute_commitment_tree_state_root, CommitmentFrontier, CommitmentTreeError}; +mod cost; mod tests; /// Key used to store the serialized commitment frontier in data storage. @@ -146,6 +147,14 @@ pub fn deserialize_ciphertext(data: &[u8]) -> Option { frontier: CommitmentFrontier, pub(crate) bulk_tree: BulkAppendTree, + /// Serialized size of the frontier as loaded from storage at + /// [`open`](Self::open) — `None` when no frontier was stored (or the tree + /// was built with [`new`](Self::new)). [`save`](Self::save) sizes the + /// rewrite against it. Deliberately NOT updated by `save`: a + /// `StorageBatch` keeps one put per key, so a session that saves twice is + /// charged for the last put only, which must describe the transition + /// from the committed frontier, not from the intermediate one. + persisted_frontier_len: Option, _memo: PhantomData, } @@ -170,6 +179,7 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { Ok(Self { frontier: CommitmentFrontier::new(), bulk_tree, + persisted_frontier_len: None, _memo: PhantomData, }) } @@ -204,12 +214,12 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { .get(COMMITMENT_TREE_DATA_KEY) .unwrap_add_cost(&mut cost); - let frontier = match data { + let (frontier, persisted_frontier_len) = match data { Ok(Some(bytes)) => match CommitmentFrontier::deserialize(&bytes) { - Ok(f) => f, + Ok(f) => (f, Some(bytes.len() as u32)), Err(e) => return Err(e).wrap_with_cost(cost), }, - Ok(None) => CommitmentFrontier::new(), + Ok(None) => (CommitmentFrontier::new(), None), Err(e) => { return Err(CommitmentTreeError::InvalidData(format!( "storage error loading frontier: {}", @@ -235,6 +245,7 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { Ok(Self { frontier, bulk_tree, + persisted_frontier_len, _memo: PhantomData, }) .wrap_with_cost(cost) @@ -335,6 +346,14 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { } }; cost.hash_node_calls += bulk_result.hash_count; + // The note's chunk-blob share — its permanent bytes — charged as + // added storage now, so the compaction blob is later reported as a + // replacement of bytes already paid for. Zero under the shipped + // accounting (GROVE_V1..V3). See `BulkAppendTree` issue #822. + cost.storage_cost.added_bytes = cost + .storage_cost + .added_bytes + .saturating_add(bulk_result.prepaid_chunk_bytes); // 2. Append cmx to Sinsemilla frontier (tracks sinsemilla_hash_calls) let sinsemilla_root = match self.frontier.append(cmx) { @@ -487,6 +506,11 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { } }; hash_count = hash_count.saturating_add(r.hash_count); + // The note's chunk-blob share, billed per entry as in `append_raw`. + cost.storage_cost.added_bytes = cost + .storage_cost + .added_bytes + .saturating_add(r.prepaid_chunk_bytes); if r.compacted { any_compacted = true; } @@ -553,14 +577,28 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { } /// Persist the current frontier state to storage. - pub fn save(&self) -> CostResult<(), CommitmentTreeError> { + /// + /// The bytes written are identical under every grove version; what the + /// version selects is how the rewrite is reported to the storage cost + /// layer — under the shipped accounting the key and the whole frontier + /// are new storage on every save; from GROVE_V4 the rewrite replaces the + /// frontier loaded at [`open`](Self::open) and only growth is added. + pub fn save(&self, grove_version: &GroveVersion) -> CostResult<(), CommitmentTreeError> { let mut cost = OperationCost::default(); let serialized = self.frontier.serialize(); + let cost_info = match cost::frontier_save_cost_info( + self.persisted_frontier_len, + serialized.len() as u32, + grove_version, + ) { + Ok(c) => c, + Err(e) => return Err(e).wrap_with_cost(cost), + }; let result = self .bulk_tree .dense_tree .storage - .put(COMMITMENT_TREE_DATA_KEY, &serialized, None, None) + .put(COMMITMENT_TREE_DATA_KEY, &serialized, None, cost_info) .unwrap_add_cost(&mut cost); match result { Ok(()) => Ok(()).wrap_with_cost(cost), @@ -600,9 +638,9 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { /// /// Delegates to [`BulkAppendTree::commit_mmr`]. Call this at the end of a /// session to persist MMR nodes buffered during compaction cycles. - pub fn commit_mmr(&mut self) -> Result<(), CommitmentTreeError> { + pub fn commit_mmr(&mut self, grove_version: &GroveVersion) -> Result<(), CommitmentTreeError> { self.bulk_tree - .commit_mmr() + .commit_mmr(grove_version) .map_err(|e| CommitmentTreeError::InvalidData(format!("MMR commit: {}", e))) } diff --git a/grovedb-commitment-tree/src/commitment_tree/tests.rs b/grovedb-commitment-tree/src/commitment_tree/tests.rs index 5e8c65ebe..8025c6106 100644 --- a/grovedb-commitment-tree/src/commitment_tree/tests.rs +++ b/grovedb-commitment-tree/src/commitment_tree/tests.rs @@ -527,7 +527,7 @@ mod storage_tests { let expected_total_count = ct.total_count(); // Save - let save_result = ct.save(); + let save_result = ct.save(GroveVersion::latest()); save_result.value.expect("save should succeed"); assert!( save_result.cost.seek_count > 0, @@ -561,7 +561,9 @@ mod storage_tests { .expect("open should succeed"); // Save empty - ct.save().value.expect("save empty should succeed"); + ct.save(GroveVersion::latest()) + .value + .expect("save empty should succeed"); // Append and save again (overwrites) ct.append( @@ -575,7 +577,9 @@ mod storage_tests { .expect("append should succeed"); let expected_root = ct.root_hash(); let total_count = ct.total_count(); - ct.save().value.expect("save non-empty should succeed"); + ct.save(GroveVersion::latest()) + .value + .expect("save non-empty should succeed"); // Re-open should return the latest (non-empty) frontier let storage = ct.bulk_tree.dense_tree.storage; @@ -622,9 +626,10 @@ mod storage_tests { let ct: CommitmentTree<_, DashMemo> = CommitmentTree { frontier: CommitmentFrontier::new(), bulk_tree, + persisted_frontier_len: None, _memo: PhantomData, }; - let result = ct.save(); + let result = ct.save(GroveVersion::latest()); assert!(result.value.is_err(), "should surface storage put error"); let err_msg = format!("{}", result.value.expect_err("should be storage error")); assert!( @@ -641,7 +646,9 @@ mod storage_tests { .value .expect("open should succeed"); - ct.save().value.expect("save empty should succeed"); + ct.save(GroveVersion::latest()) + .value + .expect("save empty should succeed"); let storage = ct.bulk_tree.dense_tree.storage; let loaded = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, storage) @@ -679,7 +686,9 @@ mod storage_tests { } let total_count = ct.total_count(); - ct.save().value.expect("save should succeed"); + ct.save(GroveVersion::latest()) + .value + .expect("save should succeed"); let storage = ct.bulk_tree.dense_tree.storage; let loaded = CommitmentTree::<_, DashMemo>::open(total_count, TEST_CHUNK_POWER, storage) @@ -1138,7 +1147,9 @@ mod storage_tests { ) .value .expect("append should succeed"); - ct.save().value.expect("save should succeed"); + ct.save(GroveVersion::latest()) + .value + .expect("save should succeed"); // 2. Re-open with total_count=0 but the frontier has tree_size=1 let storage = ct.bulk_tree.dense_tree.storage; @@ -1593,7 +1604,8 @@ mod storage_tests { .value .expect("warmup"); - ct.commit_mmr().expect("commit_mmr should flush cleanly"); + ct.commit_mmr(GroveVersion::latest()) + .expect("commit_mmr should flush cleanly"); } // ── cv_net round-trip + anchor invariance ─────────────────────────── diff --git a/grovedb-commitment-tree/src/error.rs b/grovedb-commitment-tree/src/error.rs index 41827de87..31b0c7381 100644 --- a/grovedb-commitment-tree/src/error.rs +++ b/grovedb-commitment-tree/src/error.rs @@ -13,6 +13,11 @@ pub enum CommitmentTreeError { /// A 32-byte value is not a valid Pallas field element. #[error("invalid Pallas field element")] InvalidFieldElement, + + /// An unknown grove version was supplied for a versioned accounting + /// decision. + #[error("version error: {0}")] + VersionError(String), /// The ciphertext payload length does not match the expected size for the /// configured `MemoSize`. #[error("invalid payload size: expected {expected}, got {actual}")] diff --git a/grovedb-dense-fixed-sized-merkle-tree/src/lib.rs b/grovedb-dense-fixed-sized-merkle-tree/src/lib.rs index 644480e6e..14040b388 100644 --- a/grovedb-dense-fixed-sized-merkle-tree/src/lib.rs +++ b/grovedb-dense-fixed-sized-merkle-tree/src/lib.rs @@ -24,4 +24,6 @@ mod tests; pub use error::DenseMerkleError; pub use proof::DenseTreeProof; +#[cfg(feature = "storage")] +pub use tree::SlotWriteAccounting; pub use tree::{position_key, DenseFixedSizedMerkleTree}; diff --git a/grovedb-dense-fixed-sized-merkle-tree/src/test_utils.rs b/grovedb-dense-fixed-sized-merkle-tree/src/test_utils.rs index e09a2a67a..8912910b7 100644 --- a/grovedb-dense-fixed-sized-merkle-tree/src/test_utils.rs +++ b/grovedb-dense-fixed-sized-merkle-tree/src/test_utils.rs @@ -16,6 +16,8 @@ use grovedb_storage::{Batch, RawIterator, StorageContext}; #[derive(Default)] pub(crate) struct MemStorageContext { pub data: RefCell, Vec>>, + /// Every data `put` in order, with the cost information it carried. + pub puts: RefCell, Option)>>, } impl MemStorageContext { @@ -37,8 +39,11 @@ impl<'db> StorageContext<'db> for MemStorageContext { key: K, value: &[u8], _children_sizes: ChildrenSizesWithIsSumTree, - _cost_info: Option, + cost_info: Option, ) -> CostResult<(), grovedb_storage::Error> { + self.puts + .borrow_mut() + .push((key.as_ref().to_vec(), cost_info)); self.data .borrow_mut() .insert(key.as_ref().to_vec(), value.to_vec()); diff --git a/grovedb-dense-fixed-sized-merkle-tree/src/tests.rs b/grovedb-dense-fixed-sized-merkle-tree/src/tests.rs index 0802e62f1..ac3b94bfd 100644 --- a/grovedb-dense-fixed-sized-merkle-tree/src/tests.rs +++ b/grovedb-dense-fixed-sized-merkle-tree/src/tests.rs @@ -1102,3 +1102,65 @@ fn test_verify_for_query_rejects_subquery() { "verify_for_query should reject queries with subqueries" ); } + +mod slot_write_accounting { + use grovedb_costs::storage_cost::removal::StorageRemovedBytes::NoStorageRemoval; + + use crate::{test_utils::MemStorageContext, DenseFixedSizedMerkleTree, SlotWriteAccounting}; + + /// `AsNew` (and the plain inserts) attach no cost information; the + /// commit path bills key + value as new storage. + #[test] + fn as_new_attaches_no_cost_info() { + let mut tree = DenseFixedSizedMerkleTree::new(2, MemStorageContext::new()).unwrap(); + tree.insert(&[1u8; 8]).unwrap().unwrap(); + tree.try_insert(&[2u8; 8]).unwrap().unwrap(); + tree.try_insert_no_root_with_accounting(&[3u8; 8], SlotWriteAccounting::AsNew) + .unwrap() + .unwrap(); + let puts = tree.storage.puts.borrow(); + assert_eq!(puts.len(), 3); + assert!(puts.iter().all(|(_, c)| c.is_none())); + } + + /// `AgainstCommitted` reads the slot first: an empty slot is new + /// storage; a held value makes the write a replacement (growth added, + /// shrink not credited, key not charged), and the read is billed. + #[test] + fn against_committed_sizes_the_rewrite_from_the_stored_value() { + let mut tree = DenseFixedSizedMerkleTree::new(2, MemStorageContext::new()).unwrap(); + // Nothing stored yet: new storage, as `AsNew`. + tree.try_insert_no_root_with_accounting(&[1u8; 8], SlotWriteAccounting::AgainstCommitted) + .unwrap() + .unwrap(); + tree.try_insert_with_accounting(&[2u8; 8], SlotWriteAccounting::AgainstCommitted) + .unwrap() + .unwrap(); + assert!(tree.storage.puts.borrow().iter().all(|(_, c)| c.is_none())); + + // A new cycle over the same keys. + tree.reset(); + let grow = tree + .try_insert_no_root_with_accounting(&[9u8; 16], SlotWriteAccounting::AgainstCommitted) + .unwrap() + .unwrap(); + assert_eq!(grow, Some(0)); + let shrink = tree + .try_insert_with_accounting(&[9u8; 4], SlotWriteAccounting::AgainstCommitted) + .unwrap() + .unwrap(); + assert_eq!(shrink.map(|(_, p)| p), Some(1)); + + let puts = tree.storage.puts.borrow(); + let c = puts[2].1.as_ref().expect("rewrite carries cost info"); + assert!(!c.new_node); + assert!(c.needs_value_verification); + assert_eq!(c.key_storage_cost, Default::default()); + assert_eq!(c.value_storage_cost.replaced_bytes, 9, "paid(8)"); + assert_eq!(c.value_storage_cost.added_bytes, 8, "paid(16) - paid(8)"); + let c = puts[3].1.as_ref().expect("rewrite carries cost info"); + assert_eq!(c.value_storage_cost.replaced_bytes, 5, "paid(4)"); + assert_eq!(c.value_storage_cost.added_bytes, 0); + assert_eq!(c.value_storage_cost.removed_bytes, NoStorageRemoval); + } +} diff --git a/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs b/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs index 53e8a8a30..74e986adf 100644 --- a/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs +++ b/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs @@ -1,5 +1,7 @@ #[cfg(feature = "storage")] -use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_costs::{ + storage_cost::key_value_cost::KeyValueStorageCost, CostResult, CostsExt, OperationCost, +}; #[cfg(feature = "storage")] use grovedb_storage::StorageContext; @@ -26,6 +28,39 @@ pub fn position_key(pos: u16) -> [u8; 2] { pos.to_be_bytes() } +/// How a slot write is reported to the storage cost layer. +/// +/// The tree itself never overwrites a slot: positions fill sequentially and +/// only [`reset`](DenseFixedSizedMerkleTree::reset) — used by the bulk-append +/// tree to start a new epoch over the same position keys — makes a later +/// insert land on a key that already holds a committed value. The owner, +/// which knows whether that can be the case, chooses the mode. +#[cfg(feature = "storage")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SlotWriteAccounting { + /// Issue the put with no cost information: the commit path charges the + /// key and the value as new storage. Right for a slot that has never + /// been written (and what every shipped version reports for all slots). + AsNew, + /// The slot may already hold a committed value. Read it first (one + /// billed storage read) and, if present, report the write as replacing + /// it: `replaced_bytes` is the smaller of the previous and the new paid + /// size, `added_bytes` is growth only, and shrink is not credited (no + /// refund semantics for a rolling buffer). The key, which already exists, + /// is not charged. A slot found empty is reported as [`AsNew`]. + /// + /// The read goes to the underlying storage context — committed state + /// plus the surrounding transaction — never to this session's + /// write-through cache. That is deliberate: a `StorageBatch` keeps one + /// put per key, so when a session writes the same slot twice (an epoch + /// boundary inside one batch) only the last put is charged, and it must + /// describe the transition from the committed value, not from the + /// intermediate one. + /// + /// [`AsNew`]: SlotWriteAccounting::AsNew + AgainstCommitted, +} + /// A dense fixed-sized Merkle tree with embedded storage. /// /// Positions are indexed level-order (BFS): root=0, left child=2i+1, right @@ -136,7 +171,10 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { } let position = self.count; - cost_return_on_error!(cost, self.put_value(position, value)); + cost_return_on_error!( + cost, + self.put_value(position, value, SlotWriteAccounting::AsNew) + ); self.count += 1; match self.compute_root_hash().unwrap_add_cost(&mut cost) { @@ -160,10 +198,22 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { /// Try to insert a value at the next available position. /// /// Returns `None` if the tree is full, otherwise returns - /// `Some((root_hash, position))`. + /// `Some((root_hash, position))`. The write is reported as new storage; + /// see [`try_insert_with_accounting`](Self::try_insert_with_accounting) + /// for a slot that may already hold a committed value. pub fn try_insert( &mut self, value: &[u8], + ) -> CostResult, DenseMerkleError> { + self.try_insert_with_accounting(value, SlotWriteAccounting::AsNew) + } + + /// [`try_insert`](Self::try_insert) with an explicit + /// [`SlotWriteAccounting`] for the slot write. + pub fn try_insert_with_accounting( + &mut self, + value: &[u8], + accounting: SlotWriteAccounting, ) -> CostResult, DenseMerkleError> { let mut cost = OperationCost::default(); @@ -172,7 +222,7 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { } let position = self.count; - cost_return_on_error!(cost, self.put_value(position, value)); + cost_return_on_error!(cost, self.put_value(position, value, accounting)); self.count += 1; match self.compute_root_hash().unwrap_add_cost(&mut cost) { @@ -202,6 +252,16 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { pub fn try_insert_no_root( &mut self, value: &[u8], + ) -> CostResult, DenseMerkleError> { + self.try_insert_no_root_with_accounting(value, SlotWriteAccounting::AsNew) + } + + /// [`try_insert_no_root`](Self::try_insert_no_root) with an explicit + /// [`SlotWriteAccounting`] for the slot write. + pub fn try_insert_no_root_with_accounting( + &mut self, + value: &[u8], + accounting: SlotWriteAccounting, ) -> CostResult, DenseMerkleError> { let mut cost = OperationCost::default(); @@ -210,7 +270,7 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { } let position = self.count; - cost_return_on_error!(cost, self.put_value(position, value)); + cost_return_on_error!(cost, self.put_value(position, value, accounting)); self.count += 1; Ok(Some(position)).wrap_with_cost(cost) @@ -300,7 +360,15 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { /// On success, the value is stored in the write-through cache so that /// subsequent reads (e.g., during root hash computation) can be served /// from memory even when the storage context defers writes. - fn put_value(&mut self, position: u16, value: &[u8]) -> CostResult<(), DenseMerkleError> { + /// + /// `accounting` selects the cost information attached to the put — see + /// [`SlotWriteAccounting`]. + fn put_value( + &mut self, + position: u16, + value: &[u8], + accounting: SlotWriteAccounting, + ) -> CostResult<(), DenseMerkleError> { debug_assert!( (position as usize) < self.cache.len(), "put_value called with position {} >= cache capacity {}", @@ -309,9 +377,34 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { ); let mut cost = OperationCost::default(); let key = position_key(position); + + let cost_info = match accounting { + SlotWriteAccounting::AsNew => None, + SlotWriteAccounting::AgainstCommitted => { + // Committed/transaction state only — NOT the session cache; + // see `SlotWriteAccounting::AgainstCommitted`. + let previous = match self.storage.get(key).unwrap_add_cost(&mut cost) { + Ok(v) => v, + Err(e) => { + return Err(DenseMerkleError::StoreError(format!( + "get at pos {} before overwrite: {}", + position, e + ))) + .wrap_with_cost(cost); + } + }; + previous.map(|old| { + KeyValueStorageCost::for_in_place_value_rewrite( + old.len() as u32, + value.len() as u32, + ) + }) + } + }; + let result = self .storage - .put(key, value, None, None) + .put(key, value, None, cost_info) .unwrap_add_cost(&mut cost); match result { Ok(()) => { diff --git a/grovedb-merkle-mountain-range/Cargo.toml b/grovedb-merkle-mountain-range/Cargo.toml index 36ac59a0c..f03a1b26e 100644 --- a/grovedb-merkle-mountain-range/Cargo.toml +++ b/grovedb-merkle-mountain-range/Cargo.toml @@ -14,7 +14,7 @@ documentation = "https://docs.rs/grovedb" [features] default = ["storage"] -storage = ["grovedb-storage"] +storage = ["grovedb-storage", "integer-encoding"] mem_store = [] [dependencies] @@ -23,6 +23,7 @@ bincode = { workspace = true, features = ["derive"] } grovedb-costs = { version = "5.0.1", path = "../costs" } grovedb-storage = { version = "5.0.1", path = "../storage", optional = true } grovedb-version = { version = "5.0.1", path = "../grovedb-version" } +integer-encoding = { workspace = true, optional = true } [dev-dependencies] faster-hex = "0.10.0" diff --git a/grovedb-merkle-mountain-range/src/lib.rs b/grovedb-merkle-mountain-range/src/lib.rs index 1ea9b28ba..c79bab33f 100644 --- a/grovedb-merkle-mountain-range/src/lib.rs +++ b/grovedb-merkle-mountain-range/src/lib.rs @@ -53,4 +53,4 @@ pub use mmr_store::{MMRBatch, MMRStoreReadOps, MMRStoreWriteOps}; pub use node::{blake3_merge, leaf_hash, MmrNode}; pub use proof::{MerkleProof, MmrTreeProof, VerifiedLeaves}; #[cfg(feature = "storage")] -pub use storage_adapter::MmrStore; +pub use storage_adapter::{LeafValueStorageCost, MmrStore}; diff --git a/grovedb-merkle-mountain-range/src/storage_adapter.rs b/grovedb-merkle-mountain-range/src/storage_adapter.rs index 41dd8873a..07fb3d58f 100644 --- a/grovedb-merkle-mountain-range/src/storage_adapter.rs +++ b/grovedb-merkle-mountain-range/src/storage_adapter.rs @@ -3,8 +3,15 @@ //! Provides `MmrStore`, which implements `MMRStoreReadOps` and //! `MMRStoreWriteOps` backed by a GroveDB storage context. -use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_costs::{ + storage_cost::{ + key_value_cost::KeyValueStorageCost, removal::StorageRemovedBytes::NoStorageRemoval, + StorageCost, + }, + CostResult, CostsExt, OperationCost, +}; use grovedb_storage::StorageContext; +use integer_encoding::VarInt; use crate::{ helper::{mmr_node_key_sized, MmrKeySize}, @@ -26,16 +33,44 @@ use crate::{ pub struct MmrStore<'a, C> { ctx: &'a C, key_size: MmrKeySize, + leaf_value_storage_cost: LeafValueStorageCost, +} + +/// How the value carried by a leaf node is reported to the storage cost +/// layer when the node is written by [`MMRStoreWriteOps::append`]. +/// +/// Internal nodes (hash only) are always new storage. Leaf values usually are +/// too — an `MmrTree` append stores a fresh value — but an owner that has +/// already charged part of a leaf's bytes as added storage before the flush +/// can say so, and that part is then reported as replaced rather than added. +/// The bulk-append tree does exactly this: every entry's chunk-blob share is +/// charged at its own append, so the blob written at compaction replaces +/// bytes that were paid for, and only its framing is new. +#[derive(Clone, Copy)] +pub enum LeafValueStorageCost { + /// Issue the put with no cost information: key and value are charged as + /// new storage (what every shipped version reports). + New, + /// `prepaid(value)` bytes of the leaf value were already charged as added + /// storage by the owner. The put reports them as `replaced_bytes` and the + /// remainder of the paid value size (value length plus its length varint) + /// as `added_bytes`; a `prepaid` figure above the paid size is clamped to + /// it. The key is new and charged in full. The callback must be a pure + /// function of the value bytes — it runs at flush time, which may be long + /// after the leaf was pushed. + PartlyPrepaid(fn(&[u8]) -> u32), } impl<'a, C> MmrStore<'a, C> { /// Create a new store backed by the given storage context. /// - /// Uses [`MmrKeySize::U64`] (8-byte keys) by default. + /// Uses [`MmrKeySize::U64`] (8-byte keys) by default and reports every + /// written node as new storage. pub fn new(ctx: &'a C) -> Self { Self { ctx, key_size: MmrKeySize::U64, + leaf_value_storage_cost: LeafValueStorageCost::New, } } @@ -44,7 +79,49 @@ impl<'a, C> MmrStore<'a, C> { /// Use [`MmrKeySize::U32`] for compact 4-byte keys when positions /// are guaranteed to fit in a `u32`. pub fn with_key_size(ctx: &'a C, key_size: MmrKeySize) -> Self { - Self { ctx, key_size } + Self { + ctx, + key_size, + leaf_value_storage_cost: LeafValueStorageCost::New, + } + } + + /// Select how leaf values are reported to the storage cost layer on + /// write; see [`LeafValueStorageCost`]. + pub fn with_leaf_value_storage_cost(mut self, policy: LeafValueStorageCost) -> Self { + self.leaf_value_storage_cost = policy; + self + } + + /// Cost information for writing `serialized` — a node whose value part is + /// `value` (none for an internal node) — under this store's leaf policy. + fn write_cost_info( + &self, + value: Option<&[u8]>, + serialized_len: u32, + ) -> Option { + match (self.leaf_value_storage_cost, value) { + (LeafValueStorageCost::New, _) | (_, None) => None, + (LeafValueStorageCost::PartlyPrepaid(prepaid), Some(value)) => { + // The paid size of a stored value is its length plus the + // varint encoding that length — exactly what the commit path + // verifies `added + replaced` against. + let paid = serialized_len + serialized_len.required_space() as u32; + let replaced = prepaid(value).min(paid); + Some(KeyValueStorageCost { + // Supplied without the path prefix; the storage context + // completes it for a new node. + key_storage_cost: StorageCost::default(), + value_storage_cost: StorageCost { + added_bytes: paid - replaced, + replaced_bytes: replaced, + removed_bytes: NoStorageRemoval, + }, + new_node: true, + needs_value_verification: true, + }) + } + } } } @@ -95,7 +172,8 @@ impl<'db, C: StorageContext<'db>> MMRStoreWriteOps for &MmrStore<'_, C> { .wrap_with_cost(cost); } }; - let result = self.ctx.put(key, &serialized, None, None); + let cost_info = self.write_cost_info(elem.value(), serialized.len() as u32); + let result = self.ctx.put(key, &serialized, None, cost_info); cost += result.cost; if let Err(e) = result.value { return Err(crate::Error::StoreError(format!( diff --git a/grovedb-private-document-store/src/store.rs b/grovedb-private-document-store/src/store.rs index d28bcb0bb..a1f9d8d77 100644 --- a/grovedb-private-document-store/src/store.rs +++ b/grovedb-private-document-store/src/store.rs @@ -530,9 +530,12 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { /// /// Delegates to [`BulkAppendTree::commit_mmr`]. Call this at the end of /// a session to persist MMR nodes buffered during compaction cycles. - pub fn commit_mmr(&mut self) -> Result<(), PrivateDocumentStoreError> { + pub fn commit_mmr( + &mut self, + grove_version: &GroveVersion, + ) -> Result<(), PrivateDocumentStoreError> { self.bulk_tree - .commit_mmr() + .commit_mmr(grove_version) .map_err(|e| PrivateDocumentStoreError::InvalidData(format!("MMR commit: {}", e))) } @@ -687,7 +690,9 @@ mod atomicity_tests { .unwrap() .expect("append"); } - store.commit_mmr().expect("commit mmr"); + store + .commit_mmr(GroveVersion::latest()) + .expect("commit mmr"); let storage = PrivateDocumentStore::into_storage_for_test(store); // `from_state` leaves the MMR root cache empty, so this computation @@ -965,7 +970,9 @@ mod error_path_tests { .unwrap() .expect("append"); } - store.commit_mmr().expect("commit mmr"); + store + .commit_mmr(GroveVersion::latest()) + .expect("commit mmr"); let storage = PrivateDocumentStore::into_storage_for_test(store); storage.data.borrow_mut().clear(); @@ -999,7 +1006,9 @@ mod error_path_tests { .unwrap() .expect("append"); } - store.commit_mmr().expect("commit mmr"); + store + .commit_mmr(GroveVersion::latest()) + .expect("commit mmr"); let storage = PrivateDocumentStore::into_storage_for_test(store); // Wrong chunk_power: epoch is now 4, so the stored 8-entry chunk no @@ -1059,7 +1068,9 @@ mod error_path_tests { .unwrap() .expect("append"); } - store.commit_mmr().expect("commit mmr"); + store + .commit_mmr(GroveVersion::latest()) + .expect("commit mmr"); let storage = PrivateDocumentStore::into_storage_for_test(store); // Claim 20 entries: chunks 1..=3 and their buffer slots do not exist. @@ -1116,7 +1127,9 @@ mod error_path_tests { .unwrap() .expect("append"); } - store.commit_mmr().expect("commit mmr"); + store + .commit_mmr(GroveVersion::latest()) + .expect("commit mmr"); let storage = PrivateDocumentStore::into_storage_for_test(store); storage.data.borrow_mut().clear(); @@ -1153,7 +1166,9 @@ mod error_path_tests { .unwrap() .expect("append"); } - store.commit_mmr().expect("commit mmr"); + store + .commit_mmr(GroveVersion::latest()) + .expect("commit mmr"); // Reopen before injecting the fault. The dense tree keeps a // write-through cache, so on the original handle a live buffer read is @@ -1385,7 +1400,9 @@ mod tests { .unwrap() .expect("append"); } - store.commit_mmr().expect("commit mmr"); + store + .commit_mmr(GroveVersion::latest()) + .expect("commit mmr"); let root_before = store.compute_current_state_root().expect("root"); let storage = PrivateDocumentStore::into_storage_for_test(store); @@ -1419,7 +1436,9 @@ mod tests { .unwrap() .expect("append"); } - store.commit_mmr().expect("commit mmr"); + store + .commit_mmr(GroveVersion::latest()) + .expect("commit mmr"); let storage = PrivateDocumentStore::into_storage_for_test(store); let reopened = PrivateDocumentStore::from_state(6, 16, 2, storage) diff --git a/grovedb-version/src/version/bulk_append_tree_versions.rs b/grovedb-version/src/version/bulk_append_tree_versions.rs index a9b3bfbe3..20750eb09 100644 --- a/grovedb-version/src/version/bulk_append_tree_versions.rs +++ b/grovedb-version/src/version/bulk_append_tree_versions.rs @@ -27,4 +27,22 @@ pub struct BulkAppendTreeCostVersions { /// Version 1 adds that bagging term. Shipped chunk bytes and roots are /// unaffected; what moves is the fee a compacting append is charged. pub compaction_hash_count: FeatureVersion, + /// How an append's data-storage writes are reported to the storage cost + /// layer (issue #822). + /// + /// Version 0 issues every put with no cost information, so the commit + /// path charges key + value as NEW storage: a dense-buffer slot rewritten + /// in epoch 2 and later, and the chunk blob that supersedes the buffer it + /// was built from, are both billed as permanent growth — the whole blob + /// (≈ 630 KB at `chunk_power` 11) lands on the one compacting append. + /// + /// Version 1 charges each entry's permanent bytes once, at its own + /// append: the entry's chunk-blob share (`value.len()`) is reported as + /// added storage on every append; a buffer slot that already holds a + /// committed value is reported as replaced (growth added, shrink not + /// credited) and a slot written for the first time stays fully added; the + /// compaction blob is reported as a replacement of the entry bytes it + /// supersedes, with only its framing (and the MMR internal nodes) added. + /// Stored bytes, chunks and roots are identical under both versions. + pub append_storage_accounting: FeatureVersion, } diff --git a/grovedb-version/src/version/commitment_tree_versions.rs b/grovedb-version/src/version/commitment_tree_versions.rs new file mode 100644 index 000000000..989785eaa --- /dev/null +++ b/grovedb-version/src/version/commitment_tree_versions.rs @@ -0,0 +1,30 @@ +//! Version gates for the commitment tree crate. +//! +//! As in [`super::mmr_versions`] and [`super::bulk_append_tree_versions`], +//! only cost accounting is versioned: the frontier bytes, the anchors and the +//! state roots are identical under every version. The gate reaches a live +//! fee — the shielded pool's every append rewrites the frontier — so the +//! corrected figure arrives as a new version rather than replacing the old +//! one. + +use versioned_feature_core::FeatureVersion; + +#[derive(Clone, Debug, Default)] +pub struct CommitmentTreeVersions { + pub cost: CommitmentTreeCostVersions, +} + +#[derive(Clone, Debug, Default)] +pub struct CommitmentTreeCostVersions { + /// How `CommitmentTree::save` reports the frontier rewrite to the storage + /// cost layer. + /// + /// Version 0 issues the put with no cost information, so the commit path + /// charges the key and the whole serialized frontier as NEW storage on + /// every append — even though `__ct_data__` is one value rewritten in + /// place. Version 1 reports the rewrite as a replacement of the bytes + /// loaded at open (`replaced_bytes` = the smaller of the previous and new + /// paid size, `added_bytes` = growth only, shrink not credited); the very + /// first save of a tree, which creates the key, stays fully added. + pub frontier_save_storage_accounting: FeatureVersion, +} diff --git a/grovedb-version/src/version/mod.rs b/grovedb-version/src/version/mod.rs index 148dce001..a66657659 100644 --- a/grovedb-version/src/version/mod.rs +++ b/grovedb-version/src/version/mod.rs @@ -1,4 +1,5 @@ pub mod bulk_append_tree_versions; +pub mod commitment_tree_versions; pub mod grovedb_versions; pub mod merk_versions; pub mod mmr_versions; @@ -12,7 +13,8 @@ pub use versioned_feature_core::*; use crate::version::v3::GROVE_V3; use crate::version::v4::GROVE_V4; use crate::version::{ - bulk_append_tree_versions::BulkAppendTreeVersions, grovedb_versions::GroveDBVersions, + bulk_append_tree_versions::BulkAppendTreeVersions, + commitment_tree_versions::CommitmentTreeVersions, grovedb_versions::GroveDBVersions, merk_versions::MerkVersions, mmr_versions::MmrVersions, v1::GROVE_V1, v2::GROVE_V2, }; @@ -23,6 +25,7 @@ pub struct GroveVersion { pub merk_versions: MerkVersions, pub mmr_versions: MmrVersions, pub bulk_append_tree_versions: BulkAppendTreeVersions, + pub commitment_tree_versions: CommitmentTreeVersions, } impl GroveVersion { diff --git a/grovedb-version/src/version/v1.rs b/grovedb-version/src/version/v1.rs index 43b129b2f..72bf4b887 100644 --- a/grovedb-version/src/version/v1.rs +++ b/grovedb-version/src/version/v1.rs @@ -1,6 +1,7 @@ use crate::version::grovedb_versions::GroveDBAggregateSumPathQueryMethodVersions; use crate::version::{ bulk_append_tree_versions::{BulkAppendTreeCostVersions, BulkAppendTreeVersions}, + commitment_tree_versions::{CommitmentTreeCostVersions, CommitmentTreeVersions}, grovedb_versions::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, @@ -273,6 +274,19 @@ pub const GROVE_V1: GroveVersion = GroveVersion { bulk_append_tree_versions: BulkAppendTreeVersions { cost: BulkAppendTreeCostVersions { compaction_hash_count: 0, + // Append storage accounting: the shipped figure, which bills + // every data put (buffer slot, chunk blob, MMR node) as new + // storage. Locked — released versions replay what they were + // admitted under. + append_storage_accounting: 0, + }, + }, + // Frontier save: the shipped figure, which bills the rewritten frontier + // (key and value) as new storage on every append. Locked for the same + // reason. + commitment_tree_versions: CommitmentTreeVersions { + cost: CommitmentTreeCostVersions { + frontier_save_storage_accounting: 0, }, }, }; diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index d817794ed..05372dc63 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -1,6 +1,7 @@ use crate::version::grovedb_versions::GroveDBAggregateSumPathQueryMethodVersions; use crate::version::{ bulk_append_tree_versions::{BulkAppendTreeCostVersions, BulkAppendTreeVersions}, + commitment_tree_versions::{CommitmentTreeCostVersions, CommitmentTreeVersions}, grovedb_versions::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, @@ -272,6 +273,19 @@ pub const GROVE_V2: GroveVersion = GroveVersion { bulk_append_tree_versions: BulkAppendTreeVersions { cost: BulkAppendTreeCostVersions { compaction_hash_count: 0, + // Append storage accounting: the shipped figure, which bills + // every data put (buffer slot, chunk blob, MMR node) as new + // storage. Locked — released versions replay what they were + // admitted under. + append_storage_accounting: 0, + }, + }, + // Frontier save: the shipped figure, which bills the rewritten frontier + // (key and value) as new storage on every append. Locked for the same + // reason. + commitment_tree_versions: CommitmentTreeVersions { + cost: CommitmentTreeCostVersions { + frontier_save_storage_accounting: 0, }, }, }; diff --git a/grovedb-version/src/version/v3.rs b/grovedb-version/src/version/v3.rs index 6a141dce3..ad5721c31 100644 --- a/grovedb-version/src/version/v3.rs +++ b/grovedb-version/src/version/v3.rs @@ -1,6 +1,7 @@ use crate::version::grovedb_versions::GroveDBAggregateSumPathQueryMethodVersions; use crate::version::{ bulk_append_tree_versions::{BulkAppendTreeCostVersions, BulkAppendTreeVersions}, + commitment_tree_versions::{CommitmentTreeCostVersions, CommitmentTreeVersions}, grovedb_versions::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, @@ -287,6 +288,19 @@ pub const GROVE_V3: GroveVersion = GroveVersion { bulk_append_tree_versions: BulkAppendTreeVersions { cost: BulkAppendTreeCostVersions { compaction_hash_count: 0, + // Append storage accounting: the shipped figure, which bills + // every data put (buffer slot, chunk blob, MMR node) as new + // storage. Locked — released versions replay what they were + // admitted under. + append_storage_accounting: 0, + }, + }, + // Frontier save: the shipped figure, which bills the rewritten frontier + // (key and value) as new storage on every append. Locked for the same + // reason. + commitment_tree_versions: CommitmentTreeVersions { + cost: CommitmentTreeCostVersions { + frontier_save_storage_accounting: 0, }, }, }; diff --git a/grovedb-version/src/version/v4.rs b/grovedb-version/src/version/v4.rs index c0f812e6d..e275ea3e3 100644 --- a/grovedb-version/src/version/v4.rs +++ b/grovedb-version/src/version/v4.rs @@ -99,6 +99,23 @@ //! admission bound: raising it ungated would make already-committed //! shield transitions re-validate as under-funded and brick sync. //! +//! - `bulk_append_tree_versions.cost.append_storage_accounting: 1` and +//! `commitment_tree_versions.cost.frontier_save_storage_accounting: 1` — +//! the append-only family (`BulkAppendTree`, `CommitmentTree`, +//! `PrivateDocumentStore`) reports write churn as replacement instead of +//! as new storage (issue #822). Each entry's permanent bytes — its share +//! of the eventual chunk blob — are charged as `added_bytes` once, at its +//! own append; a dense-buffer slot that already holds a committed value +//! (epoch 2 onward) is reported as `replaced_bytes` (growth added, shrink +//! not credited); the compaction blob is reported as a replacement of the +//! entry bytes it supersedes, so only its framing and the MMR internal +//! nodes are added; and the in-place frontier rewrite is a replacement of +//! the bytes loaded at open. V1..V3 keep issuing every data put with no +//! cost information, which bills key + value as new storage every time — +//! ≈ 2× the bytes that persist, with the whole ≈ 630 KB blob landing on +//! one append per epoch at `chunk_power` 11. Stored bytes, roots and +//! proofs are identical under both; gated because the figures are fees. +//! //! Note that `GroveVersion::latest()` resolves to this version, so anything //! defaulting to "latest" — tests, benchmarks, tools — exercises every gate //! listed above rather than V3 behaviour. @@ -118,6 +135,7 @@ use crate::version::grovedb_versions::GroveDBAggregateSumPathQueryMethodVersions; use crate::version::{ bulk_append_tree_versions::{BulkAppendTreeCostVersions, BulkAppendTreeVersions}, + commitment_tree_versions::{CommitmentTreeCostVersions, CommitmentTreeVersions}, grovedb_versions::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, @@ -413,6 +431,20 @@ pub const GROVE_V4: GroveVersion = GroveVersion { bulk_append_tree_versions: BulkAppendTreeVersions { cost: BulkAppendTreeCostVersions { compaction_hash_count: 1, + // Append storage accounting: each entry's permanent bytes are + // charged once (its chunk-blob share, at its own append); buffer + // slot rewrites and the compaction blob are reported as + // replacements of the bytes they supersede instead of as new + // storage (issue #822). Stored bytes and roots are unchanged. + append_storage_accounting: 1, + }, + }, + // Frontier save: the in-place rewrite of `__ct_data__` is reported as a + // replacement of the bytes loaded at open, with only growth added + // (issue #822). Frontier bytes and anchors are unchanged. + commitment_tree_versions: CommitmentTreeVersions { + cost: CommitmentTreeCostVersions { + frontier_save_storage_accounting: 1, }, }, }; diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index aa63ca788..4e8b43ee3 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -281,8 +281,14 @@ impl GroveOp { item_cost.add_cost(OperationCost { seek_count: 1, // 1 buffer entry write storage_cost: StorageCost { - added_bytes: entry_size, - replaced_bytes: 0, + // The value's chunk-blob share, charged at every + // append (issue #822), plus its buffer slot when + // written for the first time. + added_bytes: entry_size.saturating_mul(2), + // The slot rewrite from epoch 2 on, plus the + // compaction blob — a replacement of the epoch's + // prepaid entry bytes — amortized per append. + replaced_bytes: entry_size.saturating_mul(2), removed_bytes: StorageRemovedBytes::NoStorageRemoval, }, storage_loaded_bytes: 0, @@ -323,7 +329,10 @@ impl GroveOp { let epoch_entries: u32 = 1u32 << chunk_power.min(16) as u32; // Amortized over one epoch: every entry is written once to // the buffer, and once more into the chunk blob when the - // epoch compacts. + // epoch compacts. Under the GROVE_V4 accounting (issue + // #822) the blob share is exactly what each append is + // charged as added storage, and the blob itself lands as a + // replacement of those prepaid bytes. let amortized_compaction_bytes = entry_size; // The dense-buffer root walk costs two hashes per filled // position and runs on every append, so across an epoch it @@ -349,16 +358,25 @@ impl GroveOp { // append to a non-counted store. const NON_COUNTED_WRAPPER_BYTE: u32 = 1; item_cost.add_cost(OperationCost { - // 1 buffer entry write + the root walk's reads. - seek_count: 1u32.saturating_add(avg_dense_reads), + // 1 buffer entry write + the read of the slot's committed + // value that sizes the rewrite + the root walk's reads. + seek_count: 2u32.saturating_add(avg_dense_reads), storage_cost: StorageCost { + // The buffer slot (charged as new — it is in epoch + // 1; later it is a rewrite, replaced below) and the + // entry's chunk-blob share. added_bytes: entry_size .saturating_add(amortized_compaction_bytes) .saturating_add(NON_COUNTED_WRAPPER_BYTE), - replaced_bytes: 0, + // The slot rewrite from epoch 2 on, and the + // compaction blob — a replacement of the epoch's + // prepaid entry bytes — amortized per append. + replaced_bytes: entry_size.saturating_add(amortized_compaction_bytes), removed_bytes: StorageRemovedBytes::NoStorageRemoval, }, - storage_loaded_bytes: (avg_dense_reads as u64) + // The slot's committed value (one entry) + the root + // walk's reads. + storage_loaded_bytes: (avg_dense_reads as u64 + 1) .saturating_mul(entry_size as u64), hash_node_calls: avg_dense_hashes .saturating_add(ROOT_AND_CONFIG_HASHES) diff --git a/grovedb/src/batch/estimated_costs/mod.rs b/grovedb/src/batch/estimated_costs/mod.rs index c38f68419..327c9d70d 100644 --- a/grovedb/src/batch/estimated_costs/mod.rs +++ b/grovedb/src/batch/estimated_costs/mod.rs @@ -123,6 +123,10 @@ const CT_ELEMENT_LOAD_BASE: u32 = 256; /// bound; making the two estimators differ would make them silently /// non-interchangeable, which is a consensus fault for admission /// control (see issue #812). +/// +/// The storage terms follow the GROVE_V4 accounting of the append-only +/// family (issue #822), which charges each note's permanent bytes once +/// and reports write churn as replacement — see the field comments. #[cfg(feature = "minimal")] pub(in crate::batch) fn commitment_tree_insert_op_cost( payload_len: u32, @@ -137,9 +141,11 @@ pub(in crate::batch) fn commitment_tree_insert_op_cost( // overflow the shift. let epoch_size: u32 = 1u32 << chunk_power.min(PHYSICAL_MAX_CHUNK_POWER); - // Chunk-blob serialization overhead per entry (length prefix) and - // per blob (entry count, MMR leaf node framing). - const CHUNK_ENTRY_OVERHEAD: u64 = 16; + // Chunk-blob framing per blob: the fixed-format header (format byte, + // entry count, entry size) inside the MMR leaf envelope (flag, hash, + // length) — 9 + 37 — with margin. Every note in a commitment tree has + // the same size, so its blobs always take the fixed format, which + // carries no per-entry framing. const CHUNK_BLOB_OVERHEAD: u64 = 64; // An MMR internal node: 1 (flag) + 32 (hash). const MMR_INTERNAL_NODE_SIZE: u64 = 33; @@ -147,16 +153,35 @@ pub(in crate::batch) fn commitment_tree_insert_op_cost( // The epoch term multiplies the entry size by up to 2^16, which // overflows u32 for hand-built ops with oversized payloads (the op // is public; the apply path only rejects wrong-sized payloads - // later). Sum in u64 and saturate at u32::MAX — a wrapped - // added_bytes would silently UNDER-estimate, the exact failure this - // model exists to prevent, while a saturated one merely - // over-reserves for an op the apply would reject anyway. + // later). Sum in u64 and saturate at u32::MAX — a wrapped figure + // would silently UNDER-estimate, the exact failure this model exists + // to prevent, while a saturated one merely over-reserves for an op + // the apply would reject anyway. + // + // Added storage — what this append makes the database permanently + // larger by: + // - the note's dense-buffer slot when written for the first time + // (epoch 1), key included; later epochs rewrite it (replaced below), + // - the note's share of the eventual chunk blob (its own bytes), + // charged at every append so the blob is a replacement when it lands, + // - the frontier's very first save (key + value); later saves only + // add growth (at most one ommer), which this term dominates, + // - on compaction: the blob's framing beyond the prepaid entry bytes, + // and the MMR merge cascade's internal nodes. let added_bytes_u64: u64 = (entry_size + PER_PUT_OVERHEAD as u64) + + entry_size + (MAX_FRONTIER_SIZE as u64 + PER_PUT_OVERHEAD as u64) - + (epoch_size as u64 * (entry_size + CHUNK_ENTRY_OVERHEAD) - + CHUNK_BLOB_OVERHEAD - + PER_PUT_OVERHEAD as u64) + + (CHUNK_BLOB_OVERHEAD + PER_PUT_OVERHEAD as u64) + FRONTIER_DEPTH as u64 * (MMR_INTERNAL_NODE_SIZE + PER_PUT_OVERHEAD as u64); + // Replaced storage — bytes rewritten over bytes that were already + // paid for: + // - the note's buffer slot from epoch 2 on, + // - the re-serialized frontier (up to MAX_FRONTIER_SIZE), + // - on compaction: the whole epoch's entry bytes, which the chunk blob + // supersedes and every append already charged as added. + let replaced_bytes_u64: u64 = (entry_size + PER_PUT_OVERHEAD as u64) + + (MAX_FRONTIER_SIZE as u64 + PER_PUT_OVERHEAD as u64) + + epoch_size as u64 * entry_size; OperationCost { // 2 reads (CommitmentTree element + frontier) and up to @@ -164,20 +189,10 @@ pub(in crate::batch) fn commitment_tree_insert_op_cost( // MMR internal nodes). seek_count: 5 + FRONTIER_DEPTH, storage_cost: StorageCost { - // Data-storage writes are charged as added bytes (the - // commit path has no previous-size information for them, - // and dense/MMR keys are new within an epoch), so the whole - // write volume lands here: - // - the note entry into the dense buffer, - // - the re-serialized frontier (grows toward - // MAX_FRONTIER_SIZE), - // - on compaction: the epoch's chunk blob (every entry is - // re-written once into the blob) and the MMR merge - // cascade's internal nodes. added_bytes: u32::try_from(added_bytes_u64).unwrap_or(u32::MAX), // The parent-Merk node replacement is charged by the - // replace_tree part; the append itself replaces nothing. - replaced_bytes: 0, + // replace_tree part; this is the append's own churn. + replaced_bytes: u32::try_from(replaced_bytes_u64).unwrap_or(u32::MAX), removed_bytes: StorageRemovedBytes::NoStorageRemoval, }, // Reads: the stored CommitmentTree element (fixed serialized diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index eb8b8d247..59d6cafc7 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -270,7 +270,12 @@ impl GroveOp { seek_count: MAX_WRITES + MAX_READS, storage_cost: StorageCost { added_bytes: value_size + MAX_COMPACTION_BLOB, - replaced_bytes: 0, + // GROVE_V4 accounting (issue #822): a rewritten + // buffer slot and the compaction blob (a replacement + // of the epoch's prepaid entry bytes) are reported + // as replaced, so the bound carries the same volume + // on that side too. + replaced_bytes: value_size + MAX_COMPACTION_BLOB, removed_bytes: StorageRemovedBytes::NoStorageRemoval, }, storage_loaded_bytes: (33 * MAX_READS) as u64, @@ -357,9 +362,12 @@ impl GroveOp { .saturating_mul(entry_size) .saturating_add(MMR_SERIALIZATION_OVERHEAD); item_cost.add_cost(OperationCost { + // +1: the read of the slot's committed value that sizes + // the rewrite (GROVE_V4 accounting, issue #822). seek_count: MAX_WRITES .saturating_add(MAX_MMR_READS) - .saturating_add(MAX_DENSE_READS), + .saturating_add(MAX_DENSE_READS) + .saturating_add(1), storage_cost: StorageCost { // +1 for the NonCounted wrapper byte a preserved // wrapper adds to the replaced parent element; the op @@ -368,15 +376,21 @@ impl GroveOp { added_bytes: entry_size .saturating_add(max_compaction_blob) .saturating_add(1), - replaced_bytes: 0, + // GROVE_V4 accounting: a rewritten buffer slot and + // the compaction blob (a replacement of the epoch's + // prepaid entry bytes) are reported as replaced, so + // the bound carries the same volume on that side. + replaced_bytes: entry_size.saturating_add(max_compaction_blob), removed_bytes: StorageRemovedBytes::NoStorageRemoval, }, // Each dense read returns one entry, so the bytes those // reads load scale with the committed entry size. This - // product exceeds u32, hence the u64 arithmetic. + // product exceeds u32, hence the u64 arithmetic. The +1 + // entry is the slot's committed value read before the + // rewrite. storage_loaded_bytes: (33 * MAX_MMR_READS) as u64 + max_compaction_blob as u64 - + (MAX_DENSE_READS as u64).saturating_mul(entry_size as u64), + + (MAX_DENSE_READS as u64 + 1).saturating_mul(entry_size as u64), hash_node_calls: MAX_HASH_CALLS, sinsemilla_hash_calls: 0, }) @@ -1861,8 +1875,9 @@ mod tests { /// with an oversized payload (the op type is public; the apply path only /// rejects wrong-sized payloads later) drives the physical-ceiling epoch /// term past u32. The estimate must saturate at u32::MAX — never panic - /// in debug builds nor wrap in release builds, since a wrapped - /// added_bytes would silently UNDER-estimate. + /// in debug builds nor wrap in release builds, since a wrapped figure + /// would silently UNDER-estimate. The epoch term is the compaction + /// blob, which the V4 accounting reports as replaced (issue #822). #[test] fn test_commitment_tree_insert_worst_case_cost_oversized_payload_saturates() { let grove_version = GroveVersion::latest(); @@ -1885,9 +1900,16 @@ mod tests { .cost_as_result() .expect("expected worst case cost for oversized payload"); assert_eq!( - cost.storage_cost.added_bytes, + cost.storage_cost.replaced_bytes, u32::MAX, "oversized-payload estimate must saturate, not wrap", ); + // The per-note added term (slot + blob share + frontier + framing) + // is epoch-independent and stays far from saturation. + assert!( + cost.storage_cost.added_bytes < 1_000_000, + "added_bytes is per-note, not epoch-scaled: {}", + cost.storage_cost.added_bytes + ); } } diff --git a/grovedb/src/operations/bulk_append_tree.rs b/grovedb/src/operations/bulk_append_tree.rs index cc77e6608..b7303e3a7 100644 --- a/grovedb/src/operations/bulk_append_tree.rs +++ b/grovedb/src/operations/bulk_append_tree.rs @@ -90,12 +90,19 @@ impl GroveDb { ); cost.hash_node_calls += result.hash_count; + // The value's chunk-blob share — its permanent bytes — is billed at + // its own append (zero under the shipped accounting); the compaction + // blob is then a replacement of bytes already paid for (issue #822). + cost.storage_cost.added_bytes = cost + .storage_cost + .added_bytes + .saturating_add(result.prepaid_chunk_bytes); let new_state_root = result.state_root; let new_total_count = tree.total_count; // Flush MMR overlay to storage (through the batch) - cost_return_on_error_no_add!(cost, tree.commit_mmr().map_err(map_bulk_err)); + cost_return_on_error_no_add!(cost, tree.commit_mmr(grove_version).map_err(map_bulk_err)); // Drop tree (and its embedded storage context) before opening merk drop(tree); @@ -543,6 +550,12 @@ impl GroveDb { tree.append(value, grove_version).map_err(map_bulk_err) ); cost.hash_node_calls += result.hash_count; + // The value's chunk-blob share, billed per append as in the + // direct path (issue #822). + cost.storage_cost.added_bytes = cost + .storage_cost + .added_bytes + .saturating_add(result.prepaid_chunk_bytes); } // Compute final state root @@ -555,7 +568,10 @@ impl GroveDb { let current_total_count = tree.total_count; // Flush MMR overlay to storage (through the batch) - cost_return_on_error_no_add!(cost, tree.commit_mmr().map_err(map_bulk_err)); + cost_return_on_error_no_add!( + cost, + tree.commit_mmr(grove_version).map_err(map_bulk_err) + ); // Drop tree (and its embedded storage context) drop(tree); diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index 1b217eed4..815166050 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -161,7 +161,10 @@ impl GroveDb { ); // 4. Save frontier to storage - cost_return_on_error!(&mut cost, ct.save().map(|r| r.map_err(map_ct_err))); + cost_return_on_error!( + &mut cost, + ct.save(grove_version).map(|r| r.map_err(map_ct_err)) + ); let new_sinsemilla_root = append_result.sinsemilla_root; let bulk_state_root = append_result.bulk_state_root; @@ -177,7 +180,7 @@ impl GroveDb { ); // Flush MMR overlay to storage (through the batch) - cost_return_on_error_no_add!(cost, ct.commit_mmr().map_err(map_ct_err)); + cost_return_on_error_no_add!(cost, ct.commit_mmr(grove_version).map_err(map_ct_err)); // Drop ct (and its storage context) before opening merk drop(ct); @@ -551,7 +554,10 @@ impl GroveDb { } // Save frontier to storage - cost_return_on_error!(&mut cost, ct.save().map(|r| r.map_err(map_ct_err))); + cost_return_on_error!( + &mut cost, + ct.save(grove_version).map(|r| r.map_err(map_ct_err)) + ); // Read state for the replacement op let bulk_state_root = cost_return_on_error_no_add!( @@ -561,7 +567,7 @@ impl GroveDb { let current_total_count = ct.total_count(); // Flush MMR overlay to storage (through the batch) - cost_return_on_error_no_add!(cost, ct.commit_mmr().map_err(map_ct_err)); + cost_return_on_error_no_add!(cost, ct.commit_mmr(grove_version).map_err(map_ct_err)); // Drop ct (and its storage context) drop(ct); diff --git a/grovedb/src/operations/private_document_store.rs b/grovedb/src/operations/private_document_store.rs index d644c94aa..d9d51c3d1 100644 --- a/grovedb/src/operations/private_document_store.rs +++ b/grovedb/src/operations/private_document_store.rs @@ -199,7 +199,7 @@ impl GroveDb { let new_total_count = store.total_count(); // Flush MMR overlay to storage (through the batch). - cost_return_on_error_no_add!(cost, store.commit_mmr().map_err(map_pds_err)); + cost_return_on_error_no_add!(cost, store.commit_mmr(grove_version).map_err(map_pds_err)); // Drop the store (and its storage context) before opening merk. drop(store); @@ -557,7 +557,10 @@ impl GroveDb { let current_total_count = store.total_count(); // Flush MMR overlay to storage (through the batch). - cost_return_on_error_no_add!(cost, store.commit_mmr().map_err(map_pds_err)); + cost_return_on_error_no_add!( + cost, + store.commit_mmr(grove_version).map_err(map_pds_err) + ); // Drop the store (and its storage context). drop(store); diff --git a/grovedb/src/tests/append_storage_accounting_tests.rs b/grovedb/src/tests/append_storage_accounting_tests.rs new file mode 100644 index 000000000..967fc8a36 --- /dev/null +++ b/grovedb/src/tests/append_storage_accounting_tests.rs @@ -0,0 +1,709 @@ +//! Storage accounting of the append-only tree family (issue #822). +//! +//! From GROVE_V4 an append charges each entry's permanent bytes once and +//! reports write churn as replacement: +//! +//! - every append charges the entry's chunk-blob share (its own bytes) as +//! `added_bytes`; +//! - a dense-buffer slot that already holds a committed value (epoch 2 on) +//! is `replaced_bytes`, growth added, shrink not credited; +//! - the compaction blob replaces the epoch's prepaid entry bytes, so only +//! its framing and the MMR internal nodes are added; +//! - the commitment tree's frontier rewrite replaces the frontier loaded at +//! open, growth added. +//! +//! V1..V3 issue every data put with no cost information — key + value as +//! new storage, every time. Stored bytes and roots are identical. +//! +//! These tests run the real operations against RocksDB twice — once under +//! GROVE_V4 and once under GROVE_V4 with the two accounting gates switched +//! off — so that the difference between the two costs is exactly the +//! accounting change and nothing else (the parent-Merk update, the hash +//! counts and the Sinsemilla work are identical on both sides). The legacy +//! figures themselves are then pinned by the model below. + +use grovedb_commitment_tree::{ + CommitmentFrontier, DashMemo, NoteBytesData, TransmittedNoteCiphertext, +}; +use grovedb_costs::OperationCost; +use grovedb_version::version::{ + v1::GROVE_V1, v2::GROVE_V2, v3::GROVE_V3, v4::GROVE_V4, GroveVersion, +}; + +use crate::{ + batch::QualifiedGroveDbOp, + tests::{common::EMPTY_PATH, make_empty_grovedb, TempGroveDb}, + Element, +}; + +/// A stored note entry: cmx (32) || rho (32) || cv_net (32) || DashMemo +/// ciphertext payload (216). +const NOTE_ENTRY: u32 = 312; + +/// Paid key bytes of a dense-buffer slot: 32-byte prefix + 2-byte position +/// + 1 length byte. +const SLOT_KEY_PAID: u32 = 35; +/// Paid key bytes of the frontier: prefix + `__ct_data__` (11) + 1. +const FRONTIER_KEY_PAID: u32 = 44; + +fn varint_len(mut n: u32) -> u32 { + let mut len = 1; + while n >= 0x80 { + n >>= 7; + len += 1; + } + len +} + +/// The paid size of a stored value: its length plus the varint of it. +fn paid(len: u32) -> u32 { + len + varint_len(len) +} + +/// Serialized frontier size once the note at `position` is the latest leaf: +/// 1 (flag) + 8 (position) + 32 (leaf) + 1 (ommer count) + 32 per ommer, +/// and a frontier at leaf index `p` holds `popcount(p)` ommers. +fn frontier_len(position: u64) -> u32 { + 42 + 32 * position.count_ones() +} + +/// GROVE_V4 with both accounting gates switched off: what V1..V3 report, +/// inside the V4 envelope so nothing else differs. +fn legacy_accounting() -> GroveVersion { + let mut version = GROVE_V4.clone(); + version + .bulk_append_tree_versions + .cost + .append_storage_accounting = 0; + version + .commitment_tree_versions + .cost + .frontier_save_storage_accounting = 0; + version +} + +/// Expected `(added, replaced)` difference, V4 minus legacy, for the append +/// that lands at `position` in a tree with `2^chunk_power` entries per epoch +/// and fixed `entry_len`-byte entries; `with_frontier` for a commitment +/// tree, which also rewrites its frontier. +fn expected_delta( + position: u64, + chunk_power: u8, + entry_len: u32, + with_frontier: bool, +) -> (i64, i64) { + let epoch = 1u64 << chunk_power; + let mut added: i64 = 0; + let mut replaced: i64 = 0; + + // The entry's chunk-blob share, charged at every append. + added += entry_len as i64; + + if position % epoch == epoch - 1 { + // Compacting append: the overflow entry goes straight into the + // blob (no slot write). Legacy charges the whole blob as added; V4 + // reports the epoch's entry bytes as replaced and only the framing + // (identical on both sides) as added. + let entry_bytes = (epoch * entry_len as u64) as i64; + added -= entry_bytes; + replaced += entry_bytes; + } else if position >= epoch { + // Slot rewrite: legacy charges key + value as new; V4 replaces the + // value (same size: nothing added) and does not charge the key. + added -= (SLOT_KEY_PAID + paid(entry_len)) as i64; + replaced += paid(entry_len) as i64; + } + // else: a fresh slot is new storage on both sides. + + if with_frontier && position > 0 { + // Legacy: key + whole frontier added on every save. V4: replaces the + // frontier loaded at open, adds growth only. + let new = paid(frontier_len(position)); + let old = paid(frontier_len(position - 1)); + added += new.saturating_sub(old) as i64 - (FRONTIER_KEY_PAID + new) as i64; + replaced += new.min(old) as i64; + } + // else: the first save creates the key on both sides. + + (added, replaced) +} + +fn delta(v4: &OperationCost, legacy: &OperationCost) -> (i64, i64) { + ( + v4.storage_cost.added_bytes as i64 - legacy.storage_cost.added_bytes as i64, + v4.storage_cost.replaced_bytes as i64 - legacy.storage_cost.replaced_bytes as i64, + ) +} + +/// Everything except the storage figures must agree: the accounting gates +/// move bytes between `added` and `replaced`, nothing else. +fn assert_only_storage_differs(v4: &OperationCost, legacy: &OperationCost, what: &str) { + assert_eq!(v4.seek_count, legacy.seek_count, "{what}: seek_count"); + assert_eq!( + v4.storage_loaded_bytes, legacy.storage_loaded_bytes, + "{what}: storage_loaded_bytes" + ); + assert_eq!( + v4.hash_node_calls, legacy.hash_node_calls, + "{what}: hash_node_calls" + ); + assert_eq!( + v4.sinsemilla_hash_calls, legacy.sinsemilla_hash_calls, + "{what}: sinsemilla_hash_calls" + ); + assert_eq!( + v4.storage_cost.removed_bytes, legacy.storage_cost.removed_bytes, + "{what}: removed_bytes" + ); +} + +// ── Commitment tree fixtures ────────────────────────────────────────── + +fn test_cmx(index: u32) -> [u8; 32] { + let mut bytes = [0u8; 32]; + bytes[..4].copy_from_slice(&index.to_le_bytes()); + bytes[31] &= 0x7f; + bytes +} + +fn test_ciphertext(index: u32) -> TransmittedNoteCiphertext { + let mut epk_bytes = [0u8; 32]; + epk_bytes[..4].copy_from_slice(&index.to_le_bytes()); + let mut enc_data = [0u8; 104]; + enc_data[..4].copy_from_slice(&index.to_le_bytes()); + let mut out_ciphertext = [0u8; 80]; + out_ciphertext[..4].copy_from_slice(&index.to_le_bytes()); + TransmittedNoteCiphertext::from_parts(epk_bytes, NoteBytesData(enc_data), out_ciphertext) +} + +fn rho(index: u32) -> [u8; 32] { + let mut rho = [0u8; 32]; + rho[..4].copy_from_slice(&index.to_le_bytes()); + rho[4] = 0xAA; + rho +} + +fn cv_net(index: u32) -> [u8; 32] { + let mut cv = [0u8; 32]; + cv[..4].copy_from_slice(&index.to_le_bytes()); + cv[4] = 0xCC; + cv +} + +fn ct_op(index: u32) -> QualifiedGroveDbOp { + QualifiedGroveDbOp::commitment_tree_insert_op_typed( + vec![b"pool".to_vec()], + test_cmx(index), + rho(index), + cv_net(index), + &test_ciphertext(index), + ) +} + +fn ct_db(chunk_power: u8, version: &GroveVersion) -> TempGroveDb { + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"pool", + Element::empty_commitment_tree(chunk_power).expect("valid chunk_power"), + None, + None, + version, + ) + .unwrap() + .expect("insert commitment tree"); + db +} + +fn ct_insert(db: &TempGroveDb, index: u32, version: &GroveVersion) -> OperationCost { + let ctx = db.commitment_tree_insert( + EMPTY_PATH, + b"pool", + test_cmx(index), + rho(index), + cv_net(index), + test_ciphertext(index), + None, + version, + ); + ctx.value.expect("commitment tree insert"); + ctx.cost +} + +fn root_hash(db: &TempGroveDb, version: &GroveVersion) -> [u8; 32] { + db.root_hash(None, version).unwrap().expect("root hash") +} + +fn assert_verifies(db: &TempGroveDb, version: &GroveVersion) { + let issues = db + .verify_grovedb(None, true, false, version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "integrity issues: {issues:?}"); +} + +// ── Tests ───────────────────────────────────────────────────────────── + +/// The frontier-size model the expectations rely on matches the real +/// serialization at every position used below. +#[test] +fn frontier_size_model_matches_serialization() { + let mut frontier = CommitmentFrontier::new(); + for position in 0..64u32 { + frontier + .append(test_cmx(position)) + .unwrap() + .expect("append to frontier"); + assert_eq!( + frontier.serialize().len() as u32, + frontier_len(position as u64), + "position {position}" + ); + } +} + +/// The gates are off in every released version and on in V4. +#[test] +fn accounting_gates_are_locked_before_v4() { + for version in [&GROVE_V1, &GROVE_V2, &GROVE_V3] { + assert_eq!( + version + .bulk_append_tree_versions + .cost + .append_storage_accounting, + 0 + ); + assert_eq!( + version + .commitment_tree_versions + .cost + .frontier_save_storage_accounting, + 0 + ); + } + assert_eq!( + GROVE_V4 + .bulk_append_tree_versions + .cost + .append_storage_accounting, + 1 + ); + assert_eq!( + GROVE_V4 + .commitment_tree_versions + .cost + .frontier_save_storage_accounting, + 1 + ); +} + +/// Commitment tree at chunk_power 4, one direct append per position across +/// two and a half epochs: every append's `(added, replaced)` moves by +/// exactly the model — fresh slots in epoch 1, slot rewrites from epoch 2, +/// the blob-as-replacement at positions 15, 31 and 47, and the frontier +/// rewrite (growth at `2^k - 1`, shrink at `2^k`, first save at 0) — while +/// nothing else in the cost changes and both trees stay byte-identical. +#[test] +fn commitment_tree_append_storage_accounting_matches_model_across_epochs() { + const CHUNK_POWER: u8 = 4; + let legacy = legacy_accounting(); + let legacy_db = ct_db(CHUNK_POWER, &legacy); + let v4_db = ct_db(CHUNK_POWER, &GROVE_V4); + + for position in 0..40u64 { + let legacy_cost = ct_insert(&legacy_db, position as u32, &legacy); + let v4_cost = ct_insert(&v4_db, position as u32, &GROVE_V4); + + assert_eq!( + delta(&v4_cost, &legacy_cost), + expected_delta(position, CHUNK_POWER, NOTE_ENTRY, true), + "position {position}: v4 {v4_cost:?}\nlegacy {legacy_cost:?}" + ); + assert_only_storage_differs(&v4_cost, &legacy_cost, &format!("position {position}")); + assert_eq!( + root_hash(&v4_db, &GROVE_V4), + root_hash(&legacy_db, &legacy), + "position {position}: the accounting must not touch stored bytes" + ); + } + assert_verifies(&v4_db, &GROVE_V4); + assert_verifies(&legacy_db, &legacy); +} + +/// The frontier rewrite in isolation, at chunk_power 11 where nothing +/// compacts: at `2^k - 1` the frontier gains an ommer (replaced at the old +/// size, 32 bytes added), at `2^k` it collapses to 74 bytes (replaced at the +/// new size, nothing credited), and in between it is replaced in full. +#[test] +fn frontier_rewrite_replaces_previous_size_and_adds_only_growth() { + const CHUNK_POWER: u8 = 11; + let legacy = legacy_accounting(); + let legacy_db = ct_db(CHUNK_POWER, &legacy); + let v4_db = ct_db(CHUNK_POWER, &GROVE_V4); + + for position in 0..34u64 { + let legacy_cost = ct_insert(&legacy_db, position as u32, &legacy); + let v4_cost = ct_insert(&v4_db, position as u32, &GROVE_V4); + let (d_added, d_replaced) = delta(&v4_cost, &legacy_cost); + + // Strip the (epoch-1, fresh-slot) share so only the frontier is left. + let frontier_added = d_added - NOTE_ENTRY as i64; + let frontier_replaced = d_replaced; + if position == 0 { + assert_eq!( + (frontier_added, frontier_replaced), + (0, 0), + "first save: new on both" + ); + continue; + } + let new = paid(frontier_len(position)); + let old = paid(frontier_len(position - 1)); + assert_eq!( + frontier_replaced, + new.min(old) as i64, + "position {position}: replaced the smaller of previous/new paid size" + ); + assert_eq!( + frontier_added, + new.saturating_sub(old) as i64 - (FRONTIER_KEY_PAID + new) as i64, + "position {position}: legacy key + full value added; v4 growth only" + ); + if position >= 4 && position.is_power_of_two() { + // popcount drops from k >= 2 to 1 (positions 1 and 2 keep it at 1). + assert!(new < old, "position {position} collapses the frontier"); + assert_eq!(new.saturating_sub(old), 0, "shrink is not credited"); + } else if (position + 1).is_power_of_two() { + // popcount rises by one: 32 more bytes (plus one more length byte + // when the varint widens, e.g. 106 -> 138 at position 7). + assert_eq!( + frontier_len(position) - frontier_len(position - 1), + 32, + "position {position} gains one ommer" + ); + assert!(new - old >= 32, "position {position}: growth is added"); + } + } +} + +/// The epoch boundary at Dash Platform's chunk_power 11: the compacting +/// append is no longer a ~630 KB `added_bytes` spike. V4 adds the note's +/// own 312 bytes plus a few hundred bytes of framing, and reports the +/// 2048 × 312 entry bytes the blob supersedes as replaced. +#[test] +fn epoch_boundary_at_chunk_power_11_is_not_an_added_bytes_spike() { + const CHUNK_POWER: u8 = 11; + const EPOCH: u32 = 1 << CHUNK_POWER as u32; + let legacy = legacy_accounting(); + let legacy_db = ct_db(CHUNK_POWER, &legacy); + let v4_db = ct_db(CHUNK_POWER, &GROVE_V4); + + // Seeding 2047 notes walks the dense buffer O(n^2); do both trees at + // once. + std::thread::scope(|scope| { + for (db, version) in [(&legacy_db, &legacy), (&v4_db, &GROVE_V4)] { + scope.spawn(move || { + let seed: Vec<_> = (0..EPOCH - 1).map(ct_op).collect(); + db.apply_batch(seed, None, None, version) + .unwrap() + .expect("seed to one before the boundary"); + }); + } + }); + + let legacy_cost = ct_insert(&legacy_db, EPOCH - 1, &legacy); + let v4_cost = ct_insert(&v4_db, EPOCH - 1, &GROVE_V4); + + let blob_entry_bytes = (EPOCH * NOTE_ENTRY) as u64; // 638_976 + assert!( + legacy_cost.storage_cost.added_bytes as u64 > blob_entry_bytes, + "legacy bills the whole blob as new storage: {legacy_cost:?}" + ); + assert!( + v4_cost.storage_cost.added_bytes < 2_000, + "v4 adds the note's share plus framing only: {v4_cost:?}" + ); + assert!( + v4_cost.storage_cost.replaced_bytes as u64 >= blob_entry_bytes, + "v4 reports the superseded entry bytes as replaced: {v4_cost:?}" + ); + assert_eq!( + delta(&v4_cost, &legacy_cost), + expected_delta((EPOCH - 1) as u64, CHUNK_POWER, NOTE_ENTRY, true) + ); + assert_only_storage_differs(&v4_cost, &legacy_cost, "boundary"); + assert_eq!(root_hash(&v4_db, &GROVE_V4), root_hash(&legacy_db, &legacy)); +} + +/// Over a whole epoch of steady state (epoch 2 at chunk_power 4) the V4 +/// `added_bytes` add up to the bytes that persist — one blob share per +/// note plus the blob framing and the MMR node — while the legacy figures +/// add up to that PLUS a second copy of every note (the slot rewrites) and +/// the blob: ≈ 2× the physical growth. +#[test] +fn added_bytes_over_an_epoch_match_physical_growth() { + const CHUNK_POWER: u8 = 4; + const EPOCH: u64 = 1 << CHUNK_POWER; + let legacy = legacy_accounting(); + let legacy_db = ct_db(CHUNK_POWER, &legacy); + let v4_db = ct_db(CHUNK_POWER, &GROVE_V4); + for position in 0..EPOCH { + ct_insert(&legacy_db, position as u32, &legacy); + ct_insert(&v4_db, position as u32, &GROVE_V4); + } + + let mut v4_added: u64 = 0; + let mut legacy_added: u64 = 0; + let mut expected_added_delta: i64 = 0; + for position in EPOCH..2 * EPOCH { + legacy_added += ct_insert(&legacy_db, position as u32, &legacy) + .storage_cost + .added_bytes as u64; + v4_added += ct_insert(&v4_db, position as u32, &GROVE_V4) + .storage_cost + .added_bytes as u64; + expected_added_delta += expected_delta(position, CHUNK_POWER, NOTE_ENTRY, true).0; + } + assert_eq!(v4_added as i64 - legacy_added as i64, expected_added_delta); + + // Legacy over the epoch: 15 slot rewrites (key + note) + the whole blob + // + 16 frontier saves (key + value) + the parent-Merk update; V4: 16 + // blob shares + blob framing + frontier growth + the parent-Merk update. + // Strip the common parts the model does not see (parent Merk, MMR node, + // frontier) by comparing the two sums: legacy carries an extra + // 15 × (35 + 314) + (blob - framing = 16 × 312) - 16 × 312 (shares) + // + frontier keys/values. + let slot_rewrites = 15 * (SLOT_KEY_PAID + paid(NOTE_ENTRY)) as u64; + assert!( + legacy_added - v4_added >= slot_rewrites, + "legacy charges every note twice over its life: legacy {legacy_added}, v4 {v4_added}" + ); +} + +/// An epoch boundary inside ONE batch on a fresh tree: every slot is +/// written twice in the same session, but a `StorageBatch` keeps one put +/// per key and the slot is new in committed storage, so it is charged once +/// as new — never as a rewrite of a value that was never committed. Both +/// blobs replace their epochs' prepaid bytes; the shares net out exactly. +#[test] +fn epoch_boundary_inside_one_batch_charges_slots_once_as_new() { + const CHUNK_POWER: u8 = 2; // epoch 4, capacity 3 + let legacy = legacy_accounting(); + let legacy_db = ct_db(CHUNK_POWER, &legacy); + let v4_db = ct_db(CHUNK_POWER, &GROVE_V4); + + let ops = || (0..8u32).map(ct_op).collect::>(); + let legacy_ctx = legacy_db.apply_batch(ops(), None, None, &legacy); + legacy_ctx.value.expect("legacy batch"); + let v4_ctx = v4_db.apply_batch(ops(), None, None, &GROVE_V4); + v4_ctx.value.expect("v4 batch"); + + // Eight shares (8 × 312) exactly cover the two blobs' entry bytes + // (2 × 4 × 312) that legacy bills as added and V4 as replaced; the + // three slots are new on both sides (committed storage held nothing); + // the single frontier save is the first (new on both sides). + let two_blobs_entry_bytes = 2 * 4 * NOTE_ENTRY as i64; + assert_eq!( + delta(&v4_ctx.cost, &legacy_ctx.cost), + (0, two_blobs_entry_bytes), + "v4 {:?}\nlegacy {:?}", + v4_ctx.cost, + legacy_ctx.cost + ); + assert_only_storage_differs(&v4_ctx.cost, &legacy_ctx.cost, "batch"); + assert_eq!(root_hash(&v4_db, &GROVE_V4), root_hash(&legacy_db, &legacy)); + assert_verifies(&v4_db, &GROVE_V4); +} + +/// A plain `BulkAppendTree` with VARIABLE-size values: a slot rewrite is +/// sized against the value the slot held in committed storage (growth +/// added, shrink not credited), the share is the value's own length, and a +/// variable-format blob still replaces exactly the entry bytes. +#[test] +fn bulk_append_tree_accounting_with_variable_size_values() { + const CHUNK_POWER: u8 = 2; // epoch 4, capacity 3 + let legacy = legacy_accounting(); + let make = |version: &GroveVersion| { + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"bulk", + Element::empty_bulk_append_tree(CHUNK_POWER).expect("valid chunk_power"), + None, + None, + version, + ) + .unwrap() + .expect("insert bulk append tree"); + db + }; + let legacy_db = make(&legacy); + let v4_db = make(&GROVE_V4); + + // Epoch 1 fixed, epoch 2 variable, epoch 3 fixed again. + let sizes: [u32; 12] = [8, 8, 8, 8, 8, 16, 4, 8, 8, 8, 8, 8]; + // Committed value length per slot. + let mut slots: [Option; 3] = [None; 3]; + let mut epoch_bytes: u32 = 0; + + for (position, &len) in sizes.iter().enumerate() { + let value = vec![position as u8 + 1; len as usize]; + let legacy_ctx = legacy_db.bulk_append(EMPTY_PATH, b"bulk", value.clone(), None, &legacy); + legacy_ctx.value.expect("legacy append"); + let v4_ctx = v4_db.bulk_append(EMPTY_PATH, b"bulk", value, None, &GROVE_V4); + v4_ctx.value.expect("v4 append"); + + let mut added: i64 = len as i64; // the share + let mut replaced: i64 = 0; + epoch_bytes += len; + if position % 4 == 3 { + // Compaction: the blob replaces the epoch's entry bytes. + added -= epoch_bytes as i64; + replaced += epoch_bytes as i64; + epoch_bytes = 0; + } else { + let slot = position % 4; + if let Some(previous) = slots[slot] { + added += paid(len).saturating_sub(paid(previous)) as i64 + - (SLOT_KEY_PAID + paid(len)) as i64; + replaced += paid(len).min(paid(previous)) as i64; + } + slots[slot] = Some(len); + } + assert_eq!( + delta(&v4_ctx.cost, &legacy_ctx.cost), + (added, replaced), + "position {position} (len {len}): v4 {:?}\nlegacy {:?}", + v4_ctx.cost, + legacy_ctx.cost + ); + assert_only_storage_differs( + &v4_ctx.cost, + &legacy_ctx.cost, + &format!("position {position}"), + ); + } + assert_eq!(root_hash(&v4_db, &GROVE_V4), root_hash(&legacy_db, &legacy)); + assert_verifies(&v4_db, &GROVE_V4); +} + +/// `PrivateDocumentStore` (fixed entry size) follows the same model. Its +/// append propagates the dense tree's costs, so V4 also bills the read that +/// sizes the slot rewrite: one seek per buffered append, loading the +/// committed entry from epoch 2 on. +#[test] +fn private_document_store_accounting_matches_model() { + const CHUNK_POWER: u8 = 2; // epoch 4 + const ENTRY: u32 = 24; + let legacy = legacy_accounting(); + let make = |version: &GroveVersion| { + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"docs", + Element::empty_private_document_store(ENTRY, CHUNK_POWER).expect("valid config"), + None, + None, + version, + ) + .unwrap() + .expect("insert private document store"); + db + }; + let legacy_db = make(&legacy); + let v4_db = make(&GROVE_V4); + + for position in 0..12u64 { + let entry = vec![position as u8 + 1; ENTRY as usize]; + let legacy_ctx = legacy_db.private_document_store_insert( + EMPTY_PATH, + b"docs", + entry.clone(), + None, + &legacy, + ); + legacy_ctx.value.expect("legacy append"); + let v4_ctx = + v4_db.private_document_store_insert(EMPTY_PATH, b"docs", entry, None, &GROVE_V4); + v4_ctx.value.expect("v4 append"); + + assert_eq!( + delta(&v4_ctx.cost, &legacy_ctx.cost), + expected_delta(position, CHUNK_POWER, ENTRY, false), + "position {position}: v4 {:?}\nlegacy {:?}", + v4_ctx.cost, + legacy_ctx.cost + ); + let compacting = position % 4 == 3; + let (extra_seeks, extra_loaded) = if compacting { + (0, 0) + } else if position >= 4 { + (1, ENTRY as u64) + } else { + (1, 0) + }; + assert_eq!( + v4_ctx.cost.seek_count, + legacy_ctx.cost.seek_count + extra_seeks, + "position {position}: the slot read before a buffered write" + ); + assert_eq!( + v4_ctx.cost.storage_loaded_bytes, + legacy_ctx.cost.storage_loaded_bytes + extra_loaded, + "position {position}: the committed entry loaded to size the rewrite" + ); + assert_eq!(v4_ctx.cost.hash_node_calls, legacy_ctx.cost.hash_node_calls); + } + assert_eq!(root_hash(&v4_db, &GROVE_V4), root_hash(&legacy_db, &legacy)); + assert_verifies(&v4_db, &GROVE_V4); +} + +/// The standalone `MmrTree` and dense tree never rewrite a key (every +/// append lands on a fresh position), so their accounting is untouched: +/// identical costs with the gates on and off. +#[test] +fn standalone_mmr_and_dense_trees_are_unaffected() { + let legacy = legacy_accounting(); + let make = |version: &GroveVersion| { + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"mmr", + Element::empty_mmr_tree(), + None, + None, + version, + ) + .unwrap() + .expect("insert mmr tree"); + db.insert( + EMPTY_PATH, + b"dense", + Element::empty_dense_tree(3), + None, + None, + version, + ) + .unwrap() + .expect("insert dense tree"); + db + }; + let legacy_db = make(&legacy); + let v4_db = make(&GROVE_V4); + for i in 0..6u8 { + let value = vec![i + 1; 10 + i as usize]; + let a = legacy_db + .mmr_tree_append(EMPTY_PATH, b"mmr", value.clone(), None, &legacy) + .cost; + let b = v4_db + .mmr_tree_append(EMPTY_PATH, b"mmr", value.clone(), None, &GROVE_V4) + .cost; + assert_eq!(a, b, "mmr append {i}"); + let a = legacy_db + .dense_tree_insert(EMPTY_PATH, b"dense", value.clone(), None, &legacy) + .cost; + let b = v4_db + .dense_tree_insert(EMPTY_PATH, b"dense", value, None, &GROVE_V4) + .cost; + assert_eq!(a, b, "dense insert {i}"); + } +} diff --git a/grovedb/src/tests/commitment_tree_cost_bound_tests.rs b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs index d95ec51b3..0391d2268 100644 --- a/grovedb/src/tests/commitment_tree_cost_bound_tests.rs +++ b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs @@ -330,9 +330,17 @@ fn test_commitment_tree_insert_declared_chunk_power_tightens_estimate() { value.expect("compaction append should succeed"); // Far tighter than the worst-case physical-ceiling assumption - // (2^4 vs 2^16 epoch)... + // (2^4 vs 2^16 epoch). The epoch-scaled storage term is the compaction + // blob, which the V4 accounting reports as a replacement of the epoch's + // prepaid entry bytes (issue #822) — so it is `replaced_bytes` that + // carries the scale; `added_bytes` is per-note and epoch-independent. assert!( - declared.storage_cost.added_bytes < worst.storage_cost.added_bytes / 100, + declared.storage_cost.replaced_bytes < worst.storage_cost.replaced_bytes / 100, + "declared estimate should be far tighter than the physical-ceiling worst case; declared \ + {declared:?}\nworst {worst:?}", + ); + assert!( + declared.hash_node_calls < worst.hash_node_calls / 100, "declared estimate should be far tighter than the physical-ceiling worst case; declared \ {declared:?}\nworst {worst:?}", ); diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index 027b0d37a..b5c5e63d7 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -11,6 +11,7 @@ mod aggregate_count_and_sum_query_tests; mod aggregate_count_query_tests; mod aggregate_sum_carrier_query_tests; mod aggregate_sum_query_tests; +mod append_storage_accounting_tests; mod batch_coverage_tests; mod batch_delete_tree_tests; mod batch_indexed_fresh_create_tests; diff --git a/storage/src/rocksdb_storage/storage_context/context_tx.rs b/storage/src/rocksdb_storage/storage_context/context_tx.rs index 64ea3d943..36ee06a7b 100644 --- a/storage/src/rocksdb_storage/storage_context/context_tx.rs +++ b/storage/src/rocksdb_storage/storage_context/context_tx.rs @@ -33,6 +33,7 @@ use grovedb_costs::{ cost_return_on_error, storage_cost::key_value_cost::KeyValueStorageCost, ChildrenSizesWithIsSumTree, CostResult, CostsExt, OperationCost, }; +use integer_encoding::VarInt; use rocksdb::{ColumnFamily, DBRawIteratorWithThreadMode}; use super::{batch::PrefixedMultiContextBatchPart, make_prefixed_key, PrefixedRocksDbRawIterator}; @@ -136,12 +137,23 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbTransactionContext<'db> { .wrap_with_cost(OperationCost::default()) } }; - batch.put( - make_prefixed_key(&self.prefix, key), - value.to_vec(), - children_sizes, - cost_info, - ); + let prefixed_key = make_prefixed_key(&self.prefix, key); + + // Same contract as the `Batch` implementations: a caller describing a + // NEW node supplies the key cost without the path prefix — it does + // not know the prefix — and this layer, which does, completes it. + // Left incomplete, the commit's `verify_key_storage_cost` would + // reject the put. An update (`new_node: false`) is not verified on + // the key and is passed through unchanged. + let cost_info = cost_info.map(|mut key_value_storage_cost| { + if key_value_storage_cost.new_node { + key_value_storage_cost.key_storage_cost.added_bytes += + (prefixed_key.len() + prefixed_key.len().required_space()) as u32; + } + key_value_storage_cost + }); + + batch.put(prefixed_key, value.to_vec(), children_sizes, cost_info); Ok(()).wrap_with_cost(OperationCost::default()) } diff --git a/storage/src/rocksdb_storage/tests.rs b/storage/src/rocksdb_storage/tests.rs index 44e0079f6..3d8b2c6c1 100644 --- a/storage/src/rocksdb_storage/tests.rs +++ b/storage/src/rocksdb_storage/tests.rs @@ -984,6 +984,91 @@ mod batch_transaction { ); } + /// A direct `put` through the transactional context that describes a + /// NEW node must have its key cost completed with the path prefix by the + /// context — the caller cannot know the prefix — exactly as the `Batch` + /// implementations do. An update (`new_node: false`) is passed through + /// unchanged. Both must then survive the commit-time verification. + #[test] + fn test_transactional_put_completes_new_node_key_cost() { + use grovedb_costs::storage_cost::{ + key_value_cost::KeyValueStorageCost, removal::StorageRemovedBytes::NoStorageRemoval, + StorageCost, + }; + use integer_encoding::VarInt; + + let storage = TempStorage::new(); + let transaction = storage.start_transaction(); + let batch = StorageBatch::new(); + let context = storage + .get_transactional_storage_context( + [b"ayya"].as_ref().into(), + Some(&batch), + &transaction, + ) + .unwrap(); + + // A new node: key cost supplied WITHOUT the prefix (zero), value cost + // split into a replaced and an added part. + let value = vec![7u8; 100]; + let paid_value = value.len() as u32 + value.len().required_space() as u32; + context + .put( + b"new", + &value, + None, + Some(KeyValueStorageCost { + key_storage_cost: StorageCost::default(), + value_storage_cost: StorageCost { + added_bytes: 11, + replaced_bytes: paid_value - 11, + removed_bytes: NoStorageRemoval, + }, + new_node: true, + needs_value_verification: true, + }), + ) + .unwrap() + .expect("put new node"); + + // An update: key cost zero, value fully replaced. + context + .put( + b"old", + &value, + None, + Some(KeyValueStorageCost { + key_storage_cost: StorageCost::default(), + value_storage_cost: StorageCost { + added_bytes: 0, + replaced_bytes: paid_value, + removed_bytes: NoStorageRemoval, + }, + new_node: false, + needs_value_verification: true, + }), + ) + .unwrap() + .expect("put update"); + + let commit = storage.commit_multi_context_batch(batch, Some(&transaction)); + commit.value.expect("commit must pass cost verification"); + let cost = commit.cost; + + // prefix (32) + "new" (3) = 35, + 1 byte of length. + let new_key_paid = 32 + 3 + 1; + assert_eq!( + cost.storage_cost.added_bytes, + new_key_paid + 11, + "new node: prefixed key + the added part of the value" + ); + assert_eq!( + cost.storage_cost.replaced_bytes, + (paid_value - 11) + paid_value, + "replaced parts of both values; no key cost for the update" + ); + } + #[test] fn test_db_batch_in_transaction_merged_into_context_batch() { let storage = TempStorage::new(); From 546e5768438898e1b0061ad85d5f706d714a2b31 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 22 Aug 2026 08:58:35 +0700 Subject: [PATCH 2/4] chore(bench): pass grove version to CommitmentTree::commit_mmr in seeding bench The commit_mmr signature change (#822) missed the criterion bench, which only the all-targets CI check compiles. Co-Authored-By: Claude Fable 5 --- grovedb-commitment-tree/benches/seeding.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grovedb-commitment-tree/benches/seeding.rs b/grovedb-commitment-tree/benches/seeding.rs index 3e6356961..1d3a201a4 100644 --- a/grovedb-commitment-tree/benches/seeding.rs +++ b/grovedb-commitment-tree/benches/seeding.rs @@ -94,7 +94,7 @@ fn main() { // Flush the MMR overlay into the storage batch now that we're done // appending. Must come before dropping `ct` (which would lose the overlay) // and before committing the storage batch (which writes to disk). - ct.commit_mmr().expect("commit_mmr"); + ct.commit_mmr(GroveVersion::latest()).expect("commit_mmr"); let seed_elapsed = t_seed.elapsed(); From 5fba355d8bf5f79e660221525eb8b3b66b30d266 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 22 Aug 2026 08:59:28 +0700 Subject: [PATCH 3/4] fix(bulk): gate append storage accounting behind the storage feature The new cost dispatch imported storage-only types (SlotWriteAccounting, LeafValueStorageCost) unconditionally, which broke grovedb's --no-default-features --features verify build (CI's "Check verify feature" step). The dispatch only serves the storage-backed append paths, so it is compiled with them. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/cost/mod.rs | 5 +++++ grovedb-bulk-append-tree/src/cost/v0.rs | 7 ++++++- grovedb-bulk-append-tree/src/cost/v1.rs | 10 +++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/grovedb-bulk-append-tree/src/cost/mod.rs b/grovedb-bulk-append-tree/src/cost/mod.rs index 144d00c82..180e2e536 100644 --- a/grovedb-bulk-append-tree/src/cost/mod.rs +++ b/grovedb-bulk-append-tree/src/cost/mod.rs @@ -22,7 +22,9 @@ mod v0; mod v1; +#[cfg(feature = "storage")] use grovedb_dense_fixed_sized_merkle_tree::SlotWriteAccounting; +#[cfg(feature = "storage")] use grovedb_merkle_mountain_range::LeafValueStorageCost; use grovedb_version::{error::GroveVersionError, version::GroveVersion}; @@ -30,6 +32,7 @@ use crate::BulkAppendError; /// How an append's data-storage writes are reported to the storage cost /// layer. Selected per grove version by [`append_storage_accounting`]. +#[cfg(feature = "storage")] #[derive(Clone, Copy)] pub(crate) struct AppendStorageAccounting { /// How the dense-buffer slot write is reported. @@ -41,6 +44,7 @@ pub(crate) struct AppendStorageAccounting { prepay_chunk_share: bool, } +#[cfg(feature = "storage")] impl AppendStorageAccounting { /// The entry's chunk-blob share to charge as added storage at its append: /// its own bytes, or nothing when the blob is charged in full at @@ -55,6 +59,7 @@ impl AppendStorageAccounting { } /// The storage accounting an append uses under `grove_version`. +#[cfg(feature = "storage")] pub(crate) fn append_storage_accounting( grove_version: &GroveVersion, ) -> Result { diff --git a/grovedb-bulk-append-tree/src/cost/v0.rs b/grovedb-bulk-append-tree/src/cost/v0.rs index 576d55385..841617dfd 100644 --- a/grovedb-bulk-append-tree/src/cost/v0.rs +++ b/grovedb-bulk-append-tree/src/cost/v0.rs @@ -7,9 +7,13 @@ //! Locked: GROVE_V1..V3 are released and CommitmentTree has been billing this //! figure on mainnet. +#[cfg(feature = "storage")] use grovedb_dense_fixed_sized_merkle_tree::SlotWriteAccounting; -use grovedb_merkle_mountain_range::{hash_count_for_push, LeafValueStorageCost}; +use grovedb_merkle_mountain_range::hash_count_for_push; +#[cfg(feature = "storage")] +use grovedb_merkle_mountain_range::LeafValueStorageCost; +#[cfg(feature = "storage")] use super::AppendStorageAccounting; pub(super) fn compaction_hash_count(leaf_count: u64) -> u32 { @@ -23,6 +27,7 @@ pub(super) fn compaction_hash_count(leaf_count: u64) -> u32 { /// /// Locked: GROVE_V1..V3 are released and the shielded pool has been billed /// this way on mainnet. +#[cfg(feature = "storage")] pub(super) fn append_storage_accounting() -> AppendStorageAccounting { AppendStorageAccounting { slot_write: SlotWriteAccounting::AsNew, diff --git a/grovedb-bulk-append-tree/src/cost/v1.rs b/grovedb-bulk-append-tree/src/cost/v1.rs index a3472c2a8..6a51fe3c2 100644 --- a/grovedb-bulk-append-tree/src/cost/v1.rs +++ b/grovedb-bulk-append-tree/src/cost/v1.rs @@ -2,12 +2,15 @@ //! //! Adds the peak-bagging merges [`v0`](super::v0) omitted. Used from GROVE_V4. +#[cfg(feature = "storage")] use grovedb_dense_fixed_sized_merkle_tree::SlotWriteAccounting; -use grovedb_merkle_mountain_range::{ - hash_count_for_push, hash_count_for_root_bagging, LeafValueStorageCost, -}; +#[cfg(feature = "storage")] +use grovedb_merkle_mountain_range::LeafValueStorageCost; +use grovedb_merkle_mountain_range::{hash_count_for_push, hash_count_for_root_bagging}; +#[cfg(feature = "storage")] use super::AppendStorageAccounting; +#[cfg(feature = "storage")] use crate::chunk::chunk_blob_entry_bytes; pub(super) fn compaction_hash_count(leaf_count: u64, mmr_size_after_push: u64) -> u32 { @@ -29,6 +32,7 @@ pub(super) fn compaction_hash_count(leaf_count: u64, mmr_size_after_push: u64) - /// - The compaction blob is reported as a replacement of the entry bytes it /// supersedes (all prepaid), leaving only its framing — and the MMR /// internal nodes — as added storage. +#[cfg(feature = "storage")] pub(super) fn append_storage_accounting() -> AppendStorageAccounting { AppendStorageAccounting { slot_write: SlotWriteAccounting::AgainstCommitted, From 6376ae82cfd02301a65794212d464a2daa2baa3d Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 22 Aug 2026 09:05:27 +0700 Subject: [PATCH 4/4] test: cover the slot-read fault and unknown frontier version arms Codecov flagged the dense tree's failing-read branch before an overwrite and CommitmentTree::save's unknown-version arm; the dense test context gains read fault injection to reach the former. Co-Authored-By: Claude Fable 5 --- .../src/commitment_tree/tests.rs | 24 +++++++++++++++++++ .../src/test_utils.rs | 13 +++++++++- .../src/tests.rs | 18 ++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/grovedb-commitment-tree/src/commitment_tree/tests.rs b/grovedb-commitment-tree/src/commitment_tree/tests.rs index 8025c6106..9123ef137 100644 --- a/grovedb-commitment-tree/src/commitment_tree/tests.rs +++ b/grovedb-commitment-tree/src/commitment_tree/tests.rs @@ -617,6 +617,30 @@ mod storage_tests { ); } + /// An unknown frontier-save accounting version is rejected before any + /// write. + #[test] + fn test_save_rejects_unknown_accounting_version() { + let bulk_tree = BulkAppendTree::new(TEST_CHUNK_POWER, FailingDataStorageContext) + .expect("bulk tree new should succeed"); + let ct: CommitmentTree<_, DashMemo> = CommitmentTree { + frontier: CommitmentFrontier::new(), + bulk_tree, + persisted_frontier_len: Some(74), + _memo: PhantomData, + }; + let mut bad = GroveVersion::latest().clone(); + bad.commitment_tree_versions + .cost + .frontier_save_storage_accounting = 99; + let result = ct.save(&bad); + assert!( + matches!(result.value, Err(CommitmentTreeError::VersionError(_))), + "unknown version must be rejected, not written: {:?}", + result.value + ); + } + #[test] fn test_save_storage_error_surfaces() { // FailingDataStorageContext.get fails, so open() would fail. diff --git a/grovedb-dense-fixed-sized-merkle-tree/src/test_utils.rs b/grovedb-dense-fixed-sized-merkle-tree/src/test_utils.rs index 8912910b7..591982db7 100644 --- a/grovedb-dense-fixed-sized-merkle-tree/src/test_utils.rs +++ b/grovedb-dense-fixed-sized-merkle-tree/src/test_utils.rs @@ -1,6 +1,9 @@ //! Test utilities: in-memory StorageContext implementations. -use std::{cell::RefCell, collections::HashMap}; +use std::{ + cell::{Cell, RefCell}, + collections::HashMap, +}; use grovedb_costs::{ storage_cost::key_value_cost::KeyValueStorageCost, ChildrenSizesWithIsSumTree, CostContext, @@ -18,6 +21,8 @@ pub(crate) struct MemStorageContext { pub data: RefCell, Vec>>, /// Every data `put` in order, with the cost information it carried. pub puts: RefCell, Option)>>, + /// When set, every `get` returns a storage error (fault injection). + pub fail_get: Cell, } impl MemStorageContext { @@ -31,6 +36,12 @@ impl<'db> StorageContext<'db> for MemStorageContext { type RawIterator = MemRawIterator; fn get>(&self, key: K) -> CostResult>, grovedb_storage::Error> { + if self.fail_get.get() { + return Err(grovedb_storage::Error::StorageError( + "injected read failure".to_string(), + )) + .wrap_with_cost(OperationCost::default()); + } Ok(self.data.borrow().get(key.as_ref()).cloned()).wrap_with_cost(OperationCost::default()) } diff --git a/grovedb-dense-fixed-sized-merkle-tree/src/tests.rs b/grovedb-dense-fixed-sized-merkle-tree/src/tests.rs index ac3b94bfd..c7d7215a3 100644 --- a/grovedb-dense-fixed-sized-merkle-tree/src/tests.rs +++ b/grovedb-dense-fixed-sized-merkle-tree/src/tests.rs @@ -1163,4 +1163,22 @@ mod slot_write_accounting { assert_eq!(c.value_storage_cost.added_bytes, 0); assert_eq!(c.value_storage_cost.removed_bytes, NoStorageRemoval); } + + /// A storage fault on the read that sizes the rewrite surfaces as an + /// error, and nothing is written. + #[test] + fn against_committed_surfaces_a_failing_read() { + let mut tree = DenseFixedSizedMerkleTree::new(2, MemStorageContext::new()).unwrap(); + tree.storage.fail_get.set(true); + let err = tree + .try_insert_no_root_with_accounting(&[1u8; 8], SlotWriteAccounting::AgainstCommitted) + .unwrap() + .expect_err("the read before the overwrite failed"); + assert!( + err.to_string().contains("before overwrite"), + "error should name the read: {err}" + ); + assert!(tree.storage.puts.borrow().is_empty(), "nothing written"); + assert_eq!(tree.count(), 0); + } }