From ac2b1db356cc2aa8f615372e96cabb75f8228145 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 22 Aug 2026 08:16:36 +0700 Subject: [PATCH] fix: report the append-only family's write churn as replaced bytes, not new storage Every data-storage write of CommitmentTree / BulkAppendTree (and so PrivateDocumentStore) was issued with no cost info, so the commit path charged key + value as added bytes: the compaction's chunk blob was billed as ~epoch_size x entry of NEW storage on the one append that compacts, although it supersedes the buffer entries it was built from (whose position keys the next epoch simply overwrites); buffer writes from the second epoch on were billed as new although they overwrite a stale slot; and the frontier was billed in full on every append although it is one value rewritten in place. Measured on Dash Platform's shielded notes: ~1,200 B metered per note against ~550 B that persist, with a ~630 KB spike on one arbitrary transaction per epoch (issue #822). Storage accounting v1, gated by the new bulk_append_tree_versions.cost.storage_accounting slot (0 on V1..V3, 1 on V4), charges each entry's permanent bytes once, as added, by the append that creates it (entry + varint + an amortized chunk-framing share), and reports the churn as replaced: the chunk blob as replacement of the pre-paid buffer bytes (the compacting append pays its own entry, which never enters the buffer, explicitly), the frontier rewrite as replacement of its previous serialization with only growth as added. MMR nodes are staged in an overlay and written at commit_mmr, so the blob's report travels with its staged position until then. The transactional context's direct put now adds the prefixed-key bytes for a new key exactly as the Merk batch path does (nothing outside Merk passed cost info before). The V4 CommitmentTreeInsert estimator and the PrivateDocumentStoreInsert arm follow: the epoch scale moves to a replaced-bytes term and the added term no longer scales with the epoch; the estimate-dominates property tests hold at every position including the compaction boundary. Stored bytes, chunks, roots and proofs are identical under both versions, which a test pins by comparing V3 and V4 roots after the same appends. Fixes #822 Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/Cargo.toml | 1 + grovedb-bulk-append-tree/src/cost/mod.rs | 97 ++++++++ grovedb-bulk-append-tree/src/tree/append.rs | 101 ++++++-- grovedb-bulk-append-tree/src/tree/mod.rs | 7 + grovedb-commitment-tree/Cargo.toml | 1 + .../src/commitment_tree/mod.rs | 79 ++++++- .../src/commitment_tree/tests.rs | 29 ++- grovedb-commitment-tree/src/error.rs | 4 + .../src/tree.rs | 43 +++- .../src/storage_adapter.rs | 37 ++- grovedb-version/src/tests.rs | 18 ++ .../src/version/bulk_append_tree_versions.rs | 24 ++ grovedb-version/src/version/v1.rs | 1 + grovedb-version/src/version/v2.rs | 1 + grovedb-version/src/version/v3.rs | 1 + grovedb-version/src/version/v4.rs | 1 + .../estimated_costs/average_case_costs.rs | 33 ++- grovedb/src/batch/estimated_costs/mod.rs | 54 +++-- .../batch/estimated_costs/worst_case_costs.rs | 15 +- grovedb/src/operations/commitment_tree.rs | 10 +- .../append_only_storage_accounting_tests.rs | 219 ++++++++++++++++++ .../tests/commitment_tree_cost_bound_tests.rs | 6 +- grovedb/src/tests/mod.rs | 1 + .../storage_context/context_tx.rs | 21 +- 24 files changed, 725 insertions(+), 79 deletions(-) create mode 100644 grovedb/src/tests/append_only_storage_accounting_tests.rs diff --git a/grovedb-bulk-append-tree/Cargo.toml b/grovedb-bulk-append-tree/Cargo.toml index 4583b0f91..6da6a3ac9 100644 --- a/grovedb-bulk-append-tree/Cargo.toml +++ b/grovedb-bulk-append-tree/Cargo.toml @@ -27,6 +27,7 @@ grovedb-merkle-mountain-range = { version = "5.0.1", path = "../grovedb-merkle-m grovedb-dense-fixed-sized-merkle-tree = { version = "5.0.1", path = "../grovedb-dense-fixed-sized-merkle-tree", default-features = false } grovedb-query = { version = "5.0.1", path = "../grovedb-query" } grovedb-costs = { version = "5.0.1", path = "../costs" } +integer-encoding = { workspace = true } grovedb-version = { version = "5.0.1", path = "../grovedb-version" } grovedb-storage = { version = "5.0.1", path = "../storage", optional = true } blake3 = { workspace = true } diff --git a/grovedb-bulk-append-tree/src/cost/mod.rs b/grovedb-bulk-append-tree/src/cost/mod.rs index 7b3a412db..d9f8f4b1e 100644 --- a/grovedb-bulk-append-tree/src/cost/mod.rs +++ b/grovedb-bulk-append-tree/src/cost/mod.rs @@ -9,10 +9,107 @@ mod v0; mod v1; +use grovedb_costs::storage_cost::{key_value_cost::KeyValueStorageCost, StorageCost}; use grovedb_version::{error::GroveVersionError, version::GroveVersion}; +use integer_encoding::VarInt; use crate::BulkAppendError; +/// Per-entry framing the chunk blob adds over the raw entry bytes (length +/// prefix and per-entry overhead), charged to each append as its amortized +/// share of the blob under storage accounting v1 — so that a compacting +/// append's blob put can be reported as replacement of bytes that were +/// already paid for. Kept equal to the estimator's per-entry chunk overhead +/// so the bound and the actual agree on what is pre-paid. +pub const CHUNK_ENTRY_AMORTIZED_BYTES: u32 = 16; + +/// Which storage-accounting report the append-only writes use; see +/// `BulkAppendTreeCostVersions::storage_accounting`. +pub(crate) fn storage_accounting_version( + grove_version: &GroveVersion, +) -> Result { + match grove_version + .bulk_append_tree_versions + .cost + .storage_accounting + { + v @ (0 | 1) => Ok(v), + version => Err(BulkAppendError::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "BulkAppendTree storage accounting".to_string(), + known_versions: vec![0, 1], + received: version, + } + .to_string(), + )), + } +} + +/// Cost info for writing one entry into the dense buffer under storage +/// accounting v1: the entry's bytes (plus the value-length varint the +/// storage layer would have counted) and its amortized share of the chunk +/// blob's framing are charged as **added** — each entry's permanent bytes, +/// paid once, by the append that creates it. `new_key` is whether this +/// buffer position has never been written (first epoch); from the second +/// epoch on the position key already exists and only the value is new. +pub(crate) fn buffer_entry_cost_info(value_len: u32, new_key: bool) -> KeyValueStorageCost { + let added = entry_charge_bytes(value_len); + KeyValueStorageCost { + // The key's own bytes are appended by the storage context when + // `new_node` (it alone knows the prefix). + key_storage_cost: StorageCost::default(), + value_storage_cost: StorageCost { + added_bytes: added, + replaced_bytes: 0, + removed_bytes: Default::default(), + }, + new_node: new_key, + needs_value_verification: false, + } +} + +/// Cost info for writing the chunk blob a compaction produces, under +/// storage accounting v1. +/// +/// The blob supersedes the buffer entries it was built from; their bytes +/// (entry + varint + amortized framing share) were already paid as added +/// by the appends that wrote them — `pre_paid_bytes` — so they are +/// reported as **replaced**. The value that triggered the compaction never +/// entered the buffer (it goes straight into the blob), so this append +/// pays for it here, as added, exactly as a buffered append would have +/// (`compacting_entry_bytes`, the same entry + varint + framing share). +/// Any residual the blob carries beyond the pre-paid bytes (its own +/// header; nothing, when the compact fixed-size format undercuts the +/// per-entry framing pre-payment) is added too. The blob's key is a new +/// MMR position. +pub(crate) fn chunk_blob_cost_info( + blob_len: u32, + pre_paid_bytes: u32, + compacting_entry_bytes: u32, +) -> KeyValueStorageCost { + let total = blob_len.saturating_add(blob_len.required_space() as u32); + let replaced = total.min(pre_paid_bytes); + let residual = total.saturating_sub(pre_paid_bytes); + KeyValueStorageCost { + key_storage_cost: StorageCost::default(), + value_storage_cost: StorageCost { + added_bytes: compacting_entry_bytes.saturating_add(residual), + replaced_bytes: replaced, + removed_bytes: Default::default(), + }, + new_node: true, + needs_value_verification: false, + } +} + +/// The bytes an append is charged (as added) for one buffered entry: +/// entry + value-length varint + its amortized chunk-framing share. +pub(crate) fn entry_charge_bytes(value_len: u32) -> u32 { + value_len + .saturating_add(value_len.required_space() as u32) + .saturating_add(CHUNK_ENTRY_AMORTIZED_BYTES) +} + /// Hashes to report for a compacting append. /// /// `leaf_count` is the MMR leaf count BEFORE the push (what diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index 89819e2a1..ff653e063 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -1,5 +1,6 @@ //! Append and compaction logic for BulkAppendTree. +use grovedb_costs::storage_cost::key_value_cost::KeyValueStorageCost; use grovedb_costs::{CostResult, CostsExt, OperationCost}; use grovedb_merkle_mountain_range::{mmr_size_to_leaf_count, MmrKeySize, MmrNode, MmrStore, MMR}; use grovedb_storage::StorageContext; @@ -9,7 +10,14 @@ use super::{ capacity_for_height, hash::compute_state_root, AppendNoStateRootResult, AppendResult, BulkAppendTree, }; -use crate::{chunk::serialize_chunk_blob, cost::compaction_hash_count, BulkAppendError}; +use crate::{ + chunk::serialize_chunk_blob, + cost::{ + buffer_entry_cost_info, chunk_blob_cost_info, compaction_hash_count, entry_charge_bytes, + storage_accounting_version, + }, + BulkAppendError, +}; impl<'db, S: StorageContext<'db>> BulkAppendTree { /// Create a new empty tree. @@ -23,6 +31,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { total_count: 0, dense_tree, mmr_overlay: Vec::new(), + pending_blob_cost_infos: std::collections::BTreeMap::new(), // Empty tree → empty MMR → zero root. last_mmr_root: Some([0u8; 32]), }) @@ -49,6 +58,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { total_count, dense_tree, mmr_overlay: Vec::new(), + pending_blob_cost_infos: std::collections::BTreeMap::new(), // Lazy: the restored MMR may not be readable until an append occurs, // so don't compute the root here. The first append fills the cache. last_mmr_root: None, @@ -100,9 +110,14 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { let global_position = self.total_count; // 1. Try to insert into the dense tree buffer. - let try_result = self.dense_tree.try_insert(value).unwrap().map_err(|e| { - BulkAppendError::StorageError(format!("dense tree insert failed: {}", e)) - })?; + let cost_info = self.buffer_entry_cost_info(value, grove_version)?; + let try_result = self + .dense_tree + .try_insert_with_cost_info(value, cost_info) + .unwrap() + .map_err(|e| { + BulkAppendError::StorageError(format!("dense tree insert failed: {}", e)) + })?; let compacted = match try_result { Some((_dense_root, _position)) => { @@ -162,9 +177,13 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { let mut cost = OperationCost::default(); let global_position = self.total_count; + let cost_info = match self.buffer_entry_cost_info(value, grove_version) { + Ok(c) => c, + Err(e) => return Err(e).wrap_with_cost(cost), + }; let try_result = match self .dense_tree - .try_insert_no_root(value) + .try_insert_no_root_with_cost_info(value, cost_info) .unwrap_add_cost(&mut cost) { Ok(r) => r, @@ -282,6 +301,29 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { Ok(compute_state_root(&mmr_root, &dense_root)).wrap_with_cost(cost) } + /// Storage cost info for the next buffer entry write, per the + /// version's storage accounting: `None` (v0) lets the storage layer + /// report key + value as added; v1 reports the entry's permanent bytes + /// plus its amortized blob-framing share as added, marking the + /// position key new only in the first epoch (later epochs overwrite a + /// stale slot at an existing key). + fn buffer_entry_cost_info( + &self, + value: &[u8], + grove_version: &GroveVersion, + ) -> Result, BulkAppendError> { + match storage_accounting_version(grove_version)? { + 0 => Ok(None), + _ => { + let first_epoch = self.total_count < self.epoch_size(); + Ok(Some(buffer_entry_cost_info( + value.len() as u32, + first_epoch, + ))) + } + } + } + /// Compact all dense tree entries plus a new value into a chunk blob /// and append to the chunk MMR. Resets the dense tree. /// Returns `(hash_count, mmr_root)`. @@ -341,6 +383,19 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { entries.push(value); } + // Under storage accounting v1 the blob is reported as replacement of + // the buffer bytes it supersedes: every BUFFERED entry's bytes plus + // its amortized framing share were charged as added by the append + // that wrote it. The value that triggers the compaction never enters + // the buffer — it goes straight into the blob — so it is not + // pre-paid and stays in the blob's added residual, which is how this + // append pays for its own entry. + let pre_paid_bytes: u32 = entries + .iter() + .map(|e| entry_charge_bytes(e.len() as u32)) + .fold(0u32, u32::saturating_add); + let compacting_entry_bytes = entry_charge_bytes(new_value.len() as u32); + // Add the new value that didn't fit entries.push(new_value.to_vec()); @@ -349,6 +404,15 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { Ok(b) => b, Err(e) => return Err(e).wrap_with_cost(cost), }; + let blob_cost_info = match storage_accounting_version(grove_version) { + Ok(0) => None, + Ok(_) => Some(chunk_blob_cost_info( + blob.len() as u32, + pre_paid_bytes, + compacting_entry_bytes, + )), + Err(e) => return Err(e).wrap_with_cost(cost), + }; // `MmrNode::leaf` hashes the blob eagerly. cost.hash_node_calls = cost.hash_node_calls.saturating_add(1); let leaf = MmrNode::leaf(blob); @@ -370,11 +434,19 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { MMR::new_with_overlay(mmr_size, &mmr_store, std::mem::take(&mut self.mmr_overlay)); let push_result = mmr.push(leaf, grove_version).unwrap_add_cost(&mut cost); - if let Err(e) = push_result { - // Restore overlay before returning error - self.mmr_overlay = mmr.batch.take_overlay(); - return Err(BulkAppendError::MmrError(format!("MMR push failed: {}", e))) - .wrap_with_cost(cost); + let leaf_pos = match push_result { + Ok(pos) => pos, + Err(e) => { + // Restore overlay before returning error + self.mmr_overlay = mmr.batch.take_overlay(); + return Err(BulkAppendError::MmrError(format!("MMR push failed: {}", e))) + .wrap_with_cost(cost); + } + }; + // The leaf is only staged here; it is written at `commit_mmr`, + // so its storage report travels with it until then. + if let Some(cost_info) = blob_cost_info { + self.pending_blob_cost_infos.insert(leaf_pos, cost_info); } let root_result = mmr.get_root(grove_version).unwrap_add_cost(&mut cost); @@ -459,16 +531,19 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { if self.mmr_overlay.is_empty() { return Ok(()); } - let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32); + let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32) + .with_put_cost_infos(std::mem::take(&mut self.pending_blob_cost_infos)); let mut mmr = MMR::new_with_overlay( self.mmr_size(), &mmr_store, std::mem::take(&mut self.mmr_overlay), ); if let Err(e) = mmr.commit().unwrap() { - // Restore overlay before returning error so retries/get_mmr_root - // still see the staged nodes. + // Restore overlay (and the unconsumed storage reports) before + // returning error so retries/get_mmr_root still see the staged + // nodes. self.mmr_overlay = mmr.batch.take_overlay(); + self.pending_blob_cost_infos = mmr_store.take_put_cost_infos(); return Err(BulkAppendError::MmrError(format!( "MMR commit failed: {}", e diff --git a/grovedb-bulk-append-tree/src/tree/mod.rs b/grovedb-bulk-append-tree/src/tree/mod.rs index bcea9a521..64e7d81db 100644 --- a/grovedb-bulk-append-tree/src/tree/mod.rs +++ b/grovedb-bulk-append-tree/src/tree/mod.rs @@ -20,6 +20,7 @@ pub use fetch::{BufferQueryResult, ChunkQueryResult}; #[cfg(all(test, feature = "storage"))] mod tests; +use grovedb_costs::storage_cost::key_value_cost::KeyValueStorageCost; use grovedb_dense_fixed_sized_merkle_tree::DenseFixedSizedMerkleTree; use grovedb_merkle_mountain_range::MmrNode; @@ -92,6 +93,12 @@ pub struct BulkAppendTree { /// lifetimes (compaction cycles) so that reads can find recently-pushed /// nodes without a storage round-trip. pub(crate) mmr_overlay: Vec<(u64, Vec)>, + /// Storage cost info for chunk-blob leaves staged in `mmr_overlay` but + /// not yet committed, keyed by MMR position. Under storage accounting + /// v1 a compaction's blob is reported as replacement of the buffer + /// bytes it supersedes; since MMR nodes are written at `commit_mmr`, + /// the report travels with the staged node until then. + pub(crate) pending_blob_cost_infos: std::collections::BTreeMap, /// Cached MMR root, refreshed only when a compaction mutates the MMR. /// /// The MMR is only touched on compaction (every `epoch_size` appends), so diff --git a/grovedb-commitment-tree/Cargo.toml b/grovedb-commitment-tree/Cargo.toml index caa1e7ed8..71e4a2481 100644 --- a/grovedb-commitment-tree/Cargo.toml +++ b/grovedb-commitment-tree/Cargo.toml @@ -22,6 +22,7 @@ incrementalmerkletree = "0.8" shardtree = { version = "0.6", optional = true } rusqlite = { version = "0.38", features = ["bundled"], optional = true } grovedb-costs = { version = "5.0.1", path = "../costs" } +integer-encoding = { workspace = true } grovedb-version = { version = "5.0.1", path = "../grovedb-version" } grovedb-storage = { version = "5.0.1", path = "../storage", optional = true } grovedb-bulk-append-tree = { version = "5.0.1", path = "../grovedb-bulk-append-tree", default-features = false } diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index 9f674e583..62e252042 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -12,9 +12,13 @@ use std::marker::PhantomData; use grovedb_bulk_append_tree::BulkAppendTree; -use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_costs::{ + storage_cost::{key_value_cost::KeyValueStorageCost, StorageCost}, + CostResult, CostsExt, OperationCost, +}; use grovedb_storage::StorageContext; -use grovedb_version::version::GroveVersion; +use grovedb_version::{error::GroveVersionError, version::GroveVersion}; +use integer_encoding::VarInt; use orchard::{ memo::{DashMemo, MemoSize}, note::TransmittedNoteCiphertext, @@ -28,6 +32,27 @@ mod tests; /// Key used to store the serialized commitment frontier in data storage. pub const COMMITMENT_TREE_DATA_KEY: &[u8] = b"__ct_data__"; +/// Cost info for persisting the frontier under storage accounting v1: the +/// previous serialization's bytes are replaced, only growth is added; the +/// first save of a tree that never stored a frontier is a new key. +fn frontier_save_cost_info(new_len: u32, previous_len: Option) -> KeyValueStorageCost { + let total = new_len.saturating_add(new_len.required_space() as u32); + let previous_total = previous_len + .map(|l| l.saturating_add(l.required_space() as u32)) + .unwrap_or(0); + let replaced = total.min(previous_total); + KeyValueStorageCost { + key_storage_cost: StorageCost::default(), + value_storage_cost: StorageCost { + added_bytes: total - replaced, + replaced_bytes: replaced, + removed_bytes: Default::default(), + }, + new_node: previous_len.is_none(), + needs_value_verification: false, + } +} + /// Result of appending to a [`CommitmentTree`]. #[derive(Debug, Clone)] pub struct CommitmentAppendResult { @@ -146,6 +171,11 @@ pub fn deserialize_ciphertext(data: &[u8]) -> Option { frontier: CommitmentFrontier, pub(crate) bulk_tree: BulkAppendTree, + /// Serialized length of the frontier as last persisted (loaded at open, + /// refreshed by `save`), or `None` when no frontier has been stored + /// yet. Lets `save` report the rewrite as replacement of the previous + /// serialization under storage accounting v1. + stored_frontier_len: Option, _memo: PhantomData, } @@ -170,6 +200,7 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { Ok(Self { frontier: CommitmentFrontier::new(), bulk_tree, + stored_frontier_len: None, _memo: PhantomData, }) } @@ -204,12 +235,12 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { .get(COMMITMENT_TREE_DATA_KEY) .unwrap_add_cost(&mut cost); - let frontier = match data { + let (frontier, stored_frontier_len) = match data { Ok(Some(bytes)) => match CommitmentFrontier::deserialize(&bytes) { - Ok(f) => f, + Ok(f) => (f, Some(bytes.len() as u32)), Err(e) => return Err(e).wrap_with_cost(cost), }, - Ok(None) => CommitmentFrontier::new(), + Ok(None) => (CommitmentFrontier::new(), None), Err(e) => { return Err(CommitmentTreeError::InvalidData(format!( "storage error loading frontier: {}", @@ -235,6 +266,7 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { Ok(Self { frontier, bulk_tree, + stored_frontier_len, _memo: PhantomData, }) .wrap_with_cost(cost) @@ -553,17 +585,48 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { } /// Persist the current frontier state to storage. - pub fn save(&self) -> CostResult<(), CommitmentTreeError> { + /// + /// The frontier is one value rewritten in place on every append. Under + /// storage accounting v1 (`bulk_append_tree_versions.cost + /// .storage_accounting`) the write is reported as replacement of its + /// previous serialization, with only growth as added bytes; v0 lets the + /// storage layer report the whole value as added every time. + pub fn save(&mut self, grove_version: &GroveVersion) -> CostResult<(), CommitmentTreeError> { let mut cost = OperationCost::default(); let serialized = self.frontier.serialize(); + let cost_info = match grove_version + .bulk_append_tree_versions + .cost + .storage_accounting + { + 0 => None, + 1 => Some(frontier_save_cost_info( + serialized.len() as u32, + self.stored_frontier_len, + )), + version => { + return Err(CommitmentTreeError::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "CommitmentTree frontier storage accounting".to_string(), + known_versions: vec![0, 1], + received: version, + } + .to_string(), + )) + .wrap_with_cost(cost) + } + }; let result = self .bulk_tree .dense_tree .storage - .put(COMMITMENT_TREE_DATA_KEY, &serialized, None, None) + .put(COMMITMENT_TREE_DATA_KEY, &serialized, None, cost_info) .unwrap_add_cost(&mut cost); match result { - Ok(()) => Ok(()).wrap_with_cost(cost), + Ok(()) => { + self.stored_frontier_len = Some(serialized.len() as u32); + Ok(()).wrap_with_cost(cost) + } Err(e) => Err(CommitmentTreeError::InvalidData(format!( "storage error saving frontier: {}", e diff --git a/grovedb-commitment-tree/src/commitment_tree/tests.rs b/grovedb-commitment-tree/src/commitment_tree/tests.rs index 5e8c65ebe..055051f64 100644 --- a/grovedb-commitment-tree/src/commitment_tree/tests.rs +++ b/grovedb-commitment-tree/src/commitment_tree/tests.rs @@ -527,7 +527,7 @@ mod storage_tests { let expected_total_count = ct.total_count(); // Save - let save_result = ct.save(); + let save_result = ct.save(GroveVersion::latest()); save_result.value.expect("save should succeed"); assert!( save_result.cost.seek_count > 0, @@ -561,7 +561,9 @@ mod storage_tests { .expect("open should succeed"); // Save empty - ct.save().value.expect("save empty should succeed"); + ct.save(GroveVersion::latest()) + .value + .expect("save empty should succeed"); // Append and save again (overwrites) ct.append( @@ -575,7 +577,9 @@ mod storage_tests { .expect("append should succeed"); let expected_root = ct.root_hash(); let total_count = ct.total_count(); - ct.save().value.expect("save non-empty should succeed"); + ct.save(GroveVersion::latest()) + .value + .expect("save non-empty should succeed"); // Re-open should return the latest (non-empty) frontier let storage = ct.bulk_tree.dense_tree.storage; @@ -619,12 +623,13 @@ mod storage_tests { // Construct directly to test save() error path. let bulk_tree = BulkAppendTree::new(TEST_CHUNK_POWER, FailingDataStorageContext) .expect("bulk tree new should succeed"); - let ct: CommitmentTree<_, DashMemo> = CommitmentTree { + let mut ct: CommitmentTree<_, DashMemo> = CommitmentTree { frontier: CommitmentFrontier::new(), bulk_tree, + stored_frontier_len: None, _memo: PhantomData, }; - let result = ct.save(); + let result = ct.save(GroveVersion::latest()); assert!(result.value.is_err(), "should surface storage put error"); let err_msg = format!("{}", result.value.expect_err("should be storage error")); assert!( @@ -637,11 +642,13 @@ mod storage_tests { #[test] fn test_save_empty_and_reopen() { let ctx = MockDataStorageContext::new(); - let ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) + let mut ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) .value .expect("open should succeed"); - ct.save().value.expect("save empty should succeed"); + ct.save(GroveVersion::latest()) + .value + .expect("save empty should succeed"); let storage = ct.bulk_tree.dense_tree.storage; let loaded = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, storage) @@ -679,7 +686,9 @@ mod storage_tests { } let total_count = ct.total_count(); - ct.save().value.expect("save should succeed"); + ct.save(GroveVersion::latest()) + .value + .expect("save should succeed"); let storage = ct.bulk_tree.dense_tree.storage; let loaded = CommitmentTree::<_, DashMemo>::open(total_count, TEST_CHUNK_POWER, storage) @@ -1138,7 +1147,9 @@ mod storage_tests { ) .value .expect("append should succeed"); - ct.save().value.expect("save should succeed"); + ct.save(GroveVersion::latest()) + .value + .expect("save should succeed"); // 2. Re-open with total_count=0 but the frontier has tree_size=1 let storage = ct.bulk_tree.dense_tree.storage; diff --git a/grovedb-commitment-tree/src/error.rs b/grovedb-commitment-tree/src/error.rs index 41827de87..118e85715 100644 --- a/grovedb-commitment-tree/src/error.rs +++ b/grovedb-commitment-tree/src/error.rs @@ -10,6 +10,10 @@ pub enum CommitmentTreeError { /// Data read from storage is invalid or corrupt. #[error("invalid frontier data: {0}")] InvalidData(String), + + /// A grove version selected an unknown accounting variant. + #[error("version error: {0}")] + VersionError(String), /// A 32-byte value is not a valid Pallas field element. #[error("invalid Pallas field element")] InvalidFieldElement, diff --git a/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs b/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs index 53e8a8a30..d96196537 100644 --- a/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs +++ b/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs @@ -1,5 +1,7 @@ #[cfg(feature = "storage")] -use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_costs::{ + storage_cost::key_value_cost::KeyValueStorageCost, CostResult, CostsExt, OperationCost, +}; #[cfg(feature = "storage")] use grovedb_storage::StorageContext; @@ -136,7 +138,7 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { } let position = self.count; - cost_return_on_error!(cost, self.put_value(position, value)); + cost_return_on_error!(cost, self.put_value(position, value, None)); self.count += 1; match self.compute_root_hash().unwrap_add_cost(&mut cost) { @@ -164,6 +166,19 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { pub fn try_insert( &mut self, value: &[u8], + ) -> CostResult, DenseMerkleError> { + self.try_insert_with_cost_info(value, None) + } + + /// [`try_insert`](Self::try_insert) with caller-supplied storage cost + /// info for the entry write. `None` keeps the storage layer's default + /// report (key + value as added bytes); the bulk-append tree passes + /// `Some` under its versioned storage accounting to describe the write + /// as what it is (a new entry, or an overwrite of a stale slot). + pub fn try_insert_with_cost_info( + &mut self, + value: &[u8], + cost_info: Option, ) -> CostResult, DenseMerkleError> { let mut cost = OperationCost::default(); @@ -172,7 +187,7 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { } let position = self.count; - cost_return_on_error!(cost, self.put_value(position, value)); + cost_return_on_error!(cost, self.put_value(position, value, cost_info)); self.count += 1; match self.compute_root_hash().unwrap_add_cost(&mut cost) { @@ -202,6 +217,17 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { pub fn try_insert_no_root( &mut self, value: &[u8], + ) -> CostResult, DenseMerkleError> { + self.try_insert_no_root_with_cost_info(value, None) + } + + /// [`try_insert_no_root`](Self::try_insert_no_root) with caller-supplied + /// storage cost info for the entry write — see + /// [`try_insert_with_cost_info`](Self::try_insert_with_cost_info). + pub fn try_insert_no_root_with_cost_info( + &mut self, + value: &[u8], + cost_info: Option, ) -> CostResult, DenseMerkleError> { let mut cost = OperationCost::default(); @@ -210,7 +236,7 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { } let position = self.count; - cost_return_on_error!(cost, self.put_value(position, value)); + cost_return_on_error!(cost, self.put_value(position, value, cost_info)); self.count += 1; Ok(Some(position)).wrap_with_cost(cost) @@ -300,7 +326,12 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { /// On success, the value is stored in the write-through cache so that /// subsequent reads (e.g., during root hash computation) can be served /// from memory even when the storage context defers writes. - fn put_value(&mut self, position: u16, value: &[u8]) -> CostResult<(), DenseMerkleError> { + fn put_value( + &mut self, + position: u16, + value: &[u8], + cost_info: Option, + ) -> CostResult<(), DenseMerkleError> { debug_assert!( (position as usize) < self.cache.len(), "put_value called with position {} >= cache capacity {}", @@ -311,7 +342,7 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { let key = position_key(position); let result = self .storage - .put(key, value, None, None) + .put(key, value, None, cost_info) .unwrap_add_cost(&mut cost); match result { Ok(()) => { diff --git a/grovedb-merkle-mountain-range/src/storage_adapter.rs b/grovedb-merkle-mountain-range/src/storage_adapter.rs index 41dd8873a..412dd102c 100644 --- a/grovedb-merkle-mountain-range/src/storage_adapter.rs +++ b/grovedb-merkle-mountain-range/src/storage_adapter.rs @@ -3,7 +3,11 @@ //! Provides `MmrStore`, which implements `MMRStoreReadOps` and //! `MMRStoreWriteOps` backed by a GroveDB storage context. -use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use std::{cell::RefCell, collections::BTreeMap}; + +use grovedb_costs::{ + storage_cost::key_value_cost::KeyValueStorageCost, CostResult, CostsExt, OperationCost, +}; use grovedb_storage::StorageContext; use crate::{ @@ -26,6 +30,14 @@ use crate::{ pub struct MmrStore<'a, C> { ctx: &'a C, key_size: MmrKeySize, + /// Storage cost info to attach to specific node positions when they are + /// written (consumed on use). Lets a caller that knows what a leaf + /// replaces — the bulk-append tree's chunk blob supersedes the buffer + /// entries it was built from — report it as such instead of as new + /// bytes, even though MMR nodes are staged in an overlay and written + /// later. Positions without an entry get the storage layer's default + /// report (new bytes). + put_cost_infos: RefCell>, } impl<'a, C> MmrStore<'a, C> { @@ -36,6 +48,7 @@ impl<'a, C> MmrStore<'a, C> { Self { ctx, key_size: MmrKeySize::U64, + put_cost_infos: RefCell::new(BTreeMap::new()), } } @@ -44,7 +57,24 @@ impl<'a, C> MmrStore<'a, C> { /// Use [`MmrKeySize::U32`] for compact 4-byte keys when positions /// are guaranteed to fit in a `u32`. pub fn with_key_size(ctx: &'a C, key_size: MmrKeySize) -> Self { - Self { ctx, key_size } + Self { + ctx, + key_size, + put_cost_infos: RefCell::new(BTreeMap::new()), + } + } + + /// Attach storage cost info to the writes of specific node positions; + /// each entry is consumed by the write of its position. + pub fn with_put_cost_infos(self, cost_infos: BTreeMap) -> Self { + *self.put_cost_infos.borrow_mut() = cost_infos; + self + } + + /// Take back any cost infos whose positions were not written (so a + /// caller can re-stage them after a failed commit). + pub fn take_put_cost_infos(&self) -> BTreeMap { + std::mem::take(&mut *self.put_cost_infos.borrow_mut()) } } @@ -81,6 +111,7 @@ impl<'db, C: StorageContext<'db>> MMRStoreWriteOps for &MmrStore<'_, C> { let mut cost = OperationCost::default(); for (i, elem) in elems.into_iter().enumerate() { let node_pos = pos + i as u64; + let cost_info = self.put_cost_infos.borrow_mut().remove(&node_pos); let key = match mmr_node_key_sized(node_pos, self.key_size) { Ok(k) => k, Err(e) => return Err(e).wrap_with_cost(cost), @@ -95,7 +126,7 @@ impl<'db, C: StorageContext<'db>> MMRStoreWriteOps for &MmrStore<'_, C> { .wrap_with_cost(cost); } }; - let result = self.ctx.put(key, &serialized, None, None); + let result = self.ctx.put(key, &serialized, None, cost_info); cost += result.cost; if let Err(e) = result.value { return Err(crate::Error::StoreError(format!( diff --git a/grovedb-version/src/tests.rs b/grovedb-version/src/tests.rs index 2f43ac135..8a6d818ab 100644 --- a/grovedb-version/src/tests.rs +++ b/grovedb-version/src/tests.rs @@ -663,3 +663,21 @@ fn bulk_append_cost_versions_are_pinned_per_protocol_version() { shipped figure" ); } + +#[test] +fn bulk_append_storage_accounting_is_pinned_per_protocol_version() { + for (name, version) in [ + ("GROVE_V1", &GROVE_V1), + ("GROVE_V2", &GROVE_V2), + ("GROVE_V3", &GROVE_V3), + ] { + assert_eq!( + version.bulk_append_tree_versions.cost.storage_accounting, 0, + "{name} must keep the shipped all-added storage report" + ); + } + assert_eq!( + GROVE_V4.bulk_append_tree_versions.cost.storage_accounting, 1, + "GROVE_V4 reports the append-only family's churn as replaced bytes" + ); +} diff --git a/grovedb-version/src/version/bulk_append_tree_versions.rs b/grovedb-version/src/version/bulk_append_tree_versions.rs index a9b3bfbe3..13f346b30 100644 --- a/grovedb-version/src/version/bulk_append_tree_versions.rs +++ b/grovedb-version/src/version/bulk_append_tree_versions.rs @@ -27,4 +27,28 @@ pub struct BulkAppendTreeCostVersions { /// Version 1 adds that bagging term. Shipped chunk bytes and roots are /// unaffected; what moves is the fee a compacting append is charged. pub compaction_hash_count: FeatureVersion, + /// How the append-only family's data-storage writes are reported to the + /// fee layer (the dense-buffer entry write, the compaction chunk blob, + /// and — for `CommitmentTree` — the frontier rewrite). + /// + /// Version 0 (shipped, V1..V3) issues every data put with no cost info, + /// so the commit path charges key + value as `added_bytes`: the chunk + /// blob is billed as ~`epoch_size × entry` of NEW storage on the one + /// append that compacts (although it supersedes the buffer entries it + /// was built from), buffer writes from the second epoch on are billed + /// as new (although they overwrite last epoch's stale value at the same + /// position key), and the frontier is billed in full on every append + /// (although it is one value rewritten in place). Metered storage ends + /// up ~2× the bytes that persist, concentrated on one arbitrary tx per + /// epoch. + /// + /// Version 1 (V4+) charges each entry's permanent bytes once, at the + /// append that creates it (entry + its amortized share of the chunk + /// blob's framing, as `added_bytes`), and reports the churn as + /// `replaced_bytes`: the chunk blob as replacement of the buffer bytes + /// it supersedes, the frontier rewrite as replacement of its previous + /// serialization (growth only is added). Stored bytes, chunks, roots and + /// proofs are identical under both versions; only the cost report + /// moves. + pub storage_accounting: FeatureVersion, } diff --git a/grovedb-version/src/version/v1.rs b/grovedb-version/src/version/v1.rs index 43b129b2f..832177b4e 100644 --- a/grovedb-version/src/version/v1.rs +++ b/grovedb-version/src/version/v1.rs @@ -273,6 +273,7 @@ pub const GROVE_V1: GroveVersion = GroveVersion { bulk_append_tree_versions: BulkAppendTreeVersions { cost: BulkAppendTreeCostVersions { compaction_hash_count: 0, + storage_accounting: 0, }, }, }; diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index d817794ed..831907eb4 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -272,6 +272,7 @@ pub const GROVE_V2: GroveVersion = GroveVersion { bulk_append_tree_versions: BulkAppendTreeVersions { cost: BulkAppendTreeCostVersions { compaction_hash_count: 0, + storage_accounting: 0, }, }, }; diff --git a/grovedb-version/src/version/v3.rs b/grovedb-version/src/version/v3.rs index 6a141dce3..3152a2db3 100644 --- a/grovedb-version/src/version/v3.rs +++ b/grovedb-version/src/version/v3.rs @@ -287,6 +287,7 @@ pub const GROVE_V3: GroveVersion = GroveVersion { bulk_append_tree_versions: BulkAppendTreeVersions { cost: BulkAppendTreeCostVersions { compaction_hash_count: 0, + storage_accounting: 0, }, }, }; diff --git a/grovedb-version/src/version/v4.rs b/grovedb-version/src/version/v4.rs index c0f812e6d..d6c77cd6d 100644 --- a/grovedb-version/src/version/v4.rs +++ b/grovedb-version/src/version/v4.rs @@ -413,6 +413,7 @@ pub const GROVE_V4: GroveVersion = GroveVersion { bulk_append_tree_versions: BulkAppendTreeVersions { cost: BulkAppendTreeCostVersions { compaction_hash_count: 1, + storage_accounting: 1, }, }, }; diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index aa63ca788..27734fbbb 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -321,10 +321,17 @@ impl GroveOp { // need no separate declaration. let entry_size = entry.len() as u32; let epoch_entries: u32 = 1u32 << chunk_power.min(16) as u32; - // Amortized over one epoch: every entry is written once to - // the buffer, and once more into the chunk blob when the - // epoch compacts. - let amortized_compaction_bytes = entry_size; + // Storage accounting v1 (the only report at GROVE_V4, where + // the store exists): an append ADDS its entry plus an + // amortized share of the chunk blob's framing; when the + // epoch compacts, the blob is REPLACEMENT of the pre-paid + // entries, and only its residual header is added. + let amortized_compaction_bytes: u32 = 16; + const CHUNK_BLOB_RESIDUAL: u32 = 64 + 50; + // The compacting append's replaced bytes: the whole epoch's + // entries with their framing share. + let compaction_replaced_bytes: u32 = + epoch_entries.saturating_mul(entry_size.saturating_add(16)); // The dense-buffer root walk costs two hashes per filled // position and runs on every append, so across an epoch it // averages half the buffer. @@ -354,8 +361,9 @@ impl GroveOp { storage_cost: StorageCost { added_bytes: entry_size .saturating_add(amortized_compaction_bytes) + .saturating_add(CHUNK_BLOB_RESIDUAL) .saturating_add(NON_COUNTED_WRAPPER_BYTE), - replaced_bytes: 0, + replaced_bytes: compaction_replaced_bytes, removed_bytes: StorageRemovedBytes::NoStorageRemoval, }, storage_loaded_bytes: (avg_dense_reads as u64) @@ -2146,13 +2154,18 @@ mod tests { big.hash_node_calls, small.hash_node_calls ); - // Each entry is written TWICE across its lifetime: once into the - // dense buffer and once more into the chunk blob when the epoch - // compacts. The amortized per-append charge is therefore 2x the - // entry size, so doubling the entry grows added_bytes by 2 x 64. + // Each entry's permanent bytes are charged once, as added, by the + // append that writes it (plus a fixed amortized framing share); the + // chunk blob the epoch compacts into is replacement of those + // pre-paid bytes. Doubling the entry therefore grows added_bytes by + // exactly 64, and the compaction's replaced bound by 2^4 x 64. assert_eq!( cost_large.storage_cost.added_bytes - cost.storage_cost.added_bytes, - 128 + 64 + ); + assert_eq!( + cost_large.storage_cost.replaced_bytes - cost.storage_cost.replaced_bytes, + 16 * 64 ); } diff --git a/grovedb/src/batch/estimated_costs/mod.rs b/grovedb/src/batch/estimated_costs/mod.rs index c38f68419..ac4987570 100644 --- a/grovedb/src/batch/estimated_costs/mod.rs +++ b/grovedb/src/batch/estimated_costs/mod.rs @@ -104,6 +104,16 @@ const CT_ELEMENT_LOAD_BASE: u32 = 256; /// root recompute), and a full epoch compaction (chunk-blob write plus /// MMR merge cascade). /// +/// Storage bytes follow the append-only family's storage accounting v1 +/// (`bulk_append_tree_versions.cost.storage_accounting`, the only report +/// GROVE_V4 — where this arm is selected — uses): an append **adds** its +/// entry plus its amortized share of the chunk blob's framing, and any +/// growth of the frontier; the compaction's blob and the frontier rewrite +/// are **replaced** bytes (the blob supersedes the pre-paid buffer entries, +/// the frontier overwrites its previous serialization), so the bound +/// carries a replaced-bytes term sized for a full epoch and the frontier at +/// its maximum, and its added-bytes term no longer scales with the epoch. +/// /// `chunk_power` is the tree's epoch scale: the average-case estimator /// requires it declared in the tree's own layer in the estimation paths /// (`TreeType::CommitmentTree(chunk_power)`) and errors when it is @@ -143,20 +153,32 @@ pub(in crate::batch) fn commitment_tree_insert_op_cost( const CHUNK_BLOB_OVERHEAD: u64 = 64; // An MMR internal node: 1 (flag) + 32 (hash). const MMR_INTERNAL_NODE_SIZE: u64 = 33; - - // The epoch term multiplies the entry size by up to 2^16, which - // overflows u32 for hand-built ops with oversized payloads (the op - // is public; the apply path only rejects wrong-sized payloads - // later). Sum in u64 and saturate at u32::MAX — a wrapped - // added_bytes would silently UNDER-estimate, the exact failure this - // model exists to prevent, while a saturated one merely - // over-reserves for an op the apply would reject anyway. - let added_bytes_u64: u64 = (entry_size + PER_PUT_OVERHEAD as u64) - + (MAX_FRONTIER_SIZE as u64 + PER_PUT_OVERHEAD as u64) - + (epoch_size as u64 * (entry_size + CHUNK_ENTRY_OVERHEAD) - + CHUNK_BLOB_OVERHEAD - + PER_PUT_OVERHEAD as u64) + // The frontier's serialization grows by at most one 32-byte ommer per + // append (popcount rises by at most one), and its first-ever save of + // a one-leaf frontier is 42 bytes; 64 bounds both, with the varint. + const FRONTIER_ADDED_PER_APPEND: u64 = 64; + + // Added bytes: the entry with its amortized blob-framing share, any + // frontier growth, the blob's own residual (header beyond what the + // entries pre-paid), and the MMR merge cascade's internal nodes. None + // of these scale with the epoch: the epoch-sized blob copy is a + // replacement (below), not new storage. + let added_bytes_u64: u64 = (entry_size + CHUNK_ENTRY_OVERHEAD + PER_PUT_OVERHEAD as u64) + + (FRONTIER_ADDED_PER_APPEND + PER_PUT_OVERHEAD as u64) + + (CHUNK_BLOB_OVERHEAD + PER_PUT_OVERHEAD as u64) + FRONTIER_DEPTH as u64 * (MMR_INTERNAL_NODE_SIZE + PER_PUT_OVERHEAD as u64); + // Replaced bytes: on the compacting append, the whole epoch's entries + // (each with its framing share) are superseded by the blob; on every + // append the previous frontier serialization is overwritten. The + // epoch term multiplies the entry size by up to 2^16, which overflows + // u32 for hand-built ops with oversized payloads (the op is public; + // the apply path only rejects wrong-sized payloads later). Sum in u64 + // and saturate at u32::MAX — a wrapped figure would silently + // UNDER-estimate, the exact failure this model exists to prevent, + // while a saturated one merely over-reserves for an op the apply + // would reject anyway. + let replaced_bytes_u64: u64 = epoch_size as u64 * (entry_size + CHUNK_ENTRY_OVERHEAD) + + (MAX_FRONTIER_SIZE as u64 + PER_PUT_OVERHEAD as u64); OperationCost { // 2 reads (CommitmentTree element + frontier) and up to @@ -175,9 +197,9 @@ pub(in crate::batch) fn commitment_tree_insert_op_cost( // re-written once into the blob) and the MMR merge // cascade's internal nodes. added_bytes: u32::try_from(added_bytes_u64).unwrap_or(u32::MAX), - // The parent-Merk node replacement is charged by the - // replace_tree part; the append itself replaces nothing. - replaced_bytes: 0, + // The blob copy and the frontier rewrite; the parent-Merk node + // replacement is charged by the replace_tree part. + replaced_bytes: u32::try_from(replaced_bytes_u64).unwrap_or(u32::MAX), removed_bytes: StorageRemovedBytes::NoStorageRemoval, }, // Reads: the stored CommitmentTree element (fixed serialized diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index eb8b8d247..9ffc9ca9d 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -1860,9 +1860,11 @@ mod tests { /// Boundary test for the saturating epoch arithmetic: a hand-built op /// with an oversized payload (the op type is public; the apply path only /// rejects wrong-sized payloads later) drives the physical-ceiling epoch - /// term past u32. The estimate must saturate at u32::MAX — never panic - /// in debug builds nor wrap in release builds, since a wrapped - /// added_bytes would silently UNDER-estimate. + /// term — the compaction's replaced bytes — past u32. The estimate must + /// saturate at u32::MAX — never panic in debug builds nor wrap in + /// release builds, since a wrapped figure would silently + /// UNDER-estimate. The per-append added term does not scale with the + /// epoch and stays far below the ceiling. #[test] fn test_commitment_tree_insert_worst_case_cost_oversized_payload_saturates() { let grove_version = GroveVersion::latest(); @@ -1885,9 +1887,14 @@ mod tests { .cost_as_result() .expect("expected worst case cost for oversized payload"); assert_eq!( - cost.storage_cost.added_bytes, + cost.storage_cost.replaced_bytes, u32::MAX, "oversized-payload estimate must saturate, not wrap", ); + assert!( + cost.storage_cost.added_bytes < u32::MAX, + "the added term is per-append and must not reach the ceiling: {}", + cost.storage_cost.added_bytes + ); } } diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index 1b217eed4..c422cc38e 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -161,7 +161,10 @@ impl GroveDb { ); // 4. Save frontier to storage - cost_return_on_error!(&mut cost, ct.save().map(|r| r.map_err(map_ct_err))); + cost_return_on_error!( + &mut cost, + ct.save(grove_version).map(|r| r.map_err(map_ct_err)) + ); let new_sinsemilla_root = append_result.sinsemilla_root; let bulk_state_root = append_result.bulk_state_root; @@ -551,7 +554,10 @@ impl GroveDb { } // Save frontier to storage - cost_return_on_error!(&mut cost, ct.save().map(|r| r.map_err(map_ct_err))); + cost_return_on_error!( + &mut cost, + ct.save(grove_version).map(|r| r.map_err(map_ct_err)) + ); // Read state for the replacement op let bulk_state_root = cost_return_on_error_no_add!( diff --git a/grovedb/src/tests/append_only_storage_accounting_tests.rs b/grovedb/src/tests/append_only_storage_accounting_tests.rs new file mode 100644 index 000000000..b499a0c5d --- /dev/null +++ b/grovedb/src/tests/append_only_storage_accounting_tests.rs @@ -0,0 +1,219 @@ +//! Storage accounting of the append-only family (issue #822). +//! +//! Under `bulk_append_tree_versions.cost.storage_accounting = 1` (GROVE_V4) +//! each entry's permanent bytes are charged as **added** by the append that +//! creates it, and the churn — the compaction's chunk blob (which supersedes +//! the pre-paid buffer entries) and the frontier rewrite (one value +//! overwritten in place) — is reported as **replaced**. Under the shipped +//! report (V1..V3) every data put is key + value as added, so the +//! compacting append is billed the whole epoch's bytes as new storage. +//! +//! The assertions are structural (ratios and floors) rather than exact byte +//! counts, so they pin the accounting model without freezing framing +//! constants; the exact figures are covered by the estimate-dominates +//! property tests, which must keep holding under the new report. + +use grovedb_commitment_tree::{DashMemo, NoteBytesData, TransmittedNoteCiphertext}; +use grovedb_costs::{storage_cost::StorageCost, CostContext}; +use grovedb_version::version::{v3::GROVE_V3, v4::GROVE_V4, GroveVersion}; + +use crate::{ + batch::QualifiedGroveDbOp, + tests::{common::EMPTY_PATH, make_empty_grovedb, TempGroveDb}, + Element, +}; + +/// cmx (32) + rho (32) + cv_net (32) + 216-byte DashMemo ciphertext. +const ENTRY_SIZE: u32 = 312; +const CHUNK_POWER: u8 = 4; +const EPOCH: u32 = 1 << CHUNK_POWER as u32; + +fn test_cmx(index: u32) -> [u8; 32] { + let mut bytes = [0u8; 32]; + bytes[..4].copy_from_slice(&index.to_le_bytes()); + bytes[31] &= 0x7f; + bytes +} + +fn test_ciphertext(index: u32) -> TransmittedNoteCiphertext { + let mut epk_bytes = [0u8; 32]; + epk_bytes[..4].copy_from_slice(&index.to_le_bytes()); + let mut enc_data = [0u8; 104]; + enc_data[..4].copy_from_slice(&index.to_le_bytes()); + let mut out_ciphertext = [0u8; 80]; + out_ciphertext[..4].copy_from_slice(&index.to_le_bytes()); + TransmittedNoteCiphertext::from_parts(epk_bytes, NoteBytesData(enc_data), out_ciphertext) +} + +fn ct_op(index: u32) -> QualifiedGroveDbOp { + let mut rho = [0u8; 32]; + rho[..4].copy_from_slice(&index.to_le_bytes()); + rho[4] = 0xAA; + let mut cv_net = [0u8; 32]; + cv_net[..4].copy_from_slice(&index.to_le_bytes()); + cv_net[4] = 0xCC; + QualifiedGroveDbOp::commitment_tree_insert_op_typed( + vec![b"pool".to_vec()], + test_cmx(index), + rho, + cv_net, + &test_ciphertext(index), + ) +} + +/// A fresh db with an empty commitment tree at `pool`, plus the per-append +/// storage cost of applying `count` sequential inserts under `grove_version`. +fn per_append_storage(grove_version: &GroveVersion, count: u32) -> (TempGroveDb, Vec) { + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"pool", + Element::empty_commitment_tree(CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert commitment tree"); + let mut costs = Vec::with_capacity(count as usize); + for index in 0..count { + let CostContext { value, cost } = + db.apply_batch(vec![ct_op(index)], None, None, grove_version); + value.expect("append should succeed"); + costs.push(cost.storage_cost); + } + (db, costs) +} + +/// v1: the compacting append reports the epoch's entries as replaced, not +/// as an epoch-sized spike of new storage; its added bytes stay in the same +/// band as an ordinary append. +#[test] +fn v1_compaction_is_replacement_not_added_storage() { + let (_db, costs) = per_append_storage(&GROVE_V4, EPOCH + 2); + let compacting = &costs[(EPOCH - 1) as usize]; + let ordinary = &costs[(EPOCH - 2) as usize]; + + assert!( + compacting.replaced_bytes >= EPOCH * ENTRY_SIZE, + "the compacting append must report the whole epoch's entries as replaced: \ + replaced {} < {}", + compacting.replaced_bytes, + EPOCH * ENTRY_SIZE + ); + assert!( + compacting.added_bytes < compacting.replaced_bytes / 4, + "the compacting append's added bytes must not scale with the epoch: added {} vs \ + replaced {}", + compacting.added_bytes, + compacting.replaced_bytes + ); + // An ordinary append adds its entry (+ framing share); the compacting + // append adds that plus the blob residual and a few MMR nodes — same + // order of magnitude, never an epoch multiple. + assert!( + compacting.added_bytes < 4 * ordinary.added_bytes, + "compacting added {} vs ordinary added {}", + compacting.added_bytes, + ordinary.added_bytes + ); +} + +/// v1: every append — first epoch and later — is charged its entry as added +/// (the entry's permanent bytes, paid once by the append that creates it), +/// and from the second append on the frontier rewrite shows up as replaced. +#[test] +fn v1_every_append_pays_its_entry_once_and_frontier_rewrites_are_replaced() { + let (_db, costs) = per_append_storage(&GROVE_V4, 2 * EPOCH + 2); + for (index, cost) in costs.iter().enumerate() { + assert!( + cost.added_bytes >= ENTRY_SIZE, + "append #{index} must add at least its entry: added {}", + cost.added_bytes + ); + } + // Second-epoch appends overwrite a stale buffer slot at an existing key: + // they still add the entry but not a new key, so they cannot exceed the + // first epoch's corresponding append. + for i in 0..(EPOCH - 1) as usize { + let first = costs[i].added_bytes; + let second = costs[i + EPOCH as usize].added_bytes; + assert!( + second <= first, + "second-epoch append #{} added {} > first-epoch #{} added {}", + i + EPOCH as usize, + second, + i, + first + ); + } + // The frontier is one value rewritten in place: after the first save + // every append replaces at least the minimal serialized frontier. + const MIN_FRONTIER_LEN: u32 = 42; + for (index, cost) in costs.iter().enumerate().skip(1) { + assert!( + cost.replaced_bytes >= MIN_FRONTIER_LEN, + "append #{index} must report the frontier rewrite as replaced: replaced {}", + cost.replaced_bytes + ); + } +} + +/// v0 (V1..V3, the shipped report): the compacting append is billed the +/// epoch's bytes as new storage — pinned so the legacy figure cannot drift +/// under live versions. +#[test] +fn v0_compaction_is_billed_as_new_storage() { + let (_db, costs) = per_append_storage(&GROVE_V3, EPOCH + 1); + let compacting = &costs[(EPOCH - 1) as usize]; + let ordinary = &costs[(EPOCH - 2) as usize]; + assert!( + compacting.added_bytes >= EPOCH * ENTRY_SIZE, + "v0 must bill the blob as added: added {} < {}", + compacting.added_bytes, + EPOCH * ENTRY_SIZE + ); + assert!( + compacting.added_bytes > 8 * ordinary.added_bytes, + "v0 compaction spike expected: {} vs {}", + compacting.added_bytes, + ordinary.added_bytes + ); +} + +/// The two reports differ only in how bytes are labelled: stored bytes, +/// chunks and roots are identical, so the root hash after the same appends +/// agrees across versions. +#[test] +fn accounting_versions_leave_state_and_roots_identical() { + let (db_v3, _) = per_append_storage(&GROVE_V3, 2 * EPOCH + 3); + let (db_v4, _) = per_append_storage(&GROVE_V4, 2 * EPOCH + 3); + let root_v3 = db_v3.root_hash(None, &GROVE_V3).unwrap().expect("root v3"); + let root_v4 = db_v4.root_hash(None, &GROVE_V4).unwrap().expect("root v4"); + assert_eq!( + root_v3, root_v4, + "storage accounting must not change stored bytes or roots" + ); +} + +/// Across a full epoch, v1's total added bytes are within a small factor of +/// the bytes that persist (entries + framing + a few MMR/frontier bytes), +/// whereas v0 charges roughly twice that by billing the blob copy as new. +#[test] +fn v1_epoch_total_added_tracks_permanent_bytes_while_v0_double_counts() { + let (_, v4) = per_append_storage(&GROVE_V4, EPOCH); + let (_, v3) = per_append_storage(&GROVE_V3, EPOCH); + let total = |costs: &[StorageCost]| -> u64 { costs.iter().map(|c| c.added_bytes as u64).sum() }; + let permanent_floor = (EPOCH * ENTRY_SIZE) as u64; + let v4_total = total(&v4); + let v3_total = total(&v3); + assert!( + v4_total >= permanent_floor && v4_total < permanent_floor * 3 / 2, + "v1 epoch total added {v4_total} should be within 1.5x of the permanent entry bytes \ + {permanent_floor}" + ); + assert!( + v3_total >= 2 * permanent_floor, + "v0 epoch total added {v3_total} should double-count the entries ({permanent_floor})" + ); +} diff --git a/grovedb/src/tests/commitment_tree_cost_bound_tests.rs b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs index d95ec51b3..370aeb777 100644 --- a/grovedb/src/tests/commitment_tree_cost_bound_tests.rs +++ b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs @@ -330,9 +330,11 @@ fn test_commitment_tree_insert_declared_chunk_power_tightens_estimate() { value.expect("compaction append should succeed"); // Far tighter than the worst-case physical-ceiling assumption - // (2^4 vs 2^16 epoch)... + // (2^4 vs 2^16 epoch). The epoch scale lives in the replaced-bytes + // term (the compaction's blob supersedes the epoch's entries); the + // added-bytes term is per-append and does not scale with the epoch. assert!( - declared.storage_cost.added_bytes < worst.storage_cost.added_bytes / 100, + declared.storage_cost.replaced_bytes < worst.storage_cost.replaced_bytes / 100, "declared estimate should be far tighter than the physical-ceiling worst case; declared \ {declared:?}\nworst {worst:?}", ); diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index 027b0d37a..0046468a5 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -38,6 +38,7 @@ mod private_document_store_tests; // and `provable_count_provable_sum_indexed_tree_tests`; the generic-write // rejection cases it uniquely held are ported to // `generic_writes_against_pcit_primary_are_rejected`. +mod append_only_storage_accounting_tests; mod axis_descent_proof_tests; mod count_offset_paginated_tests; mod count_sum_tree_tests; diff --git a/storage/src/rocksdb_storage/storage_context/context_tx.rs b/storage/src/rocksdb_storage/storage_context/context_tx.rs index 64ea3d943..12e5a2d52 100644 --- a/storage/src/rocksdb_storage/storage_context/context_tx.rs +++ b/storage/src/rocksdb_storage/storage_context/context_tx.rs @@ -33,6 +33,7 @@ use grovedb_costs::{ cost_return_on_error, storage_cost::key_value_cost::KeyValueStorageCost, ChildrenSizesWithIsSumTree, CostResult, CostsExt, OperationCost, }; +use integer_encoding::VarInt; use rocksdb::{ColumnFamily, DBRawIteratorWithThreadMode}; use super::{batch::PrefixedMultiContextBatchPart, make_prefixed_key, PrefixedRocksDbRawIterator}; @@ -136,12 +137,20 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbTransactionContext<'db> { .wrap_with_cost(OperationCost::default()) } }; - batch.put( - make_prefixed_key(&self.prefix, key), - value.to_vec(), - children_sizes, - cost_info, - ); + let prefixed_key = make_prefixed_key(&self.prefix, key); + // Mirror the batch path: a caller describing a NEW key cannot know + // the prefix this context adds, so the prefixed key's bytes are + // accounted here. (Merk never reaches this arm — it writes through + // `Batch::put`, which does the same — so this only affects callers + // that pass their own cost info for data-storage puts.) + let cost_info = cost_info.map(|mut key_value_storage_cost| { + if key_value_storage_cost.new_node { + key_value_storage_cost.key_storage_cost.added_bytes += + (prefixed_key.len() + prefixed_key.len().required_space()) as u32; + } + key_value_storage_cost + }); + batch.put(prefixed_key, value.to_vec(), children_sizes, cost_info); Ok(()).wrap_with_cost(OperationCost::default()) }