Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 72 additions & 1 deletion costs/src/storage_cost/key_value_cost.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ use crate::{
};

/// Storage only operation costs separated by key and value
#[derive(PartialEq, Clone, Eq, Default)]
#[derive(Debug, PartialEq, Clone, Eq, Default)]
pub struct KeyValueStorageCost {
/// Key storage_cost costs
pub key_storage_cost: StorageCost,
Expand Down Expand Up @@ -110,6 +110,33 @@ impl KeyValueStorageCost {
}
}

/// Cost of rewriting an existing value in place — `previous_len` bytes
/// stored under a key that already exists, overwritten with `new_len`
/// bytes — with no refund semantics.
///
/// The paid size of a value is its length plus the varint encoding that
/// length, which is what the commit path verifies `added + replaced`
/// against. The key is not charged (it exists); `replaced_bytes` is the
/// smaller of the previous and the new paid size; `added_bytes` is the
/// growth beyond the previous size, if any; a shrink is NOT credited as
/// removed bytes — used for rolling data (a reused buffer slot, a
/// frontier rewritten on every append) where a refund would mean paying
/// someone back for bytes nobody owns.
pub fn for_in_place_value_rewrite(previous_len: u32, new_len: u32) -> Self {
let paid_previous = previous_len + previous_len.required_space() as u32;
let paid_new = new_len + new_len.required_space() as u32;
KeyValueStorageCost {
key_storage_cost: StorageCost::default(),
value_storage_cost: StorageCost {
added_bytes: paid_new.saturating_sub(paid_previous),
replaced_bytes: paid_new.min(paid_previous),
removed_bytes: NoStorageRemoval,
},
new_node: false,
needs_value_verification: true,
}
}

/// Returns the total removed bytes between the key removed bytes and the
/// value removed bytes
pub fn combined_removed_bytes(self) -> StorageRemovedBytes {
Expand Down Expand Up @@ -138,3 +165,47 @@ impl AddAssign for KeyValueStorageCost {
self.needs_value_verification &= rhs.needs_value_verification;
}
}

#[cfg(test)]
mod in_place_rewrite_tests {
use super::*;

#[test]
fn same_size_is_fully_replaced() {
let c = KeyValueStorageCost::for_in_place_value_rewrite(312, 312);
assert_eq!(c.value_storage_cost.replaced_bytes, 314);
assert_eq!(c.value_storage_cost.added_bytes, 0);
assert_eq!(c.value_storage_cost.removed_bytes, NoStorageRemoval);
assert_eq!(c.key_storage_cost, StorageCost::default());
assert!(!c.new_node);
assert!(c.needs_value_verification);
// Verifies against the paid size of the new value.
assert!(c.value_storage_cost.verify(314).is_ok());
}

#[test]
fn growth_is_added_on_top_of_the_previous_size() {
// 74 -> 106 bytes (a frontier gaining one ommer).
let c = KeyValueStorageCost::for_in_place_value_rewrite(74, 106);
assert_eq!(c.value_storage_cost.replaced_bytes, 75);
assert_eq!(c.value_storage_cost.added_bytes, 32);
assert!(c.value_storage_cost.verify(107).is_ok());
}

#[test]
fn shrink_is_replaced_at_the_new_size_and_not_credited() {
let c = KeyValueStorageCost::for_in_place_value_rewrite(1066, 74);
assert_eq!(c.value_storage_cost.replaced_bytes, 75);
assert_eq!(c.value_storage_cost.added_bytes, 0);
assert_eq!(c.value_storage_cost.removed_bytes, NoStorageRemoval);
assert!(c.value_storage_cost.verify(75).is_ok());
}

#[test]
fn varint_width_change_counts_as_growth() {
// 127 -> 128 bytes crosses a varint boundary: paid 128 -> 130.
let c = KeyValueStorageCost::for_in_place_value_rewrite(127, 128);
assert_eq!(c.value_storage_cost.replaced_bytes, 128);
assert_eq!(c.value_storage_cost.added_bytes, 2);
}
}
35 changes: 35 additions & 0 deletions docs/crates/costs.md
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,41 @@ Count trees add overhead for maintaining element counts:
- Additional 8 bytes for count storage
- Propagation costs through parent trees

#### Append-only trees (BulkAppendTree, CommitmentTree, PrivateDocumentStore)

The append-only family writes three kinds of data rows: dense-buffer slots
(one per position, keys reused every epoch), the chunk blob an epoch is
compacted into (plus the MMR internal nodes), and — for the commitment
tree — the frontier, one value rewritten on every append. How those writes
are reported to the fee layer is version-gated
(`bulk_append_tree_versions.cost.append_storage_accounting`,
`commitment_tree_versions.cost.frontier_save_storage_accounting`):

| write | GROVE_V1..V3 (v0) | GROVE_V4 (v1, issue #822) |
|---|---|---|
| buffer slot, first epoch | key + value `added` | key + value `added` |
| buffer slot, epoch ≥ 2 | key + value `added` | value `replaced` (growth `added`, shrink not credited); key not charged |
| entry's chunk-blob share | — (blob charged at compaction) | entry bytes `added` at the entry's own append |
| compaction blob | key + whole blob `added` | entry bytes `replaced`, framing + key `added` |
| MMR internal nodes | `added` | `added` |
| frontier rewrite | key + value `added` every save | value `replaced`, growth `added`; first save fully `added` |

Under v0 every note is billed roughly twice over its life (slot + blob) and
the whole blob (≈ 630 KB at `chunk_power` 11) lands on one append per
epoch. Under v1 each entry's permanent bytes are charged once, at its own
append, and the rest of the churn is replacement; the sum of `added_bytes`
over an epoch matches the database growth. `removed_bytes` stays
`NoStorageRemoval` in both — a rolling buffer refunds nobody. Stored bytes,
roots and proofs are identical under both versions.

Mechanically: the dense tree's `SlotWriteAccounting::AgainstCommitted`
reads the slot's committed value before the write (the read is billed) and
attaches `KeyValueStorageCost::for_in_place_value_rewrite`; the MMR
`MmrStore` takes a `LeafValueStorageCost::PartlyPrepaid` policy that the
bulk tree feeds `chunk_blob_entry_bytes`; `BulkAppendTree` appends report
`prepaid_chunk_bytes` for the caller to bill (already included in the cost
of the `CostResult`-returning `append_deferred_roots`).

## Cost Context

The `CostContext` trait provides functional combinators for cost-aware operations:
Expand Down
88 changes: 88 additions & 0 deletions grovedb-bulk-append-tree/src/chunk.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,50 @@ pub fn serialize_chunk_blob(entries: &[Vec<u8>]) -> Result<Vec<u8>, BulkAppendEr
}
}

/// The number of entry bytes a chunk blob carries — the sum of the lengths of
/// its entries, excluding the format byte, the headers and the per-entry
/// length prefixes.
///
/// This is the part of a compaction blob the bulk-append tree has already
/// charged as added storage, one entry at a time, by the time the blob is
/// written; the storage adapter reports it as replaced. A blob that does not
/// parse yields `0`, which charges the whole write as new storage — the safe
/// direction — rather than failing a flush over a cost figure.
pub fn chunk_blob_entry_bytes(blob: &[u8]) -> u32 {
let Some((&format, data)) = blob.split_first() else {
return 0;
};
match format {
FORMAT_FIXED => {
// [count: u32 BE] [entry_size: u32 BE] [entries...]
if data.len() < 8 {
return 0;
}
let payload = data.len() - 8;
u32::try_from(payload).unwrap_or(u32::MAX)
Comment thread
QuantumExplorer marked this conversation as resolved.
Resolved
}
FORMAT_VARIABLE => {
// [len: u32 BE] [entry] repeated.
let mut entry_bytes: u64 = 0;
let mut offset = 0;
while offset < data.len() {
let Some(len_bytes) = data.get(offset..offset + 4) else {
return 0;
};
let len = u32::from_be_bytes(len_bytes.try_into().expect("4 bytes")) as usize;
offset += 4;
if offset + len > data.len() {
return 0;
}
entry_bytes += len as u64;
offset += len;
}
u32::try_from(entry_bytes).unwrap_or(u32::MAX)
}
_ => 0,
}
}

/// Deserialize a chunk blob into individual entries.
///
/// Handles both fixed-size and variable-size formats based on the leading
Expand Down Expand Up @@ -336,3 +380,47 @@ mod tests {
assert!(matches!(err, BulkAppendError::CorruptedData(_)));
}
}

#[cfg(test)]
mod entry_bytes_tests {
use super::*;

#[test]
fn fixed_format_entry_bytes_is_count_times_size() {
let blob = serialize_chunk_blob(&[vec![1u8; 8], vec![2u8; 8], vec![3u8; 8]]).unwrap();
assert_eq!(blob[0], FORMAT_FIXED);
assert_eq!(chunk_blob_entry_bytes(&blob), 24);
assert_eq!(blob.len(), 9 + 24);
}

#[test]
fn variable_format_entry_bytes_excludes_length_prefixes() {
let blob = serialize_chunk_blob(&[vec![1u8; 8], vec![2u8; 16], vec![3u8; 4]]).unwrap();
assert_eq!(blob[0], FORMAT_VARIABLE);
assert_eq!(chunk_blob_entry_bytes(&blob), 28);
assert_eq!(blob.len(), 1 + 3 * 4 + 28);
}

#[test]
fn malformed_blobs_prepay_nothing() {
assert_eq!(chunk_blob_entry_bytes(&[]), 0);
assert_eq!(
chunk_blob_entry_bytes(&[0x7f, 1, 2, 3]),
0,
"unknown format"
);
assert_eq!(
chunk_blob_entry_bytes(&[FORMAT_FIXED, 0, 0, 0]),
0,
"short header"
);
let mut truncated = serialize_chunk_blob(&[vec![1u8; 8], vec![2u8; 16]]).unwrap();
truncated.truncate(truncated.len() - 3);
assert_eq!(chunk_blob_entry_bytes(&truncated), 0, "truncated entry");
assert_eq!(
chunk_blob_entry_bytes(&[FORMAT_VARIABLE, 0, 0]),
0,
"truncated prefix"
);
}
}
76 changes: 72 additions & 4 deletions grovedb-bulk-append-tree/src/cost/mod.rs
Original file line number Diff line number Diff line change
@@ -1,18 +1,86 @@
//! Versioned cost accounting for the bulk-append tree.
//!
//! Only the reported hash count is versioned; chunk bytes, roots and stored
//! state are identical under every version. This gate reaches a live fee —
//! Only cost reporting is versioned; chunk bytes, roots and stored state are
//! identical under every version. These gates reach live fees —
//! `CommitmentTree` adds the compaction hash count straight into its own
//! `hash_node_calls` — so the corrected figure arrives as a new version rather
//! than replacing the old one.
//! `hash_node_calls`, and every data put's cost information becomes storage
//! fees at commit — so corrected figures arrive as new versions rather than
//! replacing the old ones.
//!
//! Two gates live here:
//!
//! - `compaction_hash_count` — the hashes a compacting append reports.
//! - `append_storage_accounting` — how an append's data-storage writes are
//! reported to the storage cost layer (issue #822). v0 issues every put
//! with no cost information, so each is charged as new storage: a buffer
//! slot rewritten in epoch 2+, and the chunk blob that supersedes the
//! buffer, are both billed as permanent growth. v1 charges each entry's
//! permanent bytes once — its chunk-blob share, at its own append — and
//! reports the slot rewrite and the compaction blob as replacements of
//! the bytes they supersede.

mod v0;
mod v1;

#[cfg(feature = "storage")]
use grovedb_dense_fixed_sized_merkle_tree::SlotWriteAccounting;
#[cfg(feature = "storage")]
use grovedb_merkle_mountain_range::LeafValueStorageCost;
use grovedb_version::{error::GroveVersionError, version::GroveVersion};

use crate::BulkAppendError;

/// How an append's data-storage writes are reported to the storage cost
/// layer. Selected per grove version by [`append_storage_accounting`].
#[cfg(feature = "storage")]
#[derive(Clone, Copy)]
pub(crate) struct AppendStorageAccounting {
/// How the dense-buffer slot write is reported.
pub slot_write: SlotWriteAccounting,
/// How the chunk-blob leaf is reported when the MMR overlay is flushed.
pub chunk_leaf: LeafValueStorageCost,
/// Whether the entry's chunk-blob share is charged as added storage at
/// its own append (so the later blob write can be a replacement).
prepay_chunk_share: bool,
}

#[cfg(feature = "storage")]
impl AppendStorageAccounting {
/// The entry's chunk-blob share to charge as added storage at its append:
/// its own bytes, or nothing when the blob is charged in full at
/// compaction instead.
pub fn prepaid_chunk_bytes(&self, value_len: usize) -> u32 {
if self.prepay_chunk_share {
u32::try_from(value_len).unwrap_or(u32::MAX)
} else {
0
}
}
}

/// The storage accounting an append uses under `grove_version`.
#[cfg(feature = "storage")]
pub(crate) fn append_storage_accounting(
grove_version: &GroveVersion,
) -> Result<AppendStorageAccounting, BulkAppendError> {
match grove_version
.bulk_append_tree_versions
.cost
.append_storage_accounting
{
0 => Ok(v0::append_storage_accounting()),
1 => Ok(v1::append_storage_accounting()),
version => Err(BulkAppendError::VersionError(
GroveVersionError::UnknownVersionMismatch {
method: "BulkAppendTree append storage accounting".to_string(),
known_versions: vec![0, 1],
received: version,
}
.to_string(),
)),
}
}

/// Hashes to report for a compacting append.
///
/// `leaf_count` is the MMR leaf count BEFORE the push (what
Expand Down
23 changes: 23 additions & 0 deletions grovedb-bulk-append-tree/src/cost/v0.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,31 @@
//! Locked: GROVE_V1..V3 are released and CommitmentTree has been billing this
//! figure on mainnet.

#[cfg(feature = "storage")]
use grovedb_dense_fixed_sized_merkle_tree::SlotWriteAccounting;
use grovedb_merkle_mountain_range::hash_count_for_push;
#[cfg(feature = "storage")]
use grovedb_merkle_mountain_range::LeafValueStorageCost;

#[cfg(feature = "storage")]
use super::AppendStorageAccounting;

pub(super) fn compaction_hash_count(leaf_count: u64) -> u32 {
hash_count_for_push(leaf_count)
}

/// Shipped storage accounting: every data put is issued with no cost
/// information, so the commit path bills key + value as new storage — the
/// slot rewrite, the chunk blob and the MMR nodes alike — and nothing is
/// charged at append time beyond the slot put itself.
///
/// Locked: GROVE_V1..V3 are released and the shielded pool has been billed
/// this way on mainnet.
#[cfg(feature = "storage")]
pub(super) fn append_storage_accounting() -> AppendStorageAccounting {
AppendStorageAccounting {
slot_write: SlotWriteAccounting::AsNew,
chunk_leaf: LeafValueStorageCost::New,
prepay_chunk_share: false,
}
}
29 changes: 29 additions & 0 deletions grovedb-bulk-append-tree/src/cost/v1.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,41 @@
//!
//! Adds the peak-bagging merges [`v0`](super::v0) omitted. Used from GROVE_V4.

#[cfg(feature = "storage")]
use grovedb_dense_fixed_sized_merkle_tree::SlotWriteAccounting;
#[cfg(feature = "storage")]
use grovedb_merkle_mountain_range::LeafValueStorageCost;
use grovedb_merkle_mountain_range::{hash_count_for_push, hash_count_for_root_bagging};

#[cfg(feature = "storage")]
use super::AppendStorageAccounting;
#[cfg(feature = "storage")]
use crate::chunk::chunk_blob_entry_bytes;

pub(super) fn compaction_hash_count(leaf_count: u64, mmr_size_after_push: u64) -> u32 {
// Derived from the MMR shape rather than read back out of the accumulated
// `OperationCost`, so this stays correct regardless of whether
// `MMR::get_root`'s own charge is enabled for the caller's version — the
// two gates are independent.
hash_count_for_push(leaf_count).saturating_add(hash_count_for_root_bagging(mmr_size_after_push))
}

/// Churn-as-replacement storage accounting (issue #822). Used from GROVE_V4.
///
/// - The entry's chunk-blob share (its own bytes) is charged as added
/// storage at its append — these are the bytes that persist.
/// - The buffer slot write is sized against the value the slot already holds
/// in committed storage: a rewrite (epoch 2 onward) is replaced, growth is
/// added, shrink is not credited; a slot written for the first time stays
/// fully added.
/// - The compaction blob is reported as a replacement of the entry bytes it
/// supersedes (all prepaid), leaving only its framing — and the MMR
/// internal nodes — as added storage.
#[cfg(feature = "storage")]
pub(super) fn append_storage_accounting() -> AppendStorageAccounting {
AppendStorageAccounting {
slot_write: SlotWriteAccounting::AgainstCommitted,
chunk_leaf: LeafValueStorageCost::PartlyPrepaid(chunk_blob_entry_bytes),
prepay_chunk_share: true,
}
}
Loading
Loading